diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..778edabc --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +line_length=79 +multi_line_output=3 +include_trailing_comma=True +indent=' ' +sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..4f8c6e48 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +version: 2 + +build: + image: latest + +python: + version: 3.7 + install: + - requirements: requirements.txt + - method: pip + path: . + system_packages: true diff --git a/.travis.yml b/.travis.yml index 48894e4f..058dc4e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,10 @@ script: - make cov-ci - python setup.py check -rms +matrix: + allow_failures: + - python: "nightly" + env: global: - PYTHON=python @@ -58,4 +62,4 @@ deploy: on: tags: true all_branches: true - python: 3.5 + python: 3.5 \ No newline at end of file diff --git a/ACKS.txt b/ACKS.txt deleted file mode 100644 index e8638412..00000000 --- a/ACKS.txt +++ /dev/null @@ -1,18 +0,0 @@ -Acknowledgements ----------------- - -A number of people have contributed to *aiopg* by reporting problems, -suggesting improvements or submitting changes. Some of these people are: - -Alexander -Eugene Krevenets -Fantix King <> -Lena Kryvonos -Low Kian Seong -Marco Paolini -Michal Kuffa -Nikolay Novik -Petr Viktorin -R. Davis Murrey -Ryan Hodge -Theron Luhn diff --git a/CHANGES.txt b/CHANGES.txt index e86019d4..2eca2cca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,84 +1,81 @@ -CHANGES -------- - 0.16.0 (2019-01-25) ^^^^^^^^^^^^^^^^^^^ -* Fix select priority name (#525) +* Fix select priority name `#525 `_ -* Rename `psycopg2` to `psycopg2-binary` to fix deprecation warning (#507) +* Rename `psycopg2` to `psycopg2-binary` to fix deprecation warning `#507 `_ -* Fix #189 hstore when using ReadDictCursor (#512) +* Fix `#189 `_ hstore when using ReadDictCursor `#512 `_ -* close cannot be used while an asynchronous query is underway (#452) +* close cannot be used while an asynchronous query is underway `#452 `_ -* sqlalchemy adapter trx begin allow transaction_mode (#498) +* sqlalchemy adapter trx begin allow transaction_mode `#498 `_ 0.15.0 (2018-08-14) ^^^^^^^^^^^^^^^^^^^ -* Support Python 3.7 (#437) +* Support Python 3.7 `#437 `_ 0.14.0 (2018-05-10) ^^^^^^^^^^^^^^^^^^^ -* Add ``get_dialect`` func to have ability to pass ``json_serializer`` #451 +* Add ``get_dialect`` func to have ability to pass ``json_serializer`` `#451 `_ 0.13.2 (2018-01-03) ^^^^^^^^^^^^^^^^^^^ -* Fixed compatibility with SQLAlchemy 1.2.0 #412 +* Fixed compatibility with SQLAlchemy 1.2.0 `#412 `_ -* Added support for transaction isolation levels #219 +* Added support for transaction isolation levels `#219 `_ 0.13.1 (2017-09-10) ^^^^^^^^^^^^^^^^^^^ -* Added connection poll recycling logic #373 +* Added connection poll recycling logic `#373 `_ 0.13.0 (2016-12-02) ^^^^^^^^^^^^^^^^^^^ -* Add `async with` support to `.begin_nested()` #208 +* Add `async with` support to `.begin_nested()` `#208 `_ -* Fix connection.cancel() #212 #223 +* Fix connection.cancel() `#212 `_ `#223 `_ -* Raise informative error on unexpected connection closing #191 +* Raise informative error on unexpected connection closing `#191 `_ -* Added support for python types columns issues #217 +* Added support for python types columns issues `#217 `_ -* Added support for default values in SA table issues #206 +* Added support for default values in SA table issues `#206 `_ 0.12.0 (2016-10-09) ^^^^^^^^^^^^^^^^^^^ -* Add an on_connect callback parameter to pool #141 +* Add an on_connect callback parameter to pool `#141 `_ -* Fixed connection to work under both windows and posix based systems #142 +* Fixed connection to work under both windows and posix based systems `#142 `_ 0.11.0 (2016-09-12) ^^^^^^^^^^^^^^^^^^^ -* Immediately remove callbacks from a closed file descriptor #139 +* Immediately remove callbacks from a closed file descriptor `#139 `_ * Drop Python 3.3 support 0.10.0 (2016-07-16) ^^^^^^^^^^^^^^^^^^^ -* Refactor tests to use dockerized Postgres server #107 +* Refactor tests to use dockerized Postgres server `#107 `_ -* Reduce default pool minsize to 1 #106 +* Reduce default pool minsize to 1 `#106 `_ -* Explicitly enumerate packages in setup.py #85 +* Explicitly enumerate packages in setup.py `#85 `_ -* Remove expired connections from pool on acquire #116 +* Remove expired connections from pool on acquire `#116 `_ -* Don't crash when Connection is GC'ed #124 +* Don't crash when Connection is GC'ed `#124 `_ * Use loop.create_future() if available @@ -86,34 +83,34 @@ CHANGES ^^^^^^^^^^^^^^^^^^ * Make pool.release return asyncio.Future, so we can wait on it in - `__aexit__` #102 + `__aexit__` `#102 `_ -* Add support for uuid type #103 +* Add support for uuid type `#103 `_ 0.9.1 (2016-01-17) ^^^^^^^^^^^^^^^^^^ -* Documentation update #101 +* Documentation update `#101 `_ 0.9.0 (2016-01-14) ^^^^^^^^^^^^^^^^^^ -* Add async context managers for transactions #91 +* Add async context managers for transactions `#91 `_ -* Support async iterator in ResultProxy #92 +* Support async iterator in ResultProxy `#92 `_ -* Add async with for engine #90 +* Add async with for engine `#90 `_ 0.8.0 (2015-12-31) ^^^^^^^^^^^^^^^^^^ -* Add PostgreSQL notification support #58 +* Add PostgreSQL notification support `#58 `_ -* Support pools with unlimited size #59 +* Support pools with unlimited size `#59 `_ -* Cancel current DB operation on asyncio timeout #66 +* Cancel current DB operation on asyncio timeout `#66 `_ -* Add async with support for Pool, Connection, Cursor #88 +* Add async with support for Pool, Connection, Cursor `#88 `_ 0.7.0 (2015-04-22) ^^^^^^^^^^^^^^^^^^ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9b3fa333..b1e4edca 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -4,25 +4,33 @@ Instruction for contributors Developer environment --------------------- -First clone the git repo:: +First clone the git repo: - $ git clone git@github.com:aio-libs/aiopg.git - $ cd aiopg +.. code-block:: shell + + $ git clone git@github.com:aio-libs/aiopg.git + $ cd aiopg After that you need to create and activate a virtual environment. I recommend using :term:`virtualenvwrapper` but just :term:`virtualenv` or -:term:`venv` will also work. For ``virtualenvwrapper``:: +:term:`venv` will also work. For :term:`virtualenvwrapper`: + +.. code-block:: shell + + $ mkvirtualenv aiopg -p `which python3` - $ mkvirtualenv aiopg -p `which python3` +For `venv` (for example; put the directory wherever you want): -For ``venv`` (for example; put the directory wherever you want):: +.. code-block:: shell - $ python3 -m venv ../venv_directory - $ source ../venv_directory/bin/activate + $ python3 -m venv ../venv_directory + $ source ../venv_directory/bin/activate -Just as when doing a normal install, you need the :term:`libpq` library:: +Just as when doing a normal install, you need the :term:`libpq` library: - $ sudo apt-get install libpq-dev +.. code-block:: shell + + $ sudo apt-get install libpq-dev **UPD** @@ -35,37 +43,47 @@ No local Postgres server needed. In the virtual environment you need to install *aiopg* itself and some additional development tools (the development tools are needed for running -the test suite and other development tasks):: +the test suite and other development tasks) + +.. code-block:: shell - $ pip install -Ue . - $ pip install -Ur requirements.txt + $ pip install -Ue . + $ pip install -Ur requirements.txt That's all. -To run all of the *aiopg* tests do:: +To run all of the *aiopg* tests do: - $ make test +.. code-block:: shell + + $ make test This command runs :term:`pep8` and :term:`pyflakes` first and then executes the *aiopg* unit tests. When you are working on solving an issue you will probably want to run -some specific test, not the whole suite:: +some specific test, not the whole suite: + +.. code-block:: shell - $ py.test -s -k test_initial_empty + $ py.test -s -k test_initial_empty For debug sessions I prefer to use :term:`ipdb`, which is installed -as part of the development tools. Insert the following line into your +as part of the development tools. Insert the following line into your code in the place where you want to start interactively debugging the -execution process:: +execution process: - import ipdb; ipdb.set_trace() +.. code-block:: py3 + + import ipdb; ipdb.set_trace() The library is reasonably well covered by tests. There is a make -target for generating the coverage report:: +target for generating the coverage report: + +.. code-block:: shell - $ make cov + $ make cov Contribution diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 00000000..da44b9c8 --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1,12 @@ +* Alexander +* Eugene Krevenets +* Fantix King +* Lena Kryvonos +* Low Kian Seong +* Marco Paolini +* Michal Kuffa +* Nikolay Novik +* Petr Viktorin +* R\. David Murray +* Ryan Hodge +* Theron Luhn diff --git a/MAINTAINERS.txt b/MAINTAINERS.txt index a5e49f96..e5f5b28f 100644 --- a/MAINTAINERS.txt +++ b/MAINTAINERS.txt @@ -1,8 +1,3 @@ -Maintainers ------------ - -The list of *aiopg* maintainers. - -Andrew Svetlov -Alexey Popravka -Alexey Firsov +* Andrew Svetlov +* Alexey Firsov +* Alexey Popravka diff --git a/MANIFEST.in b/MANIFEST.in index e255ba40..1836f6a4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include LICENSE.txt include CHANGES.txt include README.rst +include MAINTAINERS.txt graft aiopg global-exclude *.pyc exclude tests/** diff --git a/Makefile b/Makefile index 61c1971f..c885e551 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,26 @@ # Some simple testing tasks (sorry, UNIX only). doc: - cd docs && make html + cd docs && rm -rf _build/html && make html @echo "open file://`pwd`/docs/_build/html/index.html" -pep: - pep8 aiopg examples tests +isort: + isort -rc aiopg + isort -rc tests + isort -rc examples -flake: - extra=$$(python -c "import sys;sys.stdout.write('--exclude tests/pep492 --builtins=StopAsyncIteration') if sys.version_info[:3] < (3, 5, 0) else sys.stdout.write('examples')"); \ - flake8 aiopg tests $$extra +flake: .flake + +.flake: $(shell find aiopg -type f) \ + $(shell find tests -type f) \ + $(shell find examples -type f) + flake8 aiopg tests examples + python setup.py check -rms + @if ! isort -c -rc aiopg tests examples; then \ + echo "Import sort errors, run 'make isort' to fix them!!!"; \ + isort --diff -rc aiopg tests examples; \ + false; \ + fi test: flake pytest -q tests @@ -24,6 +35,9 @@ cov cover coverage: flake cov-ci: flake py.test -svvv -rs --cov --cov-report=term tests --pg_tag all +clean-pip: + pip freeze | grep -v "^-e" | xargs pip uninstall -y + clean: find . -name __pycache__ |xargs rm -rf find . -type f -name '*.py[co]' -delete @@ -38,4 +52,4 @@ clean: rm -rf docs/_build rm -rf .tox -.PHONY: all pep test vtest cov clean +.PHONY: all isort flake test vtest cov clean clean-pip diff --git a/README.rst b/README.rst index adb95d7c..50cafa56 100644 --- a/README.rst +++ b/README.rst @@ -73,9 +73,6 @@ Example of SQLAlchemy optional integration loop = asyncio.get_event_loop() loop.run_until_complete(go()) -For ``yield from`` based code, see the ``./examples`` folder, files with -``old_style`` part in their names. - .. _PostgreSQL: http://www.postgresql.org/ .. _asyncio: http://docs.python.org/3.4/library/asyncio.html @@ -83,5 +80,6 @@ Please use:: $ make test -for executing the project's unittests. See CONTRIBUTING.rst for details +for executing the project's unittests. +See https://aiopg.readthedocs.io/en/stable/contributing.html for details on how to set up your environment to run the tests. diff --git a/aiopg/__init__.py b/aiopg/__init__.py index 248fef2e..d9c62da7 100644 --- a/aiopg/__init__.py +++ b/aiopg/__init__.py @@ -7,6 +7,7 @@ from .cursor import Cursor from .pool import create_pool, Pool from .transaction import IsolationLevel, Transaction +from .utils import get_running_loop warnings.filterwarnings( 'always', '.*', @@ -15,9 +16,9 @@ append=False ) -__all__ = ('connect', 'create_pool', 'Connection', 'Cursor', 'Pool', - 'version', 'version_info', 'DEFAULT_TIMEOUT', 'IsolationLevel', - 'Transaction') +__all__ = ('connect', 'create_pool', 'get_running_loop', + 'Connection', 'Cursor', 'Pool', 'version', 'version_info', + 'DEFAULT_TIMEOUT', 'IsolationLevel', 'Transaction') __version__ = '0.16.0' @@ -28,26 +29,30 @@ def _parse_version(ver): - RE = (r'^(?P\d+)\.(?P\d+)\.' - '(?P\d+)((?P[a-z]+)(?P\d+)?)?$') + RE = ( + r'^' + r'(?P\d+)\.(?P\d+)\.(?P\d+)' + r'((?P[a-z]+)(?P\d+)?)?' + r'$' + ) match = re.match(RE, ver) try: major = int(match.group('major')) minor = int(match.group('minor')) micro = int(match.group('micro')) - levels = {'c': 'candidate', + levels = {'rc': 'candidate', 'a': 'alpha', 'b': 'beta', None: 'final'} releaselevel = levels[match.group('releaselevel')] serial = int(match.group('serial')) if match.group('serial') else 0 return VersionInfo(major, minor, micro, releaselevel, serial) - except Exception: - raise ImportError("Invalid package version {}".format(ver)) + except Exception as e: + raise ImportError("Invalid package version {}".format(ver)) from e version_info = _parse_version(__version__) # make pyflakes happy (connect, create_pool, Connection, Cursor, Pool, DEFAULT_TIMEOUT, - IsolationLevel, Transaction) + IsolationLevel, Transaction, get_running_loop) diff --git a/aiopg/connection.py b/aiopg/connection.py index f10b76a9..4c5feef5 100755 --- a/aiopg/connection.py +++ b/aiopg/connection.py @@ -7,18 +7,14 @@ import traceback import warnings import weakref +from collections.abc import Mapping import psycopg2 from psycopg2 import extras -from psycopg2.extensions import ( - POLL_OK, - POLL_READ, - POLL_WRITE, - POLL_ERROR, -) +from psycopg2.extensions import POLL_ERROR, POLL_OK, POLL_READ, POLL_WRITE from .cursor import Cursor -from .utils import _ContextManager, create_future +from .utils import _ContextManager, create_future, get_running_loop __all__ = ('connect',) @@ -29,65 +25,25 @@ WSAENOTSOCK = 10038 -async def _enable_hstore(conn): - cur = await conn.cursor() - await cur.execute("""\ - SELECT t.oid, typarray - FROM pg_type t JOIN pg_namespace ns - ON typnamespace = ns.oid - WHERE typname = 'hstore'; - """) - rv0, rv1 = [], [] - async for oids in cur: - if isinstance(oids, dict): - rv0.append(oids['oid']) - rv1.append(oids['typarray']) - else: - rv0.append(oids[0]) - rv1.append(oids[1]) - - cur.close() - return tuple(rv0), tuple(rv1) - - -def connect(dsn=None, *, timeout=TIMEOUT, loop=None, enable_json=True, +def connect(dsn=None, *, timeout=TIMEOUT, enable_json=True, enable_hstore=True, enable_uuid=True, echo=False, **kwargs): """A factory for connecting to PostgreSQL. The coroutine accepts all parameters that psycopg2.connect() does - plus optional keyword-only `loop` and `timeout` parameters. + plus optional keyword-only `timeout` parameters. Returns instantiated Connection object. """ - coro = _connect(dsn=dsn, timeout=timeout, loop=loop, - enable_json=enable_json, enable_hstore=enable_hstore, - enable_uuid=enable_uuid, echo=echo, **kwargs) - return _ContextManager(coro) + coro = Connection( + dsn, timeout, bool(echo), + enable_hstore=enable_hstore, + enable_uuid=enable_uuid, + enable_json=enable_json, + **kwargs + ) - -async def _connect(dsn=None, *, timeout=TIMEOUT, loop=None, enable_json=True, - enable_hstore=True, enable_uuid=True, echo=False, **kwargs): - if loop is None: - loop = asyncio.get_event_loop() - - waiter = create_future(loop) - conn = Connection(dsn, loop, timeout, waiter, bool(echo), **kwargs) - try: - await conn._poll(waiter, timeout) - except Exception: - conn.close() - raise - if enable_json: - extras.register_default_json(conn._conn) - if enable_uuid: - extras.register_uuid(conn_or_curs=conn._conn) - if enable_hstore: - oids = await _enable_hstore(conn) - if oids is not None: - oid, array_oid = oids - extras.register_hstore(conn._conn, oid=oid, array_oid=array_oid) - return conn + return _ContextManager(coro) def _is_bad_descriptor_error(os_error): @@ -107,25 +63,35 @@ class Connection: _source_traceback = None - def __init__(self, dsn, loop, timeout, waiter, echo, **kwargs): - self._loop = loop - self._conn = psycopg2.connect(dsn, async_=True, **kwargs) + def __init__( + self, dsn, timeout, echo, + *, enable_json=True, enable_hstore=True, + enable_uuid=True, **kwargs + ): + self._enable_json = enable_json + self._enable_hstore = enable_hstore + self._enable_uuid = enable_uuid + self._loop = get_running_loop(kwargs.pop('loop', None) is not None) + self._waiter = create_future(self._loop) + + kwargs['async_'] = kwargs.pop('async', True) + self._conn = psycopg2.connect(dsn, **kwargs) + self._dsn = self._conn.dsn assert self._conn.isexecuting(), "Is conn an async at all???" self._fileno = self._conn.fileno() self._timeout = timeout self._last_usage = self._loop.time() - self._waiter = waiter self._writing = False self._cancelling = False self._cancellation_waiter = None self._echo = echo self._cursor_instance = None - self._notifies = asyncio.Queue(loop=loop) + self._notifies = asyncio.Queue(loop=self._loop) self._weakref = weakref.ref(self) self._loop.add_reader(self._fileno, self._ready, self._weakref) - if loop.get_debug(): + if self._loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) @staticmethod @@ -276,10 +242,10 @@ def cursor(self, name=None, cursor_factory=None, connection will be closed """ self._last_usage = self._loop.time() - core = self._cursor(name=name, cursor_factory=cursor_factory, + coro = self._cursor(name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, timeout=timeout) - return _ContextManager(core) + return _ContextManager(coro) async def _cursor(self, name=None, cursor_factory=None, scrollable=None, withhold=False, timeout=None): @@ -333,11 +299,12 @@ def closed_cursor(self): if not self._cursor_instance: return True - return self._cursor_instance.closed + return bool(self._cursor_instance.closed) def free_cursor(self): if not self.closed_cursor: self._cursor_instance.close() + self._cursor_instance = None def close(self): self._close() @@ -557,6 +524,53 @@ def notifies(self): """Return notification queue.""" return self._notifies + async def _get_oids(self): + cur = await self.cursor() + rv0, rv1 = [], [] + try: + await cur.execute( + "SELECT t.oid, typarray " + "FROM pg_type t JOIN pg_namespace ns ON typnamespace = ns.oid " + "WHERE typname = 'hstore';" + ) + + async for oids in cur: + if isinstance(oids, Mapping): + rv0.append(oids['oid']) + rv1.append(oids['typarray']) + else: + rv0.append(oids[0]) + rv1.append(oids[1]) + finally: + cur.close() + + return tuple(rv0), tuple(rv1) + + async def _connect(self): + try: + await self._poll(self._waiter, self._timeout) + except Exception: + self.close() + raise + if self._enable_json: + extras.register_default_json(self._conn) + if self._enable_uuid: + extras.register_uuid(conn_or_curs=self._conn) + if self._enable_hstore: + oids = await self._get_oids() + if oids is not None: + oid, array_oid = oids + extras.register_hstore( + self._conn, + oid=oid, + array_oid=array_oid + ) + + return self + + def __await__(self): + return self._connect().__await__() + async def __aenter__(self): return self diff --git a/aiopg/cursor.py b/aiopg/cursor.py index 72446608..c8a60a37 100644 --- a/aiopg/cursor.py +++ b/aiopg/cursor.py @@ -1,10 +1,9 @@ import asyncio -import warnings import psycopg2 from .log import logger -from .transaction import Transaction, IsolationLevel +from .transaction import IsolationLevel, Transaction from .utils import _TransactionBeginContextManager @@ -363,17 +362,6 @@ def timeout(self): """Return default timeout for cursor operations.""" return self._timeout - def __iter__(self): - warnings.warn("Iteration over cursor is deprecated", - DeprecationWarning, - stacklevel=2) - while True: - row = yield from self.fetchone().__await__() - if row is None: - return - else: - yield row - def __aiter__(self): return self diff --git a/aiopg/log.py b/aiopg/log.py index 23a7074a..c3c25e76 100644 --- a/aiopg/log.py +++ b/aiopg/log.py @@ -2,6 +2,5 @@ import logging - # Name the logger after the package. logger = logging.getLogger(__package__) diff --git a/aiopg/pool.py b/aiopg/pool.py index fc91b1c6..a5e1d6fe 100644 --- a/aiopg/pool.py +++ b/aiopg/pool.py @@ -1,58 +1,40 @@ import asyncio import collections -import sys import warnings from psycopg2.extensions import TRANSACTION_STATUS_IDLE -from .connection import connect, TIMEOUT +from .connection import TIMEOUT, connect from .utils import ( - _PoolContextManager, + _PoolAcquireContextManager, _PoolConnectionContextManager, + _PoolContextManager, _PoolCursorContextManager, - _PoolAcquireContextManager, - ensure_future, create_future, + ensure_future, + get_running_loop, ) -PY_341 = sys.version_info >= (3, 4, 1) - def create_pool(dsn=None, *, minsize=1, maxsize=10, - loop=None, timeout=TIMEOUT, pool_recycle=-1, + timeout=TIMEOUT, pool_recycle=-1, enable_json=True, enable_hstore=True, enable_uuid=True, echo=False, on_connect=None, **kwargs): - coro = _create_pool(dsn=dsn, minsize=minsize, maxsize=maxsize, loop=loop, - timeout=timeout, pool_recycle=pool_recycle, - enable_json=enable_json, enable_hstore=enable_hstore, - enable_uuid=enable_uuid, echo=echo, - on_connect=on_connect, **kwargs) - return _PoolContextManager(coro) - + coro = Pool.from_pool_fill( + dsn, minsize, maxsize, timeout, + enable_json=enable_json, enable_hstore=enable_hstore, + enable_uuid=enable_uuid, echo=echo, on_connect=on_connect, + pool_recycle=pool_recycle, **kwargs + ) -async def _create_pool(dsn=None, *, minsize=1, maxsize=10, - loop=None, timeout=TIMEOUT, pool_recycle=-1, - enable_json=True, enable_hstore=True, enable_uuid=True, - echo=False, on_connect=None, - **kwargs): - if loop is None: - loop = asyncio.get_event_loop() - - pool = Pool(dsn, minsize, maxsize, loop, timeout, - enable_json=enable_json, enable_hstore=enable_hstore, - enable_uuid=enable_uuid, echo=echo, on_connect=on_connect, - pool_recycle=pool_recycle, **kwargs) - if minsize > 0: - async with pool._cond: - await pool._fill_free_pool(False) - return pool + return _PoolContextManager(coro) class Pool(asyncio.AbstractServer): """Connection pool""" - def __init__(self, dsn, minsize, maxsize, loop, timeout, *, + def __init__(self, dsn, minsize, maxsize, timeout, *, enable_json, enable_hstore, enable_uuid, echo, on_connect, pool_recycle, **kwargs): if minsize < 0: @@ -61,7 +43,7 @@ def __init__(self, dsn, minsize, maxsize, loop, timeout, *, raise ValueError("maxsize should be not less than minsize") self._dsn = dsn self._minsize = minsize - self._loop = loop + self._loop = get_running_loop(kwargs.pop('loop', None) is not None) self._timeout = timeout self._recycle = pool_recycle self._enable_json = enable_json @@ -72,7 +54,7 @@ def __init__(self, dsn, minsize, maxsize, loop, timeout, *, self._conn_kwargs = kwargs self._acquiring = 0 self._free = collections.deque(maxlen=maxsize or None) - self._cond = asyncio.Condition(loop=loop) + self._cond = asyncio.Condition(loop=self._loop) self._used = set() self._terminated = set() self._closing = False @@ -162,6 +144,18 @@ def acquire(self): coro = self._acquire() return _PoolAcquireContextManager(coro, self) + @classmethod + async def from_pool_fill(cls, *args, **kwargs): + """constructor for filling the free pool with connections, + the number is controlled by the minsize parameter + """ + self = cls(*args, **kwargs) + if self._minsize > 0: + async with self._cond: + await self._fill_free_pool(False) + + return self + async def _acquire(self): if self._closing: raise RuntimeError("Cannot acquire connection after closing pool") @@ -197,7 +191,7 @@ async def _fill_free_pool(self, override_min): self._acquiring += 1 try: conn = await connect( - self._dsn, loop=self._loop, timeout=self._timeout, + self._dsn, timeout=self._timeout, enable_json=self._enable_json, enable_hstore=self._enable_hstore, enable_uuid=self._enable_uuid, @@ -215,7 +209,7 @@ async def _fill_free_pool(self, override_min): self._acquiring += 1 try: conn = await connect( - self._dsn, loop=self._loop, timeout=self._timeout, + self._dsn, timeout=self._timeout, enable_json=self._enable_json, enable_hstore=self._enable_hstore, enable_uuid=self._enable_uuid, @@ -262,22 +256,12 @@ def release(self, conn): async def cursor(self, name=None, cursor_factory=None, scrollable=None, withhold=False, *, timeout=None): - """XXX""" conn = await self.acquire() cur = await conn.cursor(name=name, cursor_factory=cursor_factory, scrollable=scrollable, withhold=withhold, timeout=timeout) return _PoolCursorContextManager(self, conn, cur) - def __enter__(self): - raise RuntimeError( - '"await" should be used as context manager expression') - - def __exit__(self, *args): - # This must exist because __enter__ exists, even though that - # always raises; that's how the with-statement works. - pass # pragma: nocover - def __await__(self): # This is not a coroutine. It is meant to enable the idiom: # @@ -294,6 +278,15 @@ def __await__(self): conn = yield from self._acquire().__await__() return _PoolConnectionContextManager(self, conn) + def __enter__(self): + raise RuntimeError( + '"await" should be used as context manager expression') + + def __exit__(self, *args): + # This must exist because __enter__ exists, even though that + # always raises; that's how the with-statement works. + pass # pragma: nocover + async def __aenter__(self): return self diff --git a/aiopg/sa/connection.py b/aiopg/sa/connection.py index b831aa52..604f84fe 100644 --- a/aiopg/sa/connection.py +++ b/aiopg/sa/connection.py @@ -3,10 +3,14 @@ from sqlalchemy.sql.dml import UpdateBase from . import exc -from .result import ResultProxy -from .transaction import (RootTransaction, Transaction, - NestedTransaction, TwoPhaseTransaction) from ..utils import _SAConnectionContextManager, _TransactionContextManager +from .result import ResultProxy +from .transaction import ( + NestedTransaction, + RootTransaction, + Transaction, + TwoPhaseTransaction, +) class SAConnection: @@ -129,11 +133,7 @@ async def scalar(self, query, *multiparams, **params): @property def closed(self): """The readonly property that returns True if connections is closed.""" - return self._connection is None or self._connection.closed - - @property - def info(self): - return self._connection.info + return self.connection is None or self.connection.closed @property def connection(self): @@ -196,14 +196,14 @@ async def _begin_impl(self, isolation_level, readonly, deferrable): if deferrable: stmt += ' DEFERRABLE' - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute(stmt) finally: cur.close() async def _commit_impl(self): - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute('COMMIT') finally: @@ -211,7 +211,7 @@ async def _commit_impl(self): self._transaction = None async def _rollback_impl(self): - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute('ROLLBACK') finally: @@ -245,7 +245,7 @@ async def _savepoint_impl(self, name=None): self._savepoint_seq += 1 name = 'aiopg_sa_savepoint_%s' % self._savepoint_seq - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute('SAVEPOINT ' + name) return name @@ -253,7 +253,7 @@ async def _savepoint_impl(self, name=None): cur.close() async def _rollback_to_savepoint_impl(self, name, parent): - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute('ROLLBACK TO SAVEPOINT ' + name) finally: @@ -261,7 +261,7 @@ async def _rollback_to_savepoint_impl(self, name, parent): self._transaction = parent async def _release_savepoint_impl(self, name, parent): - cur = await self._connection.cursor() + cur = await self._get_cursor() try: await cur.execute('RELEASE SAVEPOINT ' + name) finally: @@ -332,7 +332,7 @@ async def close(self): After .close() is called, the SAConnection is permanently in a closed state, and will allow no further operations. """ - if self._connection is None: + if self.connection is None: return if self._transaction is not None: diff --git a/aiopg/sa/engine.py b/aiopg/sa/engine.py index e4375a3c..7d75b560 100644 --- a/aiopg/sa/engine.py +++ b/aiopg/sa/engine.py @@ -1,12 +1,11 @@ -import asyncio import json import aiopg +from ..connection import TIMEOUT +from ..utils import _PoolAcquireContextManager, _PoolContextManager from .connection import SAConnection from .exc import InvalidRequestError -from ..connection import TIMEOUT -from ..utils import _PoolContextManager, _PoolAcquireContextManager try: from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 @@ -49,9 +48,8 @@ def get_dialect(json_serializer=json.dumps, json_deserializer=lambda x: x): _dialect = get_dialect() -def create_engine(dsn=None, *, minsize=1, maxsize=10, loop=None, - dialect=_dialect, timeout=TIMEOUT, pool_recycle=-1, - **kwargs): +def create_engine(dsn=None, *, minsize=1, maxsize=10, dialect=_dialect, + timeout=TIMEOUT, pool_recycle=-1, **kwargs): """A coroutine for Engine creation. Returns Engine instance with embedded connection pool. @@ -60,19 +58,18 @@ def create_engine(dsn=None, *, minsize=1, maxsize=10, loop=None, """ coro = _create_engine(dsn=dsn, minsize=minsize, maxsize=maxsize, - loop=loop, dialect=dialect, timeout=timeout, + dialect=dialect, timeout=timeout, pool_recycle=pool_recycle, **kwargs) return _EngineContextManager(coro) -async def _create_engine(dsn=None, *, minsize=1, maxsize=10, loop=None, - dialect=_dialect, timeout=TIMEOUT, pool_recycle=-1, - **kwargs): - if loop is None: - loop = asyncio.get_event_loop() - pool = await aiopg.create_pool(dsn, minsize=minsize, maxsize=maxsize, - loop=loop, timeout=timeout, - pool_recycle=pool_recycle, **kwargs) +async def _create_engine(dsn=None, *, minsize=1, maxsize=10, dialect=_dialect, + timeout=TIMEOUT, pool_recycle=-1, **kwargs): + + pool = await aiopg.create_pool( + dsn, minsize=minsize, maxsize=maxsize, + timeout=timeout, pool_recycle=pool_recycle, **kwargs + ) conn = await pool.acquire() try: real_dsn = conn.dsn diff --git a/aiopg/sa/result.py b/aiopg/sa/result.py index 89ba8c37..d58bf7b6 100644 --- a/aiopg/sa/result.py +++ b/aiopg/sa/result.py @@ -7,7 +7,6 @@ class RowProxy(Mapping): - __slots__ = ('_result_proxy', '_row', '_processors', '_keymap') def __init__(self, result_proxy, row, processors, keymap): @@ -83,7 +82,7 @@ class ResultMetaData(object): """Handle cursor.description, applying additional info from an execution context.""" - def __init__(self, result_proxy, metadata): + def __init__(self, result_proxy, cursor_description): self._processors = processors = [] map_type, map_column_name = self.result_map(result_proxy._result_map) @@ -109,7 +108,7 @@ def __init__(self, result_proxy, metadata): assert not dialect.description_encoding, \ "psycopg in py3k should not use this" - for i, rec in enumerate(metadata): + for i, rec in enumerate(cursor_description): colname = rec[0] coltype = rec[1] @@ -186,8 +185,7 @@ def _key_fallback(self, key, raiseerr=True): # isn't a column/label name overlap. # this check isn't currently available if the row # was unpickled. - if (result is not None and - result[1] is not None): + if result is not None and result[1] is not None: for obj in result[1]: if key._compare_name_for_result(obj): break @@ -233,19 +231,13 @@ class ResultProxy: def __init__(self, connection, cursor, dialect, result_map=None): self._dialect = dialect - self._closed = False self._result_map = result_map self._cursor = cursor self._connection = connection self._rowcount = cursor.rowcount - - if cursor.description is not None: - self._metadata = ResultMetaData(self, cursor.description) - self._weak = weakref.ref(self, lambda wr: cursor.close()) - else: - self._metadata = None - self.close() - self._weak = None + self._metadata = None + self._weak = None + self._init_metadata() @property def dialect(self): @@ -294,6 +286,15 @@ def rowcount(self): """ return self._rowcount + def _init_metadata(self): + cursor_description = self.cursor.description + if cursor_description is not None: + self._metadata = ResultMetaData(self, cursor_description) + self._weak = weakref.ref(self, lambda wr: self.cursor.close()) + else: + self.close() + self._weak = None + @property def returns_rows(self): """True if this ResultProxy returns rows. @@ -305,7 +306,10 @@ def returns_rows(self): @property def closed(self): - return self._closed + if self._cursor is None: + return True + + return bool(self._cursor.closed) def close(self): """Close this ResultProxy. @@ -325,9 +329,8 @@ def close(self): * cursor.description is None. """ - if not self._closed: - self._closed = True - self._cursor.close() + if not self.closed: + self.cursor.close() # allow consistent errors self._cursor = None self._weak = None @@ -361,7 +364,7 @@ def _process_rows(self, rows): async def fetchall(self): """Fetch all rows, just like DB-API cursor.fetchall().""" try: - rows = await self._cursor.fetchall() + rows = await self.cursor.fetchall() except AttributeError: self._non_result() else: @@ -376,7 +379,7 @@ async def fetchone(self): Else the cursor is automatically closed and None is returned. """ try: - row = await self._cursor.fetchone() + row = await self.cursor.fetchone() except AttributeError: self._non_result() else: @@ -395,9 +398,9 @@ async def fetchmany(self, size=None): """ try: if size is None: - rows = await self._cursor.fetchmany() + rows = await self.cursor.fetchmany() else: - rows = await self._cursor.fetchmany(size) + rows = await self.cursor.fetchmany(size) except AttributeError: self._non_result() else: diff --git a/aiopg/transaction.py b/aiopg/transaction.py index 46bd4ef5..ecd22ec6 100644 --- a/aiopg/transaction.py +++ b/aiopg/transaction.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod import psycopg2 + from aiopg.utils import _TransactionPointContextManager __all__ = ('IsolationLevel', 'Transaction') diff --git a/aiopg/utils.py b/aiopg/utils.py index 28a7f5a5..c288c74c 100644 --- a/aiopg/utils.py +++ b/aiopg/utils.py @@ -1,13 +1,47 @@ -from collections.abc import Coroutine import asyncio +import sys +import warnings +from collections.abc import Coroutine + import psycopg2 +from .log import logger try: ensure_future = asyncio.ensure_future except AttributeError: ensure_future = getattr(asyncio, 'async') +if sys.version_info >= (3, 7, 0): + __get_running_loop = asyncio.get_running_loop +else: + def __get_running_loop() -> asyncio.AbstractEventLoop: + loop = asyncio.get_event_loop() + if not loop.is_running(): + raise RuntimeError('no running event loop') + return loop + + +def get_running_loop(is_warn: bool = False) -> asyncio.AbstractEventLoop: + loop = __get_running_loop() + + if is_warn: + warnings.warn( + 'aiopg always uses "aiopg.get_running_loop", ' + 'look the documentation.', + DeprecationWarning, + stacklevel=3 + ) + + if loop.get_debug(): + logger.warning( + 'aiopg always uses "aiopg.get_running_loop", ' + 'look the documentation.', + exc_info=True + ) + + return loop + def create_future(loop): try: @@ -89,7 +123,6 @@ async def __aexit__(self, exc_type, exc, tb): class _TransactionPointContextManager(_ContextManager): - async def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: await self._obj.rollback_savepoint() @@ -100,7 +133,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class _TransactionBeginContextManager(_ContextManager): - async def __aexit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: await self._obj.rollback() @@ -111,7 +143,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): class _TransactionContextManager(_ContextManager): - async def __aexit__(self, exc_type, exc, tb): if exc_type: await self._obj.rollback() diff --git a/docs/_static/aiopg-icon.png b/docs/_static/aiopg-icon.png new file mode 100644 index 00000000..278be652 Binary files /dev/null and b/docs/_static/aiopg-icon.png differ diff --git a/docs/changes.rst b/docs/changes.rst new file mode 100644 index 00000000..4c00139d --- /dev/null +++ b/docs/changes.rst @@ -0,0 +1,8 @@ +.. _aiopg-changes: + +========= +Changelog +========= + + +.. include:: ../CHANGES.txt diff --git a/docs/conf.py b/docs/conf.py index 007658fc..a79ef224 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,8 +12,11 @@ # # All configuration values have a default; values that are commented out # serve to show the default. +import datetime +import os +import os.path +import re -import re, os, os.path def get_release(): regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") @@ -37,12 +40,12 @@ def get_version(release): # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -52,10 +55,11 @@ def get_version(release): 'sphinx.ext.intersphinx', 'sphinxcontrib.asyncio'] -intersphinx_mapping = {'python': ('http://docs.python.org/3', None), - 'sqlalchemy': ('http://docs.sqlalchemy.org/en/rel_0_9/', - None), - 'psycopg2': ('http://initd.org/psycopg/docs/', None)} +intersphinx_mapping = { + 'python': ('http://docs.python.org/3', None), + 'sqlalchemy': ('http://docs.sqlalchemy.org/en/latest', None), + 'psycopg2-binary': ('http://initd.org/psycopg/docs/', None), +} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -64,14 +68,16 @@ def get_version(release): source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'aiopg' -copyright = '2014-2016, Andrew Svetlov' +date = datetime.date.today() + +copyright = '2014-{year}, Andrew Svetlov, Alexey Firsov'.format(year=date.year) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -84,13 +90,13 @@ def get_version(release): # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -98,65 +104,68 @@ def get_version(release): # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False highlight_language = 'python3' - # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -on_rtd = os.environ.get('READTHEDOCS') == 'True' - -if on_rtd: - html_theme = 'default' -else: - html_theme = 'pyramid' +html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'logo': 'aiopg-icon.png', + 'description': 'aiopg - Postgres integration with asyncio', + 'github_user': 'aio-libs', + 'github_repo': 'aiopg', + 'github_button': True, + 'github_type': 'star', + 'github_banner': True, + 'canonical_url': 'https://aiopg.readthedocs.io/en/stable/', +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +html_title = 'Welcome to AIOPG' # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -166,93 +175,95 @@ def get_version(release): # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - +html_sidebars = { + '**': [ + 'about.html', 'navigation.html', 'searchbox.html', + ] +} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'aiopgdoc' - # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'aiopg.tex', 'aiopg Documentation', - 'Andrew Svetlov', 'manual'), + ('index', 'aiopg.tex', 'aiopg Documentation', + 'Andrew Svetlov, Alexey Firsov', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -261,11 +272,11 @@ def get_version(release): # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'aiopg', 'aiopg Documentation', - ['Andrew Svetlov'], 1) + ['Andrew Svetlov', 'Alexey Firsov'], 1) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -274,19 +285,20 @@ def get_version(release): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'aiopg', 'aiopg Documentation', - 'Andrew Svetlov', 'aiopg', 'One line description of project.', - 'Miscellaneous'), + ('index', 'aiopg', 'aiopg Documentation', + 'Andrew Svetlov, Alexey Firsov', + 'aiopg', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False diff --git a/docs/core.rst b/docs/core.rst index af7f7012..1aff9da8 100644 --- a/docs/core.rst +++ b/docs/core.rst @@ -28,7 +28,7 @@ Example:: ret = await cur.fetchall() -.. cofunction:: connect(dsn=None, *, loop=None, timeout=60.0, \ +.. cofunction:: connect(dsn=None, *, timeout=60.0, \ enable_json=True, enable_hstore=True, \ enable_uuid=True, \ echo=False, \ @@ -39,9 +39,7 @@ Example:: Make a connection to :term:`PostgreSQL` server. The function accepts all parameters that :func:`psycopg2.connect` - does plus optional keyword-only *loop* and *timeout* parameters. - - :param loop: asyncio event loop instance or ``None`` for default one. + does plus optional keyword-only *timeout* parameter. :param float timeout: default timeout (in seconds) for connection operations. @@ -90,7 +88,7 @@ Example:: Creates a new cursor object using the connection. The only *cursor_factory* can be specified, all other - parameters are not supported by :term:`psycopg2` in + parameters are not supported by :term:`psycopg2-binary` in asynchronous mode yet. The *cursor_factory* argument can be used to create @@ -103,7 +101,7 @@ Example:: parameter is not `None`. *name*, *scrollable* and *withhold* parameters are not supported - by :term:`psycopg2` in asynchronous mode. + by :term:`psycopg2-binary` in asynchronous mode. :returns: :class:`Cursor` instance. @@ -130,6 +128,16 @@ Example:: The readonly property that returns ``True`` if connections is closed. + .. method:: free_cursor() + + Call method :meth:`Cursor.closed` + for current instance :class:`Connection` + + .. attribute:: closed_cursor + + Return attribute :attr:`Cursor.closed` + for current instance :class:`Connection`. + .. attribute:: echo Return *echo mode* status. Log all executed queries to logger @@ -170,7 +178,7 @@ Example:: .. note:: - :term:`psycopg2` doesn't allow to change *autocommit* mode in + :term:`psycopg2-binary` doesn't allow to change *autocommit* mode in asynchronous mode. .. attribute:: encoding @@ -179,7 +187,7 @@ Example:: .. note:: - :term:`psycopg2` doesn't allow to change encoding in + :term:`psycopg2-binary` doesn't allow to change encoding in asynchronous mode. .. attribute:: isolation_level @@ -720,17 +728,14 @@ The basic usage is:: .. cofunction:: create_pool(dsn=None, *, minsize=1, maxsize=10,\ enable_json=True, enable_hstore=True, \ enable_uuid=True, echo=False, on_connect=None, \ - loop=None, timeout=60.0, **kwargs) + timeout=60.0, **kwargs) :coroutine: :async-with: Create a pool of connections to :term:`PostgreSQL` database. The function accepts all parameters that :func:`psycopg2.connect` - does plus optional keyword-only parameters *loop*, *minsize*, *maxsize*. - - :param loop: is an optional *event loop* instance, - :func:`asyncio.get_event_loop` is used if *loop* is not specified. + does plus optional keyword-only parameters *minsize*, *maxsize*. :param int minsize: minimum size of the *pool*, ``1`` by default. @@ -819,6 +824,14 @@ The basic usage is:: A read-only float representing default timeout for operations for connections from pool. + .. comethod:: from_pool_fill(*args, **kwargs) + :coroutine: + :classmethod: + + The method is a :ref:`coroutine `. + Constructor for filling the free pool with connections, + the number is controlled by the :attr:`minsize` parameter + .. method:: clear() A :ref:`coroutine ` that closes all *free* connections @@ -887,7 +900,7 @@ The basic usage is:: connection and returns *context manager*. The only *cursor_factory* can be specified, all other - parameters are not supported by :term:`psycopg2` in + parameters are not supported by :term:`psycopg2-binary` in asynchronous mode yet. The *cursor_factory* argument can be used to create @@ -900,7 +913,7 @@ The basic usage is:: is not `None`. *name*, *scrollable* and *withhold* parameters are not supported - by :term:`psycopg2` in asynchronous mode. + by :term:`psycopg2-binary` in asynchronous mode. The usage is:: @@ -918,7 +931,7 @@ Exceptions Any call to library function, method or property can raise an exception. :mod:`aiopg` doesn't define any exception class itself, it reuses -:ref:`DBAPI Exceptions ` from :mod:`psycopg2` +:ref:`DBAPI Exceptions ` from :mod:`psycopg2-binary` .. _aiopg-core-transactions: @@ -926,7 +939,7 @@ Any call to library function, method or property can raise an exception. Transactions ============ -While :term:`psycopg2` asynchronous connections have to be in *autocommit mode* it is still +While :term:`psycopg2-binary` asynchronous connections have to be in *autocommit mode* it is still possible to use SQL transactions executing **BEGIN** and **COMMIT** statements manually as `Psycopg Asynchronous Support docs`_ . @@ -954,7 +967,7 @@ For pushing data to server please wrap json dict into data = {'a': 1, 'b': 'str'} await cur.execute("INSERT INTO tbl (val) VALUES (%s)", [Json(data)]) -On receiving data from json column :term:`psycopg2` autoconvers result +On receiving data from json column :term:`psycopg2-binary` autoconvers result into python :class:`dict` object:: await cur.execute("SELECT val FROM tbl") diff --git a/docs/essays.rst b/docs/essays.rst new file mode 100644 index 00000000..43eb20b6 --- /dev/null +++ b/docs/essays.rst @@ -0,0 +1,8 @@ +Essays +====== + + +.. toctree:: + + one_cursor + run_loop diff --git a/docs/examples.rst b/docs/examples.rst index bda71bcc..62e11c4c 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -9,11 +9,6 @@ Below is a list of examples from `aiopg/examples Every example is a correct tiny python program. -.. _aiopg-examples-new-style: - -async/await style -================= - .. _aiopg-examples-simple: Low-level API @@ -38,40 +33,49 @@ Simple sqlalchemy usage .. literalinclude:: ../examples/simple_sa.py -.. _aiopg-examples-sa-complex: +.. _aiopg-examples-sa-default-field: -Complex sqlalchemy queries ---------------------------- +Default value field sqlalchemy usage +------------------------------------ -.. literalinclude:: ../examples/sa.py +.. literalinclude:: ../examples/default_field_sa.py -.. _aiopg-examples-old-style: +.. _aiopg-examples-sa-types-field: -yield from/@coroutine style -============================ +Types field sqlalchemy usage +---------------------------- -.. _aiopg-examples-simple-old-style: +.. literalinclude:: ../examples/types_field_sa.py -Old style Low-level API ------------------------ -.. literalinclude:: ../examples/simple_old_style.py +.. _aiopg-examples-sa-named-field: + +Named field sqlalchemy usage +---------------------------- + +.. literalinclude:: ../examples/named_field_sa.py + + +.. _aiopg-examples-sa-complex: + +Complex sqlalchemy queries +--------------------------- + +.. literalinclude:: ../examples/sa.py -.. _aiopg-examples-notify-old-style: +.. _aiopg-examples-sa-simple-transaction: -Usage of LISTEN/NOTIFY commands using old-style API ---------------------------------------------------- +Simple transaction in sqlalchemy +-------------------------------- -.. literalinclude:: ../examples/notify_old_style.py +.. literalinclude:: ../examples/simple_sa_transaction.py -Simple sqlalchemy usage commands using old-style API ----------------------------------------------------- -.. literalinclude:: ../examples/simple_sa_oldstyle.py +.. _aiopg-examples-sa-isolation-transaction: -Complex sqlalchemy queries commands using old-style API -------------------------------------------------------- +Isolation transaction in sqlalchemy +----------------------------------- -.. literalinclude:: ../examples/sa_oldstyle.py +.. literalinclude:: ../examples/isolation_sa_transaction.py diff --git a/docs/glossary.rst b/docs/glossary.rst index 07e9b33d..e1d4ce4c 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -1,9 +1,9 @@ .. _glossary: -******** +======== Glossary -******** +======== .. if you add new entries, keep the alphabetical sorting! @@ -38,13 +38,23 @@ Glossary http://www.postgresql.org/ - psycopg2 + PostgreSQL Error Codes - A PostgreSQL database adapter for the Python programming - language. psycopg2 was written with the aim of being very small - and fast, and stable as a rock. + All messages emitted by the PostgreSQL server are assigned + five-character error codes that follow the + SQL standard's conventions for “SQLSTATE” codes. - http://initd.org/psycopg/ + https://www.postgresql.org/docs/current/errcodes-appendix.html#ERRCODES-TABLE + + psycopg2-binary + + Psycopg is the most popular PostgreSQL database adapter for + the Python programming language. + Its main features are the complete implementation of + the Python DB API 2.0 specification and the thread safety + (several threads can share the same connection). + + https://pypi.org/project/psycopg2-binary/ pyflakes diff --git a/docs/index.rst b/docs/index.rst index 66cc603c..3d2a6f60 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,17 +3,42 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -aiopg -================================= - -.. _GitHub: https://github.com/aio-libs/aiopg -.. _asyncio: http://docs.python.org/3.4/library/asyncio.html +================ +Welcome to AIOPG +================ **aiopg** is a library for accessing a :term:`PostgreSQL` database from the asyncio_ (PEP-3156/tulip) framework. It wraps asynchronous features of the Psycopg database driver. +Current version is |release|. + +.. image:: https://travis-ci.com/aio-libs/aiopg.svg?branch=master + :target: https://travis-ci.com/aio-libs/aiopg + :alt: Travis CI status + +.. image:: https://codecov.io/github/aio-libs/aiopg/coverage.svg?branch=master + :target: https://codecov.io/github/aio-libs/aiopg + :alt: Code coverage status + +.. image:: https://badge.fury.io/py/aiopg.svg + :target: https://badge.fury.io/py/aiopg + :alt: Latest PyPI package version + +.. _GitHub: https://github.com/aio-libs/aiopg +.. _asyncio: http://docs.python.org/3.4/library/asyncio.html + +.. warning:: + 1. Only supports ``python >= 3.5.2`` + + 2. Only support syntax ``async/await`` + + 3. :ref:`aiopg-one-cursor` + + 4. :ref:`aiopg-run-loop` + + Features -------- @@ -27,13 +52,13 @@ Features Basics ------ -The library uses :mod:`psycopg2` connections in **asynchronous** mode +The library uses :mod:`psycopg2-binary` connections in **asynchronous** mode internally. -Literally it is an (almost) transparent wrapper for psycopg2 +Literally it is an (almost) transparent wrapper for psycopg2-binary connection and cursor, but with only exception. -You should use ``yield from conn.f()`` instead of just call ``conn.f()`` for +You should use ``await conn.f()`` instead of just call ``conn.f()`` for every method. Properties are unchanged, so ``conn.prop`` is correct as well as @@ -62,31 +87,12 @@ See example:: For documentation about connection and cursor methods/properties please go to psycopg docs: http://initd.org/psycopg/docs/ -.. note:: psycopg2 creates new connections with ``autocommit=True`` +.. note:: psycopg2-binary creates new connections with ``autocommit=True`` option in asynchronous mode. Autocommitting cannot be disabled. See :ref:`aiopg-core-transactions` about transaction usage in *autocommit mode*. -.. note:: - - Throughout this documentation, examples utilize the `async/await` syntax - introduced by :pep:`492` that is only valid for Python 3.5+. - - If you are using Python 3.4, please replace ``await`` with - ``yield from`` and ``async def`` with a ``@coroutine`` decorator. - For example, this:: - - async def coro(...): - ret = await f() - - shoud be replaced by:: - - @asyncio.coroutine - def coro(...): - ret = yield from f() - - see also :ref:`aiopg-examples-old-style` examples. SQLAlchemy and aiopg -------------------- @@ -134,26 +140,26 @@ Installation pip3 install aiopg -.. note:: :mod:`aiopg` requires :term:`psycopg2` library. - - You can use standard one from your distro like:: +.. note:: :mod:`aiopg` requires :term:`psycopg2-binary` library. - $ sudo apt-get install python3-psycopg2 - - but if you like to use virtual environments + You can use global environment or you use like to use virtual environments (:term:`virtualenvwrapper`, :term:`virtualenv` or :term:`venv`) you - probably have to install :term:`libpq` development package:: + probably have to install :term:`libpq` development package + + .. code-block:: shell - $ sudo apt-get install libpq-dev + $ sudo apt-get install libpq-dev Also you probably want to use :mod:`aiopg.sa`. .. _aiozmq-install-sqlalchemy: :mod:`aiopg.sa` module is **optional** and requires -:term:`sqlalchemy`. You can install *sqlalchemy* by running:: +:term:`sqlalchemy`. You can install *sqlalchemy* by running + +.. code-block:: shell - pip3 install sqlalchemy + $ pip3 install aiopg[sa] Source code ----------- @@ -178,8 +184,8 @@ Feel free to post your questions and ideas here. Dependencies ------------ -- Python 3.3 and :mod:`asyncio` or Python 3.4+ -- psycopg2 +- Python 3.5.2+ +- psycopg2-binary - aiopg.sa requires :term:`sqlalchemy`. Authors and License @@ -200,6 +206,7 @@ Contents: examples contributing glossary + misc Indices and tables ================== diff --git a/docs/misc.rst b/docs/misc.rst new file mode 100644 index 00000000..e313bf07 --- /dev/null +++ b/docs/misc.rst @@ -0,0 +1,14 @@ +.. _aiopg-misc: + +Miscellaneous +============= + +Helpful pages. + +.. toctree:: + :name: misc + + essays + team + glossary + changes diff --git a/docs/one_cursor.rst b/docs/one_cursor.rst new file mode 100644 index 00000000..52150295 --- /dev/null +++ b/docs/one_cursor.rst @@ -0,0 +1,66 @@ +.. _aiopg-one-cursor: + +One connection, one cursor or forced close +========================================== + +Rationale +--------- + +The :meth:`aiopg.sa.SAConnection.execute` method creates a new cursor +each time it is called. But since we release the :ref:`aiopg-sa-connection`, +weak references to the :ref:`aiopg-core-cursor` remain, which means that the closing +does not occur. + +The code must be restructured so that instead the pool is transferred +so that each execution has its own :ref:`aiopg-core-cursor`. + +Example: + +.. code-block:: py3 + + async with engine.acquire() as conn: + assert res.cursor.closed is False + + res = await conn.execute(tbl.select()) + assert res.cursor.closed is False + + row = await res.fetchone() + assert res.cursor.closed is False + + assert res.cursor.closed is False + +After exiting `async with` the :ref:`aiopg-sa-connection` was closed, +but the :ref:`aiopg-core-cursor` remained open. + +Implementation +-------------- + +It was decided to select one connection, one :ref:`aiopg-core-cursor`. +For the interface :ref:`aiopg-core-connection` interface +over :term:`psycopg2-binary`, +we added the :meth:`aiopg.Connection.free_cursor` +method to clean the cursor if it is open. + +The :attr:`aiopg.Connection.free_cursor` method is called in several places: + + * at the time call method :meth:`aiopg.Connection.cursor` + * at the time call method :meth:`aiopg.Connection.close` + * at the time call method :meth:`aiopg.Pool.release` + * at the time call method :meth:`aiopg.sa.SAConnection.execute` + * at the time call method :meth:`aiopg.Engine.release` + +.. warning:: + At the time call method :meth:`aiopg.Connection.cursor` + or :meth:`aiopg.sa.SAConnection.execute`, + if the current :ref:`aiopg-core-connection` have + a open :ref:`aiopg-core-cursor`, a warning will be issued before closing. + + .. code-block:: py3 + + warnings.warn( + ('You can only have one cursor per connection. ' + 'The cursor for connection will be ' + 'closed forcibly {!r}.' + ).format(self), + ResourceWarning + ) diff --git a/docs/run_loop.rst b/docs/run_loop.rst new file mode 100644 index 00000000..6b70d0a8 --- /dev/null +++ b/docs/run_loop.rst @@ -0,0 +1,69 @@ +.. _aiopg-run-loop: + +Only use get_running_loop +========================= + +Rationale +--------- + +:func:`asyncio.get_event_loop()` returns the +running loop :class:`asyncio.AbstractEventLoop` instead of **default**, +which may be different, e.g. + +.. code-block:: py3 + + loop = asyncio.new_event_loop() + loop.run_until_complete(f()) + +.. note:: + + :func:`asyncio.set_event_loop` was not called and default + loop :class:`asyncio.AbstractEventLoop` + is not equal to actually executed one. + + +Implementation +-------------- + +For the version below ``python3.7`` we added this implementation. + +.. code-block:: py3 + + if sys.version_info >= (3, 7, 0): + __get_running_loop = asyncio.get_running_loop + else: + def __get_running_loop() -> asyncio.AbstractEventLoop: + loop = asyncio.get_event_loop() + if not loop.is_running(): + raise RuntimeError('no running event loop') + return loop + +This allows you to get a loop :class:`asyncio.AbstractEventLoop` correctly +and causes :exc:`DeprecationWarning` if you explicitly +pass a loop :class:`asyncio.AbstractEventLoop` for class or method: + + * :class:`aiopg.Pool` and :meth:`aiopg.create_pool` + * :class:`aiopg.Connection` and :meth:`aiopg.connect` + * :meth:`aiopg.sa.create_engine` + +.. code-block:: py3 + + def get_running_loop(is_warn=False): + loop = __get_running_loop() + + if is_warn: + warnings.warn( + 'aiopg always uses "aiopg.get_running_loop", ' + 'look the documentation.', + DeprecationWarning, + stacklevel=3 + ) + + if loop.get_debug(): + logger.warning( + 'aiopg always uses "aiopg.get_running_loop", ' + 'look the documentation.', + exc_info=True + ) + + return loop diff --git a/docs/sa.rst b/docs/sa.rst index 1cd2ef24..dccf539b 100644 --- a/docs/sa.rst +++ b/docs/sa.rst @@ -17,7 +17,9 @@ strings too annoying. Fortunately we can use excellent :ref:`core_toplevel` as **SQL query builder**. -Example:: +Example + +.. code-block:: py3 import asyncio from aiopg.sa import create_engine @@ -69,7 +71,7 @@ Also we provide SQL transactions support. Please take a look on Engine ------ -.. cofunction:: create_engine(dsn=None, *, minsize=1, maxsize=10, loop=None, \ +.. cofunction:: create_engine(dsn=None, *, minsize=1, maxsize=10, \ dialect=dialect, timeout=60, **kwargs) :coroutine: :async-with: @@ -81,7 +83,7 @@ Engine .. data:: dialect - An instance of :term:`SQLAlchemy` dialect set up for :term:`psycopg2` usage. + An instance of :term:`SQLAlchemy` dialect set up for :term:`psycopg2-binary` usage. An :class:`sqlalchemy.engine.interfaces.Dialect` instance. @@ -118,7 +120,7 @@ Engine .. seealso:: - `psycopg2 connection.dsn + `psycopg2-binary connection.dsn `_ attribute. @@ -198,6 +200,7 @@ Engine .. warning:: The method is not a :ref:`coroutine `. +.. _aiopg-sa-connection: Connection ---------- @@ -254,6 +257,17 @@ Connection :returns: :class:`ResultProxy` instance with results of SQL query execution. + .. seealso:: + + * Simple examples sqlalchemy style :ref:`aiopg-examples-sa-simple` + + * Examples sqlalchemy default field :ref:`aiopg-examples-sa-default-field` + + * Examples sqlalchemy type field :ref:`aiopg-examples-sa-types-field` + + * Examples sqlalchemy name field :ref:`aiopg-examples-sa-named-field` + + .. comethod:: scalar(query, *multiparams, **params) Executes a *SQL* *query* and returns a scalar value. @@ -295,10 +309,13 @@ Connection .. seealso:: - :meth:`.SAConnection.begin_nested` - use a SAVEPOINT + * Simple examples :ref:`aiopg-examples-sa-simple-transaction` + + * Examples with isolation level :ref:`aiopg-examples-sa-isolation-transaction` + + * :meth:`.SAConnection.begin_nested` - use a SAVEPOINT - :meth:`.SAConnection.begin_twophase` - use a two phase (XA) - transaction + * :meth:`.SAConnection.begin_twophase` - use a two phase (XA) transaction .. comethod:: begin_nested() :coroutine: @@ -315,7 +332,11 @@ Connection .. seealso:: - :meth:`.SAConnection.begin`, :meth:`.SAConnection.begin_twophase`. + * Simple examples :ref:`aiopg-examples-sa-simple-transaction` + + * :meth:`.SAConnection.begin` + + * :meth:`.SAConnection.begin_twophase`. .. comethod:: begin_twophase(xid=None) :coroutine: diff --git a/docs/team.rst b/docs/team.rst new file mode 100644 index 00000000..14c57426 --- /dev/null +++ b/docs/team.rst @@ -0,0 +1,19 @@ +.. _aiopg-team: + +Team AIOPG +========== + +Maintainers +----------- + +Main library developers and those who support: + +.. include:: ../MAINTAINERS.txt + +Contributors +------------ + +A number of people have contributed to *aiopg* by reporting problems, +suggesting improvements or submitting changes. Some of these people are: + +.. include:: ../CONTRIBUTORS.txt diff --git a/examples/default_field_sa.py b/examples/default_field_sa.py new file mode 100644 index 00000000..3ee5d693 --- /dev/null +++ b/examples/default_field_sa.py @@ -0,0 +1,56 @@ +import asyncio +import datetime +import uuid + +import sqlalchemy as sa +from sqlalchemy.sql.ddl import CreateTable + +from aiopg.sa import create_engine + +metadata = sa.MetaData() + +now = datetime.datetime.now + +tbl = sa.Table( + 'tbl', metadata, + sa.Column('id', sa.Integer, autoincrement=True, primary_key=True), + sa.Column('uuid', sa.String, default=lambda: str(uuid.uuid4())), + sa.Column('name', sa.String(255), default='default name'), + sa.Column('date', sa.DateTime, default=datetime.datetime.now), + sa.Column('flag', sa.Integer, default=0), + sa.Column('count_str', sa.Integer, default=sa.func.length('default')), + sa.Column('is_active', sa.Boolean, default=True), +) + + +async def insert_tbl(conn, pk, **kwargs): + await conn.execute(tbl.insert().values(**kwargs)) + row = await (await conn.execute(tbl.select())).first() + + assert row.id == pk + + for name, val in kwargs.items(): + assert row[name] == val + + await conn.execute(sa.delete(tbl)) + + +async def create_table(conn): + await conn.execute('DROP TABLE IF EXISTS tbl') + await conn.execute(CreateTable(tbl)) + + +async def go(): + async with create_engine(user='aiopg', + database='aiopg', + host='127.0.0.1', + password='passwd') as engine: + async with engine.acquire() as conn: + await create_table(conn) + async with engine.acquire() as conn: + await insert_tbl(conn, 1) + await insert_tbl(conn, 2, name='test', is_active=False, date=now()) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(go()) diff --git a/examples/isolation_sa_transaction.py b/examples/isolation_sa_transaction.py new file mode 100644 index 00000000..4c3b1038 --- /dev/null +++ b/examples/isolation_sa_transaction.py @@ -0,0 +1,160 @@ +import asyncio + +import sqlalchemy as sa +from psycopg2 import InternalError +from psycopg2.extensions import TransactionRollbackError +from sqlalchemy.sql.ddl import CreateTable + +from aiopg.sa import create_engine + +metadata = sa.MetaData() + +users = sa.Table( + 'users_sa_isolation_transaction', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String(255)) +) + + +async def create_sa_transaction_tables(conn): + await conn.execute(CreateTable(users)) + + +async def repea_sa_transaction(conn, conn2): + isolation_level = 'REPEATABLE READ' + await conn.execute(sa.insert(users).values(id=1, name='test1')) + t1 = await conn.begin(isolation_level=isolation_level) + + where = users.c.id == 1 + q_user = users.select().where(where) + user = await (await conn.execute(q_user)).fetchone() + + assert await (await conn2.execute(q_user)).fetchone() == user + + await conn.execute(sa.update(users).values({'name': 'name2'}).where(where)) + + t2 = await conn2.begin(isolation_level=isolation_level) + assert await (await conn2.execute(q_user)).fetchone() == user + + await t1.commit() + + await conn2.execute(users.insert().values({'id': 2, 'name': 'test'})) + + try: + await conn2.execute( + sa.update(users).values({'name': 't'}).where(where)) + except TransactionRollbackError as e: + assert e.pgcode == '40001' + + await t2.commit() + + assert len(await (await conn2.execute(q_user)).fetchall()) == 1 + await conn.execute(sa.delete(users)) + assert len(await (await conn.execute(users.select())).fetchall()) == 0 + + +async def serializable_sa_transaction(conn, conn2): + isolation_level = 'SERIALIZABLE' + await conn.execute(sa.insert(users).values(id=1, name='test1')) + t1 = await conn.begin(isolation_level=isolation_level) + + where = users.c.id == 1 + q_user = users.select().where(where) + user = await (await conn.execute(q_user)).fetchone() + + assert await (await conn2.execute(q_user)).fetchone() == user + + await conn.execute(sa.update(users).values({'name': 'name2'}).where(where)) + + t2 = await conn2.begin(isolation_level=isolation_level) + assert await (await conn2.execute(q_user)).fetchone() == user + + await t1.commit() + + try: + await conn2.execute(users.insert().values({'id': 2, 'name': 'test'})) + except TransactionRollbackError as e: + assert e.pgcode == '40001' + + try: + await conn2.execute(users.update().values({'name': 't'}).where(where)) + except InternalError as e: + assert e.pgcode == '25P02' + + await t2.commit() + + user = dict(await (await conn2.execute(q_user)).fetchone()) + assert user == {'name': 'name2', 'id': 1} + + await conn.execute(sa.delete(users)) + assert len(await (await conn.execute(users.select())).fetchall()) == 0 + + +async def read_only_read_sa_transaction(conn, deferrable): + await conn.execute(sa.insert(users).values(id=1, name='test1')) + t1 = await conn.begin( + isolation_level='SERIALIZABLE', + readonly=True, + deferrable=deferrable + ) + + where = users.c.id == 1 + + try: + await conn.execute(sa.update(users).values({'name': 't'}).where(where)) + except InternalError as e: + assert e.pgcode == '25006' + + await t1.commit() + + await conn.execute(sa.delete(users)) + assert len(await (await conn.execute(users.select())).fetchall()) == 0 + + +async def isolation_read_sa_transaction(conn, conn2): + await conn.execute(sa.insert(users).values(id=1, name='test1')) + t1 = await conn.begin() + + where = users.c.id == 1 + q_user = users.select().where(where) + user = await (await conn.execute(q_user)).fetchone() + + assert await (await conn2.execute(q_user)).fetchone() == user + + await conn.execute(sa.update(users).values({'name': 'name2'}).where(where)) + + t2 = await conn2.begin() + assert await (await conn2.execute(q_user)).fetchone() == user + + await t1.commit() + + await conn2.execute(sa.update(users).values(user).where(where)) + await t2.commit() + + assert await (await conn2.execute(q_user)).fetchone() == user + + await conn.execute(sa.delete(users)) + assert len(await (await conn.execute(users.select())).fetchall()) == 0 + + +async def go(): + engine = await create_engine(user='aiopg', + database='aiopg', + host='127.0.0.1', + password='passwd') + async with engine: + async with engine.acquire() as conn: + await create_sa_transaction_tables(conn) + + async with engine.acquire() as conn: + await read_only_read_sa_transaction(conn, True) + await read_only_read_sa_transaction(conn, False) + + async with engine.acquire() as conn2: + await repea_sa_transaction(conn, conn2) + await serializable_sa_transaction(conn, conn2) + await isolation_read_sa_transaction(conn, conn2) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(go()) diff --git a/examples/named_field_sa.py b/examples/named_field_sa.py new file mode 100644 index 00000000..78a705f6 --- /dev/null +++ b/examples/named_field_sa.py @@ -0,0 +1,51 @@ +import asyncio +import datetime + +import sqlalchemy as sa + +from aiopg.sa import create_engine + +metadata = sa.MetaData() + +now = datetime.datetime.now + +tbl = sa.Table( + 'tbl', metadata, + sa.Column('MyIDField', sa.Integer, key='id', primary_key=True), + sa.Column('NaMe', sa.String(255), key='name', default='default name'), +) + + +async def insert_tbl(conn, **kwargs): + await conn.execute(tbl.insert().values(**kwargs)) + row = await (await conn.execute(tbl.select())).first() + + for name, val in kwargs.items(): + assert row[name] == val + + await conn.execute(sa.delete(tbl)) + + +async def create_table(conn): + await conn.execute('DROP TABLE IF EXISTS tbl') + await conn.execute( + 'CREATE TABLE tbl (' + '"MyIDField" INTEGER NOT NULL, ' + '"NaMe" VARCHAR(255), ' + 'PRIMARY KEY ("MyIDField"))' + ) + + +async def go(): + async with create_engine(user='aiopg', + database='aiopg', + host='127.0.0.1', + password='passwd') as engine: + async with engine.acquire() as conn: + await create_table(conn) + await insert_tbl(conn, id=1) + await insert_tbl(conn, id=2, name='test') + + +loop = asyncio.get_event_loop() +loop.run_until_complete(go()) diff --git a/examples/notify.py b/examples/notify.py index 0367b20d..795620c2 100644 --- a/examples/notify.py +++ b/examples/notify.py @@ -1,4 +1,5 @@ import asyncio + import aiopg dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' diff --git a/examples/notify_old_style.py b/examples/notify_old_style.py deleted file mode 100644 index 3264f40c..00000000 --- a/examples/notify_old_style.py +++ /dev/null @@ -1,45 +0,0 @@ -import asyncio -import aiopg - -dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' - - -async def notify(conn): - cur = await conn.cursor() - try: - for i in range(5): - msg = "message {}".format(i) - print('Send ->', msg) - await cur.execute("NOTIFY channel, %s", (msg,)) - - await cur.execute("NOTIFY channel, 'finish'") - finally: - cur.close() - - -async def listen(conn): - cur = await conn.cursor() - try: - await cur.execute("LISTEN channel") - while True: - msg = await conn.notifies.get() - if msg.payload == 'finish': - return - else: - print('Receive <-', msg.payload) - finally: - cur.close() - - -async def main(): - pool = await aiopg.create_pool(dsn) - async with pool as conn1: - listener = listen(conn1) - async with pool as conn2: - notifier = notify(conn2) - await asyncio.gather(listener, notifier) - print("ALL DONE") - - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) diff --git a/examples/sa.py b/examples/sa.py index 932564ae..e94bed5b 100644 --- a/examples/sa.py +++ b/examples/sa.py @@ -1,9 +1,10 @@ import asyncio -from aiopg.sa import create_engine -import sqlalchemy as sa -import random import datetime +import random +import sqlalchemy as sa + +from aiopg.sa import create_engine metadata = sa.MetaData() diff --git a/examples/sa_oldstyle.py b/examples/sa_oldstyle.py deleted file mode 100644 index 986fa55b..00000000 --- a/examples/sa_oldstyle.py +++ /dev/null @@ -1,126 +0,0 @@ -import asyncio -from aiopg.sa import create_engine -import sqlalchemy as sa -import random -import datetime - - -metadata = sa.MetaData() - -users = sa.Table('users', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('name', sa.String(255)), - sa.Column('birthday', sa.DateTime)) - -emails = sa.Table('emails', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('user_id', None, sa.ForeignKey('users.id')), - sa.Column('email', sa.String(255), nullable=False), - sa.Column('private', sa.Boolean, nullable=False)) - - -async def create_tables(engine): - async with engine as conn: - await conn.execute('DROP TABLE IF EXISTS emails') - await conn.execute('DROP TABLE IF EXISTS users') - await conn.execute('''CREATE TABLE users ( - id serial PRIMARY KEY, - name varchar(255), - birthday timestamp)''') - await conn.execute('''CREATE TABLE emails ( - id serial, - user_id int references users(id), - email varchar(253), - private bool)''') - - -names = {'Andrew', 'Bob', 'John', 'Vitaly', 'Alex', 'Lina', 'Olga', - 'Doug', 'Julia', 'Matt', 'Jessica', 'Nick', 'Dave', 'Martin', - 'Abbi', 'Eva', 'Lori', 'Rita', 'Rosa', 'Ivy', 'Clare', 'Maria', - 'Jenni', 'Margo', 'Anna'} - - -def gen_birthday(): - now = datetime.datetime.now() - year = random.randint(now.year - 30, now.year - 20) - month = random.randint(1, 12) - day = random.randint(1, 28) - return datetime.datetime(year, month, day) - - -async def fill_data(engine): - async with engine as conn: - tr = await conn.begin() - - for name in random.sample(names, len(names)): - uid = await conn.scalar( - users.insert().values(name=name, birthday=gen_birthday())) - emails_count = int(random.paretovariate(2)) - for num in random.sample(range(10000), emails_count): - is_private = random.uniform(0, 1) < 0.8 - await conn.execute(emails.insert().values( - user_id=uid, - email='{}+{}@gmail.com'.format(name, num), - private=is_private)) - await tr.commit() - - -async def count(engine): - async with engine as conn: - c1 = await conn.scalar(users.count()) - c2 = await conn.scalar(emails.count()) - print("Population consists of", c1, "people with", - c2, "emails in total") - join = sa.join(emails, users, users.c.id == emails.c.user_id) - query = (sa.select([users.c.name]) - .select_from(join) - .where(emails.c.private == False) # noqa - .group_by(users.c.name) - .having(sa.func.count(emails.c.private) > 0)) - - print("Users with public emails:") - ret = await conn.execute(query) - for row in ret: - print(row.name) - - print() - - -async def show_julia(engine): - async with engine as conn: - print("Lookup for Julia:") - join = sa.join(emails, users, users.c.id == emails.c.user_id) - query = (sa.select([users, emails], use_labels=True) - .select_from(join).where(users.c.name == 'Julia')) - res = await conn.execute(query) - for row in res: - print(row.users_name, row.users_birthday, - row.emails_email, row.emails_private) - print() - - -async def ave_age(engine): - async with engine as conn: - query = (sa.select([sa.func.avg(sa.func.age(users.c.birthday))]) - .select_from(users)) - ave = await conn.scalar(query) - print("Average age of population is", ave, - "or ~", int(ave.days / 365), "years") - print() - - -async def go(): - engine = await create_engine(user='aiopg', - database='aiopg', - host='127.0.0.1', - password='passwd') - - await create_tables(engine) - await fill_data(engine) - await count(engine) - await show_julia(engine) - await ave_age(engine) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(go()) diff --git a/examples/simple.py b/examples/simple.py index e4941132..8befc395 100644 --- a/examples/simple.py +++ b/examples/simple.py @@ -1,4 +1,5 @@ import asyncio + import aiopg dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' diff --git a/examples/simple_old_style.py b/examples/simple_old_style.py deleted file mode 100644 index 0b72c8f2..00000000 --- a/examples/simple_old_style.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio -import aiopg - -dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' - - -async def test_select(): - pool = await aiopg.create_pool(dsn) - async with pool.cursor() as cur: - await cur.execute("SELECT 1") - ret = await cur.fetchone() - assert ret == (1,) - print("ALL DONE") - - -loop = asyncio.get_event_loop() -loop.run_until_complete(test_select()) diff --git a/examples/simple_sa.py b/examples/simple_sa.py index 2bba2dfa..43bcdb6d 100644 --- a/examples/simple_sa.py +++ b/examples/simple_sa.py @@ -1,7 +1,8 @@ import asyncio -from aiopg.sa import create_engine + import sqlalchemy as sa +from aiopg.sa import create_engine metadata = sa.MetaData() diff --git a/examples/simple_sa_oldstyle.py b/examples/simple_sa_oldstyle.py deleted file mode 100644 index 3e7e8fa4..00000000 --- a/examples/simple_sa_oldstyle.py +++ /dev/null @@ -1,37 +0,0 @@ -import asyncio -from aiopg.sa import create_engine -import sqlalchemy as sa - - -metadata = sa.MetaData() - -tbl = sa.Table('tbl', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('val', sa.String(255))) - - -async def create_table(engine): - async with engine as conn: - await conn.execute('DROP TABLE IF EXISTS tbl') - await conn.execute('''CREATE TABLE tbl ( - id serial PRIMARY KEY, - val varchar(255))''') - - -async def go(): - engine = await create_engine(user='aiopg', - database='aiopg', - host='127.0.0.1', - password='passwd') - - await create_table(engine) - async with engine as conn: - await conn.execute(tbl.insert().values(val='abc')) - - res = await conn.execute(tbl.select()) - for row in res: - print(row.id, row.val) - - -loop = asyncio.get_event_loop() -loop.run_until_complete(go()) diff --git a/examples/simple_sa_transaction.py b/examples/simple_sa_transaction.py new file mode 100644 index 00000000..d5cf7bc3 --- /dev/null +++ b/examples/simple_sa_transaction.py @@ -0,0 +1,145 @@ +import asyncio + +import sqlalchemy as sa +from sqlalchemy.schema import CreateTable + +from aiopg.sa import create_engine + +metadata = sa.MetaData() + +users = sa.Table( + 'users_sa_transaction', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String(255)) +) + + +async def create_sa_transaction_tables(conn): + await conn.execute(CreateTable(users)) + + +async def check_count_users(conn, *, count): + s_query = sa.select(users).select_from(users) + assert count == len(list(await (await conn.execute(s_query)).fetchall())) + + +async def success_transaction(conn): + await check_count_users(conn, count=0) + + async with conn.begin(): + await conn.execute(sa.insert(users).values(id=1, name='test1')) + await conn.execute(sa.insert(users).values(id=2, name='test2')) + + await check_count_users(conn, count=2) + + async with conn.begin(): + await conn.execute(sa.delete(users).where(users.c.id == 1)) + await conn.execute(sa.delete(users).where(users.c.id == 2)) + + await check_count_users(conn, count=0) + + +async def fail_transaction(conn): + await check_count_users(conn, count=0) + + trans = await conn.begin() + + try: + await conn.execute(sa.insert(users).values(id=1, name='test1')) + raise RuntimeError() + + except RuntimeError: + await trans.rollback() + else: + await trans.commit() + + await check_count_users(conn, count=0) + + +async def success_nested_transaction(conn): + await check_count_users(conn, count=0) + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=1, name='test1')) + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=2, name='test2')) + + await check_count_users(conn, count=2) + + async with conn.begin(): + await conn.execute(sa.delete(users).where(users.c.id == 1)) + await conn.execute(sa.delete(users).where(users.c.id == 2)) + + await check_count_users(conn, count=0) + + +async def fail_nested_transaction(conn): + await check_count_users(conn, count=0) + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=1, name='test1')) + + tr_f = await conn.begin_nested() + try: + await conn.execute(sa.insert(users).values(id=2, name='test2')) + raise RuntimeError() + + except RuntimeError: + await tr_f.rollback() + else: + await tr_f.commit() + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=2, name='test2')) + + await check_count_users(conn, count=2) + + async with conn.begin(): + await conn.execute(sa.delete(users).where(users.c.id == 1)) + await conn.execute(sa.delete(users).where(users.c.id == 2)) + + await check_count_users(conn, count=0) + + +async def fail_first_nested_transaction(conn): + trans = await conn.begin_nested() + + try: + await conn.execute(sa.insert(users).values(id=1, name='test1')) + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=2, name='test2')) + + async with conn.begin_nested(): + await conn.execute(sa.insert(users).values(id=3, name='test3')) + + raise RuntimeError() + + except RuntimeError: + await trans.rollback() + else: + await trans.commit() + + await check_count_users(conn, count=0) + + +async def go(): + engine = await create_engine(user='aiopg', + database='aiopg', + host='127.0.0.1', + password='passwd') + async with engine: + async with engine.acquire() as conn: + await create_sa_transaction_tables(conn) + + await success_transaction(conn) + await fail_transaction(conn) + + await success_nested_transaction(conn) + await fail_nested_transaction(conn) + await fail_first_nested_transaction(conn) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(go()) diff --git a/examples/transaction.py b/examples/transaction.py index 5641e50f..483e2ee5 100644 --- a/examples/transaction.py +++ b/examples/transaction.py @@ -1,7 +1,7 @@ import asyncio import aiopg -from aiopg.transaction import Transaction, IsolationLevel +from aiopg.transaction import IsolationLevel, Transaction dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' diff --git a/examples/transaction_oldsyle.py b/examples/transaction_oldsyle.py deleted file mode 100644 index dbccbb3d..00000000 --- a/examples/transaction_oldsyle.py +++ /dev/null @@ -1,43 +0,0 @@ -import asyncio - -import aiopg -import psycopg2 -from aiopg.transaction import Transaction, IsolationLevel - -dsn = 'dbname=aiopg user=aiopg password=passwd host=127.0.0.1' - - -async def transaction(cur, isolation_level, - readonly=False, deferrable=False): - transaction = Transaction(cur, isolation_level, readonly, deferrable) - - await transaction.begin() - try: - await cur.execute('insert into tbl values (1)') - - await transaction.savepoint() - try: - await cur.execute('insert into tbl values (3)') - await transaction.release_savepoint() - except psycopg2.Error: - await transaction.rollback_savepoint() - - await cur.execute('insert into tbl values (4)') - await transaction.commit() - - except psycopg2.Error: - await transaction.rollback() - - -async def main(): - pool = await aiopg.create_pool(dsn) - async with pool.cursor() as cur: - await transaction(cur, IsolationLevel.repeatable_read) - await transaction(cur, IsolationLevel.read_committed) - await transaction(cur, IsolationLevel.serializable) - - cur.execute('select * from tbl') - - -loop = asyncio.get_event_loop() -loop.run_until_complete(main()) diff --git a/examples/types_field_sa.py b/examples/types_field_sa.py new file mode 100644 index 00000000..997fcbb5 --- /dev/null +++ b/examples/types_field_sa.py @@ -0,0 +1,79 @@ +import asyncio + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ARRAY, ENUM, JSON +from sqlalchemy.sql.ddl import CreateTable + +from aiopg.sa import create_engine + +metadata = sa.MetaData() + + +class CustomStrList(sa.types.TypeDecorator): + impl = sa.types.String + + def __init__(self, sep=',', *args, **kwargs): + self._sep = sep + self._args = args + self._kwargs = kwargs + super().__init__(*args, **kwargs) + + def process_bind_param(self, value, dialect): + return ('{sep}'.format(sep=self._sep)).join(map(str, value)) + + def process_result_value(self, value, dialect): + if value is None: + return value + + return value.split(self._sep) + + def copy(self): + return CustomStrList(self._sep, *self._args, **self._kwargs) + + +tbl = sa.Table( + 'tbl', metadata, + sa.Column('id', sa.Integer, autoincrement=True, primary_key=True), + sa.Column('json', JSON, default=None), + sa.Column('array_int', ARRAY(sa.Integer), default=list), + sa.Column('enum', ENUM('f', 's', name='s_enum'), default='s'), + sa.Column('custom_list', CustomStrList(), default=list), +) + + +async def insert_tbl(conn, pk, **kwargs): + await conn.execute(tbl.insert().values(**kwargs)) + row = await (await conn.execute(tbl.select())).first() + + assert row.id == pk + + for name, val in kwargs.items(): + assert row[name] == val + + await conn.execute(sa.delete(tbl)) + + +async def create_table(conn): + await conn.execute('DROP TABLE IF EXISTS tbl') + await conn.execute('DROP TYPE IF EXISTS s_enum CASCADE') + await conn.execute("CREATE TYPE s_enum AS ENUM ('f', 's')") + await conn.execute(CreateTable(tbl)) + + +async def go(): + async with create_engine(user='aiopg', + database='aiopg', + host='127.0.0.1', + password='passwd') as engine: + async with engine.acquire() as conn: + await create_table(conn) + async with engine.acquire() as conn: + await insert_tbl(conn, 1) + await insert_tbl(conn, 2, json={'data': 123}) + await insert_tbl(conn, 3, array_int=[1, 3, 4]) + await insert_tbl(conn, 4, enum='f') + await insert_tbl(conn, 5, custom_list=['1', 'test', '4']) + + +loop = asyncio.get_event_loop() +loop.run_until_complete(go()) diff --git a/requirements.txt b/requirements.txt index 102bd96a..16148280 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ coverage==4.5.1 docker==3.5.1 flake8==3.5.0 +isort==4.3.4 -e .[sa] sphinx==1.8.1 tox==3.5.2 @@ -9,5 +10,6 @@ pytest-cov==2.6.0 pytest-sugar==0.9.1 pytest-timeout==1.3.2 sphinxcontrib-asyncio==0.2.0 -sqlalchemy==1.2.12 -psycopg2==2.7.5 +psycopg2-binary==2.7.7 +sqlalchemy[postgresql_psycopg2binary]==1.2.12 +ipython==7.3.0 \ No newline at end of file diff --git a/setup.py b/setup.py index c22e239e..c4e41fe4 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,21 @@ import os import re -import sys -from setuptools import setup +from setuptools import setup, find_packages install_requires = ['psycopg2-binary>=2.7.0'] extras_require = {'sa': ['sqlalchemy[postgresql_psycopg2binary]>=1.1']} -PY_VER = sys.version_info - -if PY_VER < (3, 5, 2): - raise RuntimeError("aiopg doesn't support Python earlier than 3.5.2") - def read(f): return open(os.path.join(os.path.dirname(__file__), f)).read().strip() +def get_maintainers(path='MAINTAINERS.txt'): + with open(os.path.join(os.path.dirname(__file__), path)) as f: + return ', '.join(x.strip().strip('*').strip() for x in f.readlines()) + + def read_version(): regexp = re.compile(r"^__version__\W*=\W*'([\d.abrc]+)'") init_py = os.path.join(os.path.dirname(__file__), 'aiopg', '__init__.py') @@ -29,11 +28,15 @@ def read_version(): raise RuntimeError('Cannot find version in aiopg/__init__.py') +def read_changelog(path='CHANGES.txt'): + return 'Changelog\n---------\n\n{}'.format(read(path)) + + classifiers = [ 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -41,25 +44,37 @@ def read_version(): 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Environment :: Web Environment', - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', 'Topic :: Database', 'Topic :: Database :: Front-Ends', 'Framework :: AsyncIO', ] - -setup(name='aiopg', - version=read_version(), - description='Postgres integration with asyncio.', - long_description='\n\n'.join((read('README.rst'), read('CHANGES.txt'))), - classifiers=classifiers, - platforms=['POSIX'], - author='Andrew Svetlov', - author_email='andrew.svetlov@gmail.com', - url='https://aiopg.readthedocs.io', - download_url='https://pypi.python.org/pypi/aiopg', - license='BSD', - packages=['aiopg', 'aiopg.sa'], - install_requires=install_requires, - extras_require=extras_require, - include_package_data=True) +setup( + name='aiopg', + version=read_version(), + description='Postgres integration with asyncio.', + long_description='\n\n'.join((read('README.rst'), read_changelog())), + classifiers=classifiers, + platforms=['macOS', 'POSIX', 'Windows'], + author='Andrew Svetlov', + python_requires='>=3.5.3', + project_urls={ + 'Chat: Gitter': 'https://gitter.im/aio-libs/Lobby', + 'CI: Travis': 'https://travis-ci.com/aio-libs/aiopg', + 'Coverage: codecov': 'https://codecov.io/gh/aio-libs/aiopg', + 'Docs: RTD': 'https://aiopg.readthedocs.io', + 'GitHub: issues': 'https://github.com/aio-libs/aiopg/issues', + 'GitHub: repo': 'https://github.com/aio-libs/aiopg', + }, + author_email='andrew.svetlov@gmail.com', + maintainer=get_maintainers(), + maintainer_email='virmir49@gmail.com', + url='https://aiopg.readthedocs.io', + download_url='https://pypi.python.org/pypi/aiopg', + license='BSD', + packages=find_packages(), + install_requires=install_requires, + extras_require=extras_require, + include_package_data=True +) diff --git a/tests/conftest.py b/tests/conftest.py index 4dee55d2..467e7031 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -93,12 +93,6 @@ def pytest_pyfunc_call(pyfuncitem): return True -def pytest_ignore_collect(path, config): - if 'pep492' in str(path): - if sys.version_info < (3, 5, 0): - return True - - @pytest.fixture(scope='session') def session_id(): '''Unique session identifier, random string.''' @@ -192,13 +186,12 @@ def pg_params(pg_server): def make_connection(loop, pg_params): conns = [] - async def go(*, no_loop=False, **kwargs): - nonlocal conns + async def go(**kwargs): + nonlocal conn params = pg_params.copy() params.update(kwargs) - useloop = None if no_loop else loop - conn = await aiopg.connect(loop=useloop, **params) - conn2 = await aiopg.connect(loop=useloop, **params) + conn = await aiopg.connect(**params) + conn2 = await aiopg.connect(**params) cur = await conn2.cursor() await cur.execute("DROP TABLE IF EXISTS foo") await conn2.close() @@ -212,15 +205,14 @@ async def go(*, no_loop=False, **kwargs): @pytest.fixture -def create_pool(loop, pg_params): +def create_pool(pg_params, loop): pool = None - async def go(*, no_loop=False, **kwargs): + async def go(**kwargs): nonlocal pool params = pg_params.copy() params.update(kwargs) - useloop = None if no_loop else loop - pool = await aiopg.create_pool(loop=useloop, **params) + pool = await aiopg.create_pool(**params) return pool yield go @@ -232,47 +224,36 @@ async def go(*, no_loop=False, **kwargs): @pytest.fixture def make_engine(loop, pg_params): - engine = engine_use_loop = None + engine = None - async def go(*, use_loop=True, **kwargs): - nonlocal engine, engine_use_loop + async def go(**kwargs): + nonlocal engine pg_params.update(kwargs) - if use_loop: - engine_use_loop = engine_use_loop or ( - await sa.create_engine(loop=loop, **pg_params) - ) - return engine_use_loop - else: - engine = engine or (await sa.create_engine(**pg_params)) - return engine + engine = await sa.create_engine(**pg_params) + return engine yield go - if engine_use_loop is not None: - engine_use_loop.close() - loop.run_until_complete(engine_use_loop.wait_closed()) - if engine is not None: engine.close() loop.run_until_complete(engine.wait_closed()) @pytest.fixture -def make_sa_connection(make_engine, loop): - conns = [] +def make_sa_connection(make_engine): + conn = None engine = None - async def go(*, use_loop=True, **kwargs): - nonlocal conns, engine - engine = engine or (await make_engine(use_loop=use_loop, **kwargs)) + async def go(**kwargs): + nonlocal conn, engine + engine = await make_engine(**kwargs) conn = await engine.acquire() - conns.append(conn) return conn yield go - for conn in conns: - loop.run_until_complete(conn.close()) + if conn is not None: + engine.release(conn) class _AssertWarnsContext: diff --git a/tests/pep492/test_async_await.py b/tests/test_async_await.py similarity index 84% rename from tests/pep492/test_async_await.py rename to tests/test_async_await.py index 5e2a72cb..943f3495 100644 --- a/tests/pep492/test_async_await.py +++ b/tests/test_async_await.py @@ -18,8 +18,8 @@ async def test_cursor_await(make_connection): cursor.close() -async def test_connect_context_manager(loop, pg_params): - async with aiopg.connect(loop=loop, **pg_params) as conn: +async def test_connect_context_manager(pg_params): + async with aiopg.connect(**pg_params) as conn: cursor = await conn.cursor() await cursor.execute('SELECT 42') resp = await cursor.fetchone() @@ -53,7 +53,7 @@ async def test_cursor_create_with_context_manager(make_connection): async def test_pool_context_manager_timeout(pg_params, loop): - async with aiopg.create_pool(loop=loop, **pg_params, minsize=1, + async with aiopg.create_pool(**pg_params, minsize=1, maxsize=1) as pool: cursor_ctx = await pool.cursor() with pytest.raises(psycopg2.ProgrammingError): @@ -98,8 +98,8 @@ async def test_cursor_lightweight(make_connection): assert cursor.closed -async def test_pool_context_manager(pg_params, loop): - pool = await aiopg.create_pool(loop=loop, **pg_params) +async def test_pool_context_manager(pg_params): + pool = await aiopg.create_pool(**pg_params) async with pool: conn = await pool.acquire() @@ -112,8 +112,8 @@ async def test_pool_context_manager(pg_params, loop): assert pool.closed -async def test_create_pool_context_manager(pg_params, loop): - async with aiopg.create_pool(loop=loop, **pg_params) as pool: +async def test_create_pool_context_manager(pg_params): + async with aiopg.create_pool(**pg_params) as pool: async with pool.acquire() as conn: async with conn.cursor() as cursor: await cursor.execute('SELECT 42;') @@ -139,8 +139,8 @@ async def test_cursor_aiter(make_connection): assert conn.closed -async def test_engine_context_manager(pg_params, loop): - engine = await aiopg.sa.create_engine(loop=loop, **pg_params) +async def test_engine_context_manager(pg_params): + engine = await aiopg.sa.create_engine(**pg_params) async with engine: conn = await engine.acquire() assert isinstance(conn, SAConnection) @@ -148,17 +148,17 @@ async def test_engine_context_manager(pg_params, loop): assert engine.closed -async def test_create_engine_context_manager(pg_params, loop): - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: +async def test_create_engine_context_manager(pg_params): + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: assert isinstance(conn, SAConnection) assert engine.closed -async def test_result_proxy_aiter(pg_params, loop): +async def test_result_proxy_aiter(pg_params): sql = 'SELECT generate_series(1, 5);' result = [] - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: async with conn.execute(sql) as cursor: async for v in cursor: @@ -168,10 +168,10 @@ async def test_result_proxy_aiter(pg_params, loop): assert conn.closed -async def test_transaction_context_manager(pg_params, loop): +async def test_transaction_context_manager(pg_params): sql = 'SELECT generate_series(1, 5);' result = [] - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: async with conn.begin() as tr: async with conn.execute(sql) as cursor: @@ -194,8 +194,8 @@ async def test_transaction_context_manager(pg_params, loop): assert conn.closed -async def test_transaction_context_manager_error(pg_params, loop): - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: +async def test_transaction_context_manager_error(pg_params): + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: with pytest.raises(RuntimeError) as ctx: async with conn.begin() as tr: @@ -206,8 +206,8 @@ async def test_transaction_context_manager_error(pg_params, loop): assert conn.closed -async def test_transaction_context_manager_commit_once(pg_params, loop): - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: +async def test_transaction_context_manager_commit_once(pg_params): + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: async with conn.begin() as tr: # check that in context manager we do not execute @@ -225,10 +225,10 @@ async def test_transaction_context_manager_commit_once(pg_params, loop): assert conn.closed -async def test_transaction_context_manager_nested_commit(pg_params, loop): +async def test_transaction_context_manager_nested_commit(pg_params): sql = 'SELECT generate_series(1, 5);' result = [] - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: async with conn.begin_nested() as tr1: async with conn.begin_nested() as tr2: @@ -254,10 +254,10 @@ async def test_transaction_context_manager_nested_commit(pg_params, loop): assert conn.closed -async def test_sa_connection_execute(pg_params, loop): +async def test_sa_connection_execute(pg_params): sql = 'SELECT generate_series(1, 5);' result = [] - async with aiopg.sa.create_engine(loop=loop, **pg_params) as engine: + async with aiopg.sa.create_engine(**pg_params) as engine: async with engine.acquire() as conn: async for value in conn.execute(sql): result.append(value) diff --git a/tests/pep492/test_async_transaction.py b/tests/test_async_transaction.py similarity index 99% rename from tests/pep492/test_async_transaction.py rename to tests/test_async_transaction.py index 72256015..6ad4cecd 100644 --- a/tests/pep492/test_async_transaction.py +++ b/tests/test_async_transaction.py @@ -1,5 +1,6 @@ import psycopg2 import pytest + from aiopg import IsolationLevel, Transaction diff --git a/tests/test_connection.py b/tests/test_connection.py index 814f2747..c6a189e4 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,19 +1,19 @@ import asyncio -import aiopg import gc +import socket +import sys +import time +from unittest import mock + import psycopg2 -import psycopg2.extras import psycopg2.extensions +import psycopg2.extras import pytest -import socket -import time -import sys -from aiopg.connection import Connection, TIMEOUT +import aiopg +from aiopg.connection import TIMEOUT, Connection from aiopg.cursor import Cursor from aiopg.utils import ensure_future -from unittest import mock - PY_341 = sys.version_info >= (3, 4, 1) @@ -66,7 +66,7 @@ async def test_simple_select_with_hstore(connect): async def test_default_event_loop(connect, loop): asyncio.set_event_loop(loop) - conn = await connect(no_loop=True) + conn = await connect() cur = await conn.cursor() assert isinstance(cur, Cursor) await cur.execute('SELECT 1') @@ -259,7 +259,6 @@ async def test_cancel_noop(connect): async def test_cancel_pending_op(connect, loop): - def exception_handler(loop_, context): assert context['message'] == context['exception'].pgerror assert context['future'].exception() is context['exception'] @@ -421,7 +420,7 @@ async def go(): {'connection': conn, 'message': 'Fatal error on aiopg connection: ' 'unknown answer 9999 from underlying .poll() call'} - ) + ) assert not conn._writing assert impl.close.called return waiter @@ -451,7 +450,7 @@ async def test_connect_to_unsupported_port(unused_port, loop, pg_params): pg_params['port'] = port with pytest.raises(psycopg2.OperationalError): - await aiopg.connect(loop=loop, **pg_params) + await aiopg.connect(**pg_params) async def test_binary_protocol_error(connect): @@ -523,7 +522,7 @@ async def test_echo(connect): async def test___del__(loop, pg_params, warning): exc_handler = mock.Mock() loop.set_exception_handler(exc_handler) - conn = await aiopg.connect(loop=loop, **pg_params) + conn = await aiopg.connect(**pg_params) with warning(ResourceWarning): del conn gc.collect() @@ -563,10 +562,10 @@ async def test_close_cursor_on_timeout_error(connect): conn.close() -async def test_issue_111_crash_on_connect_error(loop): +async def test_issue_111_crash_on_connect_error(): import aiopg.connection with pytest.raises(psycopg2.ProgrammingError): - await aiopg.connection.connect('baddsn:1', loop=loop) + await aiopg.connection.connect('baddsn:1') async def test_remove_reader_from_alive_fd(connect): diff --git a/tests/test_pool.py b/tests/test_pool.py index 06b10f92..d06af613 100644 --- a/tests/test_pool.py +++ b/tests/test_pool.py @@ -1,11 +1,11 @@ import asyncio from unittest import mock -import pytest +import pytest from psycopg2.extensions import TRANSACTION_STATUS_INTRANS import aiopg -from aiopg.connection import Connection, TIMEOUT +from aiopg.connection import TIMEOUT, Connection from aiopg.pool import Pool @@ -71,8 +71,7 @@ async def test_release_closed(create_pool): async def test_bad_context_manager_usage(create_pool): pool = await create_pool() with pytest.raises(RuntimeError): - with pool: - pass + with pool: pass # noqa async def test_context_manager(create_pool): @@ -212,7 +211,7 @@ async def test_release_with_invalid_status(create_pool): async def test_default_event_loop(create_pool, loop): asyncio.set_event_loop(loop) - pool = await create_pool(no_loop=True) + pool = await create_pool() assert pool._loop is loop @@ -246,7 +245,7 @@ async def test_release_with_invalid_status_wait_release(create_pool): ) -async def test__fill_free(create_pool, loop): +async def test_fill_free(create_pool, loop): pool = await create_pool(minsize=1) with (await pool): assert 0 == pool.freesize @@ -434,8 +433,8 @@ async def test_close_with_acquired_connections(create_pool, loop): await asyncio.wait_for(pool.wait_closed(), 0.1, loop=loop) -async def test___del__(loop, pg_params, warning): - pool = await aiopg.create_pool(loop=loop, **pg_params) +async def test___del__(pg_params, warning): + pool = await aiopg.create_pool(**pg_params) with warning(ResourceWarning): del pool @@ -513,7 +512,6 @@ async def sleep(conn): async def test_drop_connection_if_timedout(make_connection, create_pool, loop): - async def _kill_connections(): # Drop all connections on server conn = await make_connection() diff --git a/tests/test_sa_connection.py b/tests/test_sa_connection.py index 1ba7793d..aa95a345 100644 --- a/tests/test_sa_connection.py +++ b/tests/test_sa_connection.py @@ -1,16 +1,11 @@ -from aiopg import Cursor - from unittest import mock -import pytest -sa = pytest.importorskip("aiopg.sa") # noqa - - -from sqlalchemy import MetaData, Table, Column, Integer, String, select, func -from sqlalchemy.schema import DropTable, CreateTable - import psycopg2 +import pytest +from sqlalchemy import Column, Integer, MetaData, String, Table, func, select +from sqlalchemy.schema import CreateTable, DropTable +from aiopg import Cursor, sa meta = MetaData() tbl = Table('sa_tbl', meta, @@ -122,7 +117,6 @@ async def test_execute_sa_insert_positional_params(connect): async def test_scalar(connect): conn = await connect() - tbl.count res = await conn.scalar(select([func.count()]).select_from(tbl)) assert 1, res diff --git a/tests/test_sa_distil.py b/tests/test_sa_distil.py index 44d847ae..f7d22f25 100644 --- a/tests/test_sa_distil.py +++ b/tests/test_sa_distil.py @@ -1,8 +1,9 @@ import pytest -pytest.importorskip("aiopg.sa") # noqa from aiopg.sa.connection import _distill_params +pytest.importorskip("aiopg.sa") # noqa + def test_distill_none(): assert _distill_params(None, None) == [] diff --git a/tests/test_sa_engine.py b/tests/test_sa_engine.py index 94ea6190..1d44423b 100644 --- a/tests/test_sa_engine.py +++ b/tests/test_sa_engine.py @@ -1,11 +1,13 @@ import asyncio -from aiopg.connection import TIMEOUT -from psycopg2.extensions import parse_dsn import pytest +from psycopg2.extensions import parse_dsn +from sqlalchemy import Column, Integer, MetaData, String, Table + +from aiopg.connection import TIMEOUT + sa = pytest.importorskip("aiopg.sa") # noqa -from sqlalchemy import MetaData, Table, Column, Integer, String meta = MetaData() tbl = Table('sa_tbl3', meta, @@ -64,7 +66,7 @@ def test_freesize(engine): async def test_make_engine_with_default_loop(make_engine, loop): asyncio.set_event_loop(loop) - engine = await make_engine(use_loop=False) + engine = await make_engine() engine.close() await engine.wait_closed() diff --git a/tests/pep492/test_sa_priority_name.py b/tests/test_sa_priority_name.py similarity index 100% rename from tests/pep492/test_sa_priority_name.py rename to tests/test_sa_priority_name.py diff --git a/tests/test_sa_transaction.py b/tests/test_sa_transaction.py index 88ed7ffb..d40afd68 100644 --- a/tests/test_sa_transaction.py +++ b/tests/test_sa_transaction.py @@ -1,9 +1,9 @@ from unittest import mock import pytest -sa = pytest.importorskip("aiopg.sa") # noqa +from sqlalchemy import Column, Integer, MetaData, String, Table, func, select -from sqlalchemy import MetaData, Table, Column, Integer, String, select, func +from aiopg import sa meta = MetaData() tbl = Table('sa_tbl2', meta, diff --git a/tests/test_sa_types.py b/tests/test_sa_types.py index afd0816c..49e76c82 100644 --- a/tests/test_sa_types.py +++ b/tests/test_sa_types.py @@ -2,12 +2,11 @@ import psycopg2 import pytest -sa = pytest.importorskip("aiopg.sa") # noqa - -from sqlalchemy import MetaData, Table, Column, Integer, types +from sqlalchemy import Column, Integer, MetaData, Table, types +from sqlalchemy.dialects.postgresql import ARRAY, ENUM, HSTORE, JSON from sqlalchemy.schema import CreateTable, DropTable -from sqlalchemy.dialects.postgresql import ARRAY, JSON, HSTORE, ENUM +sa = pytest.importorskip("aiopg.sa") # noqa meta = MetaData() diff --git a/tests/test_version.py b/tests/test_version.py index 2b76e033..b16b53e7 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -14,8 +14,8 @@ def test_beta(): def test_rc(): - assert (0, 1, 2, 'candidate', 5) == _parse_version('0.1.2c5') - assert (0, 1, 2, 'candidate', 0) == _parse_version('0.1.2c') + assert (0, 1, 2, 'candidate', 5) == _parse_version('0.1.2rc5') + assert (0, 1, 2, 'candidate', 0) == _parse_version('0.1.2rc') def test_final():