Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Add type hints to schema deltas (#15497)
Browse files Browse the repository at this point in the history
Cleans-up the schema delta files:

* Removes no-op functions.
* Adds missing type hints to function parameters.
* Fixes any issues with type hints.

This also renames one (very old) schema delta to avoid a conflict
that mypy complains about.
  • Loading branch information
clokep authored Apr 27, 2023
1 parent a346b43 commit 6efa674
Show file tree
Hide file tree
Showing 30 changed files with 132 additions and 144 deletions.
1 change: 1 addition & 0 deletions changelog.d/15497.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Improve type hints.
10 changes: 0 additions & 10 deletions mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,6 @@ files =
tests/,
build_rust.py

# Note: Better exclusion syntax coming in mypy > 0.910
# https://github.com/python/mypy/pull/11329
#
# For now, set the (?x) flag enable "verbose" regexes
# https://docs.python.org/3/library/re.html#re.X
exclude = (?x)
^(
|synapse/storage/schema/
)$

[mypy-synapse.metrics._reactor_metrics]
# This module imports select.epoll. That exists on Linux, but doesn't on macOS.
# See https://github.com/matrix-org/synapse/pull/11771.
Expand Down
8 changes: 5 additions & 3 deletions synapse/storage/prepare_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from typing_extensions import Counter as CounterType

from synapse.config.homeserver import HomeServerConfig
from synapse.storage.database import LoggingDatabaseConnection
from synapse.storage.database import LoggingDatabaseConnection, LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.schema import SCHEMA_COMPAT_VERSION, SCHEMA_VERSION
from synapse.storage.types import Cursor
Expand Down Expand Up @@ -168,7 +168,9 @@ def prepare_database(


def _setup_new_database(
cur: Cursor, database_engine: BaseDatabaseEngine, databases: Collection[str]
cur: LoggingTransaction,
database_engine: BaseDatabaseEngine,
databases: Collection[str],
) -> None:
"""Sets up the physical database by finding a base set of "full schemas" and
then applying any necessary deltas, including schemas from the given data
Expand Down Expand Up @@ -289,7 +291,7 @@ def _setup_new_database(


def _upgrade_existing_database(
cur: Cursor,
cur: LoggingTransaction,
current_schema_state: _SchemaState,
database_engine: BaseDatabaseEngine,
config: Optional[HomeServerConfig],
Expand Down
13 changes: 6 additions & 7 deletions synapse/storage/schema/main/delta/20/pushers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,13 @@

import logging

from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
logger.info("Porting pushers table...")
cur.execute(
"""
Expand Down Expand Up @@ -61,8 +64,8 @@ def run_create(cur, database_engine, *args, **kwargs):
"""
)
count = 0
for row in cur.fetchall():
row = list(row)
for tuple_row in cur.fetchall():
row = list(tuple_row)
row[8] = bytes(row[8]).decode("utf-8")
row[11] = bytes(row[11]).decode("utf-8")
cur.execute(
Expand All @@ -81,7 +84,3 @@ def run_create(cur, database_engine, *args, **kwargs):
cur.execute("DROP TABLE pushers")
cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
logger.info("Moved %d pushers to new table", count)


def run_upgrade(*args, **kwargs):
pass
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/25/fts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
import json
import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand All @@ -41,7 +42,7 @@
)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
for statement in get_statements(POSTGRES_TABLE.splitlines()):
cur.execute(statement)
Expand Down Expand Up @@ -72,7 +73,3 @@ def run_create(cur, database_engine, *args, **kwargs):
)

cur.execute(sql, ("event_search", progress_json))


def run_upgrade(*args, **kwargs):
pass
8 changes: 3 additions & 5 deletions synapse/storage/schema/main/delta/27/ts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import json
import logging

from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand All @@ -25,7 +27,7 @@
)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(ALTER_TABLE.splitlines()):
cur.execute(statement)

Expand All @@ -51,7 +53,3 @@ def run_create(cur, database_engine, *args, **kwargs):
)

cur.execute(sql, ("event_origin_server_ts", progress_json))


def run_upgrade(*args, **kwargs):
pass
16 changes: 12 additions & 4 deletions synapse/storage/schema/main/delta/30/as_users.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import Dict, Iterable, List, Tuple, cast

from synapse.config.appservice import load_appservices
from synapse.config.homeserver import HomeServerConfig
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
# NULL indicates user was not registered by an appservice.
try:
cur.execute("ALTER TABLE users ADD COLUMN appservice_id TEXT")
Expand All @@ -27,9 +31,13 @@ def run_create(cur, database_engine, *args, **kwargs):
pass


def run_upgrade(cur, database_engine, config, *args, **kwargs):
def run_upgrade(
cur: LoggingTransaction,
database_engine: BaseDatabaseEngine,
config: HomeServerConfig,
) -> None:
cur.execute("SELECT name FROM users")
rows = cur.fetchall()
rows = cast(Iterable[Tuple[str]], cur.fetchall())

config_files = []
try:
Expand All @@ -39,7 +47,7 @@ def run_upgrade(cur, database_engine, config, *args, **kwargs):

appservices = load_appservices(config.server.server_name, config_files)

owned = {}
owned: Dict[str, List[str]] = {}

for row in rows:
user_id = row[0]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,17 @@

import logging

from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine

logger = logging.getLogger(__name__)


def token_to_stream_ordering(token):
def token_to_stream_ordering(token: str) -> int:
return int(token[1:].split("_")[0])


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
logger.info("Porting pushers table, delta 31...")
cur.execute(
"""
Expand Down Expand Up @@ -61,8 +64,8 @@ def run_create(cur, database_engine, *args, **kwargs):
"""
)
count = 0
for row in cur.fetchall():
row = list(row)
for tuple_row in cur.fetchall():
row = list(tuple_row)
row[12] = token_to_stream_ordering(row[12])
cur.execute(
"""
Expand All @@ -80,7 +83,3 @@ def run_create(cur, database_engine, *args, **kwargs):
cur.execute("DROP TABLE pushers")
cur.execute("ALTER TABLE pushers2 RENAME TO pushers")
logger.info("Moved %d pushers to new table", count)


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/31/search_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
import json
import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand All @@ -26,7 +27,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
return

Expand Down Expand Up @@ -56,7 +57,3 @@ def run_create(cur, database_engine, *args, **kwargs):
)

cur.execute(sql, ("event_search_order", progress_json))


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
8 changes: 3 additions & 5 deletions synapse/storage/schema/main/delta/33/event_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import json
import logging

from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand All @@ -25,7 +27,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(ALTER_TABLE.splitlines()):
cur.execute(statement)

Expand All @@ -51,7 +53,3 @@ def run_create(cur, database_engine, *args, **kwargs):
)

cur.execute(sql, ("event_fields_sender_url", progress_json))


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
12 changes: 10 additions & 2 deletions synapse/storage/schema/main/delta/33/remote_media_ts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,22 @@

import time

from synapse.config.homeserver import HomeServerConfig
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine

ALTER_TABLE = "ALTER TABLE remote_media_cache ADD COLUMN last_access_ts BIGINT"


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
cur.execute(ALTER_TABLE)


def run_upgrade(cur, database_engine, *args, **kwargs):
def run_upgrade(
cur: LoggingTransaction,
database_engine: BaseDatabaseEngine,
config: HomeServerConfig,
) -> None:
cur.execute(
"UPDATE remote_media_cache SET last_access_ts = ?",
(int(time.time() * 1000),),
Expand Down
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/34/cache_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand All @@ -34,13 +35,9 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
if not isinstance(database_engine, PostgresEngine):
return

for statement in get_statements(CREATE_TABLE.splitlines()):
cur.execute(statement)


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/34/received_txn_purge.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,16 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine

logger = logging.getLogger(__name__)


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
if isinstance(database_engine, PostgresEngine):
cur.execute("TRUNCATE received_transactions")
else:
cur.execute("DELETE FROM received_transactions")

cur.execute("CREATE INDEX received_transactions_ts ON received_transactions(ts)")


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/37/remove_auth_idx.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -68,7 +69,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(DROP_INDICES.splitlines()):
cur.execute(statement)

Expand All @@ -79,7 +80,3 @@ def run_create(cur, database_engine, *args, **kwargs):

for statement in get_statements(drop_constraint.splitlines()):
cur.execute(statement)


def run_upgrade(cur, database_engine, *args, **kwargs):
pass
9 changes: 3 additions & 6 deletions synapse/storage/schema/main/delta/42/user_dir.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@

import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.database import LoggingTransaction
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -66,7 +67,7 @@
"""


def run_create(cur, database_engine, *args, **kwargs):
def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
for statement in get_statements(BOTH_TABLES.splitlines()):
cur.execute(statement)

Expand All @@ -78,7 +79,3 @@ def run_create(cur, database_engine, *args, **kwargs):
cur.execute(statement)
else:
raise Exception("Unrecognized database engine")


def run_upgrade(*args, **kwargs):
pass
Loading

0 comments on commit 6efa674

Please sign in to comment.