From be3db859110e029f56e71509c6513cf97a910af6 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Thu, 9 May 2024 20:22:22 +0200 Subject: [PATCH 01/29] Update submodule --- app/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common b/app/common index 2962bb74..bdedd8c6 160000 --- a/app/common +++ b/app/common @@ -1 +1 @@ -Subproject commit 2962bb74aef036f1c4c33ae7a34f76e1f2cfc231 +Subproject commit bdedd8c6926a712fe6669a800e5210ef9ccf47d2 From 2570d92c392e8e91159aa94ccbd2753124f3a8d1 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Thu, 9 May 2024 20:22:31 +0200 Subject: [PATCH 02/29] Fix maps with status `-3` --- app/clients/handler.py | 19 ++++++++++++------- app/clients/versions/b388/encoder.py | 1 + 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/app/clients/handler.py b/app/clients/handler.py index fd6db009..45eb6599 100644 --- a/app/clients/handler.py +++ b/app/clients/handler.py @@ -420,14 +420,19 @@ def beatmap_info(player: Player, info: bBeatmapInfoRequest, ignore_limit: bool = map_infos: List[bBeatmapInfo] = [] for index, beatmap in maps: + if beatmap.status <= -3: + # Not submitted + continue + ranked = { - -2: 0, # Graveyard: Pending - -1: 0, # WIP: Pending - 0: 0, # Pending: Pending - 1: 1, # Ranked: Ranked - 2: 2, # Approved: Approved - 3: 2, # Qualified: Approved - 4: 2, # Loved: Approved + -3: -1, # Not submitted + -2: 0, # Graveyard: Pending + -1: 0, # WIP: Pending + 0: 0, # Pending: Pending + 1: 1, # Ranked: Ranked + 2: 2, # Approved: Approved + 3: 2, # Qualified: Approved + 4: 2, # Loved: Approved }[beatmap.status] # Get personal best in every mode for this beatmap diff --git a/app/clients/versions/b388/encoder.py b/app/clients/versions/b388/encoder.py index 3fcd5d93..65d3df77 100644 --- a/app/clients/versions/b388/encoder.py +++ b/app/clients/versions/b388/encoder.py @@ -21,6 +21,7 @@ def beatmap_info_reply(reply: bBeatmapInfoReply): for info in reply.beatmaps: # Approved status does not exist info.ranked = { + -1: -1, 0: 0, 1: 1, 2: 1 From b2d44749bc2f537c6c227e45361b51b7718fe0e9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 02:27:03 +0000 Subject: [PATCH 03/29] Bump boto3 from 1.34.98 to 1.34.103 Bumps [boto3](https://github.com/boto/boto3) from 1.34.98 to 1.34.103. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.34.98...1.34.103) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b949be1c..ff58251d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ twisted==24.3.0 redis==5.0.4 requests==2.31.0 boto3-type-annotations-with-docs==0.3.1 -boto3==1.34.98 +boto3==1.34.103 pytz==2024.1 pytimeparse==1.1.8 geoip2==4.8.0 From 988de15bf1aa02af84973d0e5f50e0536fb8c556 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Wed, 15 May 2024 18:48:22 +0200 Subject: [PATCH 04/29] Add peppy message handler --- app/clients/handler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/clients/handler.py b/app/clients/handler.py index 45eb6599..9819d758 100644 --- a/app/clients/handler.py +++ b/app/clients/handler.py @@ -206,6 +206,11 @@ def send_message(player: Player, message: bMessage): @register(RequestPacket.SEND_PRIVATE_MESSAGE) def send_private_message(sender: Player, message: bMessage): + if message.target == 'peppy': + # This could be an internal osu! anti-cheat message + officer.call(f'{sender.name} tried to message peppy: "{message.content}"') + return + if not (target := session.players.by_name(message.target)): sender.revoke_channel(message.target) return From 72be0af2ce603718ae6196ad4ff34732d2b87234 Mon Sep 17 00:00:00 2001 From: Levi Date: Thu, 16 May 2024 09:27:37 +0200 Subject: [PATCH 05/29] Fix private messages in protocol version 13 --- app/clients/versions/b20121223/decoder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/clients/versions/b20121223/decoder.py b/app/clients/versions/b20121223/decoder.py index 19393856..f7f13ef6 100644 --- a/app/clients/versions/b20121223/decoder.py +++ b/app/clients/versions/b20121223/decoder.py @@ -18,3 +18,7 @@ def wrapper(func) -> Callable: @register(RequestPacket.SEND_MESSAGE) def message(stream: StreamIn): return Reader(stream).read_message() + +@register(RequestPacket.SEND_PRIVATE_MESSAGE) +def private_message(stream: StreamIn): + return Reader(stream).read_message() From 9b0d2447b3903ba429b064497054767eff1180f4 Mon Sep 17 00:00:00 2001 From: Levi Date: Thu, 16 May 2024 09:28:46 +0200 Subject: [PATCH 06/29] Fix away messages in protocol version 13 --- app/clients/versions/b20121223/decoder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/clients/versions/b20121223/decoder.py b/app/clients/versions/b20121223/decoder.py index f7f13ef6..fe8e7c43 100644 --- a/app/clients/versions/b20121223/decoder.py +++ b/app/clients/versions/b20121223/decoder.py @@ -22,3 +22,7 @@ def message(stream: StreamIn): @register(RequestPacket.SEND_PRIVATE_MESSAGE) def private_message(stream: StreamIn): return Reader(stream).read_message() + +@register(RequestPacket.SET_AWAY_MESSAGE) +def away_message(stream: StreamIn): + return Reader(stream).read_message() From 7872f95f7aef7ed71374fec4577fb2dc57ab1492 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Thu, 16 May 2024 09:44:20 +0200 Subject: [PATCH 07/29] Fix message encoding in protocol version 13 --- app/clients/versions/b20121223/encoder.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/clients/versions/b20121223/encoder.py b/app/clients/versions/b20121223/encoder.py index aa7dee53..29f85256 100644 --- a/app/clients/versions/b20121223/encoder.py +++ b/app/clients/versions/b20121223/encoder.py @@ -20,3 +20,21 @@ def message(message: bMessage): writer = Writer() writer.write_message(message) return writer.stream.get() + +@register(ResponsePacket.TARGET_IS_SILENCED) +def target_silenced(msg: bMessage): + writer = Writer() + writer.write_message(msg) + return writer.stream.get() + +@register(ResponsePacket.USER_DM_BLOCKED) +def dm_blocked(msg: bMessage): + writer = Writer() + writer.write_message(msg) + return writer.stream.get() + +@register(ResponsePacket.INVITE) +def match_invite(msg: bMessage): + writer = Writer() + writer.write_message(msg) + return writer.stream.get() \ No newline at end of file From 42d37df5945c468b9607d6602e6f70f201ee3c60 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Thu, 16 May 2024 09:44:27 +0200 Subject: [PATCH 08/29] Update submodule --- app/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common b/app/common index bdedd8c6..2f08d6cf 160000 --- a/app/common +++ b/app/common @@ -1 +1 @@ -Subproject commit bdedd8c6926a712fe6669a800e5210ef9ccf47d2 +Subproject commit 2f08d6cf5fc59d9af0b5bc3d9c836596703d670d From 3402d31d5ef88588957a4db1a444335a401032e9 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Fri, 17 May 2024 18:46:19 +0200 Subject: [PATCH 09/29] Extend try/catch block in login request handler --- app/http.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/http.py b/app/http.py index 54a94f85..65e10e4e 100644 --- a/app/http.py +++ b/app/http.py @@ -76,14 +76,14 @@ def handle_login_request(self, request: Request) -> bytes: request.setResponseCode(403) return b'' - username, password, client_data = ( - request.content.read().decode().splitlines() - ) + try: + username, password, client_data = ( + request.content.read().decode().splitlines() + ) - ip_address = ip.resolve_ip_address_twisted(request) - client = OsuClient.from_string(client_data, ip_address) + ip_address = ip.resolve_ip_address_twisted(request) + client = OsuClient.from_string(client_data, ip_address) - try: self.player = HttpPlayer( ip_address, request.getClientAddress().port From 0f1c78c07b3bf009614499f2bb9e7468bcbc0adb Mon Sep 17 00:00:00 2001 From: Lekuru Date: Fri, 17 May 2024 18:46:23 +0200 Subject: [PATCH 10/29] Update submodule --- app/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common b/app/common index 2f08d6cf..87aa5720 160000 --- a/app/common +++ b/app/common @@ -1 +1 @@ -Subproject commit 2f08d6cf5fc59d9af0b5bc3d9c836596703d670d +Subproject commit 87aa5720fa23948789e3a38b072040565c130deb From 2ba14f6a8689d7b501bc218e5257c2c2f9d2f8c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 02:22:54 +0000 Subject: [PATCH 11/29] Bump boto3 from 1.34.103 to 1.34.108 Bumps [boto3](https://github.com/boto/boto3) from 1.34.103 to 1.34.108. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.34.103...1.34.108) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ff58251d..b94239e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ twisted==24.3.0 redis==5.0.4 requests==2.31.0 boto3-type-annotations-with-docs==0.3.1 -boto3==1.34.103 +boto3==1.34.108 pytz==2024.1 pytimeparse==1.1.8 geoip2==4.8.0 From bed43abee654e7aa2d309cd15daac5623570b479 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:20:44 +0200 Subject: [PATCH 12/29] Update submodule --- app/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common b/app/common index 87aa5720..5039812a 160000 --- a/app/common +++ b/app/common @@ -1 +1 @@ -Subproject commit 87aa5720fa23948789e3a38b072040565c130deb +Subproject commit 5039812ae764f0c9b5a2dd3759cffcee45a03498 From 7dd1c4fbda210e96bc86c3bac5984711926e5d03 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:21:57 +0200 Subject: [PATCH 13/29] Refactor jobs/tasks system --- app/__init__.py | 2 +- app/jobs/__init__.py | 44 ----------------------------- app/session.py | 4 +-- app/tasks/__init__.py | 46 +++++++++++++++++++++++++++++++ app/{jobs => tasks}/activities.py | 6 ++-- app/{jobs => tasks}/events.py | 2 +- app/{jobs => tasks}/pings.py | 8 +++--- main.py | 12 ++++---- 8 files changed, 63 insertions(+), 61 deletions(-) delete mode 100644 app/jobs/__init__.py create mode 100644 app/tasks/__init__.py rename app/{jobs => tasks}/activities.py (77%) rename app/{jobs => tasks}/events.py (87%) rename app/{jobs => tasks}/pings.py (84%) diff --git a/app/__init__.py b/app/__init__.py index 2f9678c0..3d329d48 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,4 +4,4 @@ from . import server from . import events from . import common -from . import jobs +from . import tasks diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py deleted file mode 100644 index 1924f49f..00000000 --- a/app/jobs/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ - -from concurrent.futures import ThreadPoolExecutor -from concurrent.futures._base import Future -from typing import Any, Callable, Tuple - -from . import events -from . import pings - -import logging -import time - -class Jobs(ThreadPoolExecutor): - def __init__(self, max_workers = None, thread_name_prefix: str = "job", initializer = None, initargs: Tuple[Any, ...] = ...) -> None: - super().__init__(max_workers, thread_name_prefix, initializer, initargs) - - self.logger = logging.getLogger('anchor') - - def submit(self, fn: Callable, *args, **kwargs) -> Future: - future = super().submit(fn, *args, **kwargs) - future.add_done_callback(self.__future_callback) - self.logger.info(f' - Starting job: "{fn.__name__}"') - return future - - def sleep(self, seconds: int): - for _ in range(int(seconds)): - if self._shutdown: - # Exit thread - exit() - - time.sleep(1) - - def __future_callback(self, future: Future): - if e := future.exception(): - if isinstance(e, SystemExit): - return - - self.logger.error( - f'Future {future.__repr__()} raised an exception: {e}', - exc_info=e - ) - - self.logger.debug( - f'Result for job {future.__repr__()}: {future.result()}' - ) diff --git a/app/session.py b/app/session.py index 5f534c60..9d1d70a2 100644 --- a/app/session.py +++ b/app/session.py @@ -4,7 +4,7 @@ from .clients import DefaultResponsePacket from .common.database import Postgres from .common.storage import Storage -from .jobs import Jobs +from .tasks import Tasks from typing import Callable, Dict from requests import Session @@ -44,4 +44,4 @@ storage = Storage() players = Players() matches = Matches() -jobs = Jobs() +tasks = Tasks() diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py new file mode 100644 index 00000000..2e2f0612 --- /dev/null +++ b/app/tasks/__init__.py @@ -0,0 +1,46 @@ + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures._base import Future +from typing import Callable + +from . import activities +from . import events +from . import pings + +import logging +import time + +class Tasks(ThreadPoolExecutor): + def __init__(self) -> None: + super().__init__(thread_name_prefix='task') + self.logger = logging.getLogger('anchor') + + def submit(self, fn: Callable, *args, **kwargs) -> Future: + """Submit a task to the threadpool""" + future = super().submit(fn, *args, **kwargs) + future.add_done_callback(self.future_callback) + self.logger.info(f' - Starting task: "{fn.__name__}"') + return future + + def sleep(self, seconds: int, interval: int = 1): + """Custom sleep function to check for application shutdowns""" + for _ in range(0, seconds, interval): + if self._shutdown: + exit() + + time.sleep(interval) + + def future_callback(self, future: Future): + """Callback function for a task/future""" + if e := future.exception(): + if isinstance(e, SystemExit): + return + + self.logger.error( + f'Future {future.__repr__()} raised an exception: {e}', + exc_info=e + ) + + self.logger.debug( + f'Result for task {future.__repr__()}: {future.result()}' + ) diff --git a/app/jobs/activities.py b/app/tasks/activities.py similarity index 77% rename from app/jobs/activities.py rename to app/tasks/activities.py index 87316547..562174d0 100644 --- a/app/jobs/activities.py +++ b/app/tasks/activities.py @@ -6,9 +6,9 @@ MATCH_TIMEOUT_SECONDS = MATCH_TIMEOUT_MINUTES * 60 def match_activity(): - """This job will close any matches that have not been active in the last 15 minutes.""" + """This task will close any matches that have not been active in the last 15 minutes.""" while True: - if app.session.jobs._shutdown: + if app.session.tasks._shutdown: exit() for match in app.session.matches.active: @@ -23,4 +23,4 @@ def match_activity(): ) match.close() - app.session.jobs.sleep(5) + app.session.tasks.sleep(5) diff --git a/app/jobs/events.py b/app/tasks/events.py similarity index 87% rename from app/jobs/events.py rename to app/tasks/events.py index bc6cba2f..12e6c2b5 100644 --- a/app/jobs/events.py +++ b/app/tasks/events.py @@ -5,7 +5,7 @@ def event_listener(): """This will listen for redis pubsub events and call the appropriate functions.""" events = app.session.events.listen() - if app.session.jobs._shutdown: + if app.session.tasks._shutdown: exit() for func, args, kwargs in events: diff --git a/app/jobs/pings.py b/app/tasks/pings.py similarity index 84% rename from app/jobs/pings.py rename to app/tasks/pings.py index a08697d9..353823d0 100644 --- a/app/jobs/pings.py +++ b/app/tasks/pings.py @@ -8,7 +8,7 @@ def ping(): """ - This job will handle client pings and timeouts. Pings are required for tcp clients, to keep them connected. + This task will handle client pings and timeouts. Pings are required for tcp clients, to keep them connected. For http clients, we can just check if they have responded within the timeout period, and close the connection if not. """ next_ping = (time.time() - PING_INTERVAL) @@ -40,13 +40,13 @@ def ping(): player.logger.warning('Client timed out!') player.close_connection() -def ping_job(): +def ping_task(): while True: - if app.session.jobs._shutdown: + if app.session.tasks._shutdown: exit() try: ping() time.sleep(1) except Exception as e: - officer.call(f'Ping job failed: {e}', exc_info=e) + officer.call(f'Ping task failed: {e}', exc_info=e) diff --git a/main.py b/main.py index 9977a9fd..f9156e3f 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ from app.common.logging import Console, File from app.objects.channel import Channel from app.objects.player import Player -from app.jobs import ( +from app.tasks import ( activities, events, pings @@ -57,10 +57,10 @@ def setup(): app.session.bot_player = bot_player app.session.logger.info(f' - {bot_player.name}') - app.session.logger.info('Loading jobs...') - app.session.jobs.submit(pings.ping_job) - app.session.jobs.submit(events.event_listener) - app.session.jobs.submit(activities.match_activity) + app.session.logger.info('Loading task...') + app.session.tasks.submit(pings.ping_task) + app.session.tasks.submit(events.event_listener) + app.session.tasks.submit(activities.match_activity) # Reset usercount usercount.set(0) @@ -86,7 +86,7 @@ def shutdown(): status.delete(player.id) app.session.events.submit('shutdown') - app.session.jobs.shutdown(cancel_futures=True, wait=False) + app.session.tasks.shutdown(cancel_futures=True, wait=False) def force_exit(signal, frame): app.session.logger.warning("Force exiting...") From 5c9a286d9cbe90c6f76fa61001a1274c89e967e8 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:30:47 +0200 Subject: [PATCH 14/29] Remove tcp bancho web request handler --- app/http.py | 2 +- app/tcp.py | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/app/http.py b/app/http.py index 65e10e4e..47af1a09 100644 --- a/app/http.py +++ b/app/http.py @@ -148,7 +148,7 @@ def render_POST(self, request: Request) -> bytes: return self.handle_login_request(request) if not (player := app.session.players.by_token(osu_token)): - # Tell client to reconnect immediately + # Tell client to reconnect immediately (restart packet) return b'\x56\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00' self.player = player diff --git a/app/tcp.py b/app/tcp.py index 2847fd5e..ed483b31 100644 --- a/app/tcp.py +++ b/app/tcp.py @@ -57,13 +57,6 @@ def enqueue(self, data: bytes): exc_info=e ) - def send_web_response(self): - self.enqueue('\r\n'.join([ - 'HTTP/1.1 200 OK', - 'content-type: text/html', - ANCHOR_WEB_RESPONSE - ]).encode()) - def close_connection(self, error: Exception | None = None): if not self.is_local or config.DEBUG: if error: @@ -89,12 +82,6 @@ def dataReceived(self, data: bytes): self.buffer += data.replace(b'\r', b'') self.busy = True - if data.startswith(b'GET /'): - # We received a web request - self.send_web_response() - self.close_connection() - return - if self.buffer.count(b'\n') < 3: return From 576fb88abafea606a4f956816f6ccd55f48227c9 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:41:46 +0200 Subject: [PATCH 15/29] Remove `is_bot` checks --- app/objects/player.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/objects/player.py b/app/objects/player.py index 9ca9cae9..e32164bf 100644 --- a/app/objects/player.py +++ b/app/objects/player.py @@ -387,9 +387,6 @@ def connectionLost(self, reason: Failure = Failure(ConnectionDone())): app.clients.handler.leave_match(self) def send_packet(self, packet: Enum, *args) -> None: - if self.is_bot: - return - try: stream = StreamOut() data = self.encoders[packet](*args) @@ -767,9 +764,6 @@ def check_client(self, session: Session | None = None): self.enqueue_announcement(strings.MULTIACCOUNTING_DETECTED) def packet_received(self, packet_id: int, stream: StreamIn): - if self.is_bot: - return - self.last_response = time.time() try: From 69f174e8c317bd1eaa8814079978c7a4a01e8374 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:42:07 +0200 Subject: [PATCH 16/29] Cleanup of player properties --- app/commands.py | 2 +- app/objects/player.py | 44 ++++++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/app/commands.py b/app/commands.py index c272ab12..b3196e46 100644 --- a/app/commands.py +++ b/app/commands.py @@ -1435,7 +1435,7 @@ def execute( player: Player, target: Channel | Player, command_message: str -): +) -> None: if not command_message.startswith('!'): command_message = f'!{command_message}' diff --git a/app/objects/player.py b/app/objects/player.py index e32164bf..ac7c84d9 100644 --- a/app/objects/player.py +++ b/app/objects/player.py @@ -148,24 +148,32 @@ def bot_player(cls): @property def is_bot(self) -> bool: - return self.object.is_bot if self.object else False + return ( + self.object.is_bot + if self.object else False + ) @property def silenced(self) -> bool: - if self.object.silence_end: - if self.remaining_silence > 0: - return True - else: - # User is not silenced anymore - self.unsilence() - return False - return False + if not self.object.silence_end: + return False + + if self.remaining_silence < 0: + # User is not silenced anymore + self.unsilence() + return False + + return True @property def remaining_silence(self) -> int: - if self.object.silence_end: - return self.object.silence_end.timestamp() - datetime.now().timestamp() - return 0 + if not self.object.silence_end: + return 0 + + return ( + self.object.silence_end.timestamp() - + datetime.now().timestamp() + ) @property def supporter(self) -> bool: @@ -197,11 +205,13 @@ def restricted(self) -> bool: @property def current_stats(self) -> DBStats | None: - for stats in self.stats: - if stats.mode == self.status.mode.value: - return stats - self.logger.warning('Failed to load current stats!') - return None + return next( + ( + stats for stats in self.stats + if stats.mode == self.status.mode.value + ), + None + ) @property def friends(self) -> List[int]: From 0957b745259a8f947d3303152f99977daa25f761 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:48:29 +0200 Subject: [PATCH 17/29] More cleanup in player class --- app/objects/player.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/app/objects/player.py b/app/objects/player.py index ac7c84d9..a5b1a4e9 100644 --- a/app/objects/player.py +++ b/app/objects/player.py @@ -689,14 +689,16 @@ def login_success(self): # Enqueue all public channels for channel in app.session.channels.public: - if channel.can_read(self.permissions): - # Check if channel should be autojoined - if channel.name in config.AUTOJOIN_CHANNELS: - self.enqueue_channel(channel, autojoin=True) - channel.add(self) - continue + if not channel.can_read(self.permissions): + continue - self.enqueue_channel(channel) + # Check if channel should be autojoined + if channel.name in config.AUTOJOIN_CHANNELS: + self.enqueue_channel(channel, autojoin=True) + channel.add(self) + continue + + self.enqueue_channel(channel) self.send_packet(self.packets.CHANNEL_INFO_COMPLETE) @@ -804,8 +806,9 @@ def packet_received(self, packet_id: int, stream: StreamIn): if args != None: handler_function(self, args) - else: - handler_function(self) + return + + handler_function(self) def silence(self, duration_sec: int, reason: str | None = None): if self.is_bot: @@ -1260,8 +1263,11 @@ def enqueue_announcement(self, message: str): message ) + def enqueue_server_restart(self, retry_ms: int): + self.send_packet( + self.packets.RESTART, + retry_ms + ) + def enqueue_monitor(self): self.send_packet(self.packets.MONITOR) - - def enqueue_server_restart(self, retry_ms: int): - self.send_packet(self.packets.RESTART, retry_ms) From 95678de4aa17a4c9aea0f71b882d9192a7dc5778 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:49:32 +0200 Subject: [PATCH 18/29] Remove mania support todos --- app/objects/multiplayer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/objects/multiplayer.py b/app/objects/multiplayer.py index 9dad6e14..b8d1a0fc 100644 --- a/app/objects/multiplayer.py +++ b/app/objects/multiplayer.py @@ -390,7 +390,6 @@ def change_settings(self, new_match: bMatch): if self.mode != new_match.mode: self.mode = new_match.mode self.logger.info(f'Mode: {self.mode.formatted}') - # TODO: Check osu! mania support if self.name != new_match.name: self.name = new_match.name @@ -501,8 +500,6 @@ def start(self): if slot.status == SlotStatus.NotReady: continue - # TODO: Check osu! mania support - slot.player.enqueue_match_start(self.bancho_match) if slot.status != SlotStatus.NoMap: From fa5b5df3102e5dfa655d0419bd6524a4736c523c Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:55:00 +0200 Subject: [PATCH 19/29] Add player type hints to channel class --- app/objects/channel.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/app/objects/channel.py b/app/objects/channel.py index 5d4442a4..489a25e3 100644 --- a/app/objects/channel.py +++ b/app/objects/channel.py @@ -1,8 +1,14 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from app.objects.player import Player + from app.common.database.repositories import messages from app.common.constants.strings import BAD_WORDS from app.common.objects import bMessage, bChannel from app.common.constants import Permissions +from app.objects import collections from app.common import officer import logging @@ -29,10 +35,7 @@ def __init__( self.public = public self.logger = logging.getLogger(self.name) - - from .collections import Players - - self.users = Players() + self.users = collections.Players() def __repr__(self) -> str: return f'<{self.name} - {self.topic}>' @@ -76,6 +79,7 @@ def can_write(self, perms: Permissions): def update(self): if not self.public: + # Only enqueue to users in this channel for player in self.users: player.enqueue_channel( self.bancho_channel, @@ -90,7 +94,7 @@ def update(self): autojoin=False ) - def add(self, player, no_response: bool = False): + def add(self, player: "Player", no_response: bool = False) -> None: # Update player's silence duration player.silenced @@ -114,11 +118,8 @@ def add(self, player, no_response: bool = False): self.logger.info(f'{player.name} joined') - def remove(self, player) -> None: - try: - self.users.remove(player) - except ValueError: - pass + def remove(self, player: "Player") -> None: + self.users.remove(player) if self in player.channels: player.channels.remove(self) @@ -127,7 +128,7 @@ def remove(self, player) -> None: def send_message( self, - sender, + sender: "Player", message: str, ignore_privs=False, exclude_sender=True, From 7e8894b4de5d4f9aa497bbde1b331bd2ca40aa46 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 01:57:24 +0200 Subject: [PATCH 20/29] Add database message limit --- app/clients/handler.py | 2 +- app/objects/channel.py | 2 +- app/objects/client.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/clients/handler.py b/app/clients/handler.py index 9819d758..59304f42 100644 --- a/app/clients/handler.py +++ b/app/clients/handler.py @@ -295,7 +295,7 @@ def send_private_message(sender: Player, message: bMessage): messages.create( sender.name, target.name, - message.content + message.content[:512] ) sender.update_activity() diff --git a/app/objects/channel.py b/app/objects/channel.py index 489a25e3..a4cd8fa7 100644 --- a/app/objects/channel.py +++ b/app/objects/channel.py @@ -225,5 +225,5 @@ def send_message( messages.create( sender.name, self.display_name, - message + message[:512] ) diff --git a/app/objects/client.py b/app/objects/client.py index 887b4013..916513ba 100644 --- a/app/objects/client.py +++ b/app/objects/client.py @@ -6,7 +6,6 @@ from datetime import datetime import hashlib -import utils import pytz import re From f9c76bb2e37ff28a5819351d5084c68c034a400b Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 02:02:57 +0200 Subject: [PATCH 21/29] Fix `NoneType` return in `OsuClient.from_string` --- app/objects/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/objects/client.py b/app/objects/client.py index 916513ba..ea58b656 100644 --- a/app/objects/client.py +++ b/app/objects/client.py @@ -129,9 +129,9 @@ def __init__( self.ip = ip @classmethod - def from_string(cls, line: str, ip: str): + def from_string(cls, line: str, ip: str) -> "OsuClient": if len(args := line.split('|')) < 2: - return + return OsuClient.empty() # Sent in every client version build_version = args[0] @@ -167,7 +167,7 @@ def from_string(cls, line: str, ip: str): ) @classmethod - def empty(cls): + def empty(cls) -> "OsuClient": return OsuClient( location.fetch_geolocation('127.0.0.1'), ClientVersion(OSU_VERSION.match('b1337'), 1337), From 06007a6fb1805df0b7a88788226084a13040ca33 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:18:14 +0200 Subject: [PATCH 22/29] More cleanup --- main.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index f9156e3f..b934639c 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ from twisted.internet import reactor -from app.common.database.repositories import channels +from app.common.database.repositories import channels, wrapper from app.common.cache import status, usercount from app.server import TcpBanchoFactory, HttpBanchoFactory @@ -24,9 +24,11 @@ logging.basicConfig( handlers=[Console, File], - level=logging.DEBUG + level=( + logging.DEBUG if config.DEBUG else logging.INFO + ) ) def setup(): @@ -57,7 +59,7 @@ def setup(): app.session.bot_player = bot_player app.session.logger.info(f' - {bot_player.name}') - app.session.logger.info('Loading task...') + app.session.logger.info('Loading tasks...') app.session.tasks.submit(pings.ping_task) app.session.tasks.submit(events.event_listener) app.session.tasks.submit(activities.match_activity) @@ -72,6 +74,8 @@ def setup(): def before_shutdown(*args): for player in app.session.players: + # Enqueue server restart packet to all players + # They should reconnect after 15 seconds player.enqueue_server_restart(15 * 1000) reactor.callLater(0.5, reactor.stop) @@ -88,27 +92,30 @@ def shutdown(): app.session.events.submit('shutdown') app.session.tasks.shutdown(cancel_futures=True, wait=False) - def force_exit(signal, frame): + def force_exit(*args): app.session.logger.warning("Force exiting...") os._exit(0) signal.signal(signal.SIGINT, force_exit) -def main(): - try: - http_factory = HttpBanchoFactory() - tcp_factory = TcpBanchoFactory() +def on_startup_fail(e: Exception): + app.session.logger.fatal(f'Failed to start server: "{e}"') + reactor.stop() - reactor.suggestThreadPoolSize(config.BANCHO_WORKERS) - reactor.listenTCP(config.HTTP_PORT, http_factory) +@wrapper.exception_wrapper(on_startup_fail) +def setup_servers(): + http_factory = HttpBanchoFactory() + tcp_factory = TcpBanchoFactory() - for port in config.TCP_PORTS: - reactor.listenTCP(port, tcp_factory) - except Exception as e: - app.session.logger.error(f'Failed to start server: "{e}"') - exit(1) + reactor.suggestThreadPoolSize(config.BANCHO_WORKERS) + reactor.listenTCP(config.HTTP_PORT, http_factory) + for port in config.TCP_PORTS: + reactor.listenTCP(port, tcp_factory) + +def main(): reactor.addSystemEventTrigger('before', 'startup', setup) + reactor.addSystemEventTrigger('before', 'startup', setup_servers) reactor.addSystemEventTrigger('after', 'shutdown', shutdown) reactor.run() From d734b4f06dc91cbe3231d7be24eee13ed14de288 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:22:17 +0200 Subject: [PATCH 23/29] Update submodule --- app/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/common b/app/common index 5039812a..02bbad76 160000 --- a/app/common +++ b/app/common @@ -1 +1 @@ -Subproject commit 5039812ae764f0c9b5a2dd3759cffcee45a03498 +Subproject commit 02bbad76a94159d392a460b4a5023757266b1074 From 643ebf6c5e862083c199c61616793b38f29dc5e9 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:35:37 +0200 Subject: [PATCH 24/29] Refactor exception wrapper for `resolve_channel` --- app/clients/handler.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/app/clients/handler.py b/app/clients/handler.py index 59304f42..a169312a 100644 --- a/app/clients/handler.py +++ b/app/clients/handler.py @@ -3,6 +3,7 @@ from . import DefaultRequestPacket as RequestPacket from ..common.database.objects import DBBeatmap, DBScore +from ..common.database.repositories import wrapper from ..objects.multiplayer import Match from ..objects.channel import Channel from ..objects.player import Player @@ -55,25 +56,23 @@ def wrapper(func) -> Callable: return wrapper +@wrapper.exception_wrapper() def resolve_channel(channel_name: str, player: Player) -> Optional[Channel]: - try: - if channel_name == '#spectator': - # Select spectator chat - return ( - player.spectating.spectator_chat - if player.spectating else - player.spectator_chat - ) + if channel_name == '#spectator': + # Select spectator chat + return ( + player.spectating.spectator_chat + if player.spectating else + player.spectator_chat + ) - elif channel_name == '#multiplayer': - # Select multiplayer chat - return player.match.chat + elif channel_name == '#multiplayer': + # Select multiplayer chat + return player.match.chat - # Resolve channel by name - if channel := session.channels.by_name(channel_name): - return channel - except AttributeError: - return + # Resolve channel by name + if channel := session.channels.by_name(channel_name): + return channel @register(RequestPacket.PONG) def pong(player: Player): From 0e97ab14a7fd74d10e6d2b560b95799491ab051a Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:36:00 +0200 Subject: [PATCH 25/29] Change default client version for `OsuClient.empty` --- app/objects/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/objects/client.py b/app/objects/client.py index ea58b656..20b42319 100644 --- a/app/objects/client.py +++ b/app/objects/client.py @@ -170,7 +170,7 @@ def from_string(cls, line: str, ip: str) -> "OsuClient": def empty(cls) -> "OsuClient": return OsuClient( location.fetch_geolocation('127.0.0.1'), - ClientVersion(OSU_VERSION.match('b1337'), 1337), + ClientVersion(OSU_VERSION.match('b20136969'), 20136969), ClientHash('', '', '', '', ''), 0, True, From 3b94d6fa7b9b554e0c0be80a07ea7050df49b339 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:44:41 +0200 Subject: [PATCH 26/29] Cleanup handlers --- app/clients/handler.py | 104 ++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/app/clients/handler.py b/app/clients/handler.py index a169312a..650da09d 100644 --- a/app/clients/handler.py +++ b/app/clients/handler.py @@ -88,10 +88,16 @@ def receive_updates(player: Player, filter: PresenceFilter): player.filter = filter if filter.value <= 0: + # Client set filter to "None" + # No players will be sent return - players = session.players if filter == PresenceFilter.All else \ - player.online_friends + # Account for player filter + players = ( + session.players + if filter == PresenceFilter.All + else player.online_friends + ) player.enqueue_players(players, stats_only=True) @@ -315,17 +321,18 @@ def away_message(player: Player, message: bMessage): is_private=True ) ) - else: - player.away_message = None - player.enqueue_message( - bMessage( - session.bot_player.name, - 'You are no longer marked as being away', - session.bot_player.name, - session.bot_player.id, - is_private=True - ) + return + + player.away_message = None + player.enqueue_message( + bMessage( + session.bot_player.name, + 'You are no longer marked as being away', + session.bot_player.name, + session.bot_player.id, + is_private=True ) + ) @register(RequestPacket.ADD_FRIEND) def add_friend(player: Player, target_id: int): @@ -381,9 +388,7 @@ def beatmap_info(player: Player, info: bBeatmapInfoRequest, ignore_limit: bool = if total_maps <= 0: return - player.logger.info( - f'Got {total_maps} beatmap requests' - ) + player.logger.info(f'Got {total_maps} beatmap requests') # Fetch all matching beatmaps from database with session.database.managed_session() as s: @@ -463,12 +468,12 @@ def beatmap_info(player: Player, info: bBeatmapInfoRequest, ignore_limit: bool = index, beatmap.id, beatmap.set_id, - beatmap.set_id, # thread_id + beatmap.set_id, # ThreadId ranked, - grades[0], # standard - grades[2], # fruits - grades[1], # taiko - grades[3], # mania + grades[0], # Standard + grades[2], # Fruits + grades[1], # Taiko + grades[3], # Mania beatmap.md5 ) ) @@ -498,7 +503,6 @@ def start_spectating(player: Player, player_id: int): if (player.spectating or player in target.spectators) and not player.is_tourney_client: stop_spectating(player) - # TODO: return here? player.logger.info(f'Started spectating "{target.name}".') player.spectating = target @@ -594,7 +598,6 @@ def invite(player: Player, target_id: int): return # TODO: Check invite spams - target.enqueue_invite( bMessage( player.name, @@ -756,10 +759,11 @@ def leave_match(player: Player): slot = player.match.get_slot(player) assert slot is not None - if slot.status == SlotStatus.Locked: - status = SlotStatus.Locked - else: - status = SlotStatus.Open + status = ( + SlotStatus.Locked + if slot.status == SlotStatus.Locked + else SlotStatus.Open + ) slot.reset(status) @@ -785,12 +789,12 @@ def leave_match(player: Player): player.match.beatmap_name = player.match.previous_beatmap_name if all(slot.empty for slot in player.match.slots): + # No players in match anymore -> Disband match player.enqueue_match_disband(player.match.id) for p in session.players.in_lobby: p.enqueue_match_disband(player.match.id) - # Match is empty session.matches.remove(player.match) player.match.starting = None @@ -809,25 +813,26 @@ def leave_match(player: Player): events.create(match_id, type=EventType.Disband) player.match.logger.info('Match was disbanded.') - else: - if player is player.match.host: - # Player was host, transfer to next player - for slot in player.match.slots: - if slot.status.value & SlotStatus.HasPlayer.value: - player.match.host = slot.player - player.match.host.enqueue_match_transferhost() - - events.create( - player.match.db_match.id, - type=EventType.Host, - data={ - 'previous': {'id': player.id, 'name': player.name}, - 'new': {'id': player.match.host.id, 'name': player.match.host.name} - } - ) + player.match = None + return - player.match.update() + if player is player.match.host: + # Player was host, transfer to next player + for slot in player.match.slots: + if slot.status.value & SlotStatus.HasPlayer.value: + player.match.host = slot.player + player.match.host.enqueue_match_transferhost() + + events.create( + player.match.db_match.id, + type=EventType.Host, + data={ + 'previous': {'id': player.id, 'name': player.name}, + 'new': {'id': player.match.host.id, 'name': player.match.host.name} + } + ) + player.match.update() player.match = None @register(RequestPacket.MATCH_CHANGE_SLOT) @@ -914,7 +919,7 @@ def change_mods(player: Player, mods: Mods): if player.match.freemod: if player is player.match.host: - # Onky keep SpeedMods + # Only keep SpeedMods player.match.mods = mods & Mods.SpeedMods # There is a bug, where DT and NC are enabled at the same time @@ -1016,10 +1021,11 @@ def lock(player: Player, slot_id: int): if slot.has_player: player.match.kick_player(slot.player) - if slot.status == SlotStatus.Locked: - slot.status = SlotStatus.Open - else: - slot.status = SlotStatus.Locked + slot.status = ( + SlotStatus.Open + if slot.status == SlotStatus.Locked + else SlotStatus.Locked + ) player.match.update() From 83245e07fb5aabab572130b55da669eb861c78cd Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:46:05 +0200 Subject: [PATCH 27/29] Remove utils module --- app/commands.py | 1 - utils.py | 12 ------------ 2 files changed, 13 deletions(-) delete mode 100644 utils.py diff --git a/app/commands.py b/app/commands.py index b3196e46..65667181 100644 --- a/app/commands.py +++ b/app/commands.py @@ -44,7 +44,6 @@ import config import random import shlex -import utils import time import app import os diff --git a/utils.py b/utils.py deleted file mode 100644 index 7b5f09ec..00000000 --- a/utils.py +++ /dev/null @@ -1,12 +0,0 @@ - -from twisted.python.failure import Failure -from twisted.web.http import Request - -import config -import app - -def thread_callback(error: Failure): - app.session.logger.error( - f'Failed to execute thread: {error.__str__()} ({error.getErrorMessage()})', - exc_info=error.value - ) From 9afad9a7228569513d40ed6c72fb1189a8d3faa6 Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 16:56:04 +0200 Subject: [PATCH 28/29] Add command threading support --- app/commands.py | 141 +++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 61 deletions(-) diff --git a/app/commands.py b/app/commands.py index 65667181..d0655b27 100644 --- a/app/commands.py +++ b/app/commands.py @@ -4,6 +4,7 @@ from typing import List, NamedTuple, Callable from pytimeparse.timeparse import timeparse from datetime import timedelta, datetime +from twisted.internet import threads from dataclasses import dataclass from threading import Thread @@ -38,7 +39,6 @@ from .objects.channel import Channel from .common.objects import bMessage from .objects.player import Player -from .tcp import TcpBanchoProtocol import timeago import config @@ -1350,37 +1350,39 @@ def get_command( # Regular commands for command in commands: - if trigger in command.triggers: - has_permissions = any( - group in command.groups - for group in player.groups - ) + if trigger not in command.triggers: + continue - if not has_permissions: - return - - # Try running the command - try: - response = command.callback( - Context( - player, - trigger, - target, - args - ) - ) - except Exception as e: - player.logger.error( - f'Command error: {e}', - exc_info=e - ) + has_permissions = any( + group in command.groups + for group in player.groups + ) - response = ['An error occurred while running this command.'] + if not has_permissions: + return - return CommandResponse( - response, - command.hidden + # Try running the command + try: + response = command.callback( + Context( + player, + trigger, + target, + args + ) ) + except Exception as e: + player.logger.error( + f'Command error: {e}', + exc_info=e + ) + + response = ['An error occurred while running this command.'] + + return CommandResponse( + response, + command.hidden + ) try: set_trigger, trigger, *args = trigger, *args @@ -1393,43 +1395,46 @@ def get_command( continue for command in set.commands: - if trigger in command.triggers: - has_permissions = any( - group in command.groups - for group in player.groups - ) + if trigger not in command.triggers: + continue - if not has_permissions: - continue + has_permissions = any( + group in command.groups + for group in player.groups + ) - ctx = Context( - player, - trigger, - target, - args - ) + if not has_permissions: + continue + + ctx = Context( + player, + trigger, + target, + args + ) - # Check set conditions - for condition in set.conditions: - if not condition(ctx): - break - else: - # Try running the command - try: - response = command.callback(ctx) - except Exception as e: - player.logger.error( - f'Command error: {e}', - exc_info=e - ) - - response = ['An error occurred while running this command.'] - - return CommandResponse( - response, - command.hidden + # Check set conditions + for condition in set.conditions: + if not condition(ctx): + break + + else: + # Try running the command + try: + response = command.callback(ctx) + except Exception as e: + player.logger.error( + f'Command error: {e}', + exc_info=e ) + response = ['An error occurred while running this command.'] + + return CommandResponse( + response, + command.hidden + ) + def execute( player: Player, target: Channel | Player, @@ -1438,12 +1443,26 @@ def execute( if not command_message.startswith('!'): command_message = f'!{command_message}' - command = get_command( + threads.deferToThread( + get_command, player, target, command_message + ).addCallback( + lambda result: on_command_done( + result, + player, + target, + command_message + ) ) +def on_command_done( + command: CommandResponse, + player: Player, + target: Channel | Player, + command_message: str +) -> None: if not command: return From eca8220fe908cbcb6ecd7d5fdcf42d91c32c8d9d Mon Sep 17 00:00:00 2001 From: Lekuru Date: Tue, 21 May 2024 17:00:54 +0200 Subject: [PATCH 29/29] Update release version to `1.3.0` --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index fc03e146..74a9da8f 100644 --- a/config.py +++ b/config.py @@ -53,4 +53,4 @@ DATA_PATH = os.path.abspath('.data') MULTIPLAYER_MAX_SLOTS = 8 PROTOCOL_VERSION = 18 -VERSION = '1.2.10' +VERSION = '1.3.0'