Merge pull request #284 from osuTitanic/dev

Fix rank indexing job & minimize requirements file

app/jobs/activities.py

@@ -6,6 +6,7 @@
 MATCH_TIMEOUT_SECONDS = MATCH_TIMEOUT_MINUTES * 60
 
 def match_activity():
+    """This job will close any matches that have not been active in the last 15 minutes."""
     while True:
         if app.session.jobs._shutdown:
             exit()

app/jobs/events.py

@@ -2,6 +2,7 @@
 import app
 
 def event_listener():
+    """This will listen for redis pubsub events and call the appropriate functions."""
     events = app.session.events.listen()
 
     if app.session.jobs._shutdown:

app/jobs/pings.py

@@ -7,6 +7,10 @@
 TIMEOUT_SECS  = 45
 
 def ping():
+    """
+    This job will handle client pings and timeouts. Pings are required for tcp clients, to keep them connected.
+    For http clients, we can just check if they have responded within the timeout period, and close the connection if not.
+    """
     next_ping = (time.time() - PING_INTERVAL)
 
     for player in app.session.players.tcp_clients:
@@ -26,14 +30,14 @@ def ping():
         last_response = (time.time() - player.last_response)
 
         if (last_response >= TIMEOUT_SECS):
-            player.logger.warning('Client timed out...')
+            player.logger.warning('Client timed out!')
             player.close_connection()
 
     for player in app.session.players.http_clients:
         last_response = (time.time() - player.last_response)
 
         if last_response >= TIMEOUT_SECS:
-            player.logger.warning('Client timed out...')
+            player.logger.warning('Client timed out!')
             player.close_connection()
 
 def ping_job():
@@ -43,7 +47,6 @@ def ping_job():
 
         try:
             ping()
+            time.sleep(1)
         except Exception as e:
             officer.call(f'Ping job failed: {e}', exc_info=e)
-
-        time.sleep(1)

app/jobs/ranks.py

@@ -0,0 +1,27 @@
+
+from app.common.cache import leaderboards
+from app.common.database import users
+
+import app
+
+def index_ranks():
+    """
+    This will run on startup as a background job, and check if the redis leaderboards are empty.
+    It will then try to re-index every active player in the leaderboards.
+    This should actually never happen, but it can be useful in emergency situations.
+    """
+    if not leaderboards.top_players(0):
+        with app.session.database.managed_session() as session:
+            # Redis cache was flushed
+            active_players = users.fetch_all(session=session)
+
+            app.session.logger.info(f'Indexing player ranks... ({len(active_players)})')
+
+            for player in active_players:
+                for stats in player.stats:
+                    leaderboards.update(
+                        stats,
+                        player.country.lower()
+                    )
+
+            app.session.logger.info('Index complete!')

app/objects/player.py

@@ -133,7 +133,7 @@ def bot_player(cls):
             player.object = users.fetch_by_id(1, session=session)
             player.client = OsuClient.empty()
 
-            player.id = -player.object.id # Negative UserId -> IRC Player
+            player.id = -player.object.id
             player.name = player.object.name
             player.stats  = player.object.stats
 
@@ -148,8 +148,7 @@ def bot_player(cls):
 
     @property
     def is_bot(self) -> bool:
-        # TODO: Refactor bot related code to IRC client
-        return True if self.id == -1 else False
+        return self.object.is_bot if self.object else False
 
     @property
     def silenced(self) -> bool:

main.py

@@ -11,10 +11,10 @@
 from app.objects.channel import Channel
 from app.objects.player import Player
 from app.jobs import (
-    rank_indexing,
     activities,
     events,
-    pings
+    pings,
+    ranks
 )
 
 import logging
@@ -31,8 +31,8 @@
 )
 
 def setup():
-    app.session.logger.info(ANCHOR_ASCII_ART)
-    app.session.logger.info(f'anchor-{config.VERSION}')
+    app.session.logger.info(f'{ANCHOR_ASCII_ART}')
+    app.session.logger.info(f'Running anchor-{config.VERSION}')
     os.makedirs(config.DATA_PATH, exist_ok=True)
 
     app.session.logger.info('Loading channels...')
@@ -61,7 +61,7 @@ def setup():
     app.session.logger.info('Loading jobs...')
     app.session.jobs.submit(pings.ping_job)
     app.session.jobs.submit(events.event_listener)
-    app.session.jobs.submit(rank_indexing.index_ranks)
+    app.session.jobs.submit(ranks.index_ranks)
     app.session.jobs.submit(activities.match_activity)
 
     # Reset usercount
@@ -89,14 +89,18 @@ def force_exit(signal, frame):
     signal.signal(signal.SIGINT, force_exit)
 
 def main():
-    http_factory = HttpBanchoFactory()
-    tcp_factory = TcpBanchoFactory()
-
-    reactor.suggestThreadPoolSize(config.BANCHO_WORKERS)
-    reactor.listenTCP(config.HTTP_PORT, http_factory)
-
-    for port in config.TCP_PORTS:
-        reactor.listenTCP(port, tcp_factory)
+    try:
+        http_factory = HttpBanchoFactory()
+        tcp_factory = TcpBanchoFactory()
+
+        reactor.suggestThreadPoolSize(config.BANCHO_WORKERS)
+        reactor.listenTCP(config.HTTP_PORT, http_factory)
+
+        for port in config.TCP_PORTS:
+            reactor.listenTCP(port, tcp_factory)
+    except Exception as e:
+        app.session.logger.error(f'Failed to start server: "{e}"')
+        exit(1)
 
     reactor.addSystemEventTrigger('before', 'startup', setup)
     reactor.addSystemEventTrigger('after', 'shutdown', shutdown)

requirements.txt

@@ -1,44 +1,15 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
-async-timeout==4.0.3
-attrs==23.2.0
-Automat==22.10.0
-bcrypt==4.1.2
-boto3==1.34.13
-boto3-type-annotations-with-docs==0.3.1
-botocore==1.34.13
-certifi==2023.11.17
-chardet==5.2.0
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-constantly==23.10.4
-frozenlist==1.4.1
-geoip2==4.8.0
-greenlet==3.0.3
-hyperlink==21.0.0
-idna==3.6
-incremental==22.10.0
-jmespath==1.0.1
-maxminddb==2.5.1
-multidict==6.0.4
-psycopg2==2.9.9
-python-dateutil==2.8.2
-python-dotenv==1.0.0
-python-http-client==3.3.7
-pytimeparse==1.1.8
-pytz==2023.3.post1
+titanic-pp-py==0.0.6
+psycopg2-binary==2.9.9
+sqlalchemy==2.0.25
+python-dotenv==1.0.1
+twisted==23.10.0
 redis==5.0.1
 requests==2.31.0
-s3transfer==0.10.0
+boto3-type-annotations==0.3.1
+boto3==1.34.31
+pytz==2023.4
+pytimeparse==1.1.8
+geoip2==4.8.0
+bcrypt==4.1.2
 sendgrid==6.11.0
-six==1.16.0
-SQLAlchemy==2.0.25
-starkbank-ecdsa==2.2.0
-timeago==1.0.16
-titanic-pp-py==0.0.6
-Twisted==23.10.0
-typing-extensions==4.9.0
-urllib3==1.26.18
-yarl==1.9.4
-zope.interface==6.1
+timeago==1.0.16