Skip to content

Commit

Permalink
Merge pull request #284 from osuTitanic/dev
Browse files Browse the repository at this point in the history
Fix rank indexing job & minimize requirements file
  • Loading branch information
Lekuruu authored Jan 31, 2024
2 parents 2f2e2af + fb71dbe commit 29152d3
Show file tree
Hide file tree
Showing 9 changed files with 68 additions and 89 deletions.
2 changes: 1 addition & 1 deletion app/common
1 change: 1 addition & 0 deletions app/jobs/activities.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
MATCH_TIMEOUT_SECONDS = MATCH_TIMEOUT_MINUTES * 60

def match_activity():
"""This job will close any matches that have not been active in the last 15 minutes."""
while True:
if app.session.jobs._shutdown:
exit()
Expand Down
1 change: 1 addition & 0 deletions app/jobs/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import app

def event_listener():
"""This will listen for redis pubsub events and call the appropriate functions."""
events = app.session.events.listen()

if app.session.jobs._shutdown:
Expand Down
11 changes: 7 additions & 4 deletions app/jobs/pings.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
TIMEOUT_SECS = 45

def ping():
"""
This job will handle client pings and timeouts. Pings are required for tcp clients, to keep them connected.
For http clients, we can just check if they have responded within the timeout period, and close the connection if not.
"""
next_ping = (time.time() - PING_INTERVAL)

for player in app.session.players.tcp_clients:
Expand All @@ -26,14 +30,14 @@ def ping():
last_response = (time.time() - player.last_response)

if (last_response >= TIMEOUT_SECS):
player.logger.warning('Client timed out...')
player.logger.warning('Client timed out!')
player.close_connection()

for player in app.session.players.http_clients:
last_response = (time.time() - player.last_response)

if last_response >= TIMEOUT_SECS:
player.logger.warning('Client timed out...')
player.logger.warning('Client timed out!')
player.close_connection()

def ping_job():
Expand All @@ -43,7 +47,6 @@ def ping_job():

try:
ping()
time.sleep(1)
except Exception as e:
officer.call(f'Ping job failed: {e}', exc_info=e)

time.sleep(1)
27 changes: 0 additions & 27 deletions app/jobs/rank_indexing.py

This file was deleted.

27 changes: 27 additions & 0 deletions app/jobs/ranks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@

from app.common.cache import leaderboards
from app.common.database import users

import app

def index_ranks():
"""
This will run on startup as a background job, and check if the redis leaderboards are empty.
It will then try to re-index every active player in the leaderboards.
This should actually never happen, but it can be useful in emergency situations.
"""
if not leaderboards.top_players(0):
with app.session.database.managed_session() as session:
# Redis cache was flushed
active_players = users.fetch_all(session=session)

app.session.logger.info(f'Indexing player ranks... ({len(active_players)})')

for player in active_players:
for stats in player.stats:
leaderboards.update(
stats,
player.country.lower()
)

app.session.logger.info('Index complete!')
5 changes: 2 additions & 3 deletions app/objects/player.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ def bot_player(cls):
player.object = users.fetch_by_id(1, session=session)
player.client = OsuClient.empty()

player.id = -player.object.id # Negative UserId -> IRC Player
player.id = -player.object.id
player.name = player.object.name
player.stats = player.object.stats

Expand All @@ -148,8 +148,7 @@ def bot_player(cls):

@property
def is_bot(self) -> bool:
# TODO: Refactor bot related code to IRC client
return True if self.id == -1 else False
return self.object.is_bot if self.object else False

@property
def silenced(self) -> bool:
Expand Down
30 changes: 17 additions & 13 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
from app.objects.channel import Channel
from app.objects.player import Player
from app.jobs import (
rank_indexing,
activities,
events,
pings
pings,
ranks
)

import logging
Expand All @@ -31,8 +31,8 @@
)

def setup():
app.session.logger.info(ANCHOR_ASCII_ART)
app.session.logger.info(f'anchor-{config.VERSION}')
app.session.logger.info(f'{ANCHOR_ASCII_ART}')
app.session.logger.info(f'Running anchor-{config.VERSION}')
os.makedirs(config.DATA_PATH, exist_ok=True)

app.session.logger.info('Loading channels...')
Expand Down Expand Up @@ -61,7 +61,7 @@ def setup():
app.session.logger.info('Loading jobs...')
app.session.jobs.submit(pings.ping_job)
app.session.jobs.submit(events.event_listener)
app.session.jobs.submit(rank_indexing.index_ranks)
app.session.jobs.submit(ranks.index_ranks)
app.session.jobs.submit(activities.match_activity)

# Reset usercount
Expand Down Expand Up @@ -89,14 +89,18 @@ def force_exit(signal, frame):
signal.signal(signal.SIGINT, force_exit)

def main():
http_factory = HttpBanchoFactory()
tcp_factory = TcpBanchoFactory()

reactor.suggestThreadPoolSize(config.BANCHO_WORKERS)
reactor.listenTCP(config.HTTP_PORT, http_factory)

for port in config.TCP_PORTS:
reactor.listenTCP(port, tcp_factory)
try:
http_factory = HttpBanchoFactory()
tcp_factory = TcpBanchoFactory()

reactor.suggestThreadPoolSize(config.BANCHO_WORKERS)
reactor.listenTCP(config.HTTP_PORT, http_factory)

for port in config.TCP_PORTS:
reactor.listenTCP(port, tcp_factory)
except Exception as e:
app.session.logger.error(f'Failed to start server: "{e}"')
exit(1)

reactor.addSystemEventTrigger('before', 'startup', setup)
reactor.addSystemEventTrigger('after', 'shutdown', shutdown)
Expand Down
53 changes: 12 additions & 41 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,44 +1,15 @@
aiohttp==3.9.1
aiosignal==1.3.1
async-timeout==4.0.3
attrs==23.2.0
Automat==22.10.0
bcrypt==4.1.2
boto3==1.34.13
boto3-type-annotations-with-docs==0.3.1
botocore==1.34.13
certifi==2023.11.17
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
colorama==0.4.6
constantly==23.10.4
frozenlist==1.4.1
geoip2==4.8.0
greenlet==3.0.3
hyperlink==21.0.0
idna==3.6
incremental==22.10.0
jmespath==1.0.1
maxminddb==2.5.1
multidict==6.0.4
psycopg2==2.9.9
python-dateutil==2.8.2
python-dotenv==1.0.0
python-http-client==3.3.7
pytimeparse==1.1.8
pytz==2023.3.post1
titanic-pp-py==0.0.6
psycopg2-binary==2.9.9
sqlalchemy==2.0.25
python-dotenv==1.0.1
twisted==23.10.0
redis==5.0.1
requests==2.31.0
s3transfer==0.10.0
boto3-type-annotations==0.3.1
boto3==1.34.31
pytz==2023.4
pytimeparse==1.1.8
geoip2==4.8.0
bcrypt==4.1.2
sendgrid==6.11.0
six==1.16.0
SQLAlchemy==2.0.25
starkbank-ecdsa==2.2.0
timeago==1.0.16
titanic-pp-py==0.0.6
Twisted==23.10.0
typing-extensions==4.9.0
urllib3==1.26.18
yarl==1.9.4
zope.interface==6.1
timeago==1.0.16

0 comments on commit 29152d3

Please sign in to comment.