LilSholex 5.3

- Fixed various bugs in bot logic.
- Switched over to Celery for handling background tasks.
- Improved Nginx configuration.
- Updated Django, Nginx and various other packages to their latest versions.
- Improved docker-compose configuration.
- Added new rabbitmq service with its configuration file.
- Added missing help_messages.json secret file.
- Added celery_flower service for monitoring celery tasks in web environment.
- Using Python 3.10.1 as base image of lilsholex.

Dockerfile

@@ -1,19 +1,18 @@
-FROM python:3.10.0
+FROM python:3.10.1
 # Addding requirements
 COPY requirements.txt requirements.txt
 RUN pip install -U pip && pip install -r requirements.txt --no-cache-dir
 # Setting working directory
 WORKDIR /home/sholex
+RUN mkdir static
+RUN mkdir state
+RUN mkdir healthchecks
 # Healthcheks
 COPY healthchecks/gunicorn.py healthchecks/gunicorn.py
-# Adding Static Directory
-RUN mkdir static
 # Copying source
 COPY LilSholex LilSholex
 COPY templates templates
 COPY manage.py manage.py
 COPY persianmeme persianmeme
-# Setting Volumes
-VOLUME /home/sholex/persianmeme/migrations
 # Running
-CMD gunicorn --workers=2 --bind=0.0.0.0:80 --access-logfile /dev/null --error-logfile /dev/stderr LilSholex.wsgi
+CMD gunicorn --workers=2 --bind=0.0.0.0:80 --error-logfile /dev/stderr -t 15 LilSholex.wsgi

LilSholex/__init__.py

@@ -0,0 +1,3 @@
+from .celery import celery_app
+
+__all__ = ('celery_app',)

LilSholex/celery.py

@@ -0,0 +1,7 @@
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'LilSholex.settings')
+celery_app = Celery('LilSholex')
+celery_app.config_from_object('django.conf:settings', namespace='CELERY')
+celery_app.autodiscover_tasks()

LilSholex/decorators.py

@@ -1,6 +1,7 @@
 from aiohttp import ClientError
 from requests import RequestException
-from asyncio import sleep
+from asyncio import sleep as async_sleep
+from time import sleep as sync_sleep
 from .exceptions import TooManyRequests
 
 
@@ -10,7 +11,7 @@ async def check_exception(*args, **kwargs):
             try:
                 return await func(*args, **kwargs)
             except ClientError:
-                await sleep(0.4)
+                await async_sleep(0.4)
 
     return check_exception
 
@@ -21,7 +22,7 @@ def check_exception(*args, **kwargs):
             try:
                 return func(*args, **kwargs)
             except TooManyRequests as e:
-                sleep(e.retry_after)
+                sync_sleep(e.retry_after)
             except RequestException:
                 continue
 

LilSholex/settings.py

@@ -35,8 +35,7 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-    'background_task',
-    'persianmeme.apps.PersianmemeConfig'
+    'persianmeme.apps.PersianmemeConfig',
 ]
 
 MIDDLEWARE = [
@@ -122,8 +121,6 @@
 STATIC_URL = '/static/'
 STATIC_ROOT = BASE_DIR / 'static'
 MAX_ATTEMPTS = 2
-BACKGROUND_TASK_RUN_ASYNC = True
-BACKGROUND_TASK_ASYNC_THREADS = 4
 
 # Pagination Limit Broadcast
 PAGINATION_LIMIT = 1500
@@ -146,5 +143,12 @@
 SPAM_TIME = 5
 SPAM_PENALTY = 1800
 VIOLATION_REPORT_LIMIT = 5
-VIDEO_DURATION_LIMIT = 180
-VIDEO_SIZE_LIMIT = 15728640
+VIDEO_DURATION_LIMIT = 240
+VIDEO_SIZE_LIMIT = 20971520
+# Celery
+CELERY_BROKER_URL = 'amqp://guest:guest@rabbitmq:5672/'
+CELERY_WORKER_STATE_DB = str(BASE_DIR / 'state' / 'celery_state')
+REVOKE_REVIEW_COUNTDOWN = 3600
+CHECK_MEME_COUNTDOWN = 21600
+# CSRF
+CSRF_TRUSTED_ORIGINS = [f'https://{ALLOWED_HOSTS[0]}']

LilSholex/urls.py

@@ -2,6 +2,6 @@
 from django.urls import path, include
 
 urlpatterns = [
-    path('adminpanel/', admin.site.urls),
+    path('admin/', admin.site.urls),
     path('persianmeme/', include('persianmeme.urls'))
 ]

Nginx

@@ -1,6 +1,6 @@
 FROM ubuntu:impish
-ARG NGINX_VERSION=nginx-1.21.4
-ARG OPENSSL_VERSION=openssl-3.0.0
+ARG NGINX_VERSION=nginx-1.21.5
+ARG OPENSSL_VERSION=openssl-3.0.1
 # Compiling
 RUN apt update && DEBIAN_FRONTEND="noninteractive" apt install -y build-essential \
     libpcre3 libpcre3-dev zlib1g zlib1g-dev libssl-dev python3.9 python3-pip wget  \

conf/my.cnf

@@ -1,5 +1,5 @@
 [mysqld]
-innodb_buffer_pool_chunk_size=536870912
-innodb_buffer_pool_instances=4
-innodb_buffer_pool_size=2147483648
-key_buffer_size=1073741824
+innodb_buffer_pool_chunk_size=335544320
+innodb_buffer_pool_instances=8
+innodb_buffer_pool_size=2684354560
+key_buffer_size=2684354560

conf/nginx.conf

@@ -7,13 +7,6 @@ events {
 }
 
 http {
-    server {
-        access_log off;
-        listen 80;
-        server_name localhost;
-        return 200 'I\'m Up and Running !';
-    }
-
     # Buffering
     client_max_body_size 500k;
     client_body_buffer_size 100k;
@@ -24,6 +17,7 @@ http {
     send_timeout 3s;
     tcp_nopush on;
     sendfile on;
+    sendfile_max_chunk 256m;
 
     # Includes
     include mime.types;
@@ -34,9 +28,9 @@ http {
     gzip_types text/javascript text/css;
 
     # SSL
-    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
+    ssl_conf_command Options KTLS;
     ssl_prefer_server_ciphers on;
-    ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
     ssl_session_cache shared:SSL:10m;
     ssl_session_tickets on;
     ssl_session_timeout 10m;
@@ -45,8 +39,24 @@ http {
 
     # Proxy
     proxy_set_header Host $host;
-    proxy_buffer_size 4k;
-    proxy_buffers 4 4k;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_buffers 4 2m;
+
+    # Servers
+    server_tokens off;
+
+    server {
+        access_log off;
+        listen 80;
+        server_name localhost;
+        return 200 'I\'m Up and Running !';
+    }
+
+    server {
+        listen 443 ssl default_server;
+        access_log /var/log/nginx/rejected.log;
+        ssl_reject_handshake on;
+    }
 
     server {
         listen 443 ssl http2;
@@ -74,6 +84,14 @@ http {
             proxy_pass http://gunicorn;
         }
 
+        location /flower/ {
+            expires 7d;
+            etag on;
+            add_header Cache-Control public;
+            access_log /var/log/nginx/flower.log;
+            proxy_pass http://celery_flower:5555;
+        }
+
         location /persianmeme/{persianmeme_token}/ {
             access_log off;
             keepalive_requests 100;
@@ -86,7 +104,7 @@ http {
             access_log off;
             expires 7d;
             etag on;
-            root /root;
+            root /root/lilsholex;
             try_files $uri @not_found;
         }
 

conf/rabbitmq.conf

@@ -0,0 +1 @@
+consumer_timeout = 28800000