LilSholex 5.0

- New admin actions logging system - Ability to recover voices which have been deleted by admins - New request timeout management - Using retry_after parameter to delay request retrying - New Nginx configuration supporting keepalive connection - Improved broadcasting performance - New way of showing voting results - Removed tasks related to updating voting results
Sholex-Team · Sep 3, 2021 · b84079e · b84079e
1 parent 542beba
commit b84079e
Show file tree

Hide file tree

Showing 16 changed files with 328 additions and 220 deletions.
diff --git a/LilSholex/classes.py b/LilSholex/classes.py
@@ -4,6 +4,8 @@
 from typing import Union
 from abc import ABC, abstractmethod
 from requests import Session
+from django.conf import settings
+from .exceptions import TooManyRequests
 
 
 class Base(ABC):
@@ -52,30 +54,39 @@ def send_message(
                 'reply_to_message_id': reply_to_message_id,
                 'parse_mode': parse_mode if parse_mode else str(),
                 'disable_web_page_preview': str(disable_web_page_preview)
-            }
+            },
+            timeout=settings.REQUESTS_TIMEOUT
         ) as response:
-            response = response.json()
-            if response['ok']:
-                return response['result']['message_id']
-            return 0
+            if response.status_code == 200:
+                if (result := response.json())['ok']:
+                    return result['result']['message_id']
+            elif response.status_code != 429:
+                return 0
+            raise TooManyRequests(response.json()['parameters']['retry_after'])
 
     @sync_fix
     def delete_message(self, message_id: int) -> None:
         with self._session.get(
             f'{self._BASE_URL}deleteMessage',
-            params={**self._BASE_PARAM, 'message_id': message_id}
-        ) as _:
-            return
+            params={**self._BASE_PARAM, 'message_id': message_id},
+            timeout=settings.REQUESTS_TIMEOUT
+        ) as response:
+            if response.status_code != 429:
+                return
+            raise TooManyRequests(response.json()['parameters']['retry_after'])
 
     @sync_fix
     def edit_message_text(self, message_id: int, text: str, inline_keyboard: dict = str()):
         if inline_keyboard:
             inline_keyboard = json.dumps(inline_keyboard)
         with self._session.get(
             f'{self._BASE_URL}editMessageText',
-            params={**self._BASE_PARAM, 'message_id': message_id, 'text': text, 'reply_markup': inline_keyboard}
-        ) as _:
-            return
+            params={**self._BASE_PARAM, 'message_id': message_id, 'text': text, 'reply_markup': inline_keyboard},
+            timeout=settings.REQUESTS_TIMEOUT
+        ) as response:
+            if response.status_code != 429:
+                return
+            raise TooManyRequests(response.json()['parameters']['retry_after'])
 
     @sync_fix
     def send_animation(
@@ -95,8 +106,10 @@ def send_animation(
             'reply_markup': reply_markup,
             'reply_to_message_id': reply_to_message_id,
             'parse_mode': parse_mode
-        }):
-            return
+        }, timeout=settings.REQUESTS_TIMEOUT) as response:
+            if response.status_code != 429:
+                return
+            raise TooManyRequests(response.json()['parameters']['retry_after'])
 
     @abstractmethod
     def go_back(self):

diff --git a/LilSholex/decorators.py b/LilSholex/decorators.py
@@ -1,6 +1,7 @@
 from aiohttp import ClientError
 from requests import RequestException
 from asyncio import sleep
+from .exceptions import TooManyRequests
 
 
 def async_fix(func):
@@ -19,6 +20,8 @@ def check_exception(*args, **kwargs):
         while True:
             try:
                 return func(*args, **kwargs)
+            except TooManyRequests as e:
+                sleep(e.retry_after)
             except RequestException:
                 continue
 

diff --git a/LilSholex/exceptions.py b/LilSholex/exceptions.py
@@ -0,0 +1,6 @@
+class TooManyRequests(Exception):
+    def __init__(self, retry_after: int):
+        self.retry_after = retry_after
+
+    def __str__(self):
+        return f"Too many requests, retry after {self.retry_after} seconds !"
diff --git a/LilSholex/functions.py b/LilSholex/functions.py
@@ -1,4 +1,7 @@
 from requests import Session
+from .decorators import sync_fix
+from django.conf import settings
+from .exceptions import TooManyRequests
 numbers = {
     '0': '0️⃣',
     '1': '1️⃣',
@@ -14,12 +17,17 @@
 
 
 def answer_callback_query(session: Session, token: str):
-    def answer_query(query_id, text, show_alert):
+    @sync_fix
+    def answer_query(query_id, text, show_alert, cache_time: int = 0):
         with session.get(
             f'https://api.telegram.org/bot{token}/answerCallbackQuery',
-            params={'callback_query_id': query_id, 'text': text, 'show_alert': show_alert}
-        ) as _:
-            return
+            params={'callback_query_id': query_id, 'text': text, 'show_alert': show_alert, 'cache_time': cache_time},
+            timeout=settings.REQUESTS_TIMEOUT
+        ) as response:
+            if response.status_code != 429:
+                return
+            raise TooManyRequests(response.json()['parameters']['retry_after'])
+
     return answer_query
 
 

diff --git a/conf/nginx.conf b/conf/nginx.conf
@@ -14,92 +14,78 @@ http {
         return 200 'I\'m Up and Running !';
     }
 
-    # Rate Limiting
-    limit_req_zone $binary_remote_addr zone=webhook:5m rate=30r/s;
-    limit_req_zone $binary_remote_addr zone=normal:10m rate=10r/s;
-
     # Buffering
     client_max_body_size 500k;
-    client_body_buffer_size 10k;
-    client_header_buffer_size 1k;
+    client_body_buffer_size 100k;
+    client_header_buffer_size 4k;
     client_header_timeout 2s;
     client_body_timeout 2s;
-    keepalive_timeout 2s;
-    keepalive_requests 10;
+    keepalive_timeout 0;
     send_timeout 3s;
     tcp_nopush on;
     sendfile on;
 
-    # Servers
-    server_tokens off;
-    server {
-        access_log off;
-        listen 80;
-        server_name {domain};
-        return 301 https://$server_name$uri;
-    }
-
-    server {
-        listen 443;
-        ssl_reject_handshake on;
-    }
+    # Includes
+    include mime.types;
+
+    # Gzip
+    gzip on;
+    gzip_comp_level 3;
+    gzip_types text/javascript text/css;
+
+    # SSL
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_prefer_server_ciphers on;
+    ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_tickets on;
+    ssl_session_timeout 10m;
+    ssl_dhparam /run/secrets/dhparam;
+    add_header Strict-Transport-Security "max-age=31536000" always;
+
+    # Proxy
+    proxy_set_header Host $host;
+    proxy_buffer_size 4k;
+    proxy_buffers 4 4k;
 
     server {
         listen 443 ssl http2;
         server_name {domain};
 
-        # Includes
-        include mime.types;
-
-        # Gzip
-        gzip on;
-        gzip_comp_level 3;
-        gzip_types text/javascript text/css;
-
         # SSL
-        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 
-        ssl_prefer_server_ciphers on;
-        ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
-        ssl_session_cache shared:SSL:10m;
-        ssl_session_tickets on;
-        ssl_session_timeout 10m;
         ssl_certificate /run/secrets/ssl_certificate;
         ssl_certificate_key /run/secrets/ssl_key;
-        ssl_dhparam /run/secrets/dhparam;
-        add_header Strict-Transport-Security "max-age=31536000" always;
-
-        # Proxy
-        proxy_set_header Host $host;
-        proxy_buffer_size 4k;
-        proxy_buffers 4 4k;
 
         # Routing
         location / {
-            limit_req zone=normal burst=2 nodelay;
             access_log /var/log/nginx/root.log;
             proxy_pass http://gunicorn;
-            http2_push /static/admin/css/base.css;
-            http2_push /static/admin/js/jquery.init.js;
-            http2_push /static/admin/js/vendor/jquery/jquery.min.js;
         }
 
         location /adminpanel/ {
-            limit_req zone=normal burst=2 nodelay;
             access_log /var/log/nginx/admin.log;
+            http2_push /static/admin/css/base.css;
+            http2_push /static/admin/js/jquery.init.js;
+            http2_push /static/admin/js/vendor/jquery/jquery.min.js;
+            http2_push /static/admin/fonts/Roboto-Bold-webfont.woff;
+            http2_push /static/admin/fonts/Roboto-Regular-webfont.woff;
+            http2_push /static/admin/fonts/Roboto-Light-webfont.woff;
+            http2_push /static/admin/css/responsive.css;
             proxy_pass http://gunicorn;
         }
 
         location /persianmeme/{persianmeme_token}/ {
             access_log off;
-            limit_req zone=webhook burst=20 delay=12;
+            keepalive_requests 100;
+            keepalive_timeout 100s;
             proxy_pass http://gunicorn;
         }
 
         location /static/ {
             add_header Cache-Control public;
             access_log off;
-            limit_req zone=normal burst=15 delay=10;
             expires 7d;
+            etag on;
             root /root;
             try_files $uri @not_found;
         }

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -95,7 +95,7 @@ services:
     - "443:443/tcp"
     - "80:80/tcp"
   gunicorn:
-    image: ghcr.io/sholex-team/lilsholex/lilsholex:4.9
+    image: ghcr.io/sholex-team/lilsholex/lilsholex:5.0
     networks:
       - db_django
       - nginx_lilsholex
@@ -144,7 +144,7 @@ services:
         source: persianmeme
         target: /home/sholex/persianmeme/migrations
   tasks:
-    image: ghcr.io/sholex-team/lilsholex/lilsholex:4.6
+    image: ghcr.io/sholex-team/lilsholex/lilsholex:5.0
     command: bash -c "python manage.py clear_tasks && python manage.py process_tasks --sleep 15"
     networks:
       - db_django
@@ -187,7 +187,7 @@ services:
         source: persianmeme
         target: /home/sholex/persianmeme/migrations
   broadcasts:
-    image: ghcr.io/sholex-team/lilsholex/lilsholex:4.6
+    image: ghcr.io/sholex-team/lilsholex/lilsholex:5.0
     command: "python manage.py process_broadcasts"
     networks:
       - db_django
@@ -230,7 +230,7 @@ services:
         source: persianmeme
         target: /home/sholex/persianmeme/migrations
   migrations:
-    image: ghcr.io/sholex-team/lilsholex/lilsholex:4.6
+    image: ghcr.io/sholex-team/lilsholex/lilsholex:5.0
     command: "bash -c 'yes | python manage.py makemigrations persianmeme && yes | python manage.py migrate &&\
     python manage.py collectstatic --noinput'"
     secrets:

diff --git a/persianmeme/admin.py b/persianmeme/admin.py
@@ -1,6 +1,6 @@
 from django.contrib import admin
 from persianmeme import models
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpRequest
 from django.core.serializers import serialize
 from persianmeme.functions import delete_vote_sync
 from django.conf import settings
@@ -49,7 +49,7 @@ def count_tags(obj: models.Voice):
 @admin.register(models.User)
 class User(admin.ModelAdmin):
     @admin.display(description='Ban')
-    def ban_user(self, request, queryset):
+    def ban_user(self, request: HttpRequest, queryset):
         result = queryset.update(status='b')
         if result == 0:
             self.message_user(request, 'There is no need to banned these users !')
@@ -59,7 +59,7 @@ def ban_user(self, request, queryset):
             self.message_user(request, f'{result} Users have been banned !')
 
     @admin.display(description='Full Ban')
-    def full_ban(self, request, queryset):
+    def full_ban(self, request: HttpRequest, queryset):
         result = queryset.update(status='f')
         if result == 0:
             self.message_user(request, 'There is no need to full banned these users !')
@@ -69,14 +69,14 @@ def full_ban(self, request, queryset):
             self.message_user(request, f'{result} Users have been banned !')
 
     @admin.display(description='Unban')
-    def unban_user(self, request, queryset):
+    def unban_user(self, request: HttpRequest, queryset):
         result = queryset.update(status='a')
         if result == 0:
             self.message_user(request, 'There is no need to unbanned these users !')
         elif result == 1:
             self.message_user(request, '1 User has been unbanned .')
         else:
-            self.message_user(request, f'{result} Users have been unbanned .')
+            self.message_user(request, f'{result} Users have been unbanned.')
 
     unban_user.allowed_permissions = change_permission
     ban_user.allowed_permissions = change_permission
@@ -128,7 +128,7 @@ def unban_user(self, request, queryset):
 @admin.register(models.Voice)
 class Voice(admin.ModelAdmin):
     @admin.display(description='Accept Votes')
-    def accept_vote(self, request, queryset):
+    def accept_vote(self, request: HttpRequest, queryset):
         result = [
             (target_voice, target_voice.accept(), delete_vote_sync(target_voice.message_id))
             for target_voice in queryset if target_voice.status == 'p'
@@ -142,7 +142,7 @@ def accept_vote(self, request, queryset):
             self.message_user(request, f'{result_len} Voices have been accepted !')
 
     @admin.display(description='Deny Vote')
-    def deny_vote(self, request, queryset):
+    def deny_vote(self, request: HttpRequest, queryset):
         result = [
             (target_voice, target_voice.deny(), delete_vote_sync(target_voice.message_id))
             for target_voice in queryset if target_voice.status == 'p'
@@ -158,7 +158,7 @@ def deny_vote(self, request, queryset):
                 self.message_user(request, f'{result_len} Voices have been denied !')
 
     @admin.display(description='Add Fake Deny Votes')
-    def add_fake_deny_votes(self, request, queryset):
+    def add_fake_deny_votes(self, request: HttpRequest, queryset):
         if (user_count := models.User.objects.count()) < settings.MIN_FAKE_VOTE:
             fake_min = user_count
             fake_max = user_count
@@ -180,21 +180,33 @@ def add_fake_deny_votes(self, request, queryset):
             self.message_user(request, 'Fake votes have been added to a voice !')
         else:
             self.message_user(request, f'Fake votes has been added to {faked_count} voices !')
+
+    @admin.display(description='Recover Voices')
+    def recover_voices(self, request: HttpRequest, queryset):
+        if not (recovered_voices_count := queryset.filter(status=models.Voice.Status.DELETED).update(
+                status=models.Voice.Status.ACTIVE
+        )):
+            self.message_user(request, 'There isn\'t any voice to recover !')
+        elif recovered_voices_count == 1:
+            self.message_user(request, 'One voice has been recovered.')
+        else:
+            self.message_user(request, f'{recovered_voices_count} Voices have been recovered.')
 
     add_fake_deny_votes.allowed_permissions = change_permission
+    recover_voices.allowed_permissions = change_permission
     date_hierarchy = 'date'
     list_display = (
         'id', 'name', 'sender', 'votes', 'status', 'usage_count', count_deny_votes, count_accept_votes, count_tags
     )
     list_filter = ('status', 'voice_type', 'reviewed')
     search_fields = ('name', 'sender__chat_id', 'file_id', 'file_unique_id', 'id', 'sender__user_id')
-    actions = (export_json, accept_vote, deny_vote, add_fake_deny_votes)
+    actions = (export_json, accept_vote, deny_vote, add_fake_deny_votes, recover_voices)
     list_per_page = 15
     readonly_fields = ('id', 'date')
     raw_id_fields = ('sender', 'voters', 'accept_vote', 'deny_vote', 'tags', 'assigned_admin')
     fieldsets = (
         ('Information', {'fields': (
-            'id', 'file_id', 'name', 'file_unique_id', 'date', 'sender', 'tags', 'assigned_admin'
+            'id', 'file_id', 'name', 'file_unique_id', 'date', 'sender', 'tags', 'assigned_admin', 'message_id'
         )}),
         ('Status', {'fields': (
             'status', 'votes', 'voice_type', 'voters', 'accept_vote', 'deny_vote', 'usage_count', 'reviewed'