From 7ba7bc92acb39ce1395d3e48831d2a048ad533e7 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 16 Mar 2020 10:56:04 +0530 Subject: [PATCH 01/79] Add support for Direct Download generation Signed-off-by: lzzy12 --- .gitmodules | 6 + bot/__init__.py | 2 +- bot/helper/ext_utils/exceptions.py | 6 + .../download_utils/direct_link_generator.py | 277 ++++++++++++++++++ bot/modules/mirror.py | 10 +- requirements.txt | 2 + vendor/cmrudl.py | 1 + vendor/megadown | 1 + 8 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 bot/helper/mirror_utils/download_utils/direct_link_generator.py create mode 160000 vendor/cmrudl.py create mode 160000 vendor/megadown diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..f53bcd466 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "vendor/megadown"] + path = vendor/megadown + url = https://github.com/tonikelope/megadown.git +[submodule "vendor/cmrudl.py"] + path = vendor/cmrudl.py + url = https://github.com/JrMasterModelBuilder/cmrudl.py.git diff --git a/bot/__init__.py b/bot/__init__.py index 903414716..32565c413 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -28,7 +28,7 @@ def getConfig(name: str): try: if bool(getConfig('_____REMOVE_THIS_LINE_____')): - logging.ERROR('The README.md file there to be read! Exiting now!') + logging.error('The README.md file there to be read! Exiting now!') exit() except KeyError: pass diff --git a/bot/helper/ext_utils/exceptions.py b/bot/helper/ext_utils/exceptions.py index 0181b5199..9b1078e18 100644 --- a/bot/helper/ext_utils/exceptions.py +++ b/bot/helper/ext_utils/exceptions.py @@ -15,3 +15,9 @@ class DownloadCancelled(Exception): def __init__(self, message, error=None): super().__init__(message) self.error = error + + +class DirectDownloadLinkException(Exception): + def __init__(self, message, error=None): + super().__init__(message) + self.error = error diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py new file mode 100644 index 000000000..680f35976 --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -0,0 +1,277 @@ +# Copyright (C) 2019 The Raphielscape Company LLC. +# +# Licensed under the Raphielscape Public License, Version 1.c (the "License"); +# you may not use this file except in compliance with the License. +# +""" Helper Module containing various sites direct links generators""" + +from os import popen +import re +import urllib.parse +import json +from random import choice +import requests +from bs4 import BeautifulSoup +from bot.helper.ext_utils.exceptions import DirectDownloadLinkException + + +def direct_link_generator(link: str): + """ direct links generator """ + if not link: + raise DirectDownloadLinkException("`No links found!`") + if 'drive.google.com' in link: + return gdrive(link) + elif 'zippyshare.com' in link: + return zippy_share(link) + elif 'mega.' in link: + return mega_dl(link) + elif 'yadi.sk' in link: + return yandex_disk(link) + elif 'cloud.mail.ru' in link: + return cm_ru(link) + elif 'mediafire.com' in link: + return mediafire(link) + elif 'sourceforge.net' in link: + return sourceforge(link) + elif 'osdn.net' in link: + return osdn(link) + elif 'github.com' in link: + return github(link) + elif 'androidfilehost.com' in link: + return androidfilehost(link) + else: + raise DirectDownloadLinkException(re.findall(r"\bhttps?://(.*?[^/]+)", + link)[0] + 'is not supported') + + +def gdrive(url: str) -> str: + """ GDrive direct links generator """ + drive = 'https://drive.google.com' + try: + link = re.findall(r'\bhttps?://drive\.google\.com\S+', url)[0] + except IndexError: + reply = "`No Google drive links found`\n" + return reply + file_id = '' + if link.find("view") != -1: + file_id = link.split('/')[-2] + elif link.find("open?id=") != -1: + file_id = link.split("open?id=")[1].strip() + elif link.find("uc?id=") != -1: + file_id = link.split("uc?id=")[1].strip() + url = f'{drive}/uc?export=download&id={file_id}' + download = requests.get(url, stream=True, allow_redirects=False) + cookies = download.cookies + try: + # In case of small file size, Google downloads directly + dl_url = download.headers["location"] + if 'accounts.google.com' in dl_url: # non-public file + raise DirectDownloadLinkException('`Link not public`') + except KeyError: + # In case of download warning page + page = BeautifulSoup(download.content, 'lxml') + export = drive + page.find('a', {'id': 'uc-download-link'}).get('href') + response = requests.get(export, + stream=True, + allow_redirects=False, + cookies=cookies) + dl_url = response.headers['location'] + if 'accounts.google.com' in dl_url: + raise DirectDownloadLinkException('`Link not public`') + return dl_url + + +def zippy_share(url: str) -> str: + """ ZippyShare direct links generator + Based on https://github.com/LameLemon/ziggy""" + dl_url = '' + try: + link = re.findall(r'\bhttps?://.*zippyshare\.com\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No ZippyShare links found`\n") + session = requests.Session() + base_url = re.search('http.+.com', link).group() + response = session.get(link) + page_soup = BeautifulSoup(response.content, "lxml") + scripts = page_soup.find_all("script", {"type": "text/javascript"}) + for script in scripts: + if "getElementById('dlbutton')" in script.text: + url_raw = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', + script.text).group('url') + math = re.search(r'= (?P\".+\" \+ (?P\(.+\)) .+);', + script.text).group('math') + dl_url = url_raw.replace(math, '"' + str(eval(math)) + '"') + break + dl_url = base_url + eval(dl_url) + name = urllib.parse.unquote(dl_url.split('/')[-1]) + return dl_url + + +def yandex_disk(url: str) -> str: + """ Yandex.Disk direct links generator + Based on https://github.com/wldhx/yadisk-direct""" + try: + link = re.findall(r'\bhttps?://.*yadi\.sk\S+', url)[0] + except IndexError: + reply = "`No Yandex.Disk links found`\n" + return reply + api = 'https://cloud-api.yandex.net/v1/disk/public/resources/download?public_key={}' + try: + dl_url = requests.get(api.format(link)).json()['href'] + return dl_url + except KeyError: + raise DirectDownloadLinkException("`Error: File not found / Download limit reached`\n") + + +def mega_dl(url: str) -> str: + """ MEGA.nz direct links generator + Using https://github.com/tonikelope/megadown""" + try: + link = re.findall(r'\bhttps?://.*mega.*\.nz\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No MEGA.nz links found`\n") + command = f'vendor/megadown/megadown -q -m {link}' + result = popen(command).read() + try: + data = json.loads(result) + except json.JSONDecodeError: + raise DirectDownloadLinkException("`Error: Can't extract the link`\n") + dl_url = data['url'] + return dl_url + + +def cm_ru(url: str) -> str: + """ cloud.mail.ru direct links generator + Using https://github.com/JrMasterModelBuilder/cmrudl.py""" + reply = '' + try: + link = re.findall(r'\bhttps?://.*cloud\.mail\.ru\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No cloud.mail.ru links found`\n") + command = f'vendor/cmrudl.py/cmrudl -s {link}' + result = popen(command).read() + result = result.splitlines()[-1] + try: + data = json.loads(result) + except json.decoder.JSONDecodeError: + raise DirectDownloadLinkException("`Error: Can't extract the link`\n") + dl_url = data['download'] + return dl_url + + +def mediafire(url: str) -> str: + """ MediaFire direct links generator """ + try: + link = re.findall(r'\bhttps?://.*mediafire\.com\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No MediaFire links found`\n") + page = BeautifulSoup(requests.get(link).content, 'lxml') + info = page.find('a', {'aria-label': 'Download file'}) + dl_url = info.get('href') + return dl_url + + +def sourceforge(url: str) -> str: + """ SourceForge direct links generator """ + try: + link = re.findall(r'\bhttps?://.*sourceforge\.net\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No SourceForge links found`\n") + file_path = re.findall(r'files(.*)/download', link)[0] + project = re.findall(r'projects?/(.*?)/files', link)[0] + mirrors = f'https://sourceforge.net/settings/mirror_choices?' \ + f'projectname={project}&filename={file_path}' + page = BeautifulSoup(requests.get(mirrors).content, 'html.parser') + info = page.find('ul', {'id': 'mirrorList'}).findAll('li') + for mirror in info[1:]: + dl_url = f'https://{mirror["id"]}.dl.sourceforge.net/project/{project}/{file_path}' + return dl_url + + +def osdn(url: str) -> str: + """ OSDN direct links generator """ + osdn_link = 'https://osdn.net' + try: + link = re.findall(r'\bhttps?://.*osdn\.net\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No OSDN links found`\n") + page = BeautifulSoup( + requests.get(link, allow_redirects=True).content, 'lxml') + info = page.find('a', {'class': 'mirror_link'}) + link = urllib.parse.unquote(osdn_link + info['href']) + mirrors = page.find('form', {'id': 'mirror-select-form'}).findAll('tr') + urls = [] + for data in mirrors[1:]: + mirror = data.find('input')['value'] + urls.append(re.sub(r'm=(.*)&f', f'm={mirror}&f', link)) + return urls[0] + + +def github(url: str) -> str: + """ GitHub direct links generator """ + try: + re.findall(r'\bhttps?://.*github\.com.*releases\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No GitHub Releases links found`\n") + download = requests.get(url, stream=True, allow_redirects=False) + try: + dl_url = download.headers["location"] + return dl_url + except KeyError: + raise DirectDownloadLinkException("`Error: Can't extract the link`\n") + + +def androidfilehost(url: str) -> str: + """ AFH direct links generator """ + try: + link = re.findall(r'\bhttps?://.*androidfilehost.*fid.*\S+', url)[0] + except IndexError: + raise DirectDownloadLinkException("`No AFH links found`\n") + fid = re.findall(r'\?fid=(.*)', link)[0] + session = requests.Session() + user_agent = useragent() + headers = {'user-agent': user_agent} + res = session.get(link, headers=headers, allow_redirects=True) + headers = { + 'origin': 'https://androidfilehost.com', + 'accept-encoding': 'gzip, deflate, br', + 'accept-language': 'en-US,en;q=0.9', + 'user-agent': user_agent, + 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'x-mod-sbb-ctype': 'xhr', + 'accept': '*/*', + 'referer': f'https://androidfilehost.com/?fid={fid}', + 'authority': 'androidfilehost.com', + 'x-requested-with': 'XMLHttpRequest', + } + data = { + 'submit': 'submit', + 'action': 'getdownloadmirrors', + 'fid': f'{fid}' + } + error = "`Error: Can't find Mirrors for the link`\n" + try: + req = session.post( + 'https://androidfilehost.com/libs/otf/mirrors.otf.php', + headers=headers, + data=data, + cookies=res.cookies) + mirrors = req.json()['MIRRORS'] + except (json.decoder.JSONDecodeError, TypeError): + raise DirectDownloadLinkException(error) + if not mirrors: + raise DirectDownloadLinkException(error) + return mirrors[0] + + +def useragent(): + """ + useragent random setter + """ + useragents = BeautifulSoup( + requests.get( + 'https://developers.whatismybrowser.com/' + 'useragents/explore/operating_system_name/android/').content, + 'lxml').findAll('td', {'class': 'useragent'}) + user_agent = choice(useragents) + return user_agent.text diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index ad474c9b4..cb458052a 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -13,6 +13,8 @@ from bot.helper.telegram_helper.bot_commands import BotCommands import pathlib import os +from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator +from bot.helper.ext_utils.exceptions import DirectDownloadLinkException class MirrorListener(listeners.MirrorListeners): @@ -104,7 +106,7 @@ def onUploadComplete(self, link: str): pass del download_dict[self.uid] count = len(download_dict) - sendMessage(msg,self.bot,self.update) + sendMessage(msg, self.bot, self.update) if count == 0: self.clean() else: @@ -125,6 +127,7 @@ def onUploadError(self, error): else: update_all_messages() + def _mirror(bot, update, isTar=False): message_args = update.message.text.split(' ') try: @@ -145,6 +148,11 @@ def _mirror(bot, update, isTar=False): if not bot_utils.is_url(link) and not bot_utils.is_magnet(link): sendMessage('No download source provided', bot, update) return + + try: + link = direct_link_generator(link) + except DirectDownloadLinkException as e: + LOGGER.info(f'{link}: {e.error}') listener = MirrorListener(bot, update, isTar) aria = aria2_download.AriaDownloadHelper(listener) aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') diff --git a/requirements.txt b/requirements.txt index 0a00afa29..7e3109d4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +requests python-telegram-bot==12.2.0 google-api-python-client>=1.7.11,<1.7.20 google-auth-httplib2>=0.0.3,<0.1.0 @@ -6,3 +7,4 @@ aria2p>=0.3.0,<0.10.0 python-dotenv>=0.10 tenacity>=6.0.0 python-magic +beautifulsoup4>=4.8.2,<4.8.10 \ No newline at end of file diff --git a/vendor/cmrudl.py b/vendor/cmrudl.py new file mode 160000 index 000000000..f7d75bcf7 --- /dev/null +++ b/vendor/cmrudl.py @@ -0,0 +1 @@ +Subproject commit f7d75bcf7901aee7b1430fc17366b7b6af65235e diff --git a/vendor/megadown b/vendor/megadown new file mode 160000 index 000000000..734e46fec --- /dev/null +++ b/vendor/megadown @@ -0,0 +1 @@ +Subproject commit 734e46fec67a2798bfbb5e21a75f04b90afafd65 From 57afb01b633421a67069e26d5d3ab088dbba98ca Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 19 Mar 2020 15:26:57 +0530 Subject: [PATCH 02/79] Dockerfile: Add missing dependencies Signed-off-by: lzzy12 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0b26e2d97..30d9e7daa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app RUN apt -qq update -RUN apt -qq install -y aria2 python3 python3-pip locales +RUN apt -qq install -y aria2 python3 python3-pip locales python3-lxml curl pv jq COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt COPY . . From 697017fb867a906840ae7693b213e8285e478736 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 19 Jan 2020 08:50:00 -0800 Subject: [PATCH 03/79] Dockerfile: Move to smaller alpine base image Signed-off-by: lzzy12 --- Dockerfile | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 30d9e7daa..33134f329 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,27 @@ -FROM ubuntu:18.04 +FROM python:3-alpine + +WORKDIR /bot +RUN chmod 777 /bot + +# install ca-certificates so that HTTPS works consistently +RUN apk add --no-cache --update \ + ca-certificates \ + aria2 \ + libmagic \ + python3-lxml \ + curl \ + pv \ + jq + +RUN apk add --no-cache --update --virtual .build-deps \ + build-base \ + libffi-dev \ + openssl-dev -WORKDIR /usr/src/app -RUN chmod 777 /usr/src/app -RUN apt -qq update -RUN apt -qq install -y aria2 python3 python3-pip locales python3-lxml curl pv jq COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt +RUN apk del .build-deps COPY . . -RUN chmod +x aria.sh -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +RUN chmod +x *.sh -CMD ["bash","start.sh"] +CMD ["sh", "./start.sh"] From ae9d972add9e1b4dd1bd6a8dbde40c976e79f6e2 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 17 Jan 2020 10:02:46 -0800 Subject: [PATCH 04/79] Tag the author of replied message of mirror command This closes #24 Signed-off-by: lzzy12 # Conflicts: # bot/modules/mirror.py --- bot/modules/mirror.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index cb458052a..32d03e56f 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -18,9 +18,10 @@ class MirrorListener(listeners.MirrorListeners): - def __init__(self, bot, update, isTar=False): + def __init__(self, bot, update, isTar=False, tag=None): super().__init__(bot, update) self.isTar = isTar + self.tag = tag def onDownloadStarted(self): pass @@ -100,6 +101,8 @@ def onUploadComplete(self, link: str): if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): share_url += '/' msg += f'\n\n Shareable link: here' + if self.tag is not None: + msg += f'\ncc: @{self.tag}' try: fs_utils.clean_download(download_dict[self.uid].path()) except FileNotFoundError: @@ -136,15 +139,14 @@ def _mirror(bot, update, isTar=False): link = '' LOGGER.info(link) link = link.strip() - - if len(link) == 0: - if update.message.reply_to_message is not None: - document = update.message.reply_to_message.document - if document is not None and document.mime_type == "application/x-bittorrent": - link = document.get_file().file_path - else: - sendMessage('Only torrent files can be mirrored from telegram', bot, update) - return + reply_to = update.message.reply_to_message + if reply_to is not None: + tag = reply_to.from_user.username + if len(link) == 0: + if reply_to.document is not None and reply_to.document.mime_type == "application/x-bittorrent": + link = reply_to.document.get_file().file_path + else: + tag = None if not bot_utils.is_url(link) and not bot_utils.is_magnet(link): sendMessage('No download source provided', bot, update) return @@ -153,7 +155,7 @@ def _mirror(bot, update, isTar=False): link = direct_link_generator(link) except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e.error}') - listener = MirrorListener(bot, update, isTar) + listener = MirrorListener(bot, update, isTar, tag) aria = aria2_download.AriaDownloadHelper(listener) aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') sendStatusMessage(update, bot) From 36df3aa2b48740a50824917ca0b85ad7392d1ab7 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 19 Mar 2020 15:40:43 +0530 Subject: [PATCH 05/79] Improve logic for determining true/false from config.env Signed-off-by: lzzy12 # Conflicts: # bot/modules/mirror.py --- bot/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 32565c413..4da72e0e6 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -80,11 +80,10 @@ def getConfig(name: str): INDEX_URL = None try: IS_TEAM_DRIVE = getConfig('IS_TEAM_DRIVE') - if IS_TEAM_DRIVE == 'True' or IS_TEAM_DRIVE == 'true': + if IS_TEAM_DRIVE.lower() == 'true': IS_TEAM_DRIVE = True else: IS_TEAM_DRIVE = False - except KeyError: IS_TEAM_DRIVE = False updater = tg.Updater(token=BOT_TOKEN) From afc2808accf86abb88f2ee7d3c4f3f5b7492384d Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sun, 2 Feb 2020 14:48:23 +0530 Subject: [PATCH 06/79] Rewrite tar function - shutil changes pwd while archiving. as some functions of bot require relative paths, it is neccesary to keep pwd as bot root dir. Signed-off-by: lzzy12 # Conflicts: # bot/helper/ext_utils/fs_utils.py --- bot/helper/ext_utils/fs_utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 1d0a7ed9b..019cc77ae 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -4,6 +4,7 @@ import os import pathlib import magic +import tarfile def clean_download(path: str): @@ -30,12 +31,14 @@ def exit_clean_up(signal, frame): sys.exit(1) -def tar(orig_path: str): - path = pathlib.PurePath(orig_path) - base = path.name - root = pathlib.Path(path.parent.as_posix()).absolute().as_posix() - LOGGER.info(f'Tar: orig_path: {orig_path}, base: {base}, root: {root}') - return shutil.make_archive(orig_path, 'tar', root, base) +def tar(org_path): + tar_path = org_path + ".tar" + path = pathlib.PurePath(org_path) + LOGGER.info(f'Tar: orig_path: {org_path}, tar_path: {tar_path}') + tar = tarfile.open(tar_path, "w") + tar.add(org_path, arcname=path.name) + tar.close() + return tar_path def get_mime_type(file_path): From d5cfa793aa1c2f9fb2d5a4769f7c0407b97be72c Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 19 Mar 2020 15:58:57 +0530 Subject: [PATCH 07/79] Dockerfile: Fix lxml package name Signed-off-by: lzzy12 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 33134f329..8341558f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apk add --no-cache --update \ ca-certificates \ aria2 \ libmagic \ - python3-lxml \ + py3-lxml \ curl \ pv \ jq From cfe6f2fad441abdc4f35ee18bbe3bbc1596aece0 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 19 Mar 2020 16:22:03 +0530 Subject: [PATCH 08/79] Revert "Dockerfile: Fix lxml package name" This reverts commit d5cfa793aa1c2f9fb2d5a4769f7c0407b97be72c. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8341558f3..33134f329 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN apk add --no-cache --update \ ca-certificates \ aria2 \ libmagic \ - py3-lxml \ + python3-lxml \ curl \ pv \ jq From 4207f139e4dac380f35db29c51cd3f12ef347e52 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Thu, 19 Mar 2020 16:22:27 +0530 Subject: [PATCH 09/79] Revert "Dockerfile: Move to smaller alpine base image" This reverts commit 697017fb867a906840ae7693b213e8285e478736. --- Dockerfile | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 33134f329..30d9e7daa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,16 @@ -FROM python:3-alpine - -WORKDIR /bot -RUN chmod 777 /bot - -# install ca-certificates so that HTTPS works consistently -RUN apk add --no-cache --update \ - ca-certificates \ - aria2 \ - libmagic \ - python3-lxml \ - curl \ - pv \ - jq - -RUN apk add --no-cache --update --virtual .build-deps \ - build-base \ - libffi-dev \ - openssl-dev +FROM ubuntu:18.04 +WORKDIR /usr/src/app +RUN chmod 777 /usr/src/app +RUN apt -qq update +RUN apt -qq install -y aria2 python3 python3-pip locales python3-lxml curl pv jq COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt -RUN apk del .build-deps COPY . . -RUN chmod +x *.sh +RUN chmod +x aria.sh +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 -CMD ["sh", "./start.sh"] +CMD ["bash","start.sh"] From 898977df978068f4db1874132f969fd124b2a5cf Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 20 Mar 2020 17:03:29 +0530 Subject: [PATCH 10/79] Use proper url-encoding methods to replace special characters from url Signed-off-by: lzzy12 --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 7 +++---- bot/modules/mirror.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 0731bf664..3adcb7dc8 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -7,11 +7,10 @@ from googleapiclient.errors import HttpError from googleapiclient.http import MediaFileUpload from tenacity import * - from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, DOWNLOAD_STATUS_UPDATE_INTERVAL from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type - +import requests logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) @@ -239,13 +238,13 @@ def drive_list(self, fileName): msg += f"⁍ {file.get('name')}" \ f" (folder)" if INDEX_URL is not None: - url = f'{INDEX_URL}/{file.get("name")}/' + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') msg += f' | Index URL' else: msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" if INDEX_URL is not None: - url = f'{INDEX_URL}/{file.get("name")}' + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') msg += f' | Index URL' msg += '\n' results.append(file) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 32d03e56f..0dea2e16c 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -15,6 +15,7 @@ import os from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.ext_utils.exceptions import DirectDownloadLinkException +import requests class MirrorListener(listeners.MirrorListeners): @@ -96,8 +97,7 @@ def onUploadComplete(self, link: str): msg = f'{download_dict[self.uid].name()} ({download_dict[self.uid].size()})' LOGGER.info(f'Done Uploading {download_dict[self.uid].name()}') if INDEX_URL is not None: - share_url = f'{INDEX_URL}/{download_dict[self.uid].name()}' - share_url = share_url.replace(' ', '%20') + share_url = requests.utils.requote_uri(f'{INDEX_URL}/{download_dict[self.uid].name()}') if os.path.isdir(f'{DOWNLOAD_DIR}/{self.uid}/{download_dict[self.uid].name()}'): share_url += '/' msg += f'\n\n Shareable link: here' From 94ccdcbeef04f5c9f39082deb715bbfcd05f4aee Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 21 Mar 2020 13:00:26 +0530 Subject: [PATCH 11/79] Add support for mirror of telegram files Signed-off-by: lzzy12 Add a script to generate string session for user Signed-off-by: lzzy12 Some fix ups Signed-off-by: lzzy12 Fix telegram download Signed-off-by: lzzy12 --- README.md | 5 +- bot/__init__.py | 7 +- .../download_utils/aria2_download.py | 3 +- .../download_utils/download_helper.py | 2 +- .../download_utils/telegram_downloader.py | 81 +++++++++++++++++++ .../status_utils/telegram_download_status.py | 49 +++++++++++ bot/helper/telegram_helper/message_utils.py | 2 +- bot/modules/mirror.py | 19 ++++- config_sample.env | 3 + generate_string_session.py | 6 ++ requirements.txt | 4 +- 11 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 bot/helper/mirror_utils/download_utils/telegram_downloader.py create mode 100644 bot/helper/mirror_utils/status_utils/telegram_download_status.py create mode 100644 generate_string_session.py diff --git a/README.md b/README.md index d92f9c4f0..be98fa832 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,10 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages - IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. - INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' - +- USER_SESSION_STRING : Session string generated by running: +``` +python3 generate_string_session.py +``` Note: You can limit maximum concurrent downloads by changing the value of MAX_CONCURRENT_DOWNLOADS in aria.sh. By default, it's set to 2 ## Getting Google OAuth API credential file diff --git a/bot/__init__.py b/bot/__init__.py index 4da72e0e6..5e9cc67f8 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -13,7 +13,7 @@ logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[logging.FileHandler('log.txt'), logging.StreamHandler()], - level=logging.INFO) + level=logging.WARNING) load_dotenv('config.env') @@ -69,6 +69,9 @@ def getConfig(name: str): DOWNLOAD_STATUS_UPDATE_INTERVAL = int(getConfig('DOWNLOAD_STATUS_UPDATE_INTERVAL')) OWNER_ID = int(getConfig('OWNER_ID')) AUTO_DELETE_MESSAGE_DURATION = int(getConfig('AUTO_DELETE_MESSAGE_DURATION')) + USER_SESSION_STRING = getConfig('USER_SESSION_STRING') + TELEGRAM_API = getConfig('TELEGRAM_API') + TELEGRAM_HASH = getConfig('TELEGRAM_HASH') except KeyError as e: LOGGER.error("One or more env variables missing! Exiting now") exit(1) @@ -88,4 +91,4 @@ def getConfig(name: str): IS_TEAM_DRIVE = False updater = tg.Updater(token=BOT_TOKEN) bot = updater.bot -dispatcher = updater.dispatcher +dispatcher = updater.dispatcher \ No newline at end of file diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index 2817ba9e3..e29cfdfee 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -1,4 +1,4 @@ -from bot import aria2,download_dict,download_dict_lock +from bot import aria2 from bot.helper.ext_utils.bot_utils import * from .download_helper import DownloadHelper from bot.helper.mirror_utils.status_utils.aria_download_status import AriaDownloadStatus @@ -6,6 +6,7 @@ import threading from aria2p import API + class AriaDownloadHelper(DownloadHelper): def __init__(self, listener): diff --git a/bot/helper/mirror_utils/download_utils/download_helper.py b/bot/helper/mirror_utils/download_utils/download_helper.py index 907f8b162..ebcca1444 100644 --- a/bot/helper/mirror_utils/download_utils/download_helper.py +++ b/bot/helper/mirror_utils/download_utils/download_helper.py @@ -16,7 +16,7 @@ def __init__(self): self.progress = 0.0 self.progress_string = '0.00%' self.eta = 0 # Estimated time of download complete - self.eta_string = '0s' # A listener class which have event callbacks + self.eta_string = '0s' # A listener class which have event callbacks self._resource_lock = threading.Lock() def add_download(self, link: str, path): diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py new file mode 100644 index 000000000..fa3b4609f --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -0,0 +1,81 @@ +from .download_helper import DownloadHelper +import threading +import time +from ..status_utils.telegram_download_status import TelegramDownloadStatus +from bot.helper.ext_utils.bot_utils import get_readable_file_size +from bot import LOGGER, bot, download_dict, download_dict_lock, TELEGRAM_API,\ + TELEGRAM_HASH, USER_SESSION_STRING +from pyrogram import Client + +global_lock = threading.Lock() +GLOBAL_GID = set() + +class TelegramDownloadHelper(DownloadHelper): + def __init__(self, listener): + super().__init__() + self.__listener = listener + self.__resource_lock = threading.RLock() + self.__name = "" + self.__gid = '' + self.__start_time = time.time() + self.__user_bot = Client(api_id=TELEGRAM_API, + api_hash=TELEGRAM_HASH, + session_name=USER_SESSION_STRING) + self.__user_bot.start() + + @property + def gid(self): + with self.__resource_lock: + return self.__gid + + @property + def download_speed(self): + with self.__resource_lock: + return self.downloaded_bytes / (time.time() - self.__start_time) + + def __onDownloadStart(self, name, size, file_id): + with download_dict_lock: + download_dict[self.__listener.uid] = TelegramDownloadStatus(self, self.__listener.uid) + with global_lock: + GLOBAL_GID.add(file_id) + with self.__resource_lock: + self.name = name + self.size = size + self.__gid = file_id + self.__listener.onDownloadStarted() + + def __onDownloadProgress(self, current, total): + with self.__resource_lock: + self.downloaded_bytes = current + try: + self.progress = current / self.size * 100 + except ZeroDivisionError: + return 0 + def __onDownloadComplete(self): + + self.__listener.onDownloadComplete() + + def __download(self, message, path): + self.__user_bot.download_media(message, + progress=self.__onDownloadProgress, file_name=path) + self.__onDownloadComplete() + + def add_download(self, message, path): + if message.chat.type == "private": + _message = self.__user_bot.get_messages(bot.get_me().id, message.message_id) + else: + _message = self.__user_bot.get_messages(message.chat.id, message.message_id) + media = _message.document + if media is not None: + with global_lock: + # For avoiding locking the thread lock for long time unnecessarily + download = media.file_id not in GLOBAL_GID + + if download: + self.__onDownloadStart(media.file_name, media.file_size, media.file_id) + LOGGER.info(media.file_id) + threading.Thread(target=self.__download, args=(_message, path)).start() + else: + self.__listener.onDownloadError('File already being downloaded!') + else: + self.__listener.onDownloadError('No document in the replied message') diff --git a/bot/helper/mirror_utils/status_utils/telegram_download_status.py b/bot/helper/mirror_utils/status_utils/telegram_download_status.py new file mode 100644 index 000000000..186ac96bc --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/telegram_download_status.py @@ -0,0 +1,49 @@ +from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time +from .status import Status +from bot import DOWNLOAD_DIR + + +class TelegramDownloadStatus(Status): + def __init__(self, obj, uid): + self.obj = obj + self.uid = uid + + def path(self): + return f"{DOWNLOAD_DIR}{self.uid}" + + def processed_bytes(self): + return self.obj.downloaded_bytes + + def size_raw(self): + return self.obj.size + + def size(self): + return get_readable_file_size(self.size_raw()) + + def status(self): + return MirrorStatus.STATUS_DOWNLOADING + + def name(self): + return self.obj.name + + def progress_raw(self): + return self.obj.progress + + def progress(self): + return f'{round(self.progress_raw(), 2)}%' + + def speed_raw(self): + """ + :return: Download speed in Bytes/Seconds + """ + return self.obj.download_speed + + def speed(self): + return f'{get_readable_file_size(self.speed_raw())}/s' + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' diff --git a/bot/helper/telegram_helper/message_utils.py b/bot/helper/telegram_helper/message_utils.py index ad4e6fe5a..7a1fc7e7d 100644 --- a/bot/helper/telegram_helper/message_utils.py +++ b/bot/helper/telegram_helper/message_utils.py @@ -2,7 +2,7 @@ from telegram.update import Update import time from bot import AUTO_DELETE_MESSAGE_DURATION, LOGGER, bot, \ - status_reply_dict, status_reply_dict_lock, download_dict_lock, download_dict + status_reply_dict, status_reply_dict_lock from bot.helper.ext_utils.bot_utils import get_readable_message from telegram.error import TimedOut, BadRequest from bot import bot diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0dea2e16c..ed45311f3 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -4,7 +4,7 @@ from bot.helper.mirror_utils.download_utils import aria2_download from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus from bot.helper.mirror_utils.status_utils.tar_status import TarStatus -from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL +from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock from bot.helper.ext_utils import fs_utils, bot_utils from bot import Interval, INDEX_URL from bot.helper.telegram_helper.message_utils import * @@ -14,9 +14,10 @@ import pathlib import os from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator +from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from bot.helper.ext_utils.exceptions import DirectDownloadLinkException import requests - +import threading class MirrorListener(listeners.MirrorListeners): def __init__(self, bot, update, isTar=False, tag=None): @@ -142,9 +143,19 @@ def _mirror(bot, update, isTar=False): reply_to = update.message.reply_to_message if reply_to is not None: tag = reply_to.from_user.username + document = reply_to.document if len(link) == 0: - if reply_to.document is not None and reply_to.document.mime_type == "application/x-bittorrent": - link = reply_to.document.get_file().file_path + if document is not None: + if document.file_size <= 20 * 1024 * 1024: + link = document.get_file().file_path + else: + listener = MirrorListener(bot, update, isTar, tag) + tg_downloader = TelegramDownloadHelper(listener) + tg_downloader.add_download(reply_to, f'{DOWNLOAD_DIR}{listener.uid}/') + sendStatusMessage(update, bot) + if len(Interval) == 0: + Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) + return else: tag = None if not bot_utils.is_url(link) and not bot_utils.is_magnet(link): diff --git a/config_sample.env b/config_sample.env index 2e8252e85..b0f51562a 100644 --- a/config_sample.env +++ b/config_sample.env @@ -10,3 +10,6 @@ DOWNLOAD_STATUS_UPDATE_INTERVAL = 5 AUTO_DELETE_MESSAGE_DURATION = 20 IS_TEAM_DRIVE = "" INDEX_URL = "" +USER_SESSION_STRING = "" +TELEGRAM_API = +TELEGRAM_HASH = "" \ No newline at end of file diff --git a/generate_string_session.py b/generate_string_session.py new file mode 100644 index 000000000..fd6d57c55 --- /dev/null +++ b/generate_string_session.py @@ -0,0 +1,6 @@ +from pyrogram import Client + +API_KEY = int(input("Enter API KEY: ")) +API_HASH = input("Enter API HASH: ") +with Client(':memory:', api_id=API_KEY, api_hash=API_HASH) as app: + print(app.export_session_string()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7e3109d4e..4bfde1c4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,6 @@ aria2p>=0.3.0,<0.10.0 python-dotenv>=0.10 tenacity>=6.0.0 python-magic -beautifulsoup4>=4.8.2,<4.8.10 \ No newline at end of file +beautifulsoup4>=4.8.2,<4.8.10 +Pyrogram>=0.16.0,<0.16.10 +TgCrypto>=1.1.1,<1.1.10 \ No newline at end of file From 6b60ca44919c7e3beb6358b8a08406d791900550 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sat, 21 Mar 2020 11:42:07 +0530 Subject: [PATCH 12/79] Squash of clone command Implement clone feature - clones a public/driveAccountAccessible Drive Link to bot G-Drive use BotCommands for clone command trigger --- .../mirror_utils/upload_utils/gdriveTools.py | 80 +++++++++++++++++++ bot/helper/telegram_helper/bot_commands.py | 1 + bot/modules/clone.py | 23 ++++++ 3 files changed, 104 insertions(+) create mode 100644 bot/modules/clone.py diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 3adcb7dc8..75dd238c8 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -24,6 +24,7 @@ def __init__(self, name=None, listener=None): self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" + self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" self.__listener = listener self.__service = self.authorize() self._file_uploaded_bytes = 0 @@ -52,6 +53,13 @@ def speed(self): except ZeroDivisionError: return 0 + def parseLink(self,link): + if "folders" in link or "file" in link: + node_id = link.split("?")[0].split("/")[-1].replace("/",'') + else: + node_id = link.split("=")[1].split("&")[0].replace("/",'') + return node_id + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def _on_upload_progress(self): @@ -160,6 +168,78 @@ def upload(self, file_name: str): LOGGER.info("Deleting downloaded file/folder..") return link + @retry(wait=wait_exponential(multiplier=2, min=3, max=6),stop=stop_after_attempt(5),retry=retry_if_exception_type(HttpError),before=before_log(LOGGER,logging.DEBUG)) + def copyFile(self,file_id,dest_id): + body = { + 'parents': [dest_id] + } + return self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() + + def clone(self,link): + self.transferred_size = 0 + file_id = self.parseLink(link) + msg = "" + LOGGER.info(f"File ID: {file_id}") + try: + meta = self.__service.files().get(fileId=file_id,fields="name,id,mimeType,size").execute() + except Exception as e: + return f"{str(e).replace('>','').replace('<','')}" + if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: + dir_id = self.create_directory(meta.get('name'),parent_id) + try: + result = self.cloneFolder(meta.get('name'),meta.get('name'),meta.get('id'),dir_id) + except Exception as e: + if isinstance(e,RetryError): + LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") + err = e.last_attempt.exception() + else: + err = str(e).replace('>','').replace('<','') + LOGGER.error(err) + return err + msg += f'{meta.get("name")} ({get_readable_file_size(self.transferred_size)})' + else: + file = self.copyFile(meta.get('id'),parent_id) + msg += f'{meta.get("name")} ({get_readable_file_size(int(meta.get("size")))})' + return msg + + + def cloneFolder(self,name,local_path,folder_id,parent_id): + page_token = None + q =f"'{folder_id}' in parents" + files = [] + LOGGER.info(f"Syncing: {local_path}") + new_id = None + while True: + response = self.__service.files().list(q=q, + spaces='drive', + fields='nextPageToken, files(id, name, mimeType,size)', + pageToken=page_token).execute() + for file in response.get('files', []): + files.append(file) + page_token = response.get('nextPageToken', None) + if page_token is None: + break + if len(files) == 0: + return parent_id + for file in files: + if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE: + file_path = os.path.join(local_path,file.get('name')) + current_dir_id = self.create_directory(file.get('name'),parent_id) + new_id = self.cloneFolder(file.get('name'),file_path,file.get('id'),current_dir_id) + else: + self.transferred_size += int(file.get('size')) + try: + self.copyFile(file.get('id'),parent_id) + new_id = parent_id + except Exception as e: + if isinstance(e,RetryError): + LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") + err = e.last_attempt.exception() + else: + err = e + LOGGER.error(err) + return new_id + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def create_directory(self, directory_name, parent_id): diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py index 918500d62..d58c29dca 100644 --- a/bot/helper/telegram_helper/bot_commands.py +++ b/bot/helper/telegram_helper/bot_commands.py @@ -13,6 +13,7 @@ def __init__(self): self.StatsCommand = 'stats' self.HelpCommand = 'help' self.LogCommand = 'log' + self.CloneCommand = "clone" BotCommands = _BotCommands() diff --git a/bot/modules/clone.py b/bot/modules/clone.py new file mode 100644 index 000000000..fa3323338 --- /dev/null +++ b/bot/modules/clone.py @@ -0,0 +1,23 @@ +from telegram.ext import CommandHandler, run_async +from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper +from bot.helper.telegram_helper.message_utils import * +from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.telegram_helper.bot_commands import BotCommands +from bot import dispatcher + + +@run_async +def cloneNode(bot,update): + args = update.message.text.split(" ",maxsplit=1) + if len(args) > 1: + link = args[1] + msg = sendMessage(f"Cloning: {link}",bot,update) + gd = GoogleDriveHelper() + result = gd.clone(link) + deleteMessage(bot,msg) + sendMessage(result,bot,update) + else: + sendMessage("Provide G-Drive Shareable Link to Clone.",bot,update) + +clone_handler = CommandHandler(BotCommands.CloneCommand,cloneNode,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) +dispatcher.add_handler(clone_handler) \ No newline at end of file From a44d0da9f6da111d789155f88162abd0f1079a2d Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sat, 21 Mar 2020 14:28:38 +0530 Subject: [PATCH 13/79] Implement cancellation of downloads by gid --- bot/helper/ext_utils/bot_utils.py | 9 +++++++ bot/modules/cancel_mirror.py | 44 ++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index ea8f598ad..5e42b9af4 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -54,6 +54,14 @@ def get_readable_file_size(size_in_bytes) -> str: except IndexError: return 'File too large' +def getDownloadByGid(gid): + with download_dict_lock: + for dl in download_dict.values(): + if dl.status() == MirrorStatus.STATUS_DOWNLOADING: + if dl.download().gid == gid: + return dl + return None + def get_progress_bar_string(status): completed = status.processed_bytes() / 8 @@ -95,6 +103,7 @@ def get_readable_message(): if hasattr(download, 'is_torrent'): msg += f"| P: {download.download().connections} " \ f"| S: {download.download().num_seeders}" + msg += f"\nGID: {download.download().gid}" msg += "\n\n" return msg diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index b69079d6c..fd8eae477 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -5,20 +5,36 @@ from bot.helper.ext_utils.fs_utils import clean_download from bot.helper.telegram_helper.bot_commands import BotCommands from time import sleep +from bot.helper.ext_utils.bot_utils import getDownloadByGid @run_async def cancel_mirror(bot,update): - mirror_message = update.message.reply_to_message - with download_dict_lock: - keys = download_dict.keys() - dl = download_dict[mirror_message.message_id] - if mirror_message is None or mirror_message.message_id not in keys: - if '/mirror' in mirror_message.text or '/tarmirror' in mirror_message.text: - msg = 'Message has already been cancelled' - else: - msg = 'Please reply to the /mirror message which was used to start the download to cancel it!' - sendMessage(msg, bot, update) - return + args = update.message.text.split(" ",maxsplit=1) + mirror_message = None + if len(args) > 1: + gid = args[1] + dl = getDownloadByGid(gid) + if not dl: + sendMessage(f"GID: {gid} not found.",bot,update) + return + with download_dict_lock: + keys = list(download_dict.keys()) + mirror_message = dl._listener.message + elif update.message.reply_to_message: + mirror_message = update.message.reply_to_message + with download_dict_lock: + keys = list(download_dict.keys()) + dl = download_dict[mirror_message.message_id] + if len(args) == 1: + if mirror_message is None or mirror_message.message_id not in keys: + if BotCommands.MirrorCommand in mirror_message.text or BotCommands.TarMirrorCommand in mirror_message.text: + msg = "Mirror already have been cancelled" + sendMessage(msg,bot,update) + return + else: + msg = "Please reply to the /mirror message which was used to start the download or /cancel gid to cancel it!" + sendMessage(msg,bot,update) + return if dl.status() == "Uploading": sendMessage("Upload in Progress, Don't Cancel it.", bot, update) return @@ -31,13 +47,9 @@ def cancel_mirror(bot,update): downloads = aria2.get_downloads(download.followed_by_ids) aria2.pause(downloads) aria2.pause([download]) - - elif dl.status() == "Uploading": - sendMessage("Upload in Progress, Dont Cancel it.",bot,update) - return else: dl._listener.onDownloadError("Download stopped by user!") - sleep(1) #Wait a Second For Aria2 To free Resources. + sleep(1) # Wait a Second For Aria2 To free Resources. clean_download(f'{DOWNLOAD_DIR}{mirror_message.message_id}/') From 49fe46bad344c05410b88620158e3ff16c69563d Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 22 Mar 2020 14:45:53 +0530 Subject: [PATCH 14/79] Add gid method in Status class for download classes Signed-off-by: lzzy12 --- bot/helper/ext_utils/bot_utils.py | 12 +++++---- .../download_utils/telegram_downloader.py | 25 +++++++++++-------- .../status_utils/aria_download_status.py | 4 +++ .../status_utils/telegram_download_status.py | 5 +++- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 5e42b9af4..ede4e49f4 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -1,9 +1,10 @@ -from bot import download_dict, download_dict_lock import logging import re import threading import time +from bot import download_dict, download_dict_lock + LOGGER = logging.getLogger(__name__) MAGNET_REGEX = r"magnet:\?xt=urn:btih:[a-zA-Z0-9]*" @@ -54,11 +55,12 @@ def get_readable_file_size(size_in_bytes) -> str: except IndexError: return 'File too large' + def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): if dl.status() == MirrorStatus.STATUS_DOWNLOADING: - if dl.download().gid == gid: + if dl.gid() == gid: return dl return None @@ -97,13 +99,13 @@ def get_readable_message(): msg += download.status() if download.status() != MirrorStatus.STATUS_ARCHIVING: msg += f"\n{get_progress_bar_string(download)} {download.progress()} of " \ - f"{download.size()}" \ - f" at {download.speed()}, ETA: {download.eta()} " + f"{download.size()}" \ + f" at {download.speed()}, ETA: {download.eta()} " if download.status() == MirrorStatus.STATUS_DOWNLOADING: if hasattr(download, 'is_torrent'): msg += f"| P: {download.download().connections} " \ f"| S: {download.download().num_seeders}" - msg += f"\nGID: {download.download().gid}" + msg += f"\nGID: {download.gid()}" msg += "\n\n" return msg diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index fa3b4609f..8f8a6719b 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -1,15 +1,17 @@ -from .download_helper import DownloadHelper import threading import time -from ..status_utils.telegram_download_status import TelegramDownloadStatus -from bot.helper.ext_utils.bot_utils import get_readable_file_size -from bot import LOGGER, bot, download_dict, download_dict_lock, TELEGRAM_API,\ - TELEGRAM_HASH, USER_SESSION_STRING + from pyrogram import Client +from bot import LOGGER, bot, download_dict, download_dict_lock, TELEGRAM_API, \ + TELEGRAM_HASH, USER_SESSION_STRING +from .download_helper import DownloadHelper +from ..status_utils.telegram_download_status import TelegramDownloadStatus + global_lock = threading.Lock() GLOBAL_GID = set() + class TelegramDownloadHelper(DownloadHelper): def __init__(self, listener): super().__init__() @@ -19,8 +21,8 @@ def __init__(self, listener): self.__gid = '' self.__start_time = time.time() self.__user_bot = Client(api_id=TELEGRAM_API, - api_hash=TELEGRAM_HASH, - session_name=USER_SESSION_STRING) + api_hash=TELEGRAM_HASH, + session_name=USER_SESSION_STRING) self.__user_bot.start() @property @@ -51,15 +53,16 @@ def __onDownloadProgress(self, current, total): self.progress = current / self.size * 100 except ZeroDivisionError: return 0 + def __onDownloadComplete(self): - + self.__listener.onDownloadComplete() def __download(self, message, path): self.__user_bot.download_media(message, - progress=self.__onDownloadProgress, file_name=path) + progress=self.__onDownloadProgress, file_name=path) self.__onDownloadComplete() - + def add_download(self, message, path): if message.chat.type == "private": _message = self.__user_bot.get_messages(bot.get_me().id, message.message_id) @@ -70,7 +73,7 @@ def add_download(self, message, path): with global_lock: # For avoiding locking the thread lock for long time unnecessarily download = media.file_id not in GLOBAL_GID - + if download: self.__onDownloadStart(media.file_name, media.file_size, media.file_id) LOGGER.info(media.file_id) diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index bfe81f124..52dc0ca0d 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -74,3 +74,7 @@ def download(self): def uid(self): return self.__uid + + def gid(self): + self.__update() + return self.__gid diff --git a/bot/helper/mirror_utils/status_utils/telegram_download_status.py b/bot/helper/mirror_utils/status_utils/telegram_download_status.py index 186ac96bc..846aa04da 100644 --- a/bot/helper/mirror_utils/status_utils/telegram_download_status.py +++ b/bot/helper/mirror_utils/status_utils/telegram_download_status.py @@ -1,6 +1,6 @@ +from bot import DOWNLOAD_DIR from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status -from bot import DOWNLOAD_DIR class TelegramDownloadStatus(Status): @@ -8,6 +8,9 @@ def __init__(self, obj, uid): self.obj = obj self.uid = uid + def gid(self): + return self.obj.gid + def path(self): return f"{DOWNLOAD_DIR}{self.uid}" From b4b0dcd3f832986b1e4ffc675e722b27d85b3ce1 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 22 Mar 2020 15:02:21 +0530 Subject: [PATCH 15/79] Improve logic for parsing id from gdrive url Signed-off-by: lzzy12 --- .../mirror_utils/upload_utils/gdriveTools.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 75dd238c8..d88c816fe 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -1,16 +1,20 @@ import os import pickle +import urllib.parse as urlparse +from urllib.parse import parse_qs +import requests from google.auth.transport.requests import Request from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build from googleapiclient.errors import HttpError from googleapiclient.http import MediaFileUpload from tenacity import * -from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, DOWNLOAD_STATUS_UPDATE_INTERVAL + +from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type -import requests + logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) @@ -53,12 +57,12 @@ def speed(self): except ZeroDivisionError: return 0 - def parseLink(self,link): + @staticmethod + def getIdFromUrl(link: str): if "folders" in link or "file" in link: - node_id = link.split("?")[0].split("/")[-1].replace("/",'') - else: - node_id = link.split("=")[1].split("&")[0].replace("/",'') - return node_id + return link.rsplit('/')[-1] + parsed = urlparse.urlparse(link) + return parse_qs(parsed.query)['id'][0] @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) @@ -176,8 +180,8 @@ def copyFile(self,file_id,dest_id): return self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() def clone(self,link): - self.transferred_size = 0 - file_id = self.parseLink(link) + self.transferred_size = 0 + file_id = self.getIdFromUrl(link) msg = "" LOGGER.info(f"File ID: {file_id}") try: From d6a513e77b22a6a22c2822d27415dd93a1bf2d9d Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 22 Mar 2020 15:48:14 +0530 Subject: [PATCH 16/79] Fix typo in direct_link_generator.py Signed-off-by: lzzy12 --- .../download_utils/direct_link_generator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py index 680f35976..f0f002eba 100644 --- a/bot/helper/mirror_utils/download_utils/direct_link_generator.py +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -5,13 +5,15 @@ # """ Helper Module containing various sites direct links generators""" -from os import popen +import json import re import urllib.parse -import json +from os import popen from random import choice + import requests from bs4 import BeautifulSoup + from bot.helper.ext_utils.exceptions import DirectDownloadLinkException @@ -32,7 +34,7 @@ def direct_link_generator(link: str): elif 'mediafire.com' in link: return mediafire(link) elif 'sourceforge.net' in link: - return sourceforge(link) + return sourceforge(link) elif 'osdn.net' in link: return osdn(link) elif 'github.com' in link: @@ -40,8 +42,7 @@ def direct_link_generator(link: str): elif 'androidfilehost.com' in link: return androidfilehost(link) else: - raise DirectDownloadLinkException(re.findall(r"\bhttps?://(.*?[^/]+)", - link)[0] + 'is not supported') + raise DirectDownloadLinkException(f'No Direct link function found for {link}') def gdrive(url: str) -> str: From 7d74cf29d8fe1334a1e8cf83d5b40302f8c737bd Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 22 Mar 2020 15:55:59 +0530 Subject: [PATCH 17/79] Fixed support for mirror of video and audio telegram files Signed-off-by: lzzy12 --- .../download_utils/telegram_downloader.py | 7 ++- bot/modules/mirror.py | 44 +++++++++++-------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index 8f8a6719b..4531681c5 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -68,7 +68,12 @@ def add_download(self, message, path): _message = self.__user_bot.get_messages(bot.get_me().id, message.message_id) else: _message = self.__user_bot.get_messages(message.chat.id, message.message_id) - media = _message.document + media = None + media_array = [_message.document, _message.video, _message.audio] + for i in media_array: + if i is not None: + media = i + break if media is not None: with global_lock: # For avoiding locking the thread lock for long time unnecessarily diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index ed45311f3..31538e6f7 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -1,23 +1,25 @@ +import os +import pathlib + +import requests from telegram.ext import CommandHandler, run_async -from bot.helper.mirror_utils.status_utils import listeners -from bot.helper.mirror_utils.upload_utils import gdriveTools -from bot.helper.mirror_utils.download_utils import aria2_download -from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus -from bot.helper.mirror_utils.status_utils.tar_status import TarStatus + +from bot import Interval, INDEX_URL from bot import dispatcher, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, download_dict, download_dict_lock from bot.helper.ext_utils import fs_utils, bot_utils -from bot import Interval, INDEX_URL -from bot.helper.telegram_helper.message_utils import * from bot.helper.ext_utils.bot_utils import setInterval -from bot.helper.telegram_helper.filters import CustomFilters -from bot.helper.telegram_helper.bot_commands import BotCommands -import pathlib -import os +from bot.helper.ext_utils.exceptions import DirectDownloadLinkException +from bot.helper.mirror_utils.download_utils import aria2_download from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper -from bot.helper.ext_utils.exceptions import DirectDownloadLinkException -import requests -import threading +from bot.helper.mirror_utils.status_utils import listeners +from bot.helper.mirror_utils.status_utils.tar_status import TarStatus +from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus +from bot.helper.mirror_utils.upload_utils import gdriveTools +from bot.helper.telegram_helper.bot_commands import BotCommands +from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.telegram_helper.message_utils import * + class MirrorListener(listeners.MirrorListeners): def __init__(self, bot, update, isTar=False, tag=None): @@ -142,12 +144,18 @@ def _mirror(bot, update, isTar=False): link = link.strip() reply_to = update.message.reply_to_message if reply_to is not None: + file = None tag = reply_to.from_user.username - document = reply_to.document + media_array = [reply_to.document, reply_to.video, reply_to.audio] + for i in media_array: + if i is not None: + file = i + break + if len(link) == 0: - if document is not None: - if document.file_size <= 20 * 1024 * 1024: - link = document.get_file().file_path + if file is not None: + if file.file_size <= 20 * 1024 * 1024: + link = file.get_file().file_path else: listener = MirrorListener(bot, update, isTar, tag) tg_downloader = TelegramDownloadHelper(listener) From 9369c7dc989f72c3efd19b38606b55d80487cbdf Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 22 Mar 2020 16:47:40 +0530 Subject: [PATCH 18/79] Allow mirroring tg file again once file download is complete Signed-off-by: lzzy12 --- bot/__main__.py | 18 ++++++++++-------- .../download_utils/telegram_downloader.py | 10 ++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index 0c0d2d692..0c2153149 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,17 +1,19 @@ +import shutil +import signal + from telegram.ext import CommandHandler, run_async -from bot import dispatcher, LOGGER, updater, botStartTime + +from bot import dispatcher, updater, botStartTime from bot.helper.ext_utils import fs_utils -from .helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time -import signal -import time +from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.message_utils import * -import shutil +from .helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time from .helper.telegram_helper.filters import CustomFilters -from bot.helper.telegram_helper.bot_commands import BotCommands -from .modules import authorize, list, cancel_mirror, mirror_status, mirror +from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone + @run_async -def stats(bot,update): +def stats(bot, update): currentTime = get_readable_time((time.time() - botStartTime)) total, used, free = shutil.disk_usage('.') total = get_readable_file_size(total) diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index 4531681c5..51dc2a1e7 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -3,7 +3,7 @@ from pyrogram import Client -from bot import LOGGER, bot, download_dict, download_dict_lock, TELEGRAM_API, \ +from bot import LOGGER, download_dict, download_dict_lock, TELEGRAM_API, \ TELEGRAM_HASH, USER_SESSION_STRING from .download_helper import DownloadHelper from ..status_utils.telegram_download_status import TelegramDownloadStatus @@ -55,7 +55,8 @@ def __onDownloadProgress(self, current, total): return 0 def __onDownloadComplete(self): - + with global_lock: + GLOBAL_GID.remove(self.gid) self.__listener.onDownloadComplete() def __download(self, message, path): @@ -64,10 +65,7 @@ def __download(self, message, path): self.__onDownloadComplete() def add_download(self, message, path): - if message.chat.type == "private": - _message = self.__user_bot.get_messages(bot.get_me().id, message.message_id) - else: - _message = self.__user_bot.get_messages(message.chat.id, message.message_id) + _message = self.__user_bot.get_messages(message.chat.id, message.message_id) media = None media_array = [_message.document, _message.video, _message.audio] for i in media_array: From f50981b3a20b39447dd419adf67c765213f274cb Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 23 Mar 2020 14:11:47 +0530 Subject: [PATCH 19/79] Implement cancel downloads of tg mirrors Signed-off-by: lzzy12 --- bot/__init__.py | 11 ++--- .../download_utils/telegram_downloader.py | 35 ++++++++++++---- .../status_utils/telegram_download_status.py | 3 ++ bot/modules/cancel_mirror.py | 40 ++++++++++++------- 4 files changed, 62 insertions(+), 27 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 5e9cc67f8..7190675ae 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -1,11 +1,12 @@ import logging -import aria2p -import threading import os -from dotenv import load_dotenv -import telegram.ext as tg +import threading import time +import aria2p +import telegram.ext as tg +from dotenv import load_dotenv + botStartTime = time.time() if os.path.exists('log.txt'): with open('log.txt', 'r+') as f: @@ -13,7 +14,7 @@ logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[logging.FileHandler('log.txt'), logging.StreamHandler()], - level=logging.WARNING) + level=logging.INFO) load_dotenv('config.env') diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index 51dc2a1e7..4d8c02f2b 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -1,3 +1,4 @@ +import logging import threading import time @@ -11,6 +12,8 @@ global_lock = threading.Lock() GLOBAL_GID = set() +logging.getLogger("pyrogram").setLevel(logging.WARNING) + class TelegramDownloadHelper(DownloadHelper): def __init__(self, listener): @@ -24,6 +27,7 @@ def __init__(self, listener): api_hash=TELEGRAM_HASH, session_name=USER_SESSION_STRING) self.__user_bot.start() + self.__is_cancelled = False @property def gid(self): @@ -47,12 +51,21 @@ def __onDownloadStart(self, name, size, file_id): self.__listener.onDownloadStarted() def __onDownloadProgress(self, current, total): + if self.__is_cancelled: + self.__onDownloadError('Cancelled by user!') + self.__user_bot.stop_transmission() + return with self.__resource_lock: self.downloaded_bytes = current try: self.progress = current / self.size * 100 except ZeroDivisionError: - return 0 + self.progress = 0 + + def __onDownloadError(self, error): + with global_lock: + GLOBAL_GID.remove(self.gid) + self.__listener.onDownloadError(error) def __onDownloadComplete(self): with global_lock: @@ -60,9 +73,13 @@ def __onDownloadComplete(self): self.__listener.onDownloadComplete() def __download(self, message, path): - self.__user_bot.download_media(message, - progress=self.__onDownloadProgress, file_name=path) - self.__onDownloadComplete() + download = self.__user_bot.download_media(message, + progress=self.__onDownloadProgress, file_name=path) + if download is not None: + self.__onDownloadComplete() + else: + if not self.__is_cancelled: + self.__onDownloadError('Internal error occurred') def add_download(self, message, path): _message = self.__user_bot.get_messages(message.chat.id, message.message_id) @@ -79,9 +96,13 @@ def add_download(self, message, path): if download: self.__onDownloadStart(media.file_name, media.file_size, media.file_id) - LOGGER.info(media.file_id) + LOGGER.info(f'Downloading telegram file with id: {media.file_id}') threading.Thread(target=self.__download, args=(_message, path)).start() else: - self.__listener.onDownloadError('File already being downloaded!') + self.__onDownloadError('File already being downloaded!') else: - self.__listener.onDownloadError('No document in the replied message') + self.__onDownloadError('No document in the replied message') + + def cancel_download(self): + LOGGER.info(f'Cancelling download on user request: {self.gid}') + self.__is_cancelled = True diff --git a/bot/helper/mirror_utils/status_utils/telegram_download_status.py b/bot/helper/mirror_utils/status_utils/telegram_download_status.py index 846aa04da..7647f6aec 100644 --- a/bot/helper/mirror_utils/status_utils/telegram_download_status.py +++ b/bot/helper/mirror_utils/status_utils/telegram_download_status.py @@ -50,3 +50,6 @@ def eta(self): return f'{get_readable_time(seconds)}' except ZeroDivisionError: return '-' + + def download(self): + return self.obj diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index fd8eae477..88aea78ef 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -1,21 +1,26 @@ +from time import sleep + from telegram.ext import CommandHandler, run_async -from bot.helper.telegram_helper.message_utils import * + from bot import download_dict, aria2, dispatcher, download_dict_lock, DOWNLOAD_DIR -from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.ext_utils.bot_utils import getDownloadByGid from bot.helper.ext_utils.fs_utils import clean_download from bot.helper.telegram_helper.bot_commands import BotCommands -from time import sleep -from bot.helper.ext_utils.bot_utils import getDownloadByGid +from bot.helper.telegram_helper.filters import CustomFilters +from bot.helper.telegram_helper.message_utils import * +from ..helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper + @run_async -def cancel_mirror(bot,update): - args = update.message.text.split(" ",maxsplit=1) +def cancel_mirror(bot, update): + # TODO: Rewrite this for good + args = update.message.text.split(" ", maxsplit=1) mirror_message = None if len(args) > 1: gid = args[1] dl = getDownloadByGid(gid) if not dl: - sendMessage(f"GID: {gid} not found.",bot,update) + sendMessage(f"GID: {gid} not found.", bot, update) return with download_dict_lock: keys = list(download_dict.keys()) @@ -27,13 +32,15 @@ def cancel_mirror(bot,update): dl = download_dict[mirror_message.message_id] if len(args) == 1: if mirror_message is None or mirror_message.message_id not in keys: - if BotCommands.MirrorCommand in mirror_message.text or BotCommands.TarMirrorCommand in mirror_message.text: + if BotCommands.MirrorCommand in mirror_message.text or \ + BotCommands.TarMirrorCommand in mirror_message.text: msg = "Mirror already have been cancelled" - sendMessage(msg,bot,update) + sendMessage(msg, bot, update) return else: - msg = "Please reply to the /mirror message which was used to start the download or /cancel gid to cancel it!" - sendMessage(msg,bot,update) + msg = "Please reply to the /mirror message which was " \ + "used to start the download or /cancel gid to cancel it!" + sendMessage(msg, bot, update) return if dl.status() == "Uploading": sendMessage("Upload in Progress, Don't Cancel it.", bot, update) @@ -43,10 +50,13 @@ def cancel_mirror(bot,update): return elif dl.status() != "Queued": download = dl.download() - if len(download.followed_by_ids) != 0: - downloads = aria2.get_downloads(download.followed_by_ids) - aria2.pause(downloads) - aria2.pause([download]) + if isinstance(download, TelegramDownloadHelper): + download.cancel_download() + else: + if len(download.followed_by_ids) != 0: + downloads = aria2.get_downloads(download.followed_by_ids) + aria2.pause(downloads) + aria2.pause([download]) else: dl._listener.onDownloadError("Download stopped by user!") sleep(1) # Wait a Second For Aria2 To free Resources. From 94b6be4c03821da3d3ce6f0011b68f16f6a97ab7 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 23 Mar 2020 15:41:53 +0530 Subject: [PATCH 20/79] Add licensing and authorship information for direct_link_generator.py Signed-off-by: lzzy12 --- .../download_utils/direct_link_generator.py | 5 +- .../direct_link_generator_license.md | 82 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 bot/helper/mirror_utils/download_utils/direct_link_generator_license.md diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py index f0f002eba..6ee5a9402 100644 --- a/bot/helper/mirror_utils/download_utils/direct_link_generator.py +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -3,7 +3,10 @@ # Licensed under the Raphielscape Public License, Version 1.c (the "License"); # you may not use this file except in compliance with the License. # -""" Helper Module containing various sites direct links generators""" +""" Helper Module containing various sites direct links generators. This module is copied and modified as per need +from https://github.com/AvinashReddy3108/PaperplaneExtended . I hereby take no credit of the following code other +than the modifications. See https://github.com/AvinashReddy3108/PaperplaneExtended/commits/master/userbot/modules/direct_links.py +for original authorship. """ import json import re diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator_license.md b/bot/helper/mirror_utils/download_utils/direct_link_generator_license.md new file mode 100644 index 000000000..c37c0d93d --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator_license.md @@ -0,0 +1,82 @@ + RAPHIELSCAPE PUBLIC LICENSE + Version 1.c, June 2019 + + Copyright (C) 2019 Raphielscape LLC. + Copyright (C) 2019 Devscapes Open Source Holding GmbH. + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + RAPHIELSCAPE PUBLIC LICENSE + A-1. DEFINITIONS + +0. “This License” refers to version 1.c of the Raphielscape Public License. + +1. “Copyright” also means copyright-like laws that apply to other kinds of works. + +2. “The Work" refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. + “Licensees” and “recipients” may be individuals or organizations. + +3. To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, + other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work + or a work “based on” the earlier work. + +4. Source Form. The “source form” for a work means the preferred form of the work for making modifications to it. + “Object code” means any non-source form of a work. + + The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and + (for an executable work) run the object code and to modify the work, including scripts to control those activities. + + The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + The Corresponding Source for a work in source code form is that same work. + +5. "The author" refers to "author" of the code, which is the one that made the particular code which exists inside of + the Corresponding Source. + +6. "Owner" refers to any parties which is made the early form of the Corresponding Source. + + A-2. TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. You must give any other recipients of the Work or Derivative Works a copy of this License; and + +1. You must cause any modified files to carry prominent notices stating that You changed the files; and + +2. You must retain, in the Source form of any Derivative Works that You distribute, + this license, all copyright, patent, trademark, authorships and attribution notices + from the Source form of the Work; and + +3. Respecting the author and owner of works that are distributed in any way. + + You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, +or distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + + B. DISCLAIMER OF WARRANTY + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + C. REVISED VERSION OF THIS LICENSE + + The Devscapes Open Source Holding GmbH. may publish revised and/or new versions of the +Raphielscape Public License from time to time. Such new versions will be similar in spirit +to the present version, but may differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the Program specifies that a +certain numbered version of the Raphielscape Public License "or any later version" applies to it, +you have the option of following the terms and conditions either of that numbered version or of +any later version published by the Devscapes Open Source Holding GmbH. If the Program does not specify a +version number of the Raphielscape Public License, you may choose any version ever published +by the Devscapes Open Source Holding GmbH. + + END OF LICENSE \ No newline at end of file From 997e572b69f966813dcfd00218904773d57e0585 Mon Sep 17 00:00:00 2001 From: jaskaranSM <37726998+jaskaranSM@users.noreply.github.com> Date: Mon, 23 Mar 2020 16:53:13 +0530 Subject: [PATCH 21/79] Enable TeamDrive support for clone --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index d88c816fe..18eeea3f3 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -185,7 +185,7 @@ def clone(self,link): msg = "" LOGGER.info(f"File ID: {file_id}") try: - meta = self.__service.files().get(fileId=file_id,fields="name,id,mimeType,size").execute() + meta = self.__service.files().get(supportsAllDrives=True,fileId=file_id,fields="name,id,mimeType,size").execute() except Exception as e: return f"{str(e).replace('>','').replace('<','')}" if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: From 7ad7d9388adec4584690ca9898dcfd4e160428b4 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 21 Jan 2020 03:49:15 -0800 Subject: [PATCH 22/79] Add support for service account Adding support service accounts Signed-off-by: lzzy12 Added scripts and docs for generating service accounts Signed-off-by: lzzy12 gen_sa_accounts: Save credentials with indexed file name Signed-off-by: lzzy12 gdriveTools: Avoid using oauth2 library for service accounts oauth2 library is deprecated Signed-off-by: lzzy12 --- .gitignore | 1 + README.md | 31 ++ add_to_google_group.py | 84 ++++ bot/__init__.py | 12 +- .../mirror_utils/upload_utils/gdriveTools.py | 138 ++++--- config_sample.env | 3 +- gen_sa_accounts.py | 365 ++++++++++++++++++ 7 files changed, 582 insertions(+), 52 deletions(-) create mode 100644 add_to_google_group.py create mode 100644 gen_sa_accounts.py diff --git a/.gitignore b/.gitignore index b651b326b..86facd3c1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ data* *.pickle authorized_chats.txt log.txt +accounts/* \ No newline at end of file diff --git a/README.md b/README.md index be98fa832..f0db9b9a6 100644 --- a/README.md +++ b/README.md @@ -87,3 +87,34 @@ sudo docker build . -t mirror-bot ``` sudo docker run mirror-bot ``` + +## Using service accounts for uploading to avoid user rate limit + +Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts +### Generating service accounts +Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) [How to use service account in rclone](https://rclone.org/drive/#service-account-support). +--------------------------------- +Let us create only the service accounts that we need. +**Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. + +``` +Note: 1 service account can copy around 750gb a day, 1 project makes 100 service accounts so thats 75tb a day, for most users this should easily suffice. +``` + +`python3 gen_sa_accounts.py --quick-setup 1 --new-only` + +A folder named accounts will be created which will contain keys for the service accounts created +``` +We highly recommend to zip this folder and store it somewhere safe, so that you do not have to create a new project everytime you want to deploy the bot +``` +### Adding service accounts to Google Groups: +We use Google Groups to manager our service accounts considering the +[Official limits to the members of Team Drive](https://support.google.com/a/answer/7338880?hl=en) (Limit for individuals and groups directly added as members: 600). + +1. Turn on the Directory API following [official steps](https://developers.google.com/admin-sdk/directory/v1/quickstart/python) (save the generated json file to folder `credentials`). + +2. Create group for your organization [in the Admin console](https://support.google.com/a/answer/33343?hl=en). After create a group, you will have an address for example`sa@yourdomain.com`. + +3. Run `python3 add_to_google_group.py -g sa@yourdomain.com` + +4. Now, add Google Groups (**Step 2**) to manager your service accounts, add the group address `sa@yourdomain.com` or `sa@googlegroups.com` to the Team drive or folder diff --git a/add_to_google_group.py b/add_to_google_group.py new file mode 100644 index 000000000..aaed28bf4 --- /dev/null +++ b/add_to_google_group.py @@ -0,0 +1,84 @@ +# auto rclone +# Add service accounts to groups for your organization +# +# Author Telegram https://t.me/CodyDoby +# Inbox codyd@qq.com + +from __future__ import print_function + +import os +import pickle + +import argparse +import glob +import googleapiclient.discovery +import json +import progress.bar +import time +from google.auth.transport.requests import Request +from google_auth_oauthlib.flow import InstalledAppFlow + +stt = time.time() + +parse = argparse.ArgumentParser( + description='A tool to add service accounts to groups for your organization from a folder containing credential ' + 'files.') +parse.add_argument('--path', '-p', default='accounts', + help='Specify an alternative path to the service accounts folder.') +parse.add_argument('--credentials', '-c', default='credentials/credentials.json', + help='Specify the relative path for the controller file.') +parsereq = parse.add_argument_group('required arguments') +# service-account@googlegroups.com +parsereq.add_argument('--groupaddr', '-g', help='The address of groups for your organization.', required=True) + +args = parse.parse_args() +acc_dir = args.path +gaddr = args.groupaddr +credentials = glob.glob(args.credentials) + +creds = None +if os.path.exists('credentials/token.pickle'): + with open('credentials/token.pickle', 'rb') as token: + creds = pickle.load(token) +# If there are no (valid) credentials available, let the user log in. +if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[ + 'https://www.googleapis.com/auth/admin.directory.group', + 'https://www.googleapis.com/auth/admin.directory.group.member' + ]) + # creds = flow.run_local_server(port=0) + creds = flow.run_console() + # Save the credentials for the next run + with open('credentials/token.pickle', 'wb') as token: + pickle.dump(creds, token) + +group = googleapiclient.discovery.build("admin", "directory_v1", credentials=creds) + +print(group.members()) + +batch = group.new_batch_http_request() + +sa = glob.glob('%s/*.json' % acc_dir) + +# sa = sa[0:5] + +pbar = progress.bar.Bar("Readying accounts", max=len(sa)) +for i in sa: + ce = json.loads(open(i, 'r').read())['client_email'] + + body = {"email": ce, "role": "MEMBER"} + batch.add(group.members().insert(groupKey=gaddr, body=body)) + # group.members().insert(groupKey=gaddr, body=body).execute() + + pbar.next() +pbar.finish() +print('Adding...') +batch.execute() + +print('Complete.') +hours, rem = divmod((time.time() - stt), 3600) +minutes, sec = divmod(rem, 60) +print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec)) diff --git a/bot/__init__.py b/bot/__init__.py index 7190675ae..1fd6ba9af 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -90,6 +90,16 @@ def getConfig(name: str): IS_TEAM_DRIVE = False except KeyError: IS_TEAM_DRIVE = False + +try: + USE_SERVICE_ACCOUNTS = getConfig('USE_SERVICE_ACCOUNTS') + if USE_SERVICE_ACCOUNTS.lower() == 'true': + USE_SERVICE_ACCOUNTS = True + else: + USE_SERVICE_ACCOUNTS = False +except KeyError: + USE_SERVICE_ACCOUNTS = False + updater = tg.Updater(token=BOT_TOKEN) bot = updater.bot -dispatcher = updater.dispatcher \ No newline at end of file +dispatcher = updater.dispatcher diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 18eeea3f3..62bd6ef9b 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -4,33 +4,66 @@ from urllib.parse import parse_qs import requests + from google.auth.transport.requests import Request +from google.oauth2 import service_account from google_auth_oauthlib.flow import InstalledAppFlow from googleapiclient.discovery import build from googleapiclient.errors import HttpError from googleapiclient.http import MediaFileUpload from tenacity import * -from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL +from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, DOWNLOAD_STATUS_UPDATE_INTERVAL, \ + USE_SERVICE_ACCOUNTS from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) +G_DRIVE_TOKEN_FILE = "token.pickle" +# Check https://developers.google.com/drive/scopes for all available scopes +OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] + +SERVICE_ACCOUNT_INDEX = 0 + + +def authorize(): + # Get credentials + credentials = None + if not USE_SERVICE_ACCOUNTS: + if os.path.exists(G_DRIVE_TOKEN_FILE): + with open(G_DRIVE_TOKEN_FILE, 'rb') as f: + credentials = pickle.load(f) + if credentials is None or not credentials.valid: + if credentials and credentials.expired and credentials.refresh_token: + credentials.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + 'credentials.json', OAUTH_SCOPE) + LOGGER.info(flow) + credentials = flow.run_console(port=0) + + # Save the credentials for the next run + with open(G_DRIVE_TOKEN_FILE, 'wb') as token: + pickle.dump(credentials, token) + else: + credentials = service_account.Credentials \ + .from_service_account_file(f'accounts/{SERVICE_ACCOUNT_INDEX}.json', + scopes=OAUTH_SCOPE) + return build('drive', 'v3', credentials=credentials, cache_discovery=False) + + +service = authorize() + class GoogleDriveHelper: + # Redirect URI for installed apps, can be left as is + REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" + G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" + G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" def __init__(self, name=None, listener=None): - self.__G_DRIVE_TOKEN_FILE = "token.pickle" - # Check https://developers.google.com/drive/scopes for all available scopes - self.__OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] - # Redirect URI for installed apps, can be left as is - self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" - self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" - self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" - self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" self.__listener = listener - self.__service = self.authorize() self._file_uploaded_bytes = 0 self.uploaded_bytes = 0 self.start_time = 0 @@ -74,6 +107,21 @@ def _on_upload_progress(self): self.uploaded_bytes += chunk_size self.total_time += self.update_interval + @staticmethod + def __upload_empty_file(path, file_name, mime_type, parent_id=None): + media_body = MediaFileUpload(path, + mimetype=mime_type, + resumable=False) + file_metadata = { + 'name': file_name, + 'description': 'mirror', + 'mimeType': mime_type, + } + if parent_id is not None: + file_metadata['parents'] = [parent_id] + return service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body).execute() + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def __set_permission(self, drive_id): @@ -83,11 +131,13 @@ def __set_permission(self, drive_id): 'value': None, 'withLink': True } - return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() + return service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def upload_file(self, file_path, file_name, mime_type, parent_id): + global SERVICE_ACCOUNT_INDEX + global service # File body description file_metadata = { 'name': file_name, @@ -101,13 +151,13 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): media_body = MediaFileUpload(file_path, mimetype=mime_type, resumable=False) - response = self.__service.files().create(supportsTeamDrives=True, - body=file_metadata, media_body=media_body).execute() + response = service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body).execute() if not IS_TEAM_DRIVE: self.__set_permission(response['id']) - drive_file = self.__service.files().get(supportsTeamDrives=True, - fileId=response['id']).execute() - download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) + drive_file = service.files().get(supportsTeamDrives=True, + fileId=response['id']).execute() + download_url = self.G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url media_body = MediaFileUpload(file_path, mimetype=mime_type, @@ -115,20 +165,28 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): chunksize=50 * 1024 * 1024) # Insert a file - drive_file = self.__service.files().create(supportsTeamDrives=True, - body=file_metadata, media_body=media_body) + drive_file = service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body) response = None while response is None: if self.is_cancelled: return None - self.status, response = drive_file.next_chunk() + try: + self.status, response = drive_file.next_chunk() + except HttpError as err: + if err.resp.get('content-type', '').startswith('application/json'): + reason = json.loads(err.content).get('error').get('errors')[0].get('reason') + if reason == 'userRateLimitExceeded': + SERVICE_ACCOUNT_INDEX += 1 + service = authorize() + raise err self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: self.__set_permission(response['id']) # Define file instance and get url for download - drive_file = self.__service.files().get(supportsTeamDrives=True, fileId=response['id']).execute() - download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) + drive_file = service.files().get(supportsTeamDrives=True, fileId=response['id']).execute() + download_url = self.G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url def upload(self, file_name: str): @@ -249,11 +307,11 @@ def cloneFolder(self,name,local_path,folder_id,parent_id): def create_directory(self, directory_name, parent_id): file_metadata = { "name": directory_name, - "mimeType": self.__G_DRIVE_DIR_MIME_TYPE + "mimeType": self.G_DRIVE_DIR_MIME_TYPE } if parent_id is not None: file_metadata["parents"] = [parent_id] - file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute() + file = service.files().create(supportsTeamDrives=True, body=file_metadata).execute() file_id = file.get("id") if not IS_TEAM_DRIVE: self.__set_permission(file_id) @@ -280,26 +338,6 @@ def upload_dir(self, input_directory, parent_id): new_id = parent_id return new_id - def authorize(self): - # Get credentials - credentials = None - if os.path.exists(self.__G_DRIVE_TOKEN_FILE): - with open(self.__G_DRIVE_TOKEN_FILE, 'rb') as f: - credentials = pickle.load(f) - if credentials is None or not credentials.valid: - if credentials and credentials.expired and credentials.refresh_token: - credentials.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - 'credentials.json', self.__OAUTH_SCOPE) - LOGGER.info(flow) - credentials = flow.run_console(port=0) - - # Save the credentials for the next run - with open(self.__G_DRIVE_TOKEN_FILE, 'wb') as token: - pickle.dump(credentials, token) - return build('drive', 'v3', credentials=credentials, cache_discovery=False) - def drive_list(self, fileName): msg = "" # Create Search Query for API request. @@ -307,13 +345,13 @@ def drive_list(self, fileName): page_token = None results = [] while True: - response = self.__service.files().list(supportsTeamDrives=True, - includeTeamDriveItems=True, - q=query, - spaces='drive', - fields='nextPageToken, files(id, name, mimeType, size)', - pageToken=page_token, - orderBy='modifiedTime desc').execute() + response = service.files().list(supportsTeamDrives=True, + includeTeamDriveItems=True, + q=query, + spaces='drive', + fields='nextPageToken, files(id, name, mimeType, size)', + pageToken=page_token, + orderBy='modifiedTime desc').execute() for file in response.get('files', []): if len(results) >= 20: break diff --git a/config_sample.env b/config_sample.env index b0f51562a..1ef1dc730 100644 --- a/config_sample.env +++ b/config_sample.env @@ -12,4 +12,5 @@ IS_TEAM_DRIVE = "" INDEX_URL = "" USER_SESSION_STRING = "" TELEGRAM_API = -TELEGRAM_HASH = "" \ No newline at end of file +TELEGRAM_HASH = "" +USE_SERVICE_ACCOUNTS = "" diff --git a/gen_sa_accounts.py b/gen_sa_accounts.py new file mode 100644 index 000000000..0fd1c244c --- /dev/null +++ b/gen_sa_accounts.py @@ -0,0 +1,365 @@ +import errno +import os +import pickle +import sys +from argparse import ArgumentParser +from base64 import b64decode +from glob import glob +from json import loads +from random import choice +from time import sleep + +from google.auth.transport.requests import Request +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +SCOPES = ['https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/iam'] +project_create_ops = [] +current_key_dump = [] +sleep_time = 30 + + +# Create count SAs in project +def _create_accounts(service, project, count): + batch = service.new_batch_http_request(callback=_def_batch_resp) + for i in range(count): + aid = _generate_id('mfc-') + batch.add(service.projects().serviceAccounts().create(name='projects/' + project, body={'accountId': aid, + 'serviceAccount': { + 'displayName': aid}})) + batch.execute() + + +# Create accounts needed to fill project +def _create_remaining_accounts(iam, project): + print('Creating accounts in %s' % project) + sa_count = len(_list_sas(iam, project)) + while sa_count != 100: + _create_accounts(iam, project, 100 - sa_count) + sa_count = len(_list_sas(iam, project)) + + +# Generate a random id +def _generate_id(prefix='saf-'): + chars = '-abcdefghijklmnopqrstuvwxyz1234567890' + return prefix + ''.join(choice(chars) for _ in range(25)) + choice(chars[1:]) + + +# List projects using service +def _get_projects(service): + return [i['projectId'] for i in service.projects().list().execute()['projects']] + + +# Default batch callback handler +def _def_batch_resp(id, resp, exception): + if exception is not None: + if str(exception).startswith(' 0: + current_count = len(_get_projects(cloud)) + if current_count + create_projects <= max_projects: + print('Creating %d projects' % (create_projects)) + nprjs = _create_projects(cloud, create_projects) + selected_projects = nprjs + else: + sys.exit('No, you cannot create %d new project (s).\n' + 'Please reduce value of --quick-setup.\n' + 'Remember that you can totally create %d projects (%d already).\n' + 'Please do not delete existing projects unless you know what you are doing' % ( + create_projects, max_projects, current_count)) + else: + print('Will overwrite all service accounts in existing projects.\n' + 'So make sure you have some projects already.') + input("Press Enter to continue...") + + if enable_services: + ste = [] + ste.append(enable_services) + if enable_services == '~': + ste = selected_projects + elif enable_services == '*': + ste = _get_projects(cloud) + services = [i + '.googleapis.com' for i in services] + print('Enabling services') + _enable_services(serviceusage, ste, services) + if create_sas: + stc = [] + stc.append(create_sas) + if create_sas == '~': + stc = selected_projects + elif create_sas == '*': + stc = _get_projects(cloud) + for i in stc: + _create_remaining_accounts(iam, i) + if download_keys: + try: + os.mkdir(path) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + std = [] + std.append(download_keys) + if download_keys == '~': + std = selected_projects + elif download_keys == '*': + std = _get_projects(cloud) + _create_sa_keys(iam, std, path) + if delete_sas: + std = [] + std.append(delete_sas) + if delete_sas == '~': + std = selected_projects + elif delete_sas == '*': + std = _get_projects(cloud) + for i in std: + print('Deleting service accounts in %s' % i) + _delete_sas(iam, i) + + +if __name__ == '__main__': + parse = ArgumentParser(description='A tool to create Google service accounts.') + parse.add_argument('--path', '-p', default='accounts', + help='Specify an alternate directory to output the credential files.') + parse.add_argument('--token', default='token.pickle', help='Specify the pickle token file path.') + parse.add_argument('--credentials', default='credentials.json', help='Specify the credentials file path.') + parse.add_argument('--list-projects', default=False, action='store_true', + help='List projects viewable by the user.') + parse.add_argument('--list-sas', default=False, help='List service accounts in a project.') + parse.add_argument('--create-projects', type=int, default=None, help='Creates up to N projects.') + parse.add_argument('--max-projects', type=int, default=12, help='Max amount of project allowed. Default: 12') + parse.add_argument('--enable-services', default=None, + help='Enables services on the project. Default: IAM and Drive') + parse.add_argument('--services', nargs='+', default=['iam', 'drive'], + help='Specify a different set of services to enable. Overrides the default.') + parse.add_argument('--create-sas', default=None, help='Create service accounts in a project.') + parse.add_argument('--delete-sas', default=None, help='Delete service accounts in a project.') + parse.add_argument('--download-keys', default=None, help='Download keys for all the service accounts in a project.') + parse.add_argument('--quick-setup', default=None, type=int, + help='Create projects, enable services, create service accounts and download keys. ') + parse.add_argument('--new-only', default=False, action='store_true', help='Do not use exisiting projects.') + args = parse.parse_args() + # If credentials file is invalid, search for one. + if not os.path.exists(args.credentials): + options = glob('*.json') + print('No credentials found at %s. Please enable the Drive API in:\n' + 'https://developers.google.com/drive/api/v3/quickstart/python\n' + 'and save the json file as credentials.json' % args.credentials) + if len(options) < 1: + exit(-1) + else: + i = 0 + print('Select a credentials file below.') + inp_options = [str(i) for i in list(range(1, len(options) + 1))] + options + while i < len(options): + print(' %d) %s' % (i + 1, options[i])) + i += 1 + inp = None + while True: + inp = input('> ') + if inp in inp_options: + break + if inp in options: + args.credentials = inp + else: + args.credentials = options[int(inp) - 1] + print('Use --credentials %s next time to use this credentials file.' % args.credentials) + if args.quick_setup: + opt = '*' + if args.new_only: + opt = '~' + args.services = ['iam', 'drive'] + args.create_projects = args.quick_setup + args.enable_services = opt + args.create_sas = opt + args.download_keys = opt + resp = serviceaccountfactory( + path=args.path, + token=args.token, + credentials=args.credentials, + list_projects=args.list_projects, + list_sas=args.list_sas, + create_projects=args.create_projects, + max_projects=args.max_projects, + create_sas=args.create_sas, + delete_sas=args.delete_sas, + enable_services=args.enable_services, + services=args.services, + download_keys=args.download_keys + ) + if resp is not None: + if args.list_projects: + if resp: + print('Projects (%d):' % len(resp)) + for i in resp: + print(' ' + i) + else: + print('No projects.') + elif args.list_sas: + if resp: + print('Service accounts in %s (%d):' % (args.list_sas, len(resp))) + for i in resp: + print(' %s (%s)' % (i['email'], i['uniqueId'])) + else: + print('No service accounts.') From 2bbfca0496aaa4ca24d5b9e262c81cc7abc4f4fd Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Mon, 3 Feb 2020 17:05:46 +0530 Subject: [PATCH 23/79] Refactor service account implementation Signed-off-by: lzzy12 --- .../mirror_utils/upload_utils/gdriveTools.py | 199 +++++++++--------- 1 file changed, 102 insertions(+), 97 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 62bd6ef9b..646d17ee9 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -19,50 +19,20 @@ from bot.helper.ext_utils.fs_utils import get_mime_type logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) - -G_DRIVE_TOKEN_FILE = "token.pickle" -# Check https://developers.google.com/drive/scopes for all available scopes -OAUTH_SCOPE = ["https://www.googleapis.com/auth/drive"] - SERVICE_ACCOUNT_INDEX = 0 -def authorize(): - # Get credentials - credentials = None - if not USE_SERVICE_ACCOUNTS: - if os.path.exists(G_DRIVE_TOKEN_FILE): - with open(G_DRIVE_TOKEN_FILE, 'rb') as f: - credentials = pickle.load(f) - if credentials is None or not credentials.valid: - if credentials and credentials.expired and credentials.refresh_token: - credentials.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - 'credentials.json', OAUTH_SCOPE) - LOGGER.info(flow) - credentials = flow.run_console(port=0) - - # Save the credentials for the next run - with open(G_DRIVE_TOKEN_FILE, 'wb') as token: - pickle.dump(credentials, token) - else: - credentials = service_account.Credentials \ - .from_service_account_file(f'accounts/{SERVICE_ACCOUNT_INDEX}.json', - scopes=OAUTH_SCOPE) - return build('drive', 'v3', credentials=credentials, cache_discovery=False) - - -service = authorize() - - class GoogleDriveHelper: - # Redirect URI for installed apps, can be left as is - REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" - G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" - G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" - def __init__(self, name=None, listener=None): + self.__G_DRIVE_TOKEN_FILE = "token.pickle" + # Check https://developers.google.com/drive/scopes for all available scopes + self.__OAUTH_SCOPE = ['https://www.googleapis.com/auth/drive'] + # Redirect URI for installed apps, can be left as is + self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" + self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" + self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" + self.__listener = listener + self.__service = self.authorize() self.__listener = listener self._file_uploaded_bytes = 0 self.uploaded_bytes = 0 @@ -119,8 +89,8 @@ def __upload_empty_file(path, file_name, mime_type, parent_id=None): } if parent_id is not None: file_metadata['parents'] = [parent_id] - return service.files().create(supportsTeamDrives=True, - body=file_metadata, media_body=media_body).execute() + return self.__service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body).execute() @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) @@ -131,13 +101,11 @@ def __set_permission(self, drive_id): 'value': None, 'withLink': True } - return service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() + return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def upload_file(self, file_path, file_name, mime_type, parent_id): - global SERVICE_ACCOUNT_INDEX - global service # File body description file_metadata = { 'name': file_name, @@ -151,13 +119,14 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): media_body = MediaFileUpload(file_path, mimetype=mime_type, resumable=False) - response = service.files().create(supportsTeamDrives=True, - body=file_metadata, media_body=media_body).execute() + response = self.__service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body).execute() if not IS_TEAM_DRIVE: self.__set_permission(response['id']) - drive_file = service.files().get(supportsTeamDrives=True, - fileId=response['id']).execute() - download_url = self.G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) + + drive_file = self.__service.files().get(supportsTeamDrives=True, + fileId=response['id']).execute() + download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url media_body = MediaFileUpload(file_path, mimetype=mime_type, @@ -165,8 +134,8 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): chunksize=50 * 1024 * 1024) # Insert a file - drive_file = service.files().create(supportsTeamDrives=True, - body=file_metadata, media_body=media_body) + drive_file = self.__service.files().create(supportsTeamDrives=True, + body=file_metadata, media_body=media_body) response = None while response is None: if self.is_cancelled: @@ -177,16 +146,18 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): if err.resp.get('content-type', '').startswith('application/json'): reason = json.loads(err.content).get('error').get('errors')[0].get('reason') if reason == 'userRateLimitExceeded': + global SERVICE_ACCOUNT_INDEX SERVICE_ACCOUNT_INDEX += 1 - service = authorize() + LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") + self.__service = self.authorize() raise err self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: self.__set_permission(response['id']) # Define file instance and get url for download - drive_file = service.files().get(supportsTeamDrives=True, fileId=response['id']).execute() - download_url = self.G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) + drive_file = self.__service.files().get(supportsTeamDrives=True, fileId=response['id']).execute() + download_url = self.__G_DRIVE_BASE_DOWNLOAD_URL.format(drive_file.get('id')) return download_url def upload(self, file_name: str): @@ -204,9 +175,13 @@ def upload(self, file_name: str): raise Exception('Upload has been manually cancelled') LOGGER.info("Uploaded To G-Drive: " + file_path) except Exception as e: - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - LOGGER.error(e.last_attempt.exception()) - self.__listener.onUploadError(e) + if isinstance(e, RetryError): + LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") + err = e.last_attempt.exception() + else: + err = e + LOGGER.error(err) + self.__listener.onUploadError(str(err)) return finally: self.updater.cancel() @@ -219,9 +194,13 @@ def upload(self, file_name: str): LOGGER.info("Uploaded To G-Drive: " + file_name) link = f"https://drive.google.com/folderview?id={dir_id}" except Exception as e: - LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") - LOGGER.error(e.last_attempt.exception()) - self.__listener.onUploadError(e) + if isinstance(e, RetryError): + LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") + err = e.last_attempt.exception() + else: + err = e + LOGGER.error(err) + self.__listener.onUploadError(str(err)) return finally: self.updater.cancel() @@ -230,71 +209,72 @@ def upload(self, file_name: str): LOGGER.info("Deleting downloaded file/folder..") return link - @retry(wait=wait_exponential(multiplier=2, min=3, max=6),stop=stop_after_attempt(5),retry=retry_if_exception_type(HttpError),before=before_log(LOGGER,logging.DEBUG)) - def copyFile(self,file_id,dest_id): + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), + retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) + def copyFile(self, file_id, dest_id): body = { 'parents': [dest_id] } - return self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() + return self.__service.files().copy(supportsAllDrives=True, fileId=file_id, body=body).execute() - def clone(self,link): + def clone(self, link): self.transferred_size = 0 file_id = self.getIdFromUrl(link) msg = "" LOGGER.info(f"File ID: {file_id}") try: - meta = self.__service.files().get(supportsAllDrives=True,fileId=file_id,fields="name,id,mimeType,size").execute() + meta = self.__service.files().get(supportsAllDrives=True, fileId=file_id, + fields="name,id,mimeType,size").execute() except Exception as e: - return f"{str(e).replace('>','').replace('<','')}" + return f"{str(e).replace('>', '').replace('<', '')}" if meta.get("mimeType") == self.__G_DRIVE_DIR_MIME_TYPE: - dir_id = self.create_directory(meta.get('name'),parent_id) + dir_id = self.create_directory(meta.get('name'), parent_id) try: - result = self.cloneFolder(meta.get('name'),meta.get('name'),meta.get('id'),dir_id) + result = self.cloneFolder(meta.get('name'), meta.get('name'), meta.get('id'), dir_id) except Exception as e: - if isinstance(e,RetryError): + if isinstance(e, RetryError): LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") err = e.last_attempt.exception() else: - err = str(e).replace('>','').replace('<','') + err = str(e).replace('>', '').replace('<', '') LOGGER.error(err) return err msg += f'{meta.get("name")} ({get_readable_file_size(self.transferred_size)})' else: - file = self.copyFile(meta.get('id'),parent_id) + file = self.copyFile(meta.get('id'), parent_id) msg += f'{meta.get("name")} ({get_readable_file_size(int(meta.get("size")))})' return msg - - def cloneFolder(self,name,local_path,folder_id,parent_id): + def cloneFolder(self, name, local_path, folder_id, parent_id): page_token = None - q =f"'{folder_id}' in parents" + q = f"'{folder_id}' in parents" files = [] LOGGER.info(f"Syncing: {local_path}") new_id = None while True: response = self.__service.files().list(q=q, - spaces='drive', - fields='nextPageToken, files(id, name, mimeType,size)', - pageToken=page_token).execute() + spaces='drive', + fields='nextPageToken, files(id, name, mimeType,size)', + pageToken=page_token).execute() for file in response.get('files', []): files.append(file) page_token = response.get('nextPageToken', None) if page_token is None: - break + break if len(files) == 0: return parent_id for file in files: if file.get('mimeType') == self.__G_DRIVE_DIR_MIME_TYPE: - file_path = os.path.join(local_path,file.get('name')) - current_dir_id = self.create_directory(file.get('name'),parent_id) - new_id = self.cloneFolder(file.get('name'),file_path,file.get('id'),current_dir_id) + file_path = os.path.join(local_path, file.get('name')) + current_dir_id = self.create_directory(file.get('name'), parent_id) + new_id = self.cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id) else: self.transferred_size += int(file.get('size')) try: - self.copyFile(file.get('id'),parent_id) + self.copyFile(file.get('id'), parent_id) new_id = parent_id except Exception as e: - if isinstance(e,RetryError): + if isinstance(e, RetryError): LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") err = e.last_attempt.exception() else: @@ -307,11 +287,11 @@ def cloneFolder(self,name,local_path,folder_id,parent_id): def create_directory(self, directory_name, parent_id): file_metadata = { "name": directory_name, - "mimeType": self.G_DRIVE_DIR_MIME_TYPE + "mimeType": self.__G_DRIVE_DIR_MIME_TYPE } if parent_id is not None: file_metadata["parents"] = [parent_id] - file = service.files().create(supportsTeamDrives=True, body=file_metadata).execute() + file = self.__service.files().create(supportsTeamDrives=True, body=file_metadata).execute() file_id = file.get("id") if not IS_TEAM_DRIVE: self.__set_permission(file_id) @@ -338,22 +318,48 @@ def upload_dir(self, input_directory, parent_id): new_id = parent_id return new_id + def authorize(self): + # Get credentials + credentials = None + if not USE_SERVICE_ACCOUNTS: + if os.path.exists(G_DRIVE_TOKEN_FILE): + with open(G_DRIVE_TOKEN_FILE, 'rb') as f: + credentials = pickle.load(f) + if credentials is None or not credentials.valid: + if credentials and credentials.expired and credentials.refresh_token: + credentials.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + 'credentials.json', self.__OAUTH_SCOPE) + LOGGER.info(flow) + credentials = flow.run_console(port=0) + + # Save the credentials for the next run + with open(G_DRIVE_TOKEN_FILE, 'wb') as token: + pickle.dump(credentials, token) + else: + LOGGER.info(f"Authorizing with {SERVICE_ACCOUNT_INDEX}.json service account") + credentials = service_account.Credentials.from_service_account_file( + f'accounts/{SERVICE_ACCOUNT_INDEX}.json', + scopes=self.__OAUTH_SCOPE) + return build('drive', 'v3', credentials=credentials, cache_discovery=False) + def drive_list(self, fileName): msg = "" # Create Search Query for API request. query = f"'{parent_id}' in parents and (name contains '{fileName}')" page_token = None - results = [] + count = 0 while True: - response = service.files().list(supportsTeamDrives=True, - includeTeamDriveItems=True, - q=query, - spaces='drive', - fields='nextPageToken, files(id, name, mimeType, size)', - pageToken=page_token, - orderBy='modifiedTime desc').execute() + response = self.__service.files().list(supportsTeamDrives=True, + includeTeamDriveItems=True, + q=query, + spaces='drive', + fields='nextPageToken, files(id, name, mimeType, size)', + pageToken=page_token, + orderBy='modifiedTime desc').execute() for file in response.get('files', []): - if len(results) >= 20: + if count >= 20: break if file.get( 'mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. @@ -369,9 +375,8 @@ def drive_list(self, fileName): url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') msg += f' | Index URL' msg += '\n' - results.append(file) + count += 1 page_token = response.get('nextPageToken', None) - if page_token is None: + if page_token is None or count >= 20: break - del results return msg From 681b8bd47dbfcd971a408e7dee9c5a8414e35f59 Mon Sep 17 00:00:00 2001 From: jaskaranSM <37726998+jaskaranSM@users.noreply.github.com> Date: Tue, 4 Feb 2020 21:23:56 +0530 Subject: [PATCH 24/79] Added missing property --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 646d17ee9..e142d6b1a 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -36,6 +36,7 @@ def __init__(self, name=None, listener=None): self.__listener = listener self._file_uploaded_bytes = 0 self.uploaded_bytes = 0 + self.UPDATE_INTERVAL = 5 self.start_time = 0 self.total_time = 0 self._should_update = True From 6cd59d5a1547dc4048df34cc0aa7ce8326e691ca Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Mon, 17 Feb 2020 21:09:21 +0530 Subject: [PATCH 25/79] Cleanup drive_list and minor fixes Signed-off-by: lzzy12 --- .../mirror_utils/upload_utils/gdriveTools.py | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index e142d6b1a..085383766 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -323,8 +323,8 @@ def authorize(self): # Get credentials credentials = None if not USE_SERVICE_ACCOUNTS: - if os.path.exists(G_DRIVE_TOKEN_FILE): - with open(G_DRIVE_TOKEN_FILE, 'rb') as f: + if os.path.exists(self.__G_DRIVE_TOKEN_FILE): + with open(self.__G_DRIVE_TOKEN_FILE, 'rb') as f: credentials = pickle.load(f) if credentials is None or not credentials.valid: if credentials and credentials.expired and credentials.refresh_token: @@ -336,7 +336,7 @@ def authorize(self): credentials = flow.run_console(port=0) # Save the credentials for the next run - with open(G_DRIVE_TOKEN_FILE, 'wb') as token: + with open(self.__G_DRIVE_TOKEN_FILE, 'wb') as token: pickle.dump(credentials, token) else: LOGGER.info(f"Authorizing with {SERVICE_ACCOUNT_INDEX}.json service account") @@ -349,35 +349,26 @@ def drive_list(self, fileName): msg = "" # Create Search Query for API request. query = f"'{parent_id}' in parents and (name contains '{fileName}')" - page_token = None - count = 0 - while True: - response = self.__service.files().list(supportsTeamDrives=True, - includeTeamDriveItems=True, - q=query, - spaces='drive', - fields='nextPageToken, files(id, name, mimeType, size)', - pageToken=page_token, - orderBy='modifiedTime desc').execute() - for file in response.get('files', []): - if count >= 20: - break - if file.get( - 'mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. - msg += f"⁍ {file.get('name')}" \ - f" (folder)" - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') - msg += f' | Index URL' - else: - msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" - if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') - msg += f' | Index URL' - msg += '\n' - count += 1 - page_token = response.get('nextPageToken', None) - if page_token is None or count >= 20: - break + response = self.__service.files().list(supportsTeamDrives=True, + includeTeamDriveItems=True, + q=query, + spaces='drive', + pageSize=20, + fields='files(id, name, mimeType, size)', + orderBy='modifiedTime desc').execute() + for file in response.get('files', []): + if file.get( + 'mimeType') == "application/vnd.google-apps.folder": # Detect Whether Current Entity is a Folder or File. + msg += f"⁍ {file.get('name')}" \ + f" (folder)" + if INDEX_URL is not None: + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') + msg += f' | Index URL' + else: + msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" + if INDEX_URL is not None: + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') + msg += f' | Index URL' + msg += '\n' return msg From 8d1bf5aa924d8fcfe4550c27f5884ad80a4dd2d5 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sat, 21 Mar 2020 10:31:24 +0530 Subject: [PATCH 26/79] recursively try again instead of handling switch with backoff - Handling service account switching with exponential backoff implementation limits the number of service accounts to be used. recursively retrying again after switch doesnt effect backoff. Signed-off-by: lzzy12 --- .../mirror_utils/upload_utils/gdriveTools.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 085383766..4f795c2ca 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -46,6 +46,7 @@ def __init__(self, name=None, listener=None): self.updater = None self.name = name self.update_interval = 3 + self.service_account_count = len(os.listdir("accounts")) def cancel(self): self.is_cancelled = True @@ -93,6 +94,14 @@ def __upload_empty_file(path, file_name, mime_type, parent_id=None): return self.__service.files().create(supportsTeamDrives=True, body=file_metadata, media_body=media_body).execute() + def switchServiceAccount(self): + global SERVICE_ACCOUNT_INDEX + if SERVICE_ACCOUNT_INDEX == self.service_account_count - 1: + SERVICE_ACCOUNT_INDEX = 0 + SERVICE_ACCOUNT_INDEX += 1 + LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") + self.__service = self.authorize() + @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) def __set_permission(self, drive_id): @@ -146,12 +155,13 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): except HttpError as err: if err.resp.get('content-type', '').startswith('application/json'): reason = json.loads(err.content).get('error').get('errors')[0].get('reason') - if reason == 'userRateLimitExceeded': - global SERVICE_ACCOUNT_INDEX - SERVICE_ACCOUNT_INDEX += 1 - LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") - self.__service = self.authorize() - raise err + if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': + if USE_SERVICE_ACCOUNTS: + self.switchServiceAccount() + LOGGER.info(f"Got: {reason}, Trying Again.") + self.upload_file(file_path, file_name, mime_type, parent_id) + else: + raise err self._file_uploaded_bytes = 0 # Insert new permissions if not IS_TEAM_DRIVE: From 25856bfac03041a78e59e4247b135b2b409b7eb9 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 24 Mar 2020 15:49:04 +0530 Subject: [PATCH 27/79] gdrive: Stop spamming logs at INFO log level Signed-off-by: lzzy12 --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 4f795c2ca..0154da0d8 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -75,7 +75,7 @@ def _on_upload_progress(self): if self.status is not None: chunk_size = self.status.total_size * self.status.progress() - self._file_uploaded_bytes self._file_uploaded_bytes = self.status.total_size * self.status.progress() - LOGGER.info(f'Chunk size: {get_readable_file_size(chunk_size)}') + LOGGER.debug(f'Uploading {self.name}, chunk size: {get_readable_file_size(chunk_size)}') self.uploaded_bytes += chunk_size self.total_time += self.update_interval From 6ca2f479d5f5cc591a9edf4d1467863576fedc4d Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 24 Mar 2020 16:30:03 +0530 Subject: [PATCH 28/79] Fix up errors after porting service accounts from develop Signed-off-by: lzzy12 --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 0154da0d8..8881cfae2 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -3,6 +3,7 @@ import urllib.parse as urlparse from urllib.parse import parse_qs +import json import requests from google.auth.transport.requests import Request @@ -13,7 +14,7 @@ from googleapiclient.http import MediaFileUpload from tenacity import * -from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, DOWNLOAD_STATUS_UPDATE_INTERVAL, \ +from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ USE_SERVICE_ACCOUNTS from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type @@ -31,6 +32,7 @@ def __init__(self, name=None, listener=None): self.__REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob" self.__G_DRIVE_DIR_MIME_TYPE = "application/vnd.google-apps.folder" self.__G_DRIVE_BASE_DOWNLOAD_URL = "https://drive.google.com/uc?id={}&export=download" + self.__G_DRIVE_DIR_BASE_DOWNLOAD_URL = "https://drive.google.com/drive/folders/{}" self.__listener = listener self.__service = self.authorize() self.__listener = listener @@ -46,7 +48,6 @@ def __init__(self, name=None, listener=None): self.updater = None self.name = name self.update_interval = 3 - self.service_account_count = len(os.listdir("accounts")) def cancel(self): self.is_cancelled = True @@ -79,8 +80,7 @@ def _on_upload_progress(self): self.uploaded_bytes += chunk_size self.total_time += self.update_interval - @staticmethod - def __upload_empty_file(path, file_name, mime_type, parent_id=None): + def __upload_empty_file(self, path, file_name, mime_type, parent_id=None): media_body = MediaFileUpload(path, mimetype=mime_type, resumable=False) @@ -96,7 +96,8 @@ def __upload_empty_file(path, file_name, mime_type, parent_id=None): def switchServiceAccount(self): global SERVICE_ACCOUNT_INDEX - if SERVICE_ACCOUNT_INDEX == self.service_account_count - 1: + service_account_count = len(os.listdir("accounts")) + if SERVICE_ACCOUNT_INDEX == service_account_count - 1: SERVICE_ACCOUNT_INDEX = 0 SERVICE_ACCOUNT_INDEX += 1 LOGGER.info(f"Switching to {SERVICE_ACCOUNT_INDEX}.json service account") From 08a71e4bd4a4bf71caacb4e537176df7d9dd8007 Mon Sep 17 00:00:00 2001 From: jaskaranSM <37726998+jaskaranSM@users.noreply.github.com> Date: Mon, 23 Mar 2020 16:59:09 +0530 Subject: [PATCH 29/79] TeamDrive List fix Signed-off-by: lzzy12 --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 8881cfae2..e6f81dcc1 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -112,7 +112,8 @@ def __set_permission(self, drive_id): 'value': None, 'withLink': True } - return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, body=permissions).execute() + return self.__service.permissions().create(supportsTeamDrives=True, fileId=drive_id, + body=permissions).execute() @retry(wait=wait_exponential(multiplier=2, min=3, max=6), stop=stop_after_attempt(5), retry=retry_if_exception_type(HttpError), before=before_log(LOGGER, logging.DEBUG)) @@ -251,10 +252,12 @@ def clone(self, link): err = str(e).replace('>', '').replace('<', '') LOGGER.error(err) return err - msg += f'{meta.get("name")} ({get_readable_file_size(self.transferred_size)})' + msg += f'{meta.get("name")}' \ + f' ({get_readable_file_size(self.transferred_size)})' else: file = self.copyFile(meta.get('id'), parent_id) - msg += f'{meta.get("name")} ({get_readable_file_size(int(meta.get("size")))})' + msg += f'{meta.get("name")}' \ + f' ({get_readable_file_size(int(meta.get("size")))}) ' return msg def cloneFolder(self, name, local_path, folder_id, parent_id): @@ -264,7 +267,9 @@ def cloneFolder(self, name, local_path, folder_id, parent_id): LOGGER.info(f"Syncing: {local_path}") new_id = None while True: - response = self.__service.files().list(q=q, + response = self.__service.files().list(supportsTeamDrives=True, + includeTeamDriveItems=True, + q=q, spaces='drive', fields='nextPageToken, files(id, name, mimeType,size)', pageToken=page_token).execute() From 50f819d27ef994fb48274cde6ce552206a6940ee Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 25 Mar 2020 15:19:46 +0530 Subject: [PATCH 30/79] Add a script to download SA keys in case user looses them Signed-off-by: lzzy12 --- README.md | 5 +++ download_service_accounts.py | 62 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 download_service_accounts.py diff --git a/README.md b/README.md index f0db9b9a6..30e91ff82 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages - IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. - INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' +- API_KEY : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. +- API_HASH : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - USER_SESSION_STRING : Session string generated by running: ``` python3 generate_string_session.py @@ -106,6 +108,9 @@ Note: 1 service account can copy around 750gb a day, 1 project makes 100 service A folder named accounts will be created which will contain keys for the service accounts created ``` We highly recommend to zip this folder and store it somewhere safe, so that you do not have to create a new project everytime you want to deploy the bot + +**However** If you lose it anyway, you can redownload it by running script: +python3 download_service_accounts.py project_id ``` ### Adding service accounts to Google Groups: We use Google Groups to manager our service accounts considering the diff --git a/download_service_accounts.py b/download_service_accounts.py new file mode 100644 index 000000000..4fe1abb4d --- /dev/null +++ b/download_service_accounts.py @@ -0,0 +1,62 @@ +import os +import sys +import pickle + +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request +from googleapiclient import discovery + +OAUTH_SCOPE = ['https://www.googleapis.com/auth/cloud-platform'] + + +# noinspection DuplicatedCode +def authorize(): + # Get credentials + credentials = None + if os.path.exists('token-iam.pickle'): + with open('token-iam.pickle', 'rb') as f: + credentials = pickle.load(f) + if credentials is None or not credentials.valid: + if credentials and credentials.expired and credentials.refresh_token: + credentials.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + 'credentials.json', OAUTH_SCOPE) + credentials = flow.run_console(port=0) + + # Save the credentials for the next run + with open('token-iam.pickle', 'wb') as token: + pickle.dump(credentials, token) + return discovery.build('iam', 'v1', credentials=credentials) + + +def get_service_accounts(project_id): + """Returns a list of service accounts in project with id == project_id""" + accounts = service.projects().serviceAccounts().list( + name=f'projects/{project_id}').execute() + return accounts['accounts'] + + +def download_keys(project_id): + accounts = get_service_accounts(project_id) + print(f'{len(accounts)} service accounts found! Dumping now') + i = 0 + for acc in accounts: + if not os.path.exists('accounts/'): + os.mkdir('accounts/') + if os.path.isfile('accounts/'): + os.remove('accounts') + os.mkdir('accounts/') + print("Dumping " + acc['email']) + key = service.projects().serviceAccounts().keys().create( + name='projects/-/serviceAccounts/' + acc['email'], body={} + ).execute() + with open(f'accounts/{i}.json', 'w') as f: + f.write(str(key)) + i += 1 + + +service = authorize() + +if __name__ == '__main__': + download_keys(sys.argv[1]) From 284404b594d2d2339c9991fd9e2a3f40e02f4dbf Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 25 Mar 2020 16:48:29 +0530 Subject: [PATCH 31/79] Revert "Add a script to download SA keys in case user looses them" This reverts commit 50f819d27ef994fb48274cde6ce552206a6940ee. --- README.md | 5 --- download_service_accounts.py | 62 ------------------------------------ 2 files changed, 67 deletions(-) delete mode 100644 download_service_accounts.py diff --git a/README.md b/README.md index 30e91ff82..f0db9b9a6 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,6 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages - IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. - INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' -- API_KEY : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. -- API_HASH : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - USER_SESSION_STRING : Session string generated by running: ``` python3 generate_string_session.py @@ -108,9 +106,6 @@ Note: 1 service account can copy around 750gb a day, 1 project makes 100 service A folder named accounts will be created which will contain keys for the service accounts created ``` We highly recommend to zip this folder and store it somewhere safe, so that you do not have to create a new project everytime you want to deploy the bot - -**However** If you lose it anyway, you can redownload it by running script: -python3 download_service_accounts.py project_id ``` ### Adding service accounts to Google Groups: We use Google Groups to manager our service accounts considering the diff --git a/download_service_accounts.py b/download_service_accounts.py deleted file mode 100644 index 4fe1abb4d..000000000 --- a/download_service_accounts.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import sys -import pickle - -from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import Request -from googleapiclient import discovery - -OAUTH_SCOPE = ['https://www.googleapis.com/auth/cloud-platform'] - - -# noinspection DuplicatedCode -def authorize(): - # Get credentials - credentials = None - if os.path.exists('token-iam.pickle'): - with open('token-iam.pickle', 'rb') as f: - credentials = pickle.load(f) - if credentials is None or not credentials.valid: - if credentials and credentials.expired and credentials.refresh_token: - credentials.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file( - 'credentials.json', OAUTH_SCOPE) - credentials = flow.run_console(port=0) - - # Save the credentials for the next run - with open('token-iam.pickle', 'wb') as token: - pickle.dump(credentials, token) - return discovery.build('iam', 'v1', credentials=credentials) - - -def get_service_accounts(project_id): - """Returns a list of service accounts in project with id == project_id""" - accounts = service.projects().serviceAccounts().list( - name=f'projects/{project_id}').execute() - return accounts['accounts'] - - -def download_keys(project_id): - accounts = get_service_accounts(project_id) - print(f'{len(accounts)} service accounts found! Dumping now') - i = 0 - for acc in accounts: - if not os.path.exists('accounts/'): - os.mkdir('accounts/') - if os.path.isfile('accounts/'): - os.remove('accounts') - os.mkdir('accounts/') - print("Dumping " + acc['email']) - key = service.projects().serviceAccounts().keys().create( - name='projects/-/serviceAccounts/' + acc['email'], body={} - ).execute() - with open(f'accounts/{i}.json', 'w') as f: - f.write(str(key)) - i += 1 - - -service = authorize() - -if __name__ == '__main__': - download_keys(sys.argv[1]) From 6181b022acf97c2442cf7acb1b5786e008b85cf3 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 25 Mar 2020 16:57:01 +0530 Subject: [PATCH 32/79] Update README.md Signed-off-by: lzzy12 --- README.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f0db9b9a6..d5f4fa61a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages - IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. - INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' +- API_KEY : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. +- API_HASH : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org - USER_SESSION_STRING : Session string generated by running: ``` python3 generate_string_session.py @@ -104,17 +106,8 @@ Note: 1 service account can copy around 750gb a day, 1 project makes 100 service `python3 gen_sa_accounts.py --quick-setup 1 --new-only` A folder named accounts will be created which will contain keys for the service accounts created -``` -We highly recommend to zip this folder and store it somewhere safe, so that you do not have to create a new project everytime you want to deploy the bot -``` -### Adding service accounts to Google Groups: -We use Google Groups to manager our service accounts considering the -[Official limits to the members of Team Drive](https://support.google.com/a/answer/7338880?hl=en) (Limit for individuals and groups directly added as members: 600). - -1. Turn on the Directory API following [official steps](https://developers.google.com/admin-sdk/directory/v1/quickstart/python) (save the generated json file to folder `credentials`). -2. Create group for your organization [in the Admin console](https://support.google.com/a/answer/33343?hl=en). After create a group, you will have an address for example`sa@yourdomain.com`. - -3. Run `python3 add_to_google_group.py -g sa@yourdomain.com` - -4. Now, add Google Groups (**Step 2**) to manager your service accounts, add the group address `sa@yourdomain.com` or `sa@googlegroups.com` to the Team drive or folder +NOTE: If you have created SAs in past from this script, you can also just re download the keys by running: +``` +python3 gen_sa_accounts.py --download-keys project_id +``` \ No newline at end of file From be9c64a9aaf24c248293849d1deb105fbd825d4e Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 24 Mar 2020 09:59:26 +0530 Subject: [PATCH 33/79] Use Service Accounts for Cloning if availble. - increase default timeout for network socket interface Signed-off-by: lzzy12 --- bot/__init__.py | 3 +++ .../mirror_utils/upload_utils/gdriveTools.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/bot/__init__.py b/bot/__init__.py index 1fd6ba9af..8cca93a5b 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -6,6 +6,9 @@ import aria2p import telegram.ext as tg from dotenv import load_dotenv +import socket + +socket.setdefaulttimeout(600) botStartTime = time.time() if os.path.exists('log.txt'): diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index e6f81dcc1..0a7879c18 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -228,7 +228,20 @@ def copyFile(self, file_id, dest_id): body = { 'parents': [dest_id] } - return self.__service.files().copy(supportsAllDrives=True, fileId=file_id, body=body).execute() + + try: + res = self.__service.files().copy(supportsAllDrives=True,fileId=file_id,body=body).execute() + return res + except HttpError as err: + if err.resp.get('content-type', '').startswith('application/json'): + reason = json.loads(err.content).get('error').get('errors')[0].get('reason') + if reason == 'userRateLimitExceeded' or reason == 'dailyLimitExceeded': + if USE_SERVICE_ACCOUNTS: + self.switchServiceAccount() + LOGGER.info(f"Got: {reason}, Trying Again.") + self.copyFile(file_id,dest_id) + else: + raise err def clone(self, link): self.transferred_size = 0 From 6d07c3c8c4ea24d2bf67c865b05355f20dec172e Mon Sep 17 00:00:00 2001 From: Shivam Jha Date: Thu, 26 Mar 2020 16:02:36 +0530 Subject: [PATCH 34/79] Stop trying to generate sourceforge direct links --- .../download_utils/direct_link_generator.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py index 6ee5a9402..71d361913 100644 --- a/bot/helper/mirror_utils/download_utils/direct_link_generator.py +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -36,8 +36,6 @@ def direct_link_generator(link: str): return cm_ru(link) elif 'mediafire.com' in link: return mediafire(link) - elif 'sourceforge.net' in link: - return sourceforge(link) elif 'osdn.net' in link: return osdn(link) elif 'github.com' in link: @@ -175,23 +173,6 @@ def mediafire(url: str) -> str: return dl_url -def sourceforge(url: str) -> str: - """ SourceForge direct links generator """ - try: - link = re.findall(r'\bhttps?://.*sourceforge\.net\S+', url)[0] - except IndexError: - raise DirectDownloadLinkException("`No SourceForge links found`\n") - file_path = re.findall(r'files(.*)/download', link)[0] - project = re.findall(r'projects?/(.*?)/files', link)[0] - mirrors = f'https://sourceforge.net/settings/mirror_choices?' \ - f'projectname={project}&filename={file_path}' - page = BeautifulSoup(requests.get(mirrors).content, 'html.parser') - info = page.find('ul', {'id': 'mirrorList'}).findAll('li') - for mirror in info[1:]: - dl_url = f'https://{mirror["id"]}.dl.sourceforge.net/project/{project}/{file_path}' - return dl_url - - def osdn(url: str) -> str: """ OSDN direct links generator """ osdn_link = 'https://osdn.net' From 0de69152228d26570e0d6d9e3a9bb79b1a1f42ce Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 27 Mar 2020 14:35:51 +0530 Subject: [PATCH 35/79] Remove redundant exception classes and fix existing ones Signed-off-by: lzzy12 --- bot/helper/ext_utils/exceptions.py | 23 +---------------------- bot/modules/mirror.py | 2 +- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/bot/helper/ext_utils/exceptions.py b/bot/helper/ext_utils/exceptions.py index 9b1078e18..25ff87fa6 100644 --- a/bot/helper/ext_utils/exceptions.py +++ b/bot/helper/ext_utils/exceptions.py @@ -1,23 +1,2 @@ -class DriveAuthError(Exception): - pass - - -class MessageDeletedError(Exception): - """ Custom Exception class for killing thread as soon as they aren't needed""" - - def __init__(self, message, error=None): - super().__init__(message) - self.error = error - - -class DownloadCancelled(Exception): - - def __init__(self, message, error=None): - super().__init__(message) - self.error = error - - class DirectDownloadLinkException(Exception): - def __init__(self, message, error=None): - super().__init__(message) - self.error = error + pass diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 31538e6f7..734c91955 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -173,7 +173,7 @@ def _mirror(bot, update, isTar=False): try: link = direct_link_generator(link) except DirectDownloadLinkException as e: - LOGGER.info(f'{link}: {e.error}') + LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag) aria = aria2_download.AriaDownloadHelper(listener) aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') From 2254444378acb925e60c1cad83b9346d56435cf4 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Fri, 27 Mar 2020 16:00:37 +0530 Subject: [PATCH 36/79] Handle KeyError exception in Telegram download error Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/telegram_downloader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index 4d8c02f2b..3de82f751 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -64,7 +64,10 @@ def __onDownloadProgress(self, current, total): def __onDownloadError(self, error): with global_lock: - GLOBAL_GID.remove(self.gid) + try: + GLOBAL_GID.remove(self.gid) + except KeyError: + pass self.__listener.onDownloadError(error) def __onDownloadComplete(self): From 3a064586093f6d1946d47a88d0e581fe7f3cb0ba Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Fri, 27 Mar 2020 19:57:53 +0530 Subject: [PATCH 37/79] Added Youtube-dl support Signed-off-by: lzzy12 --- .../youtube_dl_download_helper.py | 104 ++++++++++++++++++ .../youtube_dl_download_status.py | 54 +++++++++ bot/modules/cancel_mirror.py | 7 +- bot/modules/mirror.py | 18 ++- requirements.txt | 6 +- 5 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py create mode 100644 bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py new file mode 100644 index 000000000..96d3fe69d --- /dev/null +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -0,0 +1,104 @@ +from .download_helper import DownloadHelper +import time +from youtube_dl import YoutubeDL +import threading +from bot import LOGGER,download_dict_lock,download_dict,DOWNLOAD_DIR +from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus + +class YoutubeDLHelper(DownloadHelper): + def __init__(self, listener): + super().__init__() + self.__name = "" + self.__start_time = time.time() + self.__listener = listener + self.__gid = "" + self.opts = { + 'format': 'bestaudio/best', + 'progress_hooks':[self.__onDownloadProgress], + 'outtmpl': f"{DOWNLOAD_DIR}{self.__listener.uid}/%(title)s.%(ext)s" + } + self.ydl = YoutubeDL(self.opts) + self.__download_speed = 0 + self.download_speed_readable = "" + self.downloaded_bytes = 0 + self.size = 0 + self.is_playlist = False + self.last_downloaded = 0 + self.is_cancelled = False + self.__resource_lock = threading.RLock() + + @property + def download_speed(self): + with self.__resource_lock: + return self.__download_speed + + @property + def gid(self): + with self.__resource_lock: + return self.__gid + + def __onDownloadProgress(self,d): + if self.is_cancelled: + raise ValueError("Cancelling Download..") + if d['status'] == "finished": + if self.is_playlist: + self.last_downloaded = 0 + if self.downloaded_bytes == self.size: + self.__onDownloadComplete() + else: + self.__onDownloadComplete() + elif d['status'] == "downloading": + with self.__resource_lock: + self.progress = self.downloaded_bytes / self.size * 100 + self.__download_speed = d['speed'] + if self.is_playlist: + chunk_size = self.size * self.progress - self.last_downloaded + self.last_downloaded = self.size * self.progress + self.downloaded_bytes += chunk_size + else: + self.download_speed_readable = d['_speed_str'] + self.downloaded_bytes = d['downloaded_bytes'] + + def __onDownloadStart(self): + with download_dict_lock: + download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self,self.__listener.uid) + + def __onDownloadComplete(self): + self.__listener.onDownloadComplete() + + def __onDownloadError(self,error): + self.__listener.onDownloadError(error) + + def extractMetaData(self,link): + result = self.ydl.extract_info(link,download=False) + if 'entries' in result: + video = result['entries'][0] + for v in result['entries']: + self.size += int(v['filesize']) + self.name = result.get('title') + self.vid_id = video.get('id') + self.is_playlist = True + self.opts['outtmpl'] = f"{DOWNLOAD_DIR}{self.__listener.uid}/%(playlist)s/%(title)s.%(ext)s" + self.ydl = YoutubeDL(self.opts) + else: + video = result + self.size = int(video.get('filesize')) + self.name = video.get('title') + self.vid_id = video.get('id') + return video + + def __download(self,link): + try: + self.ydl.download([link],) + except ValueError: + LOGGER.info("Download Cancelled by User!") + self.__onDownloadError("Download Cancelled by User!") + + def add_download(self,link): + LOGGER.info(f"Downloading with YT-DL: {link}") + self.__gid = f"{self.vid_id}{self.__listener.uid}" + threading.Thread(target=self.__download,args=(link,)).start() + self.__onDownloadStart() + + def cancel_download(self): + self.is_cancelled = True \ No newline at end of file diff --git a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py new file mode 100644 index 000000000..16b3bffbb --- /dev/null +++ b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py @@ -0,0 +1,54 @@ +from bot import DOWNLOAD_DIR +from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time +from .status import Status + +class YoutubeDLDownloadStatus(Status): + def __init__(self, obj, uid): + self.obj = obj + self.uid = uid + + def gid(self): + return self.obj.gid + + def path(self): + return f"{DOWNLOAD_DIR}{self.uid}" + + def processed_bytes(self): + return self.obj.downloaded_bytes + + def size_raw(self): + return self.obj.size + + def size(self): + return get_readable_file_size(self.size_raw()) + + def status(self): + return MirrorStatus.STATUS_DOWNLOADING + + def name(self): + return self.obj.name + + def progress_raw(self): + return self.obj.progress + + def progress(self): + return f'{round(self.progress_raw(), 2)}%' + + def speed_raw(self): + """ + :return: Download speed in Bytes/Seconds + """ + return self.obj.download_speed + + def speed(self): + return f'{get_readable_file_size(self.speed_raw())}/s' + + def eta(self): + try: + seconds = (self.size_raw() - self.processed_bytes()) / self.speed_raw() + return f'{get_readable_time(seconds)}' + except ZeroDivisionError: + return '-' + + def download(self): + return self.obj diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index 88aea78ef..245ca5643 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -3,13 +3,16 @@ from telegram.ext import CommandHandler, run_async from bot import download_dict, aria2, dispatcher, download_dict_lock, DOWNLOAD_DIR -from bot.helper.ext_utils.bot_utils import getDownloadByGid from bot.helper.ext_utils.fs_utils import clean_download from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * from ..helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper +from time import sleep +from bot.helper.ext_utils.bot_utils import getDownloadByGid +from ..helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper + @run_async def cancel_mirror(bot, update): @@ -52,6 +55,8 @@ def cancel_mirror(bot, update): download = dl.download() if isinstance(download, TelegramDownloadHelper): download.cancel_download() + if isinstance(download, YoutubeDLHelper): + download.cancel_download() else: if len(download.followed_by_ids) != 0: downloads = aria2.get_downloads(download.followed_by_ids) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 734c91955..33d8ec659 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -1,6 +1,3 @@ -import os -import pathlib - import requests from telegram.ext import CommandHandler, run_async @@ -19,6 +16,9 @@ from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * +from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper +import pathlib +import os class MirrorListener(listeners.MirrorListeners): @@ -175,8 +175,16 @@ def _mirror(bot, update, isTar=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag) - aria = aria2_download.AriaDownloadHelper(listener) - aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') + try: + ydl = YoutubeDLHelper(listener) + sup_link = ydl.extractMetaData(link) + except Exception as e: + sup_link = None + if sup_link: + ydl.add_download(link) + else: + aria = aria2_download.AriaDownloadHelper(listener) + aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/requirements.txt b/requirements.txt index 4bfde1c4d..57e62e861 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,10 @@ google-auth-httplib2>=0.0.3,<0.1.0 google-auth-oauthlib>=0.4.1,<0.10.0 aria2p>=0.3.0,<0.10.0 python-dotenv>=0.10 -tenacity>=6.0.0 +tenacity>=6.0.0 python-magic beautifulsoup4>=4.8.2,<4.8.10 Pyrogram>=0.16.0,<0.16.10 -TgCrypto>=1.1.1,<1.1.10 \ No newline at end of file +TgCrypto>=1.1.1,<1.1.10 +tenacity>=6.0.0 +youtube-dl \ No newline at end of file From 9057d3fc4e1c42e86d23adbab916a77c4f6af1ae Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 28 Mar 2020 13:47:19 +0530 Subject: [PATCH 38/79] Fixed yt-dl support Signed-off-by: lzzy12 --- .../youtube_dl_download_helper.py | 48 ++++++++++--------- .../youtube_dl_download_status.py | 1 + bot/modules/mirror.py | 4 +- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 96d3fe69d..dfc12cbcd 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -2,9 +2,10 @@ import time from youtube_dl import YoutubeDL import threading -from bot import LOGGER,download_dict_lock,download_dict,DOWNLOAD_DIR +from bot import LOGGER, download_dict_lock, download_dict, DOWNLOAD_DIR from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus + class YoutubeDLHelper(DownloadHelper): def __init__(self, listener): super().__init__() @@ -13,18 +14,18 @@ def __init__(self, listener): self.__listener = listener self.__gid = "" self.opts = { - 'format': 'bestaudio/best', - 'progress_hooks':[self.__onDownloadProgress], - 'outtmpl': f"{DOWNLOAD_DIR}{self.__listener.uid}/%(title)s.%(ext)s" + 'format': 'bestaudio/best', + 'progress_hooks': [self.__onDownloadProgress], } self.ydl = YoutubeDL(self.opts) self.__download_speed = 0 - self.download_speed_readable = "" + self.download_speed_readable = '' self.downloaded_bytes = 0 self.size = 0 self.is_playlist = False self.last_downloaded = 0 self.is_cancelled = False + self.vid_id = '' self.__resource_lock = threading.RLock() @property @@ -37,7 +38,7 @@ def gid(self): with self.__resource_lock: return self.__gid - def __onDownloadProgress(self,d): + def __onDownloadProgress(self, d): if self.is_cancelled: raise ValueError("Cancelling Download..") if d['status'] == "finished": @@ -45,40 +46,41 @@ def __onDownloadProgress(self,d): self.last_downloaded = 0 if self.downloaded_bytes == self.size: self.__onDownloadComplete() - else: + else: self.__onDownloadComplete() elif d['status'] == "downloading": with self.__resource_lock: - self.progress = self.downloaded_bytes / self.size * 100 self.__download_speed = d['speed'] if self.is_playlist: - chunk_size = self.size * self.progress - self.last_downloaded - self.last_downloaded = self.size * self.progress + progress = d['downloaded_bytes'] / d['total_bytes'] + chunk_size = d['downloaded_bytes'] - self.last_downloaded + self.last_downloaded = d['total_bytes'] * progress self.downloaded_bytes += chunk_size + self.progress = (self.downloaded_bytes / self.size) * 100 else: self.download_speed_readable = d['_speed_str'] self.downloaded_bytes = d['downloaded_bytes'] - + def __onDownloadStart(self): with download_dict_lock: - download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self,self.__listener.uid) + download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self, self.__listener.uid) def __onDownloadComplete(self): self.__listener.onDownloadComplete() - def __onDownloadError(self,error): + def __onDownloadError(self, error): self.__listener.onDownloadError(error) - def extractMetaData(self,link): - result = self.ydl.extract_info(link,download=False) + def extractMetaData(self, link): + result = self.ydl.extract_info(link, download=False) if 'entries' in result: video = result['entries'][0] for v in result['entries']: - self.size += int(v['filesize']) + self.size += float(v['filesize']) self.name = result.get('title') self.vid_id = video.get('id') self.is_playlist = True - self.opts['outtmpl'] = f"{DOWNLOAD_DIR}{self.__listener.uid}/%(playlist)s/%(title)s.%(ext)s" + self.opts['o'] = f"{DOWNLOAD_DIR}{self.__listener.uid}/%(playlist)s/%(title)s.%(ext)s" self.ydl = YoutubeDL(self.opts) else: video = result @@ -87,18 +89,20 @@ def extractMetaData(self,link): self.vid_id = video.get('id') return video - def __download(self,link): + def __download(self, link): try: - self.ydl.download([link],) + self.ydl.download([link], ) + self.__onDownloadComplete() except ValueError: LOGGER.info("Download Cancelled by User!") self.__onDownloadError("Download Cancelled by User!") - def add_download(self,link): + def add_download(self, link, path): LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" - threading.Thread(target=self.__download,args=(link,)).start() + self.opts['o'] = f"{path}/%(title)s/%(title)s.%(ext)s" self.__onDownloadStart() + threading.Thread(target=self.__download, args=(link,)).start() def cancel_download(self): - self.is_cancelled = True \ No newline at end of file + self.is_cancelled = True diff --git a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py index 16b3bffbb..51f058237 100644 --- a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py +++ b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py @@ -2,6 +2,7 @@ from bot.helper.ext_utils.bot_utils import MirrorStatus, get_readable_file_size, get_readable_time from .status import Status + class YoutubeDLDownloadStatus(Status): def __init__(self, obj, uid): self.obj = obj diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 33d8ec659..0d7e194e9 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -175,13 +175,13 @@ def _mirror(bot, update, isTar=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag) + ydl = YoutubeDLHelper(listener) try: - ydl = YoutubeDLHelper(listener) sup_link = ydl.extractMetaData(link) except Exception as e: sup_link = None if sup_link: - ydl.add_download(link) + ydl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}') else: aria = aria2_download.AriaDownloadHelper(listener) aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') From 7c304d449aa9dc264d2b5102cb2471845f52a18f Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 28 Mar 2020 14:12:32 +0530 Subject: [PATCH 39/79] Fix uploading for YT-DL Signed-off-by: lzzy12 --- .../download_utils/youtube_dl_download_helper.py | 16 +++++++--------- bot/modules/mirror.py | 2 +- requirements.txt | 1 - 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index dfc12cbcd..2a590e5c2 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -44,10 +44,6 @@ def __onDownloadProgress(self, d): if d['status'] == "finished": if self.is_playlist: self.last_downloaded = 0 - if self.downloaded_bytes == self.size: - self.__onDownloadComplete() - else: - self.__onDownloadComplete() elif d['status'] == "downloading": with self.__resource_lock: self.__download_speed = d['speed'] @@ -77,15 +73,13 @@ def extractMetaData(self, link): video = result['entries'][0] for v in result['entries']: self.size += float(v['filesize']) - self.name = result.get('title') + self.name = video.get('playlist_title') self.vid_id = video.get('id') self.is_playlist = True - self.opts['o'] = f"{DOWNLOAD_DIR}{self.__listener.uid}/%(playlist)s/%(title)s.%(ext)s" - self.ydl = YoutubeDL(self.opts) else: video = result self.size = int(video.get('filesize')) - self.name = video.get('title') + self.name = f"{video.get('title')}.{video.get('ext')}" self.vid_id = video.get('id') return video @@ -100,7 +94,11 @@ def __download(self, link): def add_download(self, link, path): LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" - self.opts['o'] = f"{path}/%(title)s/%(title)s.%(ext)s" + if not self.is_playlist: + self.opts['outtmpl'] = f"{path}/%(title)s.%(ext)s" + else: + self.opts['outtmpl'] = f"{path}/%(playlist_title)s/%(title)s.%(ext)s" + self.ydl = YoutubeDL(self.opts) self.__onDownloadStart() threading.Thread(target=self.__download, args=(link,)).start() diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0d7e194e9..4b28563f8 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -181,7 +181,7 @@ def _mirror(bot, update, isTar=False): except Exception as e: sup_link = None if sup_link: - ydl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}') + ydl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}') else: aria = aria2_download.AriaDownloadHelper(listener) aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') diff --git a/requirements.txt b/requirements.txt index 57e62e861..307a6f131 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,4 @@ python-magic beautifulsoup4>=4.8.2,<4.8.10 Pyrogram>=0.16.0,<0.16.10 TgCrypto>=1.1.1,<1.1.10 -tenacity>=6.0.0 youtube-dl \ No newline at end of file From 6262b26fe51f6e722d8bd7490f3406c4c3ed81e3 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sat, 28 Mar 2020 17:37:17 +0530 Subject: [PATCH 40/79] youtube-dl: fixed downloading from sites that provide encrypted content (m3u8) --- bot/helper/ext_utils/fs_utils.py | 10 ++++++++++ .../download_utils/youtube_dl_download_helper.py | 6 ++++-- bot/modules/mirror.py | 8 +++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 019cc77ae..8f3169268 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -30,6 +30,16 @@ def exit_clean_up(signal, frame): LOGGER.warning("Force Exiting before the cleanup finishes!") sys.exit(1) +def get_path_size(path): + if os.path.isfile(path): + return os.path.getsize(path) + total_size = 0 + for root, dirs, files in os.walk(path): + for f in files: + abs_path = os.path.join(root, f) + total_size += os.path.getsize(abs_path) + return total_size + def tar(org_path): tar_path = org_path + ".tar" diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 2a590e5c2..334858d23 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -72,13 +72,15 @@ def extractMetaData(self, link): if 'entries' in result: video = result['entries'][0] for v in result['entries']: - self.size += float(v['filesize']) + if v.get('filesize'): + self.size += float(v['filesize']) self.name = video.get('playlist_title') self.vid_id = video.get('id') self.is_playlist = True else: video = result - self.size = int(video.get('filesize')) + if video.get('filesize'): + self.size = int(video.get('filesize')) self.name = f"{video.get('title')}.{video.get('ext')}" self.vid_id = video.get('id') return video diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 4b28563f8..a92623bde 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -59,10 +59,12 @@ def onDownloadComplete(self): else: path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}' up_name = pathlib.PurePath(path).name + LOGGER.info(f"Upload Name : {up_name}") + drive = gdriveTools.GoogleDriveHelper(up_name, self) + if size == 0: + size = fs_utils.get_path_size(m_path) + upload_status = UploadStatus(drive, size, self.uid) with download_dict_lock: - LOGGER.info(f"Upload Name : {up_name}") - drive = gdriveTools.GoogleDriveHelper(up_name, self) - upload_status = UploadStatus(drive, size, self.uid) download_dict[self.uid] = upload_status update_all_messages() drive.upload(up_name) From 3bf9e0ea312faf2634018d9668db37d356705bc6 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 28 Mar 2020 18:44:48 +0530 Subject: [PATCH 41/79] Use default quality provided by youtube-dl for now Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 334858d23..154198932 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -14,7 +14,6 @@ def __init__(self, listener): self.__listener = listener self.__gid = "" self.opts = { - 'format': 'bestaudio/best', 'progress_hooks': [self.__onDownloadProgress], } self.ydl = YoutubeDL(self.opts) From 54db83196d1fc307840e45320ab7267b48f1fae4 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 28 Mar 2020 20:28:36 +0530 Subject: [PATCH 42/79] Install ffmpeg in docker images Signed-off-by: lzzy12 --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30d9e7daa..d34ec7b3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,9 @@ FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app RUN apt -qq update -RUN apt -qq install -y aria2 python3 python3-pip locales python3-lxml curl pv jq +RUN apt -qq install -y aria2 python3 python3-pip \ + locales python3-lxml \ + curl pv jq ffmpeg COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt COPY . . From 141b13b52d7e4ef8895e533d5a62977e59c9e766 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 30 Mar 2020 14:55:29 +0530 Subject: [PATCH 43/79] Make ydl object local to the methods Signed-off-by: lzzy12 --- .../download_utils/youtube_dl_download_helper.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 154198932..fb970d2d4 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -16,7 +16,6 @@ def __init__(self, listener): self.opts = { 'progress_hooks': [self.__onDownloadProgress], } - self.ydl = YoutubeDL(self.opts) self.__download_speed = 0 self.download_speed_readable = '' self.downloaded_bytes = 0 @@ -67,7 +66,11 @@ def __onDownloadError(self, error): self.__listener.onDownloadError(error) def extractMetaData(self, link): - result = self.ydl.extract_info(link, download=False) + if 'hotstar' in link: + self.opts['geo-bypass-country'] = 'IN' + + with YoutubeDL(self.opts) as ydl: + result = ydl.extract_info(link, download=False) if 'entries' in result: video = result['entries'][0] for v in result['entries']: @@ -86,7 +89,8 @@ def extractMetaData(self, link): def __download(self, link): try: - self.ydl.download([link], ) + with YoutubeDL(self.opts) as ydl: + ydl.download([link]) self.__onDownloadComplete() except ValueError: LOGGER.info("Download Cancelled by User!") @@ -99,7 +103,7 @@ def add_download(self, link, path): self.opts['outtmpl'] = f"{path}/%(title)s.%(ext)s" else: self.opts['outtmpl'] = f"{path}/%(playlist_title)s/%(title)s.%(ext)s" - self.ydl = YoutubeDL(self.opts) + self.__onDownloadStart() threading.Thread(target=self.__download, args=(link,)).start() From 0ec499ff4a8a492a83aa78dc73de45ae03a761db Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 30 Mar 2020 16:32:06 +0530 Subject: [PATCH 44/79] Handle weird path naming of youtube_dl Signed-off-by: lzzy12 --- .../download_utils/youtube_dl_download_helper.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index fb970d2d4..eff6aa624 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -71,19 +71,21 @@ def extractMetaData(self, link): with YoutubeDL(self.opts) as ydl: result = ydl.extract_info(link, download=False) + name = ydl.prepare_filename(result) if 'entries' in result: video = result['entries'][0] for v in result['entries']: if v.get('filesize'): self.size += float(v['filesize']) - self.name = video.get('playlist_title') + # For playlists, ydl.prepare-filename returns the following format: -.NA + self.name = name.split(f"-{result['id']}")[0] self.vid_id = video.get('id') self.is_playlist = True else: video = result if video.get('filesize'): self.size = int(video.get('filesize')) - self.name = f"{video.get('title')}.{video.get('ext')}" + self.name = name self.vid_id = video.get('id') return video @@ -100,9 +102,9 @@ def add_download(self, link, path): LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" if not self.is_playlist: - self.opts['outtmpl'] = f"{path}/%(title)s.%(ext)s" + self.opts['outtmpl'] = f"{path}/{self.name}" else: - self.opts['outtmpl'] = f"{path}/%(playlist_title)s/%(title)s.%(ext)s" + self.opts['outtmpl'] = f"{path}/{self.name}/%(title)s.%(ext)s" self.__onDownloadStart() threading.Thread(target=self.__download, args=(link,)).start() From 3754e8b3cecd00cb688088a55d6e16a9472d4c3f Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 30 Mar 2020 17:02:31 +0530 Subject: [PATCH 45/79] Fix tg torrent file mirror broken since 7d74cf29d8fe1334a1e8cf83d5b40302f8c737bd Signed-off-by: lzzy12 --- bot/modules/mirror.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index a92623bde..da49283c3 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -156,9 +156,7 @@ def _mirror(bot, update, isTar=False): if len(link) == 0: if file is not None: - if file.file_size <= 20 * 1024 * 1024: - link = file.get_file().file_path - else: + if file.mime_type != "application/x-bittorrent": listener = MirrorListener(bot, update, isTar, tag) tg_downloader = TelegramDownloadHelper(listener) tg_downloader.add_download(reply_to, f'{DOWNLOAD_DIR}{listener.uid}/') @@ -166,6 +164,8 @@ def _mirror(bot, update, isTar=False): if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) return + else: + link = file.get_file().file_path else: tag = None if not bot_utils.is_url(link) and not bot_utils.is_magnet(link): From 52e8fbec117cb9e5841e8c49d8aec146d6f94c3c Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sun, 29 Mar 2020 13:52:41 +0530 Subject: [PATCH 46/79] Fixed direct link torrent mirroring Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index eff6aa624..c4c9a5df6 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -72,6 +72,8 @@ def extractMetaData(self, link): with YoutubeDL(self.opts) as ydl: result = ydl.extract_info(link, download=False) name = ydl.prepare_filename(result) + if result.get('direct'): + return None if 'entries' in result: video = result['entries'][0] for v in result['entries']: From 136924ffd467453cd04d44f273c8ce603ea6d4b3 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 30 Mar 2020 17:46:56 +0530 Subject: [PATCH 47/79] Handle rare bug when download is not deleted from download_dict on error Signed-off-by: lzzy12 --- bot/helper/mirror_utils/status_utils/aria_download_status.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index 52dc0ca0d..bb913c939 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -64,6 +64,7 @@ def status(self): status = MirrorStatus.STATUS_CANCELLED elif download.has_failed: status = MirrorStatus.STATUS_FAILED + self._listener.onDownloadError('Unknown Error') else: status = MirrorStatus.STATUS_DOWNLOADING return status From 9fa41f68c1860d3e0b3636e1733b9258f64ecc24 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 30 Mar 2020 18:16:54 +0530 Subject: [PATCH 48/79] Some minor Fix-ups Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 2 +- bot/modules/mirror.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index c4c9a5df6..07d99fb30 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -67,7 +67,7 @@ def __onDownloadError(self, error): def extractMetaData(self, link): if 'hotstar' in link: - self.opts['geo-bypass-country'] = 'IN' + self.opts['geo_bypass_country'] = 'IN' with YoutubeDL(self.opts) as ydl: result = ydl.extract_info(link, download=False) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index da49283c3..bdb61f7af 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -22,6 +22,7 @@ class MirrorListener(listeners.MirrorListeners): + def __init__(self, bot, update, isTar=False, tag=None): super().__init__(bot, update) self.isTar = isTar @@ -97,6 +98,9 @@ def onDownloadError(self, error): def onUploadStarted(self): pass + def onUploadProgress(self): + pass + def onUploadComplete(self, link: str): with download_dict_lock: msg = f'{download_dict[self.uid].name()} ({download_dict[self.uid].size()})' From 85d4005c16764ee9ec8129a8decc362f72787506 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 31 Mar 2020 10:50:53 +0530 Subject: [PATCH 49/79] Use netrc by default for youtube-dl Signed-off-by: lzzy12 --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 07d99fb30..8d25aca86 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -15,6 +15,7 @@ def __init__(self, listener): self.__gid = "" self.opts = { 'progress_hooks': [self.__onDownloadProgress], + 'usenetrc': True } self.__download_speed = 0 self.download_speed_readable = '' From e2c5c8beaf71bd9f7e7c63354a5f427d6ee4bd97 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 31 Mar 2020 11:15:09 +0530 Subject: [PATCH 50/79] Log youtube-dl warnings and errors in bot log Signed-off-by: lzzy12 --- .../download_utils/youtube_dl_download_helper.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 8d25aca86..70d8e7901 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -4,6 +4,21 @@ import threading from bot import LOGGER, download_dict_lock, download_dict, DOWNLOAD_DIR from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus +import logging + + +class MyLogger: + def __init__(self, obj): + self.obj = obj + + def debug(self, msg): + logging.debug(msg) + + def warning(self, msg): + logging.warning(msg) + + def error(self, msg): + logging.error(msg) class YoutubeDLHelper(DownloadHelper): @@ -15,6 +30,7 @@ def __init__(self, listener): self.__gid = "" self.opts = { 'progress_hooks': [self.__onDownloadProgress], + 'logger': MyLogger(self), 'usenetrc': True } self.__download_speed = 0 From a54fc42e1e7f0bfa90bcb396e78d2b13e4da0cf6 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 31 Mar 2020 15:05:57 +0530 Subject: [PATCH 51/79] Handle some exceptions Signed-off-by: lzzy12 --- bot/helper/ext_utils/bot_utils.py | 2 ++ .../youtube_dl_download_helper.py | 17 +++++++++++------ .../mirror_utils/upload_utils/gdriveTools.py | 16 ++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index ede4e49f4..9b811428a 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -46,6 +46,8 @@ def cancel(self): def get_readable_file_size(size_in_bytes) -> str: + if size_in_bytes is None: + return '0B' index = 0 while size_in_bytes >= 1024: size_in_bytes /= 1024 diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 70d8e7901..c111b94b8 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -2,23 +2,25 @@ import time from youtube_dl import YoutubeDL import threading -from bot import LOGGER, download_dict_lock, download_dict, DOWNLOAD_DIR +from bot import download_dict_lock, download_dict from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus import logging +LOGGER = logging.getLogger(__name__) + class MyLogger: def __init__(self, obj): self.obj = obj def debug(self, msg): - logging.debug(msg) + LOGGER.debug(msg) def warning(self, msg): - logging.warning(msg) + LOGGER.warning(msg) def error(self, msg): - logging.error(msg) + LOGGER.error(msg) class YoutubeDLHelper(DownloadHelper): @@ -67,7 +69,10 @@ def __onDownloadProgress(self, d): chunk_size = d['downloaded_bytes'] - self.last_downloaded self.last_downloaded = d['total_bytes'] * progress self.downloaded_bytes += chunk_size - self.progress = (self.downloaded_bytes / self.size) * 100 + try: + self.progress = (self.downloaded_bytes / self.size) * 100 + except ZeroDivisionError: + pass else: self.download_speed_readable = d['_speed_str'] self.downloaded_bytes = d['downloaded_bytes'] @@ -103,7 +108,7 @@ def extractMetaData(self, link): else: video = result if video.get('filesize'): - self.size = int(video.get('filesize')) + self.size = float(video.get('filesize')) self.name = name self.vid_id = video.get('id') return video diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 0a7879c18..6bb1c949f 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -14,11 +14,12 @@ from googleapiclient.http import MediaFileUpload from tenacity import * -from bot import LOGGER, parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ +from bot import parent_id, DOWNLOAD_DIR, IS_TEAM_DRIVE, INDEX_URL, \ USE_SERVICE_ACCOUNTS from bot.helper.ext_utils.bot_utils import * from bot.helper.ext_utils.fs_utils import get_mime_type +LOGGER = logging.getLogger(__name__) logging.getLogger('googleapiclient.discovery').setLevel(logging.ERROR) SERVICE_ACCOUNT_INDEX = 0 @@ -269,8 +270,12 @@ def clone(self, link): f' ({get_readable_file_size(self.transferred_size)})' else: file = self.copyFile(meta.get('id'), parent_id) - msg += f'{meta.get("name")}' \ - f' ({get_readable_file_size(int(meta.get("size")))}) ' + + msg += f'{meta.get("name")}' + try: + msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' + except TypeError: + pass return msg def cloneFolder(self, name, local_path, folder_id, parent_id): @@ -299,7 +304,10 @@ def cloneFolder(self, name, local_path, folder_id, parent_id): current_dir_id = self.create_directory(file.get('name'), parent_id) new_id = self.cloneFolder(file.get('name'), file_path, file.get('id'), current_dir_id) else: - self.transferred_size += int(file.get('size')) + try: + self.transferred_size += int(file.get('size')) + except TypeError: + pass try: self.copyFile(file.get('id'), parent_id) new_id = parent_id From 50ab740219ea35a16894e29c50db258724f6a56c Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 31 Mar 2020 17:18:07 +0530 Subject: [PATCH 52/79] Correct filename if ffmpeg conversion is invoked by yt-dl Signed-off-by: lzzy12 --- .../download_utils/youtube_dl_download_helper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index c111b94b8..295d3079c 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -5,6 +5,7 @@ from bot import download_dict_lock, download_dict from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus import logging +import re LOGGER = logging.getLogger(__name__) @@ -14,7 +15,11 @@ def __init__(self, obj): self.obj = obj def debug(self, msg): - LOGGER.debug(msg) + LOGGER.info(msg) + # Hack to fix changing changing extension + match = re.search(r'.ffmpeg..Merging formats into..(.*?).$', msg) + if match and not self.obj.is_playlist: + self.obj.name = match.group(1) def warning(self, msg): LOGGER.warning(msg) From 708e6df87aee89f4e59762d5c82e1644d10544db Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Wed, 1 Apr 2020 12:07:29 +0530 Subject: [PATCH 53/79] youtube-dl: use best format --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 295d3079c..9e13e82db 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -38,7 +38,8 @@ def __init__(self, listener): self.opts = { 'progress_hooks': [self.__onDownloadProgress], 'logger': MyLogger(self), - 'usenetrc': True + 'usenetrc': True, + 'format':"best" } self.__download_speed = 0 self.download_speed_readable = '' From b58eb87b111ddcbd8e607000b1f13e738495247d Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 4 Apr 2020 22:58:11 +0530 Subject: [PATCH 54/79] Improvements in cancelling mirrors Signed-off-by: lzzy12 --- .../download_utils/aria2_download.py | 31 ++++++++++------ bot/modules/cancel_mirror.py | 36 +++++-------------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index e29cfdfee..ecd8e3757 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -12,8 +12,8 @@ class AriaDownloadHelper(DownloadHelper): def __init__(self, listener): super().__init__() self.gid = None - self._listener = listener - self._resource_lock = threading.Lock() + self.__listener = listener + self._resource_lock = threading.RLock() def __onDownloadStarted(self, api, gid): with self._resource_lock: @@ -28,22 +28,22 @@ def __onDownloadComplete(self, api: API, gid): if api.get_download(gid).followed_by_ids: self.gid = api.get_download(gid).followed_by_ids[0] with download_dict_lock: - download_dict[self._listener.uid] = AriaDownloadStatus(self.gid, self._listener) - download_dict[self._listener.uid].is_torrent =True + download_dict[self.__listener.uid] = AriaDownloadStatus(self.gid, self.__listener) + download_dict[self.__listener.uid].is_torrent =True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') else: - self._listener.onDownloadComplete() + self.__listener.onDownloadComplete() def __onDownloadPause(self, api, gid): if self.gid == gid: LOGGER.info("Called onDownloadPause") - self._listener.onDownloadError('Download stopped by user!') + self.__listener.onDownloadError('Download stopped by user!') def __onDownloadStopped(self, api, gid): if self.gid == gid: LOGGER.info("Called on_download_stop") - self._listener.onDownloadError('Download stopped by user!') + self.__listener.onDownloadError('Download stopped by user!') def __onDownloadError(self, api, gid): with self._resource_lock: @@ -51,7 +51,7 @@ def __onDownloadError(self, api, gid): download = api.get_download(gid) error = download.error_message LOGGER.info(f"Download Error: {error}") - self._listener.onDownloadError(error) + self.__listener.onDownloadError(error) def add_download(self, link: str, path): if is_magnet(link): @@ -60,9 +60,9 @@ def add_download(self, link: str, path): download = aria2.add_uris([link], {'dir': path}) self.gid = download.gid with download_dict_lock: - download_dict[self._listener.uid] = AriaDownloadStatus(self.gid, self._listener) + download_dict[self.__listener.uid] = AriaDownloadStatus(self.gid, self.__listener) if download.error_message: - self._listener.onDownloadError(download.error_message) + self.__listener.onDownloadError(download.error_message) return LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ") aria2.listen_to_notifications(threaded=True, on_download_start=self.__onDownloadStarted, @@ -70,3 +70,14 @@ def add_download(self, link: str, path): on_download_pause=self.__onDownloadPause, on_download_stop=self.__onDownloadStopped, on_download_complete=self.__onDownloadComplete) + + def cancel_download(self): + download = aria2.get_download(self.gid) + if download.is_queued: + aria2.remove([download]) + self.__listener.onDownloadError("Cancelled by user") + return + if len(download.followed_by_ids) != 0: + downloads = aria2.get_downloads(download.followed_by_ids) + aria2.pause(downloads) + aria2.pause([download]) diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index 245ca5643..fe489438f 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -1,22 +1,18 @@ -from time import sleep - from telegram.ext import CommandHandler, run_async -from bot import download_dict, aria2, dispatcher, download_dict_lock, DOWNLOAD_DIR +from bot import download_dict, dispatcher, download_dict_lock, DOWNLOAD_DIR from bot.helper.ext_utils.fs_utils import clean_download from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.message_utils import * -from ..helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper from time import sleep from bot.helper.ext_utils.bot_utils import getDownloadByGid -from ..helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper +from ..helper.ext_utils.bot_utils import MirrorStatus @run_async def cancel_mirror(bot, update): - # TODO: Rewrite this for good args = update.message.text.split(" ", maxsplit=1) mirror_message = None if len(args) > 1: @@ -27,7 +23,7 @@ def cancel_mirror(bot, update): return with download_dict_lock: keys = list(download_dict.keys()) - mirror_message = dl._listener.message + mirror_message = dl.__listener.message elif update.message.reply_to_message: mirror_message = update.message.reply_to_message with download_dict_lock: @@ -45,25 +41,14 @@ def cancel_mirror(bot, update): "used to start the download or /cancel gid to cancel it!" sendMessage(msg, bot, update) return - if dl.status() == "Uploading": + if dl.status() == MirrorStatus.STATUS_UPLOADING: sendMessage("Upload in Progress, Don't Cancel it.", bot, update) return - elif dl.status() == "Archiving": + elif dl.status() == MirrorStatus.STATUS_ARCHIVING: sendMessage("Archival in Progress, Don't Cancel it.", bot, update) return - elif dl.status() != "Queued": - download = dl.download() - if isinstance(download, TelegramDownloadHelper): - download.cancel_download() - if isinstance(download, YoutubeDLHelper): - download.cancel_download() - else: - if len(download.followed_by_ids) != 0: - downloads = aria2.get_downloads(download.followed_by_ids) - aria2.pause(downloads) - aria2.pause([download]) else: - dl._listener.onDownloadError("Download stopped by user!") + dl.download().cancel_download() sleep(1) # Wait a Second For Aria2 To free Resources. clean_download(f'{DOWNLOAD_DIR}{mirror_message.message_id}/') @@ -73,13 +58,10 @@ def cancel_all(update, bot): with download_dict_lock: count = 0 for dlDetails in list(download_dict.values()): - if not dlDetails.status() == "Uploading" or dlDetails.status() == "Archiving": - aria2.pause([dlDetails.download()]) - count += 1 - continue - if dlDetails.status() == "Queued": + if not dlDetails.status() == MirrorStatus.STATUS_UPLOADING\ + or not dlDetails.status() == MirrorStatus.STATUS_ARCHIVING: + dlDetails.cancel_download() count += 1 - dlDetails._listener.onDownloadError("Download Manually Cancelled By user.") delete_all_messages() sendMessage(f'Cancelled {count} downloads!', update, bot) From 0536472e28cd1193eb177aacc58a6c788b8add89 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 5 Apr 2020 13:26:52 +0530 Subject: [PATCH 55/79] Expose message object from Status objects Signed-off-by: lzzy12 --- bot/helper/ext_utils/bot_utils.py | 2 +- bot/helper/mirror_utils/download_utils/aria2_download.py | 2 +- .../mirror_utils/download_utils/telegram_downloader.py | 2 +- .../download_utils/youtube_dl_download_helper.py | 2 +- bot/helper/mirror_utils/status_utils/aria_download_status.py | 5 +++-- .../mirror_utils/status_utils/telegram_download_status.py | 5 +++-- bot/helper/mirror_utils/status_utils/upload_status.py | 5 +++-- .../mirror_utils/status_utils/youtube_dl_download_status.py | 5 +++-- bot/modules/cancel_mirror.py | 5 ++--- bot/modules/mirror.py | 2 +- 10 files changed, 19 insertions(+), 16 deletions(-) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 9b811428a..4d396b95d 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -61,7 +61,7 @@ def get_readable_file_size(size_in_bytes) -> str: def getDownloadByGid(gid): with download_dict_lock: for dl in download_dict.values(): - if dl.status() == MirrorStatus.STATUS_DOWNLOADING: + if dl.status() == MirrorStatus.STATUS_DOWNLOADING or dl.status() == MirrorStatus.STATUS_WAITING: if dl.gid() == gid: return dl return None diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index ecd8e3757..c62cc0b3c 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -29,7 +29,7 @@ def __onDownloadComplete(self, api: API, gid): self.gid = api.get_download(gid).followed_by_ids[0] with download_dict_lock: download_dict[self.__listener.uid] = AriaDownloadStatus(self.gid, self.__listener) - download_dict[self.__listener.uid].is_torrent =True + download_dict[self.__listener.uid].is_torrent = True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') else: diff --git a/bot/helper/mirror_utils/download_utils/telegram_downloader.py b/bot/helper/mirror_utils/download_utils/telegram_downloader.py index 3de82f751..b35f7f3dd 100644 --- a/bot/helper/mirror_utils/download_utils/telegram_downloader.py +++ b/bot/helper/mirror_utils/download_utils/telegram_downloader.py @@ -41,7 +41,7 @@ def download_speed(self): def __onDownloadStart(self, name, size, file_id): with download_dict_lock: - download_dict[self.__listener.uid] = TelegramDownloadStatus(self, self.__listener.uid) + download_dict[self.__listener.uid] = TelegramDownloadStatus(self, self.__listener) with global_lock: GLOBAL_GID.add(file_id) with self.__resource_lock: diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 9e13e82db..5d1ee6d40 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -85,7 +85,7 @@ def __onDownloadProgress(self, d): def __onDownloadStart(self): with download_dict_lock: - download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self, self.__listener.uid) + download_dict[self.__listener.uid] = YoutubeDLDownloadStatus(self, self.__listener) def __onDownloadComplete(self): self.__listener.onDownloadComplete() diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index bb913c939..8449c83a2 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -16,7 +16,8 @@ def __init__(self, gid, listener): self.__gid = gid self.__download = get_download(gid) self.__uid = listener.uid - self._listener = listener + self.__listener = listener + self.message = listener.message self.last = None self.is_waiting = False @@ -64,7 +65,7 @@ def status(self): status = MirrorStatus.STATUS_CANCELLED elif download.has_failed: status = MirrorStatus.STATUS_FAILED - self._listener.onDownloadError('Unknown Error') + self.__listener.onDownloadError('Unknown Error') else: status = MirrorStatus.STATUS_DOWNLOADING return status diff --git a/bot/helper/mirror_utils/status_utils/telegram_download_status.py b/bot/helper/mirror_utils/status_utils/telegram_download_status.py index 7647f6aec..89f3d5234 100644 --- a/bot/helper/mirror_utils/status_utils/telegram_download_status.py +++ b/bot/helper/mirror_utils/status_utils/telegram_download_status.py @@ -4,9 +4,10 @@ class TelegramDownloadStatus(Status): - def __init__(self, obj, uid): + def __init__(self, obj, listener): self.obj = obj - self.uid = uid + self.uid = listener.uid + self.message = listener.message def gid(self): return self.obj.gid diff --git a/bot/helper/mirror_utils/status_utils/upload_status.py b/bot/helper/mirror_utils/status_utils/upload_status.py index 0da5e905e..203e660c0 100644 --- a/bot/helper/mirror_utils/status_utils/upload_status.py +++ b/bot/helper/mirror_utils/status_utils/upload_status.py @@ -4,10 +4,11 @@ class UploadStatus(Status): - def __init__(self, obj, size, uid): + def __init__(self, obj, size, listener): self.obj = obj self.__size = size - self.uid = uid + self.uid = listener.uid + self.message = listener.message def path(self): return f"{DOWNLOAD_DIR}{self.uid}" diff --git a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py index 51f058237..a29013d79 100644 --- a/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py +++ b/bot/helper/mirror_utils/status_utils/youtube_dl_download_status.py @@ -4,9 +4,10 @@ class YoutubeDLDownloadStatus(Status): - def __init__(self, obj, uid): + def __init__(self, obj, listener): self.obj = obj - self.uid = uid + self.uid = listener.uid + self.message = listener.message def gid(self): return self.obj.gid diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index fe489438f..c93988dd4 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -7,8 +7,7 @@ from bot.helper.telegram_helper.message_utils import * from time import sleep -from bot.helper.ext_utils.bot_utils import getDownloadByGid -from ..helper.ext_utils.bot_utils import MirrorStatus +from bot.helper.ext_utils.bot_utils import getDownloadByGid, MirrorStatus @run_async @@ -23,7 +22,7 @@ def cancel_mirror(bot, update): return with download_dict_lock: keys = list(download_dict.keys()) - mirror_message = dl.__listener.message + mirror_message = dl.message elif update.message.reply_to_message: mirror_message = update.message.reply_to_message with download_dict_lock: diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index bdb61f7af..0f381157b 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -64,7 +64,7 @@ def onDownloadComplete(self): drive = gdriveTools.GoogleDriveHelper(up_name, self) if size == 0: size = fs_utils.get_path_size(m_path) - upload_status = UploadStatus(drive, size, self.uid) + upload_status = UploadStatus(drive, size, self) with download_dict_lock: download_dict[self.uid] = upload_status update_all_messages() From b84d4aca2c8e52b67de800593e30bffb3c80992c Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 5 Apr 2020 13:57:49 +0530 Subject: [PATCH 56/79] Update Documentation Signed-off-by: lzzy12 --- README.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d5f4fa61a..0e8d5b60d 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,8 @@ This project is heavily inspired from @out386 's telegram bot which is written i - Docker support - Uploading To Team Drives. - Index Link support - +- Service account support # Upcoming features (TODOs): -- Mirror from Telegram files # How to deploy? Deploying is pretty much straight forward and is divided into several steps as follows: @@ -52,7 +51,8 @@ Fill up rest of the fields. Meaning of each fields are discussed below: - DOWNLOAD_STATUS_UPDATE_INTERVAL : A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least) - OWNER_ID : The Telegram user ID (not username) of the owner of the bot - AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages -- IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. +- IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. +- USE_SERVICE_ACCOUNTS: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. - INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' - API_KEY : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. - API_HASH : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org @@ -64,12 +64,14 @@ Note: You can limit maximum concurrent downloads by changing the value of MAX_CO ## Getting Google OAuth API credential file -- Visit the Google Cloud Console +- Visit the [Google Cloud Console](https://console.developers.google.com/apis/credentials) - Go to the OAuth Consent tab, fill it, and save. - Go to the Credentials tab and click Create Credentials -> OAuth Client ID - Choose Other and Create. - Use the download button to download your credentials. - Move that file to the root of mirror-bot, and rename it to credentials.json +- Visit [Google API page](https://console.developers.google.com/apis/library) +- Search for Drive and enable it if it is disabled - Finally, run the script to generate token file (token.pickle) for Google Drive: ``` pip install google-api-python-client google-auth-httplib2 google-auth-oauthlib @@ -90,11 +92,11 @@ sudo docker build . -t mirror-bot sudo docker run mirror-bot ``` -## Using service accounts for uploading to avoid user rate limit - +# Using service accounts for uploading to avoid user rate limit +For Service Account to work, you must set USE_SERVICE_ACCOUNTS="True" in config file or environment variables Many thanks to [AutoRClone](https://github.com/xyou365/AutoRclone) for the scripts -### Generating service accounts -Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) [How to use service account in rclone](https://rclone.org/drive/#service-account-support). +## Generating service accounts +Step 1. Generate service accounts [What is service account](https://cloud.google.com/iam/docs/service-accounts) --------------------------------- Let us create only the service accounts that we need. **Warning:** abuse of this feature is not the aim of autorclone and we do **NOT** recommend that you make a lot of projects, just one project and 100 sa allow you plenty of use, its also possible that overabuse might get your projects banned by google. @@ -110,4 +112,10 @@ A folder named accounts will be created which will contain keys for the service NOTE: If you have created SAs in past from this script, you can also just re download the keys by running: ``` python3 gen_sa_accounts.py --download-keys project_id -``` \ No newline at end of file +``` + +### Add all the service accounts to the Team Drive or folder +- Run: +``` +python3 add_to_team_drive.py -d SharedTeamDriveSrcID +``` From 47dd3c2b6fc87a0612718c33f020d3419b8d1f0c Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 5 Apr 2020 14:12:13 +0530 Subject: [PATCH 57/79] gdriveTools: Handle unhandled exception in clone method Signed-off-by: lzzy12 --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 6bb1c949f..8f7e8153b 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -246,7 +246,14 @@ def copyFile(self, file_id, dest_id): def clone(self, link): self.transferred_size = 0 - file_id = self.getIdFromUrl(link) + try: + file_id = self.getIdFromUrl(link) + except KeyError: + msg = "Google drive ID could not be found in the provided link" + return msg + except IndexError: + msg = "Google drive ID could not be found in the provided link" + return msg msg = "" LOGGER.info(f"File ID: {file_id}") try: From 3ef3e9ee456b1f82803bee3fefe25076ede5339e Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sun, 5 Apr 2020 14:48:34 +0530 Subject: [PATCH 58/79] Change default name of token file for Service account scripts - Also added missing script file Signed-off-by: lzzy12 --- add_to_team_drive.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ gen_sa_accounts.py | 4 +-- 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 add_to_team_drive.py diff --git a/add_to_team_drive.py b/add_to_team_drive.py new file mode 100644 index 000000000..222cbe1b1 --- /dev/null +++ b/add_to_team_drive.py @@ -0,0 +1,77 @@ +from __future__ import print_function +from google.oauth2.service_account import Credentials +import googleapiclient.discovery, json, progress.bar, glob, sys, argparse, time +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request +import os, pickle + +stt = time.time() + +parse = argparse.ArgumentParser( + description='A tool to add service accounts to a shared drive from a folder containing credential files.') +parse.add_argument('--path', '-p', default='accounts', + help='Specify an alternative path to the service accounts folder.') +parse.add_argument('--credentials', '-c', default='./credentials.json', + help='Specify the relative path for the credentials file.') +parse.add_argument('--yes', '-y', default=False, action='store_true', help='Skips the sanity prompt.') +parsereq = parse.add_argument_group('required arguments') +parsereq.add_argument('--drive-id', '-d', help='The ID of the Shared Drive.', required=True) + +args = parse.parse_args() +acc_dir = args.path +did = args.drive_id +credentials = glob.glob(args.credentials) + +try: + open(credentials[0], 'r') + print('>> Found credentials.') +except IndexError: + print('>> No credentials found.') + sys.exit(0) + +if not args.yes: + # input('Make sure the following client id is added to the shared drive as Manager:\n' + json.loads((open( + # credentials[0],'r').read()))['installed']['client_id']) + input('>> Make sure the **Google account** that has generated credentials.json\n is added into your Team Drive ' + '(shared drive) as Manager\n>> (Press any key to continue)') + +creds = None +if os.path.exists('token_sa.pickle'): + with open('token_sa.pickle', 'rb') as token: + creds = pickle.load(token) +# If there are no (valid) credentials available, let the user log in. +if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[ + 'https://www.googleapis.com/auth/admin.directory.group', + 'https://www.googleapis.com/auth/admin.directory.group.member' + ]) + # creds = flow.run_local_server(port=0) + creds = flow.run_console() + # Save the credentials for the next run + with open('token_sa.pickle', 'wb') as token: + pickle.dump(creds, token) + +drive = googleapiclient.discovery.build("drive", "v3", credentials=creds) +batch = drive.new_batch_http_request() + +aa = glob.glob('%s/*.json' % acc_dir) +pbar = progress.bar.Bar("Readying accounts", max=len(aa)) +for i in aa: + ce = json.loads(open(i, 'r').read())['client_email'] + batch.add(drive.permissions().create(fileId=did, supportsAllDrives=True, body={ + "role": "fileOrganizer", + "type": "user", + "emailAddress": ce + })) + pbar.next() +pbar.finish() +print('Adding...') +batch.execute() + +print('Complete.') +hours, rem = divmod((time.time() - stt), 3600) +minutes, sec = divmod(rem, 60) +print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec)) \ No newline at end of file diff --git a/gen_sa_accounts.py b/gen_sa_accounts.py index 0fd1c244c..ff6f144b7 100644 --- a/gen_sa_accounts.py +++ b/gen_sa_accounts.py @@ -163,7 +163,7 @@ def _delete_sas(iam, project): def serviceaccountfactory( credentials='credentials.json', - token='token.pickle', + token='token_sa.pickle', path=None, list_projects=False, list_sas=None, @@ -282,7 +282,7 @@ def serviceaccountfactory( parse = ArgumentParser(description='A tool to create Google service accounts.') parse.add_argument('--path', '-p', default='accounts', help='Specify an alternate directory to output the credential files.') - parse.add_argument('--token', default='token.pickle', help='Specify the pickle token file path.') + parse.add_argument('--token', default='token_sa.pickle', help='Specify the pickle token file path.') parse.add_argument('--credentials', default='credentials.json', help='Specify the credentials file path.') parse.add_argument('--list-projects', default=False, action='store_true', help='List projects viewable by the user.') From b2f2ee4335e198e5605d095ae6c3f6acf0190158 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 6 Apr 2020 12:31:20 +0530 Subject: [PATCH 59/79] Fix aria downloads cancellation after b58eb87b111ddcbd8e607000b1f13e738495247d Signed-off-by: lzzy12 --- README.md | 24 +++--- add_to_google_group.py | 84 ------------------- bot/helper/ext_utils/bot_utils.py | 12 +-- .../download_utils/aria2_download.py | 6 +- .../status_utils/aria_download_status.py | 26 +++--- bot/modules/cancel_mirror.py | 6 +- 6 files changed, 35 insertions(+), 123 deletions(-) delete mode 100644 add_to_google_group.py diff --git a/README.md b/README.md index 0e8d5b60d..8e14f78f3 100644 --- a/README.md +++ b/README.md @@ -45,18 +45,18 @@ cp config_sample.env config.env _____REMOVE_THIS_LINE_____=True ``` Fill up rest of the fields. Meaning of each fields are discussed below: -- BOT_TOKEN : The telegram bot token that you get from @BotFather -- GDRIVE_FOLDER_ID : This is the folder ID of the Google Drive Folder to which you want to upload all the mirrors. -- DOWNLOAD_DIR : The path to the local folder where the downloads should be downloaded to -- DOWNLOAD_STATUS_UPDATE_INTERVAL : A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least) -- OWNER_ID : The Telegram user ID (not username) of the owner of the bot -- AUTO_DELETE_MESSAGE_DURATION : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages -- IS_TEAM_DRIVE : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. -- USE_SERVICE_ACCOUNTS: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. -- INDEX_URL : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' -- API_KEY : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. -- API_HASH : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org -- USER_SESSION_STRING : Session string generated by running: +- **BOT_TOKEN** : The telegram bot token that you get from @BotFather +- **GDRIVE_FOLDER_ID** : This is the folder ID of the Google Drive Folder to which you want to upload all the mirrors. +- **DOWNLOAD_DIR** : The path to the local folder where the downloads should be downloaded to +- **DOWNLOAD_STATUS_UPDATE_INTERVAL** : A short interval of time in seconds after which the Mirror progress message is updated. (I recommend to keep it 5 seconds at least) +- **OWNER_ID** : The Telegram user ID (not username) of the owner of the bot +- **AUTO_DELETE_MESSAGE_DURATION** : Interval of time (in seconds), after which the bot deletes it's message (and command message) which is expected to be viewed instantly. Note: Set to -1 to never automatically delete messages +- **IS_TEAM_DRIVE** : (Optional field) Set to "True" if GDRIVE_FOLDER_ID is from a Team Drive else False or Leave it empty. +- **USE_SERVICE_ACCOUNTS**: (Optional field) (Leave empty if unsure) Whether to use service accounts or not. For this to work see "Using service accounts" section below. +- **INDEX_URL** : (Optional field) Refer to https://github.com/maple3142/GDIndex/ The URL should not have any trailing '/' +- **API_KEY** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org DO NOT put this in quotes. +- **API_HASH** : This is to authenticate to your telegram account for downloading Telegram files. You can get this from https://my.telegram.org +- **USER_SESSION_STRING** : Session string generated by running: ``` python3 generate_string_session.py ``` diff --git a/add_to_google_group.py b/add_to_google_group.py deleted file mode 100644 index aaed28bf4..000000000 --- a/add_to_google_group.py +++ /dev/null @@ -1,84 +0,0 @@ -# auto rclone -# Add service accounts to groups for your organization -# -# Author Telegram https://t.me/CodyDoby -# Inbox codyd@qq.com - -from __future__ import print_function - -import os -import pickle - -import argparse -import glob -import googleapiclient.discovery -import json -import progress.bar -import time -from google.auth.transport.requests import Request -from google_auth_oauthlib.flow import InstalledAppFlow - -stt = time.time() - -parse = argparse.ArgumentParser( - description='A tool to add service accounts to groups for your organization from a folder containing credential ' - 'files.') -parse.add_argument('--path', '-p', default='accounts', - help='Specify an alternative path to the service accounts folder.') -parse.add_argument('--credentials', '-c', default='credentials/credentials.json', - help='Specify the relative path for the controller file.') -parsereq = parse.add_argument_group('required arguments') -# service-account@googlegroups.com -parsereq.add_argument('--groupaddr', '-g', help='The address of groups for your organization.', required=True) - -args = parse.parse_args() -acc_dir = args.path -gaddr = args.groupaddr -credentials = glob.glob(args.credentials) - -creds = None -if os.path.exists('credentials/token.pickle'): - with open('credentials/token.pickle', 'rb') as token: - creds = pickle.load(token) -# If there are no (valid) credentials available, let the user log in. -if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - creds.refresh(Request()) - else: - flow = InstalledAppFlow.from_client_secrets_file(credentials[0], scopes=[ - 'https://www.googleapis.com/auth/admin.directory.group', - 'https://www.googleapis.com/auth/admin.directory.group.member' - ]) - # creds = flow.run_local_server(port=0) - creds = flow.run_console() - # Save the credentials for the next run - with open('credentials/token.pickle', 'wb') as token: - pickle.dump(creds, token) - -group = googleapiclient.discovery.build("admin", "directory_v1", credentials=creds) - -print(group.members()) - -batch = group.new_batch_http_request() - -sa = glob.glob('%s/*.json' % acc_dir) - -# sa = sa[0:5] - -pbar = progress.bar.Bar("Readying accounts", max=len(sa)) -for i in sa: - ce = json.loads(open(i, 'r').read())['client_email'] - - body = {"email": ce, "role": "MEMBER"} - batch.add(group.members().insert(groupKey=gaddr, body=body)) - # group.members().insert(groupKey=gaddr, body=body).execute() - - pbar.next() -pbar.finish() -print('Adding...') -batch.execute() - -print('Complete.') -hours, rem = divmod((time.time() - stt), 3600) -minutes, sec = divmod(rem, 60) -print("Elapsed Time:\n{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), sec)) diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 4d396b95d..8d0e6ce8e 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -85,14 +85,6 @@ def get_progress_bar_string(status): return p_str -def get_download_index(_list, gid): - index = 0 - for i in _list: - if i.download().gid == gid: - return index - index += 1 - - def get_readable_message(): with download_dict_lock: msg = "" @@ -105,8 +97,8 @@ def get_readable_message(): f" at {download.speed()}, ETA: {download.eta()} " if download.status() == MirrorStatus.STATUS_DOWNLOADING: if hasattr(download, 'is_torrent'): - msg += f"| P: {download.download().connections} " \ - f"| S: {download.download().num_seeders}" + msg += f"| P: {download.aria_download().connections} " \ + f"| S: {download.aria_download().num_seeders}" msg += f"\nGID: {download.gid()}" msg += "\n\n" return msg diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index c62cc0b3c..aaa299f2f 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -28,7 +28,7 @@ def __onDownloadComplete(self, api: API, gid): if api.get_download(gid).followed_by_ids: self.gid = api.get_download(gid).followed_by_ids[0] with download_dict_lock: - download_dict[self.__listener.uid] = AriaDownloadStatus(self.gid, self.__listener) + download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) download_dict[self.__listener.uid].is_torrent = True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') @@ -60,7 +60,7 @@ def add_download(self, link: str, path): download = aria2.add_uris([link], {'dir': path}) self.gid = download.gid with download_dict_lock: - download_dict[self.__listener.uid] = AriaDownloadStatus(self.gid, self.__listener) + download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) if download.error_message: self.__listener.onDownloadError(download.error_message) return @@ -73,7 +73,7 @@ def add_download(self, link: str, path): def cancel_download(self): download = aria2.get_download(self.gid) - if download.is_queued: + if download.is_waiting: aria2.remove([download]) self.__listener.onDownloadError("Cancelled by user") return diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index 8449c83a2..7425e086a 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -9,12 +9,13 @@ def get_download(gid): class AriaDownloadStatus(Status): - def __init__(self, gid, listener): + def __init__(self, obj, listener): super().__init__() self.upload_name = None self.is_archiving = False - self.__gid = gid - self.__download = get_download(gid) + self.obj = obj + self.__gid = obj.gid + self.__download = get_download(obj.gid) self.__uid = listener.uid self.__listener = listener self.message = listener.message @@ -37,28 +38,28 @@ def size_raw(self): Gets total size of the mirror file/folder :return: total size of mirror """ - return self.download().total_length + return self.aria_download().total_length def processed_bytes(self): - return self.download().completed_length + return self.aria_download().completed_length def speed(self): - return self.download().download_speed_string() + return self.aria_download().download_speed_string() def name(self): - return self.download().name + return self.aria_download().name def path(self): return f"{DOWNLOAD_DIR}{self.__uid}" def size(self): - return self.download().total_length_string() + return self.aria_download().total_length_string() def eta(self): - return self.download().eta_string() + return self.aria_download().eta_string() def status(self): - download = self.download() + download = self.aria_download() if download.is_waiting: status = MirrorStatus.STATUS_WAITING elif download.is_paused: @@ -70,10 +71,13 @@ def status(self): status = MirrorStatus.STATUS_DOWNLOADING return status - def download(self): + def aria_download(self): self.__update() return self.__download + def download(self): + return self.obj + def uid(self): return self.__uid diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index c93988dd4..53ccdfc2c 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -57,9 +57,9 @@ def cancel_all(update, bot): with download_dict_lock: count = 0 for dlDetails in list(download_dict.values()): - if not dlDetails.status() == MirrorStatus.STATUS_UPLOADING\ - or not dlDetails.status() == MirrorStatus.STATUS_ARCHIVING: - dlDetails.cancel_download() + if dlDetails.status() == MirrorStatus.STATUS_DOWNLOADING\ + or dlDetails.status() == MirrorStatus.STATUS_WAITING: + dlDetails.download().cancel_download() count += 1 delete_all_messages() sendMessage(f'Cancelled {count} downloads!', update, bot) From ed60afa9d6ebcea8899266f22941977d741cc588 Mon Sep 17 00:00:00 2001 From: Arsalan <35004378+a092devs@users.noreply.github.com> Date: Tue, 7 Apr 2020 15:06:04 +0000 Subject: [PATCH 60/79] fix typo in INDEX_URL (#54) --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 8f7e8153b..ad7c8ef00 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -412,7 +412,7 @@ def drive_list(self, fileName): msg += f"⁍ {file.get('name')} ({get_readable_file_size(int(file.get('size')))})" if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') msg += f' | Index URL' msg += '\n' return msg From 76e3db1946d488cd94f3390d5b4e22eeb0e4172c Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 6 Apr 2020 22:54:18 +0530 Subject: [PATCH 61/79] [Hotfix] Fix bug where the link have 302 request are treated as torrents Download.is_torrent introduced in aria2p v0.9.0 https://github.com/pawamoy/aria2p/issues/53 Signed-off-by: lzzy12 --- bot/helper/mirror_utils/download_utils/aria2_download.py | 6 ++++-- requirements.txt | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index aaa299f2f..22c403ca1 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -25,10 +25,12 @@ def __onDownloadStarted(self, api, gid): def __onDownloadComplete(self, api: API, gid): with self._resource_lock: if self.gid == gid: - if api.get_download(gid).followed_by_ids: - self.gid = api.get_download(gid).followed_by_ids[0] + download = api.get_download(gid) + if download.followed_by_ids: + self.gid = download.followed_by_ids[0] with download_dict_lock: download_dict[self.__listener.uid] = AriaDownloadStatus(self, self.__listener) + if download.is_torrent: download_dict[self.__listener.uid].is_torrent = True update_all_messages() LOGGER.info(f'Changed gid from {gid} to {self.gid}') diff --git a/requirements.txt b/requirements.txt index 307a6f131..69fbbe199 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ python-telegram-bot==12.2.0 google-api-python-client>=1.7.11,<1.7.20 google-auth-httplib2>=0.0.3,<0.1.0 google-auth-oauthlib>=0.4.1,<0.10.0 -aria2p>=0.3.0,<0.10.0 +aria2p>=0.9.0,<0.15.0 python-dotenv>=0.10 tenacity>=6.0.0 python-magic From e5364f96e19872257864a1073dae023851e970c1 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Mon, 13 Apr 2020 13:32:39 +0530 Subject: [PATCH 62/79] Revert "Handle rare bug when download is not deleted from download_dict on error" This reverts commit 136924ffd467453cd04d44f273c8ce603ea6d4b3. Signed-off-by: lzzy12 --- bot/helper/mirror_utils/status_utils/aria_download_status.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bot/helper/mirror_utils/status_utils/aria_download_status.py b/bot/helper/mirror_utils/status_utils/aria_download_status.py index 7425e086a..502aaf0c8 100644 --- a/bot/helper/mirror_utils/status_utils/aria_download_status.py +++ b/bot/helper/mirror_utils/status_utils/aria_download_status.py @@ -66,7 +66,6 @@ def status(self): status = MirrorStatus.STATUS_CANCELLED elif download.has_failed: status = MirrorStatus.STATUS_FAILED - self.__listener.onDownloadError('Unknown Error') else: status = MirrorStatus.STATUS_DOWNLOADING return status From e0f4c4347c75f557afdd829f3de7047315f9e9d2 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Tue, 14 Apr 2020 23:20:52 +0530 Subject: [PATCH 63/79] Separate out youtube-dl command - /watch [youtube-dl supported link] Signed-off-by: lzzy12 --- bot/__main__.py | 25 +++++----- .../youtube_dl_download_helper.py | 19 +++++--- bot/helper/telegram_helper/bot_commands.py | 3 +- bot/modules/mirror.py | 23 ++++----- bot/modules/watch.py | 47 +++++++++++++++++++ 5 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 bot/modules/watch.py diff --git a/bot/__main__.py b/bot/__main__.py index 0c2153149..efdaf4d1e 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -9,7 +9,7 @@ from bot.helper.telegram_helper.message_utils import * from .helper.ext_utils.bot_utils import get_readable_file_size, get_readable_time from .helper.telegram_helper.filters import CustomFilters -from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone +from .modules import authorize, list, cancel_mirror, mirror_status, mirror, clone, watch @run_async @@ -21,33 +21,32 @@ def stats(bot, update): free = get_readable_file_size(free) stats = f'Bot Uptime: {currentTime}\n' \ f'Total disk space: {total}\n' \ - f'Used: {used}\n' \ - f'Free: {free}' + f'Used: {used}\n' \ + f'Free: {free}' sendMessage(stats, bot, update) - @run_async -def start(bot,update): +def start(bot, update): sendMessage("This is a bot which can mirror all your links to Google drive!\n" "Type /help to get a list of available commands", bot, update) @run_async -def ping(bot,update): +def ping(bot, update): start_time = int(round(time.time() * 1000)) reply = sendMessage("Starting Ping", bot, update) - end_time = int(round(time.time()*1000)) - editMessage(f'{end_time - start_time} ms',reply) + end_time = int(round(time.time() * 1000)) + editMessage(f'{end_time - start_time} ms', reply) @run_async -def log(bot,update): +def log(bot, update): sendLogFile(bot, update) @run_async -def bot_help(bot,update): +def bot_help(bot, update): help_string = f''' /{BotCommands.HelpCommand}: To get this message @@ -55,6 +54,10 @@ def bot_help(bot,update): /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download +/{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl + +/{BotCommands.TarWatchCommand} [youtube-dl supported link]: Mirror through youtube-dl and tar before uploading + /{BotCommands.CancelMirror} : Reply to the message by which the download was initiated and that download will be cancelled /{BotCommands.StatusCommand}: Shows a status of all the downloads @@ -80,7 +83,7 @@ def main(): help_handler = CommandHandler(BotCommands.HelpCommand, bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) stats_handler = CommandHandler(BotCommands.StatsCommand, - stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + stats, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter) dispatcher.add_handler(start_handler) dispatcher.add_handler(ping_handler) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 5d1ee6d40..08f85c5f1 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -1,6 +1,6 @@ from .download_helper import DownloadHelper import time -from youtube_dl import YoutubeDL +from youtube_dl import YoutubeDL, DownloadError import threading from bot import download_dict_lock, download_dict from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus @@ -15,7 +15,7 @@ def __init__(self, obj): self.obj = obj def debug(self, msg): - LOGGER.info(msg) + LOGGER.debug(msg) # Hack to fix changing changing extension match = re.search(r'.ffmpeg..Merging formats into..(.*?).$', msg) if match and not self.obj.is_playlist: @@ -26,6 +26,7 @@ def warning(self, msg): def error(self, msg): LOGGER.error(msg) + #self.obj.onDownloadError(msg) class YoutubeDLHelper(DownloadHelper): @@ -37,9 +38,8 @@ def __init__(self, listener): self.__gid = "" self.opts = { 'progress_hooks': [self.__onDownloadProgress], - 'logger': MyLogger(self), 'usenetrc': True, - 'format':"best" + 'format': "best" } self.__download_speed = 0 self.download_speed_readable = '' @@ -90,7 +90,7 @@ def __onDownloadStart(self): def __onDownloadComplete(self): self.__listener.onDownloadComplete() - def __onDownloadError(self, error): + def onDownloadError(self, error): self.__listener.onDownloadError(error) def extractMetaData(self, link): @@ -122,15 +122,20 @@ def extractMetaData(self, link): def __download(self, link): try: with YoutubeDL(self.opts) as ydl: - ydl.download([link]) + try: + ydl.download([link]) + except DownloadError as e: + self.onDownloadError(str(e)) + return self.__onDownloadComplete() except ValueError: LOGGER.info("Download Cancelled by User!") - self.__onDownloadError("Download Cancelled by User!") + self.onDownloadError("Download Cancelled by User!") def add_download(self, link, path): LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" + self.opts['logger'] = MyLogger(self) if not self.is_playlist: self.opts['outtmpl'] = f"{path}/{self.name}" else: diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py index d58c29dca..d4e8b6473 100644 --- a/bot/helper/telegram_helper/bot_commands.py +++ b/bot/helper/telegram_helper/bot_commands.py @@ -14,6 +14,7 @@ def __init__(self): self.HelpCommand = 'help' self.LogCommand = 'log' self.CloneCommand = "clone" - + self.WatchCommand = 'watch' + self.TarWatchCommand = 'tarwatch' BotCommands = _BotCommands() diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index 0f381157b..a7718eec5 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -36,9 +36,12 @@ def onDownloadProgress(self): pass def clean(self): - Interval[0].cancel() - del Interval[0] - delete_all_messages() + try: + Interval[0].cancel() + del Interval[0] + delete_all_messages() + except IndexError: + pass def onDownloadComplete(self): with download_dict_lock: @@ -71,6 +74,8 @@ def onDownloadComplete(self): drive.upload(up_name) def onDownloadError(self, error): + error = error.replace('<', ' ') + error = error.replace('>', ' ') LOGGER.info(self.update.effective_chat.id) with download_dict_lock: try: @@ -181,16 +186,8 @@ def _mirror(bot, update, isTar=False): except DirectDownloadLinkException as e: LOGGER.info(f'{link}: {e}') listener = MirrorListener(bot, update, isTar, tag) - ydl = YoutubeDLHelper(listener) - try: - sup_link = ydl.extractMetaData(link) - except Exception as e: - sup_link = None - if sup_link: - ydl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}') - else: - aria = aria2_download.AriaDownloadHelper(listener) - aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') + aria = aria2_download.AriaDownloadHelper(listener) + aria.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/') sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/bot/modules/watch.py b/bot/modules/watch.py new file mode 100644 index 000000000..80148b8be --- /dev/null +++ b/bot/modules/watch.py @@ -0,0 +1,47 @@ +from telegram.ext import CommandHandler, run_async +from telegram import Bot, Update +from bot import Interval, INDEX_URL, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, dispatcher, LOGGER +from bot.helper.ext_utils.bot_utils import setInterval +from bot.helper.telegram_helper.message_utils import update_all_messages, sendMessage +from .mirror import MirrorListener +from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper +from bot.helper.telegram_helper.bot_commands import BotCommands +from bot.helper.telegram_helper.filters import CustomFilters + + +def _watch(bot: Bot, update: Update, args: list, isTar=False): + try: + link = args[0] + except IndexError: + sendMessage('/watch [yt_dl supported link] to mirror with youtube_dl', bot, update) + return + reply_to = update.message.reply_to_message + if reply_to is not None: + tag = reply_to.from_user.username + else: + tag = None + + listener = MirrorListener(bot, update, isTar, tag) + ydl = YoutubeDLHelper(listener) + ydl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}') + if len(Interval) == 0: + Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) + + +@run_async +def watchTar(bot: Bot, update: Update, args: list): + _watch(bot, update, args, True) + + +def watch(bot: Bot, update: Update, args: list): + _watch(bot, update, args) + + +mirror_handler = CommandHandler(BotCommands.WatchCommand, watch, + pass_args=True, + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) +tar_mirror_handler = CommandHandler(BotCommands.TarWatchCommand, watchTar, + pass_args=True, + filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) +dispatcher.add_handler(mirror_handler) +dispatcher.add_handler(tar_mirror_handler) From c6b4c542b7caa6db51f743333f684b59ceef2ee1 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 15 Apr 2020 13:54:24 +0530 Subject: [PATCH 64/79] fixup after e0f4c4347c75f557afdd829f3de7047315f9e9d2 - Also fixed copying of netrc file in the container Signed-off-by: lzzy12 --- Dockerfile | 9 +++++---- README.md | 7 +++++++ .../youtube_dl_download_helper.py | 20 ++++++++++++------- bot/modules/watch.py | 5 +++-- netrc | 0 5 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 netrc diff --git a/Dockerfile b/Dockerfile index d34ec7b3a..5fb7f5a77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,18 @@ FROM ubuntu:18.04 WORKDIR /usr/src/app RUN chmod 777 /usr/src/app -RUN apt -qq update -RUN apt -qq install -y aria2 python3 python3-pip \ +RUN apt-get -qq update +RUN apt-get -qq install -y aria2 python3 python3-pip \ locales python3-lxml \ curl pv jq ffmpeg COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt -COPY . . -RUN chmod +x aria.sh RUN locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 +COPY . . +COPY netrc /root/.netrc +RUN chmod +x aria.sh CMD ["bash","start.sh"] diff --git a/README.md b/README.md index 8e14f78f3..9fb45fcaa 100644 --- a/README.md +++ b/README.md @@ -119,3 +119,10 @@ python3 gen_sa_accounts.py --download-keys project_id ``` python3 add_to_team_drive.py -d SharedTeamDriveSrcID ``` + +# Youtube-dl authentication using .netrc file +For using your premium accounts in youtube-dl, edit the netrc file (in the root directory of this repository) according to following format: +``` +machine host login username password my_youtube_password +``` +where host is the name of extractor (eg. youtube, twitch). Multiple accounts of different hosts can be added each separated by a new line \ No newline at end of file diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 08f85c5f1..24657e994 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -21,12 +21,13 @@ def debug(self, msg): if match and not self.obj.is_playlist: self.obj.name = match.group(1) - def warning(self, msg): + @staticmethod + def warning(msg): LOGGER.warning(msg) - def error(self, msg): + @staticmethod + def error(msg): LOGGER.error(msg) - #self.obj.onDownloadError(msg) class YoutubeDLHelper(DownloadHelper): @@ -38,8 +39,9 @@ def __init__(self, listener): self.__gid = "" self.opts = { 'progress_hooks': [self.__onDownloadProgress], + 'logger': MyLogger(self), 'usenetrc': True, - 'format': "best" + 'format': "best/bestvideo+bestaudio" } self.__download_speed = 0 self.download_speed_readable = '' @@ -98,8 +100,12 @@ def extractMetaData(self, link): self.opts['geo_bypass_country'] = 'IN' with YoutubeDL(self.opts) as ydl: - result = ydl.extract_info(link, download=False) - name = ydl.prepare_filename(result) + try: + result = ydl.extract_info(link, download=False) + name = ydl.prepare_filename(result) + except DownloadError as e: + self.onDownloadError(str(e)) + return if result.get('direct'): return None if 'entries' in result: @@ -133,9 +139,9 @@ def __download(self, link): self.onDownloadError("Download Cancelled by User!") def add_download(self, link, path): + self.extractMetaData(link) LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" - self.opts['logger'] = MyLogger(self) if not self.is_playlist: self.opts['outtmpl'] = f"{path}/{self.name}" else: diff --git a/bot/modules/watch.py b/bot/modules/watch.py index 80148b8be..9754fc4eb 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -1,8 +1,8 @@ from telegram.ext import CommandHandler, run_async from telegram import Bot, Update -from bot import Interval, INDEX_URL, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, dispatcher, LOGGER +from bot import Interval, DOWNLOAD_DIR, DOWNLOAD_STATUS_UPDATE_INTERVAL, dispatcher, LOGGER from bot.helper.ext_utils.bot_utils import setInterval -from bot.helper.telegram_helper.message_utils import update_all_messages, sendMessage +from bot.helper.telegram_helper.message_utils import update_all_messages, sendMessage, sendStatusMessage from .mirror import MirrorListener from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper from bot.helper.telegram_helper.bot_commands import BotCommands @@ -24,6 +24,7 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): listener = MirrorListener(bot, update, isTar, tag) ydl = YoutubeDLHelper(listener) ydl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}') + sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) diff --git a/netrc b/netrc new file mode 100644 index 000000000..e69de29bb From 802fefdfe44ff88c8a7a33aa461e4b89120d5dfa Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Wed, 15 Apr 2020 15:00:12 +0530 Subject: [PATCH 65/79] Update README.md Signed-off-by: lzzy12 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9fb45fcaa..8e391961f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ This project is heavily inspired from @out386 's telegram bot which is written i - Uploading To Team Drives. - Index Link support - Service account support +- Mirror all youtube-dl supported links +- Mirror telegram files + # Upcoming features (TODOs): # How to deploy? From d7bc8f3c9ad9c960cf978145b57ca1d3c846a39b Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 16 Apr 2020 12:13:08 +0530 Subject: [PATCH 66/79] switch to context based callbacks --- bot/__init__.py | 2 +- bot/__main__.py | 24 +++++++++--------- .../mirror_utils/upload_utils/gdriveTools.py | 2 ++ bot/modules/authorize.py | 8 +++--- bot/modules/cancel_mirror.py | 25 +++++++++---------- bot/modules/clone.py | 8 +++--- bot/modules/list.py | 8 +++--- bot/modules/mirror.py | 8 +++--- bot/modules/mirror_status.py | 8 +++--- requirements.txt | 4 +-- 10 files changed, 49 insertions(+), 48 deletions(-) diff --git a/bot/__init__.py b/bot/__init__.py index 8cca93a5b..1f5cf4038 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -103,6 +103,6 @@ def getConfig(name: str): except KeyError: USE_SERVICE_ACCOUNTS = False -updater = tg.Updater(token=BOT_TOKEN) +updater = tg.Updater(token=BOT_TOKEN,use_context=True) bot = updater.bot dispatcher = updater.dispatcher diff --git a/bot/__main__.py b/bot/__main__.py index efdaf4d1e..695b654eb 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -13,7 +13,7 @@ @run_async -def stats(bot, update): +def stats(update,context): currentTime = get_readable_time((time.time() - botStartTime)) total, used, free = shutil.disk_usage('.') total = get_readable_file_size(total) @@ -21,32 +21,32 @@ def stats(bot, update): free = get_readable_file_size(free) stats = f'Bot Uptime: {currentTime}\n' \ f'Total disk space: {total}\n' \ - f'Used: {used}\n' \ - f'Free: {free}' - sendMessage(stats, bot, update) + f'Used: {used}\n' \ + f'Free: {free}' + sendMessage(stats, context.bot, update) @run_async -def start(bot, update): +def start(update,context): sendMessage("This is a bot which can mirror all your links to Google drive!\n" - "Type /help to get a list of available commands", bot, update) + "Type /help to get a list of available commands", context.bot, update) @run_async -def ping(bot, update): +def ping(update, context): start_time = int(round(time.time() * 1000)) - reply = sendMessage("Starting Ping", bot, update) + reply = sendMessage("Starting Ping", context.bot, update) end_time = int(round(time.time() * 1000)) editMessage(f'{end_time - start_time} ms', reply) @run_async -def log(bot, update): - sendLogFile(bot, update) +def log(update, context): + sendLogFile(context.bot, update) @run_async -def bot_help(bot, update): +def bot_help(update, context): help_string = f''' /{BotCommands.HelpCommand}: To get this message @@ -71,7 +71,7 @@ def bot_help(bot, update): /{BotCommands.LogCommand}: Get a log file of the bot. Handy for getting crash reports ''' - sendMessage(help_string, bot, update) + sendMessage(help_string, context.bot, update) def main(): diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index ad7c8ef00..d367c0ac6 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -175,6 +175,8 @@ def upload_file(self, file_path, file_name, mime_type, parent_id): return download_url def upload(self, file_name: str): + if USE_SERVICE_ACCOUNTS: + self.service_account_count = len(os.listdir("accounts")) self.__listener.onUploadStarted() file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}" file_path = f"{file_dir}/{file_name}" diff --git a/bot/modules/authorize.py b/bot/modules/authorize.py index de254f030..fedb4d1ac 100644 --- a/bot/modules/authorize.py +++ b/bot/modules/authorize.py @@ -9,7 +9,7 @@ @run_async -def authorize(bot, update): +def authorize(update,context): reply_message = update.message.reply_to_message msg = '' with open('authorized_chats.txt', 'a') as file: @@ -31,11 +31,11 @@ def authorize(bot, update): msg = 'Person Authorized to use the bot!' else: msg = 'Person already authorized' - sendMessage(msg, bot, update) + sendMessage(msg, context.bot, update) @run_async -def unauthorize(bot,update): +def unauthorize(update,context): reply_message = update.message.reply_to_message if reply_message is None: # Trying to unauthorize a chat @@ -57,7 +57,7 @@ def unauthorize(bot,update): file.truncate(0) for i in AUTHORIZED_CHATS: file.write(f'{i}\n') - sendMessage(msg, bot, update) + sendMessage(msg, context.bot, update) authorize_handler = CommandHandler(command=BotCommands.AuthorizeCommand, callback=authorize, diff --git a/bot/modules/cancel_mirror.py b/bot/modules/cancel_mirror.py index 53ccdfc2c..c25c46037 100644 --- a/bot/modules/cancel_mirror.py +++ b/bot/modules/cancel_mirror.py @@ -11,14 +11,14 @@ @run_async -def cancel_mirror(bot, update): - args = update.message.text.split(" ", maxsplit=1) +def cancel_mirror(update,context): + args = update.message.text.split(" ",maxsplit=1) mirror_message = None if len(args) > 1: gid = args[1] dl = getDownloadByGid(gid) if not dl: - sendMessage(f"GID: {gid} not found.", bot, update) + sendMessage(f"GID: {gid} not found.",context.bot,update) return with download_dict_lock: keys = list(download_dict.keys()) @@ -33,18 +33,17 @@ def cancel_mirror(bot, update): if BotCommands.MirrorCommand in mirror_message.text or \ BotCommands.TarMirrorCommand in mirror_message.text: msg = "Mirror already have been cancelled" - sendMessage(msg, bot, update) + sendMessage(msg,context.bot,update) return else: - msg = "Please reply to the /mirror message which was " \ - "used to start the download or /cancel gid to cancel it!" - sendMessage(msg, bot, update) + msg = "Please reply to the /mirror message which was used to start the download or /cancel gid to cancel it!" + sendMessage(msg,context.bot,update) return - if dl.status() == MirrorStatus.STATUS_UPLOADING: - sendMessage("Upload in Progress, Don't Cancel it.", bot, update) + if dl.status() == "Uploading": + sendMessage("Upload in Progress, Don't Cancel it.", context.bot, update) return - elif dl.status() == MirrorStatus.STATUS_ARCHIVING: - sendMessage("Archival in Progress, Don't Cancel it.", bot, update) + elif dl.status() == "Archiving": + sendMessage("Archival in Progress, Don't Cancel it.", context.bot, update) return else: dl.download().cancel_download() @@ -53,7 +52,7 @@ def cancel_mirror(bot, update): @run_async -def cancel_all(update, bot): +def cancel_all(update, context): with download_dict_lock: count = 0 for dlDetails in list(download_dict.values()): @@ -62,7 +61,7 @@ def cancel_all(update, bot): dlDetails.download().cancel_download() count += 1 delete_all_messages() - sendMessage(f'Cancelled {count} downloads!', update, bot) + sendMessage(f'Cancelled {count} downloads!', context.bot,update) cancel_mirror_handler = CommandHandler(BotCommands.CancelMirror, cancel_mirror, diff --git a/bot/modules/clone.py b/bot/modules/clone.py index fa3323338..407e3532a 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -7,15 +7,15 @@ @run_async -def cloneNode(bot,update): +def cloneNode(update,context): args = update.message.text.split(" ",maxsplit=1) if len(args) > 1: link = args[1] - msg = sendMessage(f"Cloning: {link}",bot,update) + msg = sendMessage(f"Cloning: {link}",context.bot,update) gd = GoogleDriveHelper() result = gd.clone(link) - deleteMessage(bot,msg) - sendMessage(result,bot,update) + deleteMessage(context.bot,msg) + sendMessage(result,context.bot,update) else: sendMessage("Provide G-Drive Shareable Link to Clone.",bot,update) diff --git a/bot/modules/list.py b/bot/modules/list.py index bef678c24..24a641c0a 100644 --- a/bot/modules/list.py +++ b/bot/modules/list.py @@ -7,18 +7,18 @@ from bot.helper.telegram_helper.bot_commands import BotCommands @run_async -def list_drive(bot,update): +def list_drive(update,context): message = update.message.text search = message.split(' ',maxsplit=1)[1] LOGGER.info(f"Searching: {search}") gdrive = GoogleDriveHelper(None) msg = gdrive.drive_list(search) if msg: - reply_message = sendMessage(msg, bot, update) + reply_message = sendMessage(msg, context.bot, update) else: - reply_message = sendMessage('No result found', bot, update) + reply_message = sendMessage('No result found', context.bot, update) - threading.Thread(target=auto_delete_message, args=(bot, update.message, reply_message)).start() + threading.Thread(target=auto_delete_message, args=(context.bot, update.message, reply_message)).start() list_handler = CommandHandler(BotCommands.ListCommand, list_drive,filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index a7718eec5..b9539c302 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -194,13 +194,13 @@ def _mirror(bot, update, isTar=False): @run_async -def mirror(bot, update): - _mirror(bot, update) +def mirror(update, context): + _mirror(context.bot, update) @run_async -def tar_mirror(update, bot): - _mirror(update, bot, True) +def tar_mirror(update, context): + _mirror(context.bot, update, True) mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror, diff --git a/bot/modules/mirror_status.py b/bot/modules/mirror_status.py index bc2a913f2..e4ae32494 100644 --- a/bot/modules/mirror_status.py +++ b/bot/modules/mirror_status.py @@ -9,11 +9,11 @@ import threading @run_async -def mirror_status(bot,update): +def mirror_status(update,context): message = get_readable_message() if len(message) == 0: message = "No active downloads" - reply_message = sendMessage(message, bot, update) + reply_message = sendMessage(message, context.bot, update) threading.Thread(target=auto_delete_message, args=(bot, update.message, reply_message)).start() return index = update.effective_chat.id @@ -21,8 +21,8 @@ def mirror_status(bot,update): if index in status_reply_dict.keys(): deleteMessage(bot, status_reply_dict[index]) del status_reply_dict[index] - sendStatusMessage(update,bot) - deleteMessage(bot,update.message) + sendStatusMessage(update,context.bot) + deleteMessage(context.bot,update.message) mirror_status_handler = CommandHandler(BotCommands.StatusCommand, mirror_status, diff --git a/requirements.txt b/requirements.txt index 69fbbe199..1699c9b6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests -python-telegram-bot==12.2.0 +python-telegram-bot==12.6.1 google-api-python-client>=1.7.11,<1.7.20 google-auth-httplib2>=0.0.3,<0.1.0 google-auth-oauthlib>=0.4.1,<0.10.0 @@ -10,4 +10,4 @@ python-magic beautifulsoup4>=4.8.2,<4.8.10 Pyrogram>=0.16.0,<0.16.10 TgCrypto>=1.1.1,<1.1.10 -youtube-dl \ No newline at end of file +youtube-dl From e5bb8e55ef5f0ea3331ccbc954220999d55cd0f8 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 16 Apr 2020 12:38:12 +0530 Subject: [PATCH 67/79] Clone fix and added index link support --- .../mirror_utils/upload_utils/gdriveTools.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index d367c0ac6..0aa1376fd 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -250,10 +250,7 @@ def clone(self, link): self.transferred_size = 0 try: file_id = self.getIdFromUrl(link) - except KeyError: - msg = "Google drive ID could not be found in the provided link" - return msg - except IndexError: + except (KeyError,IndexError): msg = "Google drive ID could not be found in the provided link" return msg msg = "" @@ -277,12 +274,26 @@ def clone(self, link): return err msg += f'{meta.get("name")}' \ f' ({get_readable_file_size(self.transferred_size)})' + if INDEX_URL is not None: + url = requests.utils.requote_uri(f'{INDEX_URL}/{meta.get("name")}/') + msg += f' | Index URL' else: - file = self.copyFile(meta.get('id'), parent_id) - - msg += f'{meta.get("name")}' + try: + file = self.copyFile(meta.get('id'), parent_id) + except Exception as e: + if isinstance(e, RetryError): + LOGGER.info(f"Total Attempts: {e.last_attempt.attempt_number}") + err = e.last_attempt.exception() + else: + err = str(e).replace('>', '').replace('<', '') + LOGGER.error(err) + return err + msg += f'{file.get("name")}' try: msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' + if INDEX_URL is not None: + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') + msg += f' | Index URL' except TypeError: pass return msg From 120ca7538d0830f9b4eb283c9d4649c79f78d673 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 16 Apr 2020 12:39:46 +0530 Subject: [PATCH 68/79] clone: remove forward slash --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 0aa1376fd..1e7fb423a 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -292,7 +292,7 @@ def clone(self, link): try: msg += f' ({get_readable_file_size(int(meta.get("size")))}) ' if INDEX_URL is not None: - url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}/') + url = requests.utils.requote_uri(f'{INDEX_URL}/{file.get("name")}') msg += f' | Index URL' except TypeError: pass From 8341fe08c8925290a01447d567468880c8be4b26 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 16 Apr 2020 13:00:35 +0530 Subject: [PATCH 69/79] watch: context based callbacks --- bot/modules/watch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/modules/watch.py b/bot/modules/watch.py index 9754fc4eb..6baefc357 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -30,12 +30,12 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): @run_async -def watchTar(bot: Bot, update: Update, args: list): - _watch(bot, update, args, True) +def watchTar(update, context, args): + _watch(context.bot, update, args, True) -def watch(bot: Bot, update: Update, args: list): - _watch(bot, update, args) +def watch(update, context, args): + _watch(context.bot, update, args) mirror_handler = CommandHandler(BotCommands.WatchCommand, watch, From 236b6eb30eae1ec6b8391f9801a8b21478992061 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Thu, 16 Apr 2020 13:28:44 +0530 Subject: [PATCH 70/79] use args from CallbackContext --- bot/modules/watch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/modules/watch.py b/bot/modules/watch.py index 6baefc357..ac640ea18 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -30,12 +30,12 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): @run_async -def watchTar(update, context, args): - _watch(context.bot, update, args, True) +def watchTar(update, context): + _watch(context.bot, update, context.args, True) -def watch(update, context, args): - _watch(context.bot, update, args) +def watch(update, context): + _watch(context.bot, update, context.args) mirror_handler = CommandHandler(BotCommands.WatchCommand, watch, From 8624e0174b549b8cf1c007cbe73db74bdbb1abd1 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Sat, 18 Apr 2020 14:32:01 +0530 Subject: [PATCH 71/79] Fix IO Block on Large Playlists --- .../download_utils/youtube_dl_download_helper.py | 6 ++---- bot/modules/watch.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 24657e994..985a874ca 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -1,7 +1,6 @@ from .download_helper import DownloadHelper import time from youtube_dl import YoutubeDL, DownloadError -import threading from bot import download_dict_lock, download_dict from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus import logging @@ -139,6 +138,7 @@ def __download(self, link): self.onDownloadError("Download Cancelled by User!") def add_download(self, link, path): + self.__onDownloadStart() self.extractMetaData(link) LOGGER.info(f"Downloading with YT-DL: {link}") self.__gid = f"{self.vid_id}{self.__listener.uid}" @@ -146,9 +146,7 @@ def add_download(self, link, path): self.opts['outtmpl'] = f"{path}/{self.name}" else: self.opts['outtmpl'] = f"{path}/{self.name}/%(title)s.%(ext)s" - - self.__onDownloadStart() - threading.Thread(target=self.__download, args=(link,)).start() + self.__download(link) def cancel_download(self): self.is_cancelled = True diff --git a/bot/modules/watch.py b/bot/modules/watch.py index ac640ea18..acb98ed93 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -7,6 +7,7 @@ from bot.helper.mirror_utils.download_utils.youtube_dl_download_helper import YoutubeDLHelper from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.filters import CustomFilters +import threading def _watch(bot: Bot, update: Update, args: list, isTar=False): @@ -23,7 +24,7 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): listener = MirrorListener(bot, update, isTar, tag) ydl = YoutubeDLHelper(listener) - ydl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}') + threading.Thread(target=ydl.add_download,args=(link, f'{DOWNLOAD_DIR}{listener.uid}')).start() sendStatusMessage(update, bot) if len(Interval) == 0: Interval.append(setInterval(DOWNLOAD_STATUS_UPDATE_INTERVAL, update_all_messages)) From 78b60d67152457d8c6287540ce82198cdd2b8a7b Mon Sep 17 00:00:00 2001 From: jaskaranSM <37726998+jaskaranSM@users.noreply.github.com> Date: Sun, 19 Apr 2020 14:58:35 +0530 Subject: [PATCH 72/79] import threading lib --- .../mirror_utils/download_utils/youtube_dl_download_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py index 985a874ca..ca07ec1bf 100644 --- a/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py +++ b/bot/helper/mirror_utils/download_utils/youtube_dl_download_helper.py @@ -5,6 +5,7 @@ from ..status_utils.youtube_dl_download_status import YoutubeDLDownloadStatus import logging import re +import threading LOGGER = logging.getLogger(__name__) From 63a67d7a141f62f13ddb733060c3161a695820b0 Mon Sep 17 00:00:00 2001 From: Youssif Shaaban Alsager Date: Tue, 21 Apr 2020 13:03:05 +0300 Subject: [PATCH 73/79] Add restart command (#53) * Add restart command This is handy when you need to restart the bot when something goes wrong. Currently, the bot restarts but doesn't send a message after restarting is done, feel free to improve this. It'll need something like storing bot status locally then on bot start, if restart status exists send a message. Signed-off-by: yshalsager * restart: switch to context based callbacks * restart: restrict the command to owner only Co-authored-by: Shivam Jha --- bot/__main__.py | 10 ++++++++++ bot/helper/telegram_helper/bot_commands.py | 1 + 2 files changed, 11 insertions(+) diff --git a/bot/__main__.py b/bot/__main__.py index 695b654eb..727c001aa 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,5 +1,7 @@ import shutil import signal +from os import execl +from sys import executable from telegram.ext import CommandHandler, run_async @@ -33,6 +35,11 @@ def start(update,context): @run_async +def restart(update, context): + reply = sendMessage("Restarting, Please wait!", context.bot, update) + execl(executable, executable, "-m", "bot") + + def ping(update, context): start_time = int(round(time.time() * 1000)) reply = sendMessage("Starting Ping", context.bot, update) @@ -80,6 +87,8 @@ def main(): filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) ping_handler = CommandHandler(BotCommands.PingCommand, ping, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) + restart_handler = CommandHandler(BotCommands.RestartCommand, restart, + filters=CustomFilters.owner_filter) help_handler = CommandHandler(BotCommands.HelpCommand, bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) stats_handler = CommandHandler(BotCommands.StatsCommand, @@ -87,6 +96,7 @@ def main(): log_handler = CommandHandler(BotCommands.LogCommand, log, filters=CustomFilters.owner_filter) dispatcher.add_handler(start_handler) dispatcher.add_handler(ping_handler) + dispatcher.add_handler(restart_handler) dispatcher.add_handler(help_handler) dispatcher.add_handler(stats_handler) dispatcher.add_handler(log_handler) diff --git a/bot/helper/telegram_helper/bot_commands.py b/bot/helper/telegram_helper/bot_commands.py index d4e8b6473..fee3fa23e 100644 --- a/bot/helper/telegram_helper/bot_commands.py +++ b/bot/helper/telegram_helper/bot_commands.py @@ -10,6 +10,7 @@ def __init__(self): self.AuthorizeCommand = 'authorize' self.UnAuthorizeCommand = 'unauthorize' self.PingCommand = 'ping' + self.RestartCommand = 'restart' self.StatsCommand = 'stats' self.HelpCommand = 'help' self.LogCommand = 'log' From 4410f4e8aa654869102d8aef3514ed351c6ba2cd Mon Sep 17 00:00:00 2001 From: Arsalan <35004378+a092devs@users.noreply.github.com> Date: Wed, 22 Apr 2020 06:11:51 +0000 Subject: [PATCH 74/79] fixed hardcoded /watch for custom command users (#66) --- bot/modules/watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/modules/watch.py b/bot/modules/watch.py index acb98ed93..42bdb1d70 100644 --- a/bot/modules/watch.py +++ b/bot/modules/watch.py @@ -14,7 +14,7 @@ def _watch(bot: Bot, update: Update, args: list, isTar=False): try: link = args[0] except IndexError: - sendMessage('/watch [yt_dl supported link] to mirror with youtube_dl', bot, update) + sendMessage(f'/{BotCommands.WatchCommand} [yt_dl supported link] to mirror with youtube_dl', bot, update) return reply_to = update.message.reply_to_message if reply_to is not None: From b02990d9eb805a301985c6c83fb50be978152db6 Mon Sep 17 00:00:00 2001 From: Youssif Shaaban Alsager Date: Sat, 25 Apr 2020 18:43:25 +0300 Subject: [PATCH 75/79] restart: inform the user that restarting is complete (#67) Signed-off-by: yshalsager --- bot/__main__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index 727c001aa..ced73ea62 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -1,6 +1,8 @@ import shutil import signal -from os import execl +import pickle + +from os import execl, path, remove from sys import executable from telegram.ext import CommandHandler, run_async @@ -36,7 +38,10 @@ def start(update,context): @run_async def restart(update, context): - reply = sendMessage("Restarting, Please wait!", context.bot, update) + restart_message = sendMessage("Restarting, Please wait!", context.bot, update) + # Save restart message object in order to reply to it after restarting + with open('restart.pickle', 'wb') as status: + pickle.dump(restart_message, status) execl(executable, executable, "-m", "bot") @@ -83,6 +88,13 @@ def bot_help(update, context): def main(): fs_utils.start_cleanup() + # Check if the bot is restarting + if path.exists('restart.pickle'): + with open('restart.pickle', 'rb') as status: + restart_message = pickle.load(status) + restart_message.edit_text("Restarted Successfully!") + remove('restart.pickle') + start_handler = CommandHandler(BotCommands.StartCommand, start, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) ping_handler = CommandHandler(BotCommands.PingCommand, ping, From 4d964b6e290b08b378b6a6d42e04a422922cf294 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Mon, 4 May 2020 12:25:40 +0530 Subject: [PATCH 76/79] use regex to extract fileId from GDrive links --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 1e7fb423a..9d74da8a4 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -3,6 +3,7 @@ import urllib.parse as urlparse from urllib.parse import parse_qs +import re import json import requests @@ -67,7 +68,11 @@ def speed(self): @staticmethod def getIdFromUrl(link: str): if "folders" in link or "file" in link: - return link.rsplit('/')[-1] + regex = r"https://drive\.google\.com/(drive)?/?u?/\d?/?mobile?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" + res = re.search(regex,link) + if res is None: + raise IndexError("GDrive ID not found.") + return res.group(4) parsed = urlparse.urlparse(link) return parse_qs(parsed.query)['id'][0] From 315108e7d1c8b3177e3aee296a83f28a9895ef77 Mon Sep 17 00:00:00 2001 From: jaskaranSM Date: Tue, 5 May 2020 10:51:55 +0530 Subject: [PATCH 77/79] patch regex for more links --- bot/helper/mirror_utils/upload_utils/gdriveTools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 9d74da8a4..480ddc381 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -68,11 +68,11 @@ def speed(self): @staticmethod def getIdFromUrl(link: str): if "folders" in link or "file" in link: - regex = r"https://drive\.google\.com/(drive)?/?u?/\d?/?mobile?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" + regex = r"https://drive\.google\.com/(drive)?/?u?/?\d?/?(mobile)?/?(file)?(folders)?/?d?/([-\w]+)[?+]?/?(w+)?" res = re.search(regex,link) if res is None: raise IndexError("GDrive ID not found.") - return res.group(4) + return res.group(5) parsed = urlparse.urlparse(link) return parse_qs(parsed.query)['id'][0] From 1d8c8ce9540fad227fb86fd8414220f393fb9d14 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 9 May 2020 00:05:35 +0530 Subject: [PATCH 78/79] restart: Clean downloads before restarting Signed-off-by: lzzy12 --- bot/__main__.py | 4 ++-- bot/helper/ext_utils/fs_utils.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bot/__main__.py b/bot/__main__.py index ced73ea62..712cee668 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -6,8 +6,7 @@ from sys import executable from telegram.ext import CommandHandler, run_async - -from bot import dispatcher, updater, botStartTime +from bot import dispatcher, updater, botStartTime, DOWNLOAD_DIR from bot.helper.ext_utils import fs_utils from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.message_utils import * @@ -40,6 +39,7 @@ def start(update,context): def restart(update, context): restart_message = sendMessage("Restarting, Please wait!", context.bot, update) # Save restart message object in order to reply to it after restarting + fs_utils.clean_all() with open('restart.pickle', 'wb') as status: pickle.dump(restart_message, status) execl(executable, executable, "-m", "bot") diff --git a/bot/helper/ext_utils/fs_utils.py b/bot/helper/ext_utils/fs_utils.py index 8f3169268..0b92aa0d8 100644 --- a/bot/helper/ext_utils/fs_utils.py +++ b/bot/helper/ext_utils/fs_utils.py @@ -20,16 +20,21 @@ def start_cleanup(): pass +def clean_all(): + aria2.remove_all(True) + shutil.rmtree(DOWNLOAD_DIR) + + def exit_clean_up(signal, frame): try: LOGGER.info("Please wait, while we clean up the downloads and stop running downloads") - aria2.remove_all(True) - shutil.rmtree(DOWNLOAD_DIR) + clean_all() sys.exit(0) except KeyboardInterrupt: LOGGER.warning("Force Exiting before the cleanup finishes!") sys.exit(1) + def get_path_size(path): if os.path.isfile(path): return os.path.getsize(path) From d3938784acdd8e26724530679d8d18d43d94ce36 Mon Sep 17 00:00:00 2001 From: lzzy12 Date: Sat, 9 May 2020 00:09:38 +0530 Subject: [PATCH 79/79] Remove broken direct download link extracters Signed-off-by: lzzy12 --- .gitmodules | 3 - bot/__main__.py | 13 ++- .../download_utils/direct_link_generator.py | 103 ------------------ 3 files changed, 7 insertions(+), 112 deletions(-) diff --git a/.gitmodules b/.gitmodules index f53bcd466..ea40a2d28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "vendor/megadown"] - path = vendor/megadown - url = https://github.com/tonikelope/megadown.git [submodule "vendor/cmrudl.py"] path = vendor/cmrudl.py url = https://github.com/JrMasterModelBuilder/cmrudl.py.git diff --git a/bot/__main__.py b/bot/__main__.py index 712cee668..4140a3b33 100644 --- a/bot/__main__.py +++ b/bot/__main__.py @@ -6,7 +6,7 @@ from sys import executable from telegram.ext import CommandHandler, run_async -from bot import dispatcher, updater, botStartTime, DOWNLOAD_DIR +from bot import dispatcher, updater, botStartTime from bot.helper.ext_utils import fs_utils from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.telegram_helper.message_utils import * @@ -16,7 +16,7 @@ @run_async -def stats(update,context): +def stats(update, context): currentTime = get_readable_time((time.time() - botStartTime)) total, used, free = shutil.disk_usage('.') total = get_readable_file_size(total) @@ -24,13 +24,13 @@ def stats(update,context): free = get_readable_file_size(free) stats = f'Bot Uptime: {currentTime}\n' \ f'Total disk space: {total}\n' \ - f'Used: {used}\n' \ - f'Free: {free}' + f'Used: {used}\n' \ + f'Free: {free}' sendMessage(stats, context.bot, update) @run_async -def start(update,context): +def start(update, context): sendMessage("This is a bot which can mirror all your links to Google drive!\n" "Type /help to get a list of available commands", context.bot, update) @@ -45,6 +45,7 @@ def restart(update, context): execl(executable, executable, "-m", "bot") +@run_async def ping(update, context): start_time = int(round(time.time() * 1000)) reply = sendMessage("Starting Ping", context.bot, update) @@ -100,7 +101,7 @@ def main(): ping_handler = CommandHandler(BotCommands.PingCommand, ping, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) restart_handler = CommandHandler(BotCommands.RestartCommand, restart, - filters=CustomFilters.owner_filter) + filters=CustomFilters.owner_filter) help_handler = CommandHandler(BotCommands.HelpCommand, bot_help, filters=CustomFilters.authorized_chat | CustomFilters.authorized_user) stats_handler = CommandHandler(BotCommands.StatsCommand, diff --git a/bot/helper/mirror_utils/download_utils/direct_link_generator.py b/bot/helper/mirror_utils/download_utils/direct_link_generator.py index 71d361913..84ea43a5b 100644 --- a/bot/helper/mirror_utils/download_utils/direct_link_generator.py +++ b/bot/helper/mirror_utils/download_utils/direct_link_generator.py @@ -24,12 +24,8 @@ def direct_link_generator(link: str): """ direct links generator """ if not link: raise DirectDownloadLinkException("`No links found!`") - if 'drive.google.com' in link: - return gdrive(link) elif 'zippyshare.com' in link: return zippy_share(link) - elif 'mega.' in link: - return mega_dl(link) elif 'yadi.sk' in link: return yandex_disk(link) elif 'cloud.mail.ru' in link: @@ -40,49 +36,10 @@ def direct_link_generator(link: str): return osdn(link) elif 'github.com' in link: return github(link) - elif 'androidfilehost.com' in link: - return androidfilehost(link) else: raise DirectDownloadLinkException(f'No Direct link function found for {link}') -def gdrive(url: str) -> str: - """ GDrive direct links generator """ - drive = 'https://drive.google.com' - try: - link = re.findall(r'\bhttps?://drive\.google\.com\S+', url)[0] - except IndexError: - reply = "`No Google drive links found`\n" - return reply - file_id = '' - if link.find("view") != -1: - file_id = link.split('/')[-2] - elif link.find("open?id=") != -1: - file_id = link.split("open?id=")[1].strip() - elif link.find("uc?id=") != -1: - file_id = link.split("uc?id=")[1].strip() - url = f'{drive}/uc?export=download&id={file_id}' - download = requests.get(url, stream=True, allow_redirects=False) - cookies = download.cookies - try: - # In case of small file size, Google downloads directly - dl_url = download.headers["location"] - if 'accounts.google.com' in dl_url: # non-public file - raise DirectDownloadLinkException('`Link not public`') - except KeyError: - # In case of download warning page - page = BeautifulSoup(download.content, 'lxml') - export = drive + page.find('a', {'id': 'uc-download-link'}).get('href') - response = requests.get(export, - stream=True, - allow_redirects=False, - cookies=cookies) - dl_url = response.headers['location'] - if 'accounts.google.com' in dl_url: - raise DirectDownloadLinkException('`Link not public`') - return dl_url - - def zippy_share(url: str) -> str: """ ZippyShare direct links generator Based on https://github.com/LameLemon/ziggy""" @@ -125,23 +82,6 @@ def yandex_disk(url: str) -> str: raise DirectDownloadLinkException("`Error: File not found / Download limit reached`\n") -def mega_dl(url: str) -> str: - """ MEGA.nz direct links generator - Using https://github.com/tonikelope/megadown""" - try: - link = re.findall(r'\bhttps?://.*mega.*\.nz\S+', url)[0] - except IndexError: - raise DirectDownloadLinkException("`No MEGA.nz links found`\n") - command = f'vendor/megadown/megadown -q -m {link}' - result = popen(command).read() - try: - data = json.loads(result) - except json.JSONDecodeError: - raise DirectDownloadLinkException("`Error: Can't extract the link`\n") - dl_url = data['url'] - return dl_url - - def cm_ru(url: str) -> str: """ cloud.mail.ru direct links generator Using https://github.com/JrMasterModelBuilder/cmrudl.py""" @@ -206,49 +146,6 @@ def github(url: str) -> str: raise DirectDownloadLinkException("`Error: Can't extract the link`\n") -def androidfilehost(url: str) -> str: - """ AFH direct links generator """ - try: - link = re.findall(r'\bhttps?://.*androidfilehost.*fid.*\S+', url)[0] - except IndexError: - raise DirectDownloadLinkException("`No AFH links found`\n") - fid = re.findall(r'\?fid=(.*)', link)[0] - session = requests.Session() - user_agent = useragent() - headers = {'user-agent': user_agent} - res = session.get(link, headers=headers, allow_redirects=True) - headers = { - 'origin': 'https://androidfilehost.com', - 'accept-encoding': 'gzip, deflate, br', - 'accept-language': 'en-US,en;q=0.9', - 'user-agent': user_agent, - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'x-mod-sbb-ctype': 'xhr', - 'accept': '*/*', - 'referer': f'https://androidfilehost.com/?fid={fid}', - 'authority': 'androidfilehost.com', - 'x-requested-with': 'XMLHttpRequest', - } - data = { - 'submit': 'submit', - 'action': 'getdownloadmirrors', - 'fid': f'{fid}' - } - error = "`Error: Can't find Mirrors for the link`\n" - try: - req = session.post( - 'https://androidfilehost.com/libs/otf/mirrors.otf.php', - headers=headers, - data=data, - cookies=res.cookies) - mirrors = req.json()['MIRRORS'] - except (json.decoder.JSONDecodeError, TypeError): - raise DirectDownloadLinkException(error) - if not mirrors: - raise DirectDownloadLinkException(error) - return mirrors[0] - - def useragent(): """ useragent random setter