From bbe8a19189e3d85b58602c5eb19d80f39c68e427 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 20 Jun 2023 11:29:51 +0900 Subject: [PATCH] Use another watchfile library --- INSTALLING.md | 16 +++---- scanner/requirements.txt | 2 +- scanner/scanner/monitor.py | 85 +++++++------------------------------- scanner/scanner/scanner.py | 4 +- 4 files changed, 27 insertions(+), 80 deletions(-) diff --git a/INSTALLING.md b/INSTALLING.md index ca52bed8e..f39655432 100644 --- a/INSTALLING.md +++ b/INSTALLING.md @@ -10,17 +10,16 @@ Kyoo also needs 3 files to work properly. Two of them can simply be copy-pasted Those files can be put in any directory of your choice. Those 3 files are: - - A `docker-compose.yml` (simply copy docker-compose.prod.yml from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml)). - - A `nginx.conf.template` copied from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template). - - A `.env` file that you will need to **fill**. Look at the example [.env.example](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) +- A `docker-compose.yml` (simply copy docker-compose.prod.yml from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml)). +- A `nginx.conf.template` copied from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template). +- A `.env` file that you will need to **fill**. Look at the example [.env.example](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) > If you want an explanation of what are those files, you can read the following: > The `docker-compose.yml` file describes the different services of Kyoo, where they should be downloaded and their start order. \ > The `nignx.conf.template` file describes which service will be called when accessing the URL of Kyoo. \ > The `.env` file contains all the configuration options that the services in `docker-compose.yml` will read. - To retrieve metadata, Kyoo will need to communicate with an external service. For now, that is `the movie database`. For this purpose, you will need to get an API Key. For that, go to [themoviedb.org](https://www.themoviedb.org/) and create an account, then go [here](https://www.themoviedb.org/settings/api) and copy the `API Key (v3 auth)`, paste it after the `THEMOVIEDB_APIKEY=` on the `.env` file. @@ -34,9 +33,9 @@ Congratulation, everything is now ready to use Kyoo. You can navigate to `http:/ 1. Install docker & docker-compose 2. Download the -[`docker-compose.yml`](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml), -[`nginx.conf.template`](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template) and -[`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files + [`docker-compose.yml`](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml), + [`nginx.conf.template`](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template) and + [`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files 3. Fill the `.env` file with your configuration options (and an API Key from [themoviedb.org](https://www.themoviedb.org/)) 4. Run `docker-compose up -d` @@ -48,6 +47,9 @@ unsure that your `.env` contains all the options specified in the updated `.env. After that, you will need to update Kyoo's services. For that, open a terminal in the configuration's directory and run the command `docker-compose pull`. You are now ready to restart Kyoo, you can run `docker-compose up -d`. +You can also enable automatic updates via an external tool like [watchtower](https://containrrr.dev/watchtower/). +TLDR: `docker run -d --name watchtower -e WATCHTOWER_CLEANUP=true -e WATCHTOWER_POLL_INTERVAL=86400 -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower` + # Uninstalling To uninstall Kyoo, you need to open a terminal in the configuration's directory and run `docker-compose down`. This will diff --git a/scanner/requirements.txt b/scanner/requirements.txt index b90b7693c..8dd09130b 100644 --- a/scanner/requirements.txt +++ b/scanner/requirements.txt @@ -2,4 +2,4 @@ guessit aiohttp jsons black-with-tabs -watchdog +watchfiles diff --git a/scanner/scanner/monitor.py b/scanner/scanner/monitor.py index ad35b29d7..3b83d7c75 100644 --- a/scanner/scanner/monitor.py +++ b/scanner/scanner/monitor.py @@ -1,74 +1,19 @@ -import asyncio -from functools import wraps -from watchdog.observers import Observer -from watchdog.events import ( - FileSystemEventHandler, - DirCreatedEvent, - FileCreatedEvent, - DirMovedEvent, - FileMovedEvent, - DirDeletedEvent, - FileDeletedEvent, -) - -from scanner.utils import log_errors - +import logging +from watchfiles import awatch, Change +from .utils import ProviderError from .scanner import Scanner -task_list = [] -event = asyncio.Event() - async def monitor(path: str, scanner: Scanner): - global task_list - - observer = Observer() - handler = EventHandler(scanner) - observer.schedule(handler, path, recursive=True) - observer.start() - - while True: - if any(task_list): - tl = task_list - task_list = [] - await asyncio.gather(*tl) - await event.wait() - event.clear() - # Should call .join() if the while stops one day. - # observer.join() - - -def async_event(f): - # Log errors of f and catch them to prevent the gather to throw. - f = log_errors(f) - - @wraps(f) - def internal(*args, **kwargs): - task_list.append(f(*args, **kwargs)) - event.set() - - return internal - - -class EventHandler(FileSystemEventHandler): - def __init__(self, scanner: Scanner): - self._scanner = scanner - - @async_event - async def on_created(self, event: DirCreatedEvent | FileCreatedEvent): - if event.is_directory: - return - await self._scanner.identify(event.src_path) - - # TODO: Implement the following two methods - def on_moved(self, event: DirMovedEvent | FileMovedEvent): - if event.is_directory: - # TODO: Check if this event is also called for files in the directory or not. - return - print(event.src_path, event.dest_path) - - def on_deleted(self, event: DirDeletedEvent | FileDeletedEvent): - if event.is_directory: - # TODO: Check if this event is also called for files in the directory or not. - return - print(event.src_path) + async for changes in awatch(path): + for (event, file) in changes: + try: + if event == Change.added: + await scanner.identify(file) + else: + print(f"Change {event} occured for file {file}") + except ProviderError as e: + logging.error(str(e)) + except Exception as e: + logging.exception("Unhandled error", exc_info=e) + print("end", flush=True) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 4500a9df6..f62d6f045 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -26,7 +26,7 @@ def __init__( async def scan(self, path: str): logging.info("Starting the scan. It can take some times...") self.registered = await self.get_registered_paths() - videos = filter(lambda p: p.is_file(), Path(path).rglob("*")) + videos = (str(p) for p in Path(path).rglob("*") if p.is_file()) # We batch videos by 20 because too mutch at once kinda DDOS everything. for group in batch(videos, 20): logging.info("Batch finished. Starting a new one") @@ -44,7 +44,7 @@ async def get_registered_paths(self) -> List[Path]: return list(x["path"] for x in ret["items"]) @log_errors - async def identify(self, path: Path): + async def identify(self, path: str): if path in self.registered: return