diff --git a/INSTALLING.md b/INSTALLING.md index ca52bed8e..f39655432 100644 --- a/INSTALLING.md +++ b/INSTALLING.md @@ -10,17 +10,16 @@ Kyoo also needs 3 files to work properly. Two of them can simply be copy-pasted Those files can be put in any directory of your choice. Those 3 files are: - - A `docker-compose.yml` (simply copy docker-compose.prod.yml from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml)). - - A `nginx.conf.template` copied from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template). - - A `.env` file that you will need to **fill**. Look at the example [.env.example](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) +- A `docker-compose.yml` (simply copy docker-compose.prod.yml from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml)). +- A `nginx.conf.template` copied from [here](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template). +- A `.env` file that you will need to **fill**. Look at the example [.env.example](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) > If you want an explanation of what are those files, you can read the following: > The `docker-compose.yml` file describes the different services of Kyoo, where they should be downloaded and their start order. \ > The `nignx.conf.template` file describes which service will be called when accessing the URL of Kyoo. \ > The `.env` file contains all the configuration options that the services in `docker-compose.yml` will read. - To retrieve metadata, Kyoo will need to communicate with an external service. For now, that is `the movie database`. For this purpose, you will need to get an API Key. For that, go to [themoviedb.org](https://www.themoviedb.org/) and create an account, then go [here](https://www.themoviedb.org/settings/api) and copy the `API Key (v3 auth)`, paste it after the `THEMOVIEDB_APIKEY=` on the `.env` file. @@ -34,9 +33,9 @@ Congratulation, everything is now ready to use Kyoo. You can navigate to `http:/ 1. Install docker & docker-compose 2. Download the -[`docker-compose.yml`](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml), -[`nginx.conf.template`](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template) and -[`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files + [`docker-compose.yml`](https://raw.githubusercontent.com/zoriya/Kyoo/master/docker-compose.prod.yml), + [`nginx.conf.template`](https://raw.githubusercontent.com/zoriya/Kyoo/master/nginx.conf.template) and + [`.env`](https://raw.githubusercontent.com/zoriya/Kyoo/master/.env.example) files 3. Fill the `.env` file with your configuration options (and an API Key from [themoviedb.org](https://www.themoviedb.org/)) 4. Run `docker-compose up -d` @@ -48,6 +47,9 @@ unsure that your `.env` contains all the options specified in the updated `.env. After that, you will need to update Kyoo's services. For that, open a terminal in the configuration's directory and run the command `docker-compose pull`. You are now ready to restart Kyoo, you can run `docker-compose up -d`. +You can also enable automatic updates via an external tool like [watchtower](https://containrrr.dev/watchtower/). +TLDR: `docker run -d --name watchtower -e WATCHTOWER_CLEANUP=true -e WATCHTOWER_POLL_INTERVAL=86400 -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower` + # Uninstalling To uninstall Kyoo, you need to open a terminal in the configuration's directory and run `docker-compose down`. This will diff --git a/scanner/requirements.txt b/scanner/requirements.txt index b90b7693c..8dd09130b 100644 --- a/scanner/requirements.txt +++ b/scanner/requirements.txt @@ -2,4 +2,4 @@ guessit aiohttp jsons black-with-tabs -watchdog +watchfiles diff --git a/scanner/scanner/monitor.py b/scanner/scanner/monitor.py index ad35b29d7..3b83d7c75 100644 --- a/scanner/scanner/monitor.py +++ b/scanner/scanner/monitor.py @@ -1,74 +1,19 @@ -import asyncio -from functools import wraps -from watchdog.observers import Observer -from watchdog.events import ( - FileSystemEventHandler, - DirCreatedEvent, - FileCreatedEvent, - DirMovedEvent, - FileMovedEvent, - DirDeletedEvent, - FileDeletedEvent, -) - -from scanner.utils import log_errors - +import logging +from watchfiles import awatch, Change +from .utils import ProviderError from .scanner import Scanner -task_list = [] -event = asyncio.Event() - async def monitor(path: str, scanner: Scanner): - global task_list - - observer = Observer() - handler = EventHandler(scanner) - observer.schedule(handler, path, recursive=True) - observer.start() - - while True: - if any(task_list): - tl = task_list - task_list = [] - await asyncio.gather(*tl) - await event.wait() - event.clear() - # Should call .join() if the while stops one day. - # observer.join() - - -def async_event(f): - # Log errors of f and catch them to prevent the gather to throw. - f = log_errors(f) - - @wraps(f) - def internal(*args, **kwargs): - task_list.append(f(*args, **kwargs)) - event.set() - - return internal - - -class EventHandler(FileSystemEventHandler): - def __init__(self, scanner: Scanner): - self._scanner = scanner - - @async_event - async def on_created(self, event: DirCreatedEvent | FileCreatedEvent): - if event.is_directory: - return - await self._scanner.identify(event.src_path) - - # TODO: Implement the following two methods - def on_moved(self, event: DirMovedEvent | FileMovedEvent): - if event.is_directory: - # TODO: Check if this event is also called for files in the directory or not. - return - print(event.src_path, event.dest_path) - - def on_deleted(self, event: DirDeletedEvent | FileDeletedEvent): - if event.is_directory: - # TODO: Check if this event is also called for files in the directory or not. - return - print(event.src_path) + async for changes in awatch(path): + for (event, file) in changes: + try: + if event == Change.added: + await scanner.identify(file) + else: + print(f"Change {event} occured for file {file}") + except ProviderError as e: + logging.error(str(e)) + except Exception as e: + logging.exception("Unhandled error", exc_info=e) + print("end", flush=True) diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 8e61672f7..f62d6f045 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -5,6 +5,7 @@ from aiohttp import ClientSession from pathlib import Path from guessit import guessit +from typing import List from providers.provider import Provider from providers.types.episode import Episode, PartialShow from providers.types.season import Season, SeasonTranslation @@ -24,34 +25,32 @@ def __init__( async def scan(self, path: str): logging.info("Starting the scan. It can take some times...") - videos = filter(lambda p: p.is_file(), Path(path).rglob("*")) + self.registered = await self.get_registered_paths() + videos = (str(p) for p in Path(path).rglob("*") if p.is_file()) # We batch videos by 20 because too mutch at once kinda DDOS everything. for group in batch(videos, 20): logging.info("Batch finished. Starting a new one") await asyncio.gather(*map(self.identify, group)) - async def is_registered(self, path: Path) -> bool: + async def get_registered_paths(self) -> List[Path]: # TODO: Once movies are separated from the api, a new endpoint should be created to check for paths. async with self._client.get( - f"{self._url}/episodes/count", - params={"path": f"eq:{path}"}, + f"{self._url}/episodes", + params={"limit": 0}, headers={"X-API-Key": self._api_key}, ) as r: r.raise_for_status() - ret = await r.text() - if ret != "0": - return True - return False + ret = await r.json() + return list(x["path"] for x in ret["items"]) @log_errors - async def identify(self, path: Path): + async def identify(self, path: str): + if path in self.registered: + return + raw = guessit(path, "--episode-prefer-number") - if ( - not "mimetype" in raw - or not raw["mimetype"].startswith("video") - or await self.is_registered(path) - ): + if not "mimetype" in raw or not raw["mimetype"].startswith("video"): return logging.info("Identied %s: %s", path, raw)