From 6ba0f62d4b80b2a0bd137f7abe925afd1623a92f Mon Sep 17 00:00:00 2001 From: reglim Date: Mon, 21 Nov 2022 14:06:44 +0100 Subject: [PATCH] Refactor(docat): Apply Requests from PR --- README.md | 26 +++++--------------------- docat/docat/app.py | 40 +++++++++++++--------------------------- docat/docat/models.py | 3 +-- docat/docat/utils.py | 34 +++++++++++++++++++--------------- docat/file.txt | 0 5 files changed, 38 insertions(+), 65 deletions(-) delete mode 100644 docat/file.txt diff --git a/README.md b/README.md index 9e62bb10b..77d1d4f06 100644 --- a/README.md +++ b/README.md @@ -13,26 +13,10 @@ you can optionally use volumes to persist state: ```sh # run container in background and persist data (docs, nginx configs and tokens database as well as the content index) # use 'ghcr.io/docat-org/docat:unstable' to get the latest changes -mkdir -p docat-run/db && touch docat-run/db/db.json && touch docat-run/db/index.json +mkdir -p docat-run/ docker run \ --detach \ - --volume $PWD/docat-run/doc:/var/docat/doc/ \ - --volume $PWD/docat-run/db/:/app/docat/ \ - --publish 8000:80 \ - ghcr.io/docat-org/docat -``` - -*Alternative:* Mount a dedicated directory to host `db.json` and `index.json`: - -```sh -# run container in background and persist data (docs, nginx configs and tokens database as well as the content index) -# use 'ghcr.io/docat-org/docat:unstable' to get the latest changes -mkdir -p docat-run/db && touch docat-run/db/db.json && touch docat-run/db/index.json -docker run \ - --detach \ - --volume $PWD/docat-run/doc:/var/docat/doc/ \ - --volume $PWD/docat-run/db:/var/docat/db/ \ - --env DOCAT_DB_DIR=/var/docat/db/ + --volume $PWD/docat-run/doc:/var/docat/ \ --publish 8000:80 \ ghcr.io/docat-org/docat ``` @@ -53,7 +37,7 @@ DEV_DOC_PATH="$(mktemp -d)" poetry install # run the local development version -DOCAT_SERVE_FILES=1 DOCAT_INDEX_FILES=1 DOCAT_DOC_PATH="$DEV_DOC_PATH" poetry run python -m docat +DOCAT_SERVE_FILES=1 DOCAT_DOC_PATH="$DEV_DOC_PATH" poetry run python -m docat ``` After this you need to start the frontend (inside the `web/` folder): @@ -116,13 +100,13 @@ It is possible to configure some things after the fact. Supported config options: -* headerHTML +- headerHTML ## Advanced Usage ### Hide Controls -If you would like to send link to a specific version of the documentation without the option to change the version, you can do so by clicking on the `Hide Controls` button. This will hide the control buttons and change the link, which can then be copied as usual. +If you would like to send link to a specific version of the documentation without the option to change the version, you can do so by clicking on the `Hide Controls` button. This will hide the control buttons and change the link, which can then be copied as usual. ### Indexing diff --git a/docat/docat/app.py b/docat/docat/app.py index e43a2e177..c6fe82ba0 100644 --- a/docat/docat/app.py +++ b/docat/docat/app.py @@ -11,7 +11,7 @@ import secrets import shutil from pathlib import Path -from typing import Optional, Tuple +from typing import Optional import magic from fastapi import Depends, FastAPI, File, Header, Response, UploadFile, status @@ -56,30 +56,16 @@ redoc_url="/api/redoc", ) -DOCAT_DB_DIR_STR = os.getenv("DOCAT_DB_DIR") +DOCAT_STORAGE_PATH = Path(os.getenv("DOCAT_STORAGE_PATH") or "/var/docat") +DOCAT_DB_PATH = DOCAT_STORAGE_PATH / DB_PATH +DOCAT_INDEX_PATH = DOCAT_STORAGE_PATH / INDEX_PATH +DOCAT_UPLOAD_FOLDER = DOCAT_STORAGE_PATH / UPLOAD_FOLDER -if not DOCAT_DB_DIR_STR: - # Default Database locations - DOCAT_DB_DIR = Path.cwd() - DOCAT_DB_PATH = Path(DB_PATH) - DOCAT_INDEX_PATH = Path(INDEX_PATH) -else: - # Custom Database locations - DOCAT_DB_DIR = Path(DOCAT_DB_DIR_STR) - DOCAT_DB_PATH = DOCAT_DB_DIR / "db.json" - DOCAT_INDEX_PATH = DOCAT_DB_DIR / "index.json" - - -DOCAT_DB_DIR.mkdir(parents=True, exist_ok=True) +# Create the folders if they don't exist +DOCAT_UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True) DOCAT_DB_PATH.touch() DOCAT_INDEX_PATH.touch() -#: Holds the static base path where the uploaded documentation artifacts are stored -DOCAT_UPLOAD_FOLDER = Path(os.getenv("DOCAT_DOC_PATH", UPLOAD_FOLDER)) - -if not DOCAT_DB_PATH.exists(): - DOCAT_UPLOAD_FOLDER.mkdir(parents=True, exist_ok=True) - db = TinyDB(DOCAT_DB_PATH) @@ -127,14 +113,14 @@ def get_project(project): @app.get("/api/search/", response_model=SearchResponse, status_code=status.HTTP_200_OK) def search(query: str): query = query.lower() - found_projects: list[SearchResultProject] = list() - found_versions: list[SearchResultVersion] = list() - found_files: list[SearchResultFile] = list() + found_projects: list[SearchResultProject] = [] + found_versions: list[SearchResultVersion] = [] + found_files: list[SearchResultFile] = [] index_db = TinyDB(DOCAT_INDEX_PATH) project_table = index_db.table("projects") projects = project_table.all() - all_versions: list[Tuple] = list() + all_versions: list[tuple] = [] # Collect all projects that contain the query for project in projects: @@ -331,7 +317,7 @@ def upload( if base_path.exists(): token_status = check_token_for_project(db, docat_api_key, project) if token_status.valid: - remove_docs(project, version) + remove_docs(project, version, DOCAT_UPLOAD_FOLDER) else: response.status_code = status.HTTP_401_UNAUTHORIZED return ApiResponse(message=token_status.reason) @@ -441,7 +427,7 @@ def rename(project: str, new_project_name: str, response: Response, docat_api_ke def delete(project: str, version: str, response: Response, docat_api_key: str = Header(None), db: TinyDB = Depends(get_db)): token_status = check_token_for_project(db, docat_api_key, project) if token_status.valid: - message = remove_docs(project, version) + message = remove_docs(project, version, DOCAT_UPLOAD_FOLDER) if message: response.status_code = status.HTTP_404_NOT_FOUND return ApiResponse(message=message) diff --git a/docat/docat/models.py b/docat/docat/models.py index 0dd903b6f..d095f8881 100644 --- a/docat/docat/models.py +++ b/docat/docat/models.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Optional from pydantic import BaseModel @@ -7,7 +6,7 @@ @dataclass(frozen=True) class TokenStatus: valid: bool - reason: Optional[str] = None + reason: str | None = None class ApiResponse(BaseModel): diff --git a/docat/docat/utils.py b/docat/docat/utils.py index 31100b06d..c37ec48cc 100644 --- a/docat/docat/utils.py +++ b/docat/docat/utils.py @@ -14,7 +14,7 @@ from docat.models import ProjectDetailResponse, ProjectsResponse, ProjectVersion NGINX_CONFIG_PATH = Path("/etc/nginx/locations.d") -UPLOAD_FOLDER = Path("/var/docat/doc") +UPLOAD_FOLDER = "doc" DB_PATH = "db.json" INDEX_PATH = "index.json" @@ -55,7 +55,7 @@ def extract_archive(target_file, destination): target_file.unlink() # remove the zip file -def remove_docs(project, version): +def remove_docs(project: str, version: str, upload_folder_path: Path): """ Delete documentation @@ -63,7 +63,7 @@ def remove_docs(project, version): project (str): name of the project version (str): project version """ - docs = UPLOAD_FOLDER / project / version + docs = upload_folder_path / project / version if docs.exists(): # remove the requested version # rmtree can not remove a symlink @@ -99,7 +99,7 @@ def calculate_token(password, salt): return hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, 100000).hex() -def get_all_projects(upload_folder_path: Path): +def get_all_projects(upload_folder_path: Path) -> ProjectsResponse: """ Returns all projects in the upload folder. """ @@ -120,7 +120,7 @@ def has_not_hidden_versions(project): ) -def get_project_details(upload_folder_path: Path, project_name: str): +def get_project_details(upload_folder_path: Path, project_name: str) -> ProjectDetailResponse | None: """ Returns all versions and tags for a project. """ @@ -157,8 +157,7 @@ def index_all_projects( and save it into index.json. """ # drop existing index - if index_db_path.exists(): - open(index_db_path, "w").close() + index_db_path.unlink(missing_ok=True) all_projects = get_all_projects(upload_folder_path).projects @@ -177,6 +176,9 @@ def update_file_index_for_project(upload_folder_path: Path, index_db_path: Path, project_details = get_project_details(upload_folder_path, project) + if not project_details: + return + for version in project_details.versions: update_file_index_for_project_version(upload_folder_path, index_db_path, project, version.name) @@ -198,7 +200,7 @@ def update_file_index_for_project_version(upload_folder_path: Path, index_db_pat # save the file path path = str(file.relative_to(docs_folder)) - content = get_html_content_as_str(file) if file.name.endswith(".html") else "" + content = get_html_content(file) if file.name.endswith(".html") else "" insert_file_index_into_db(index_db_path, project, version, path, content) @@ -215,11 +217,14 @@ def update_version_index_for_project(upload_folder_path: Path, index_db_path: Pa details = get_project_details(upload_folder_path, project) + if not details: + return + for version in details.versions: insert_version_into_version_index(index_db_path, project, version.name, version.tags) -def get_html_content_as_str(file_path: Path): +def get_html_content(file_path: Path) -> str: """ Returns the content of a html file as a string. """ @@ -230,12 +235,11 @@ def html_tag_visible(element): return True - with open(file_path, "r") as f: - file_content = f.read() - soup = BeautifulSoup(file_content, "html.parser") - text_content = filter(html_tag_visible, soup.findAll(string=True)) - content = " ".join(t.strip() for t in text_content).lower() - return content + file_content = file_path.read_text() + soup = BeautifulSoup(file_content, "html.parser") + text_content = filter(html_tag_visible, soup.findAll(string=True)) + content = " ".join(t.strip() for t in text_content).lower() + return content def insert_file_index_into_db(index_db_path: Path, project: str, version: str, file_path: str, content: str): diff --git a/docat/file.txt b/docat/file.txt deleted file mode 100644 index e69de29bb..000000000