feat: PageIndex uses get_ftoc for cached ftocs to reduce load time

Author: redimp

otterwiki/helper.py

@@ -11,15 +11,20 @@
 
 import os
 import re
+import os
+from hashlib import sha256
+import json
+from datetime import datetime
 from collections import namedtuple
-from otterwiki.server import app, mail, storage, Preferences
+from otterwiki.server import app, mail, storage, Preferences, db, app_renderer
 from otterwiki.gitstorage import StorageError
 from flask import flash, url_for, session
 from threading import Thread
 from flask_mail import Message
 from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
 from otterwiki.util import split_path, join_path, clean_slashes, titleSs
 from otterwiki.renderer import OtterwikiRenderer
+from otterwiki.models import Cache
 
 
 class SerializeError(ValueError):
@@ -283,3 +288,53 @@ def patchset2urlmap(patchset, rev_b, rev_a=None):
         }
         url_map[file.path] = namedtuple('UrlData', d.keys())(*d.values())
     return url_map
+
+
+def sha256sum(s: str) -> str:
+    hash = sha256()
+    hash.update(s.encode())
+    return hash.hexdigest()
+
+
+def update_ftoc_cache(filename, ftoc, mtime=None):
+    if mtime is None:
+        mtime = storage.mtime(filename)
+    hash = sha256sum(f"ftoc://{filename}")
+    value = json.dumps({"filename": filename, "ftoc": ftoc})
+    # check if key exists in Cache
+    c = Cache.query.filter(Cache.key == hash).first()
+    if c is None:
+        c = Cache()
+        c.key = hash
+    c.value = value
+    c.datetime = mtime
+    # and update in the database
+    db.session.add(c)
+    db.session.commit()
+
+
+def get_ftoc(filename, mtime=None):
+    if mtime is None:
+        mtime = storage.mtime(filename)
+    hash = sha256sum(f"ftoc://{filename}")
+    # check if hash is in the Cache
+    result = Cache.query.filter(
+        db.and_(Cache.key == hash, Cache.datetime >= mtime)
+    ).first()
+    if result is not None:
+        try:
+            value = json.loads(result.value)
+            try:
+                # check
+                if filename == value["filename"]:
+                    return value["ftoc"]
+            except KeyError:
+                pass
+        except:
+            pass
+    content = storage.load(filename)
+    # parse file contents
+    _, ftoc = app_renderer.markdown(content)
+    update_ftoc_cache(filename, ftoc, mtime)
+
+    return ftoc

otterwiki/models.py

@@ -75,6 +75,6 @@ def __repr__(self):
 
 class Cache(db.Model):
     __tablename__ = "cache"
-    key = db.Column(db.String(64), primary_key=True)
+    key = db.Column(db.String(64), index=True, primary_key=True)
     value = db.Column(db.Text)
     datetime = db.Column(TimeStamp())

otterwiki/wiki.py

@@ -43,6 +43,8 @@
     patchset2urlmap,
     get_breadcrumbs,
     upsert_pagecrumbs,
+    get_ftoc,
+    update_ftoc_cache,
 )
 from otterwiki.auth import has_permission, current_user
 from otterwiki.plugins import chain_hooks
@@ -144,10 +146,8 @@ def __init__(self, path=None):
                 f,
                 full=False,
             )
-            # read file
-            content = storage.load(f)
-            # parse file contents
-            _, ftoc = app_renderer.markdown(content)
+            ftoc = get_ftoc(f)
+
             # add headers to page toc
             # (4, '2 L <strong>bold</strong>', 1, '2 L bold', '2-l-bold')
             for i, header in enumerate(ftoc):
@@ -549,6 +549,7 @@ def view(self):
         htmlcontent, toc = app_renderer.markdown(
             self.content, page_url=self.page_url
         )
+        update_ftoc_cache(self.filename, ftoc=toc)
 
         if len(toc) > 0:
             # use first headline to overwrite pagename

tests/test_helper.py

@@ -3,6 +3,7 @@
 
 import pytest
 import flask
+from datetime import datetime
 import otterwiki
 import otterwiki.gitstorage
 
@@ -252,3 +253,31 @@ def test_get_pagename_prefixes(test_client):
             'Example',
             'Random page',
         ]
+
+
+def test_ftoc_cache(create_app, req_ctx):
+    assert create_app
+    assert req_ctx
+    from otterwiki.helper import get_ftoc, update_ftoc_cache
+
+    mtime = datetime.fromtimestamp(0)
+    filename = "ftoc.md"
+    ftoc = [
+        [0, "Ftoc", 1, "Ftoc", "ftoc"],
+        [1, "Header 2", 2, "Header 2", "header-2"],
+        [2, "Header 3", 3, "Header 3", "header-3"],
+    ]
+    update_ftoc_cache(filename=filename, ftoc=ftoc, mtime=mtime)
+    assert get_ftoc(filename=filename, mtime=mtime) == ftoc
+    # store the file
+    assert True == create_app.storage.store(
+        filename,
+        content="# Header 1\n\n## Header 2",
+        author=("John", "[email protected]"),
+        message=f"added {filename}",
+    )
+
+    assert [
+        (0, 'Header 1', 1, 'Header 1', 'header-1'),
+        (1, 'Header 2', 2, 'Header 2', 'header-2'),
+    ] == get_ftoc(filename)