v0.1.7: support http request limit

Author: selfboot

html2notion/translate/notion_import.py

@@ -5,7 +5,7 @@
 from notion_client import AsyncClient
 from notion_client.errors import RequestTimeoutError
 from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
-from ..utils import logger, test_prepare_conf, config
+from ..utils import logger, test_prepare_conf, config, rate_limit
 from ..translate.html2json import html2json_process
 
 
@@ -34,6 +34,8 @@ async def process_file(self, file_path: Path):
         logger.info(f"Create notion page: {create_result}")
         return "succ"
 
+    # https://developers.notion.com/reference/request-limits
+    # The rate limit for incoming requests per integration is an average of three requests per second. 
     # Doc of create page: https://developers.notion.com/reference/post-page
     @retry(stop=stop_after_attempt(5),
            wait=wait_exponential(multiplier=1, min=3, max=30),
@@ -47,10 +49,11 @@ async def create_new_page(self, notion_data):
         if blocks:
             notion_data.pop("children")
         first_chunk = chunks[0] if chunks else []
-        created_page = await self.notion_client.pages.create(**notion_data, children=first_chunk)
-        page_id = created_page["id"]
-        for chunk in chunks[1:]:
-            await self.notion_client.blocks.children.append(page_id, children=chunk)
+        async with rate_limit:
+            created_page = await self.notion_client.pages.create(**notion_data, children=first_chunk)
+            page_id = created_page["id"]
+            for chunk in chunks[1:]:
+                await self.notion_client.blocks.children.append(page_id, children=chunk)
         return created_page
 
 

html2notion/utils/__init__.py

@@ -1,5 +1,5 @@
 from .log import logger, setup_logger
-from .load_config import read_config, config
+from .load_config import read_config, config, rate_limit
 from pathlib import Path
 
 
@@ -11,4 +11,4 @@ def test_prepare_conf():
     logger.info(f"test_prepare_conf, log path({log_path}), conf path({conf_path})")
 
 
-__all__ = ['logger', 'setup_logger', 'config', 'read_config', 'test_prepare_conf']
+__all__ = ['logger', 'setup_logger', 'config', 'read_config', 'test_prepare_conf', 'rate_limit']

html2notion/utils/load_config.py

@@ -1,7 +1,8 @@
 import json
 import sys
 from pathlib import Path
-
+from aiolimiter import AsyncLimiter
+rate_limit = AsyncLimiter(3, 1)
 config = {}
 
 

requirements.txt

@@ -7,4 +7,5 @@ aiohttp>=3.8.4
 anyio>=3.6.2
 cos-python-sdk-v5>=1.9.23
 tenacity>=8.2.2
-rich>=13.3.4
+rich>=13.3.4
+aiolimiter>=1.0.0

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = html2notion
-version = 0.1.6
+version = 0.1.7
 author = selfboot
 author_email = [email protected]
 description = This tool can accurately convert HTML to Notion notes and is also useful for exporting Evernote notes to Notion.
@@ -26,6 +26,7 @@ install_requires =
     rich>=13.3.4
     cos-python-sdk-v5>=1.9.23
     tenacity>=8.2.2
+    aiolimiter>=1.0.0
 
 [options.entry_points]
 console_scripts =

tests/test_batchimport.py

@@ -7,6 +7,7 @@
 from tempfile import TemporaryDirectory
 from http import HTTPStatus
 from html2notion.translate.batch_import import BatchImport
+from html2notion.utils import rate_limit
 from html2notion.utils.log import log_only_local
 
 process_once_time = 0.5
@@ -41,6 +42,12 @@ def json(self):
     return MockResponse(HTTPStatus.OK, content, elapsed_time)
 
 
+async def mock_notion_create_page(notion_data, *args, **kwargs):
+    async with rate_limit:
+        await asyncio.sleep(0.01)
+        log_only_local(f"mock_notion_create_page")
+    return "succ"
+
 @pytest.fixture(params=[10, 20])
 def temp_dir_fixture(request):
     num_files = request.param
@@ -78,3 +85,20 @@ async def test_batch_process(temp_dir_fixture, concurrent_limit):
         f"total_time: {total_time}, sync_time: {sync_time}, least_time: {least_time}")
     assert total_time >= least_time
     assert total_time <= sync_time
+
+
+@pytest.mark.parametrize("concurrent_limit", [5, 10, 20])
+@pytest.mark.asyncio
+async def test_reqlimit(temp_dir_fixture, concurrent_limit):
+    dir_path = temp_dir_fixture
+    start_time = time.perf_counter()
+    with patch("html2notion.translate.notion_import.NotionImporter.create_new_page", side_effect=mock_notion_create_page):
+        batch_processor = BatchImport(dir_path, concurrent_limit=concurrent_limit)
+        responses = await batch_processor.process_directory()
+
+    end_time = time.perf_counter()
+    total_time = end_time-start_time
+    num_files = len(list(dir_path.glob('*.html')))
+    log_only_local(f"file nums: {num_files}, concurrent {concurrent_limit}, total_time: {total_time}")
+    # The time deviation within 1 second is acceptable here.
+    assert (total_time >= num_files / 3 - 1)