Merge pull request #11 from zytedata/article-rules

Add article rules.

README.rst

@@ -37,3 +37,14 @@ The rules can be imported via:
 
 It can then be used to configure the ``DUD_LOAD_RULE_PATHS`` setting of
 ``duplicate-url-discarder``.
+
+``RULE_PATHS`` contains all files shipped in this package. If you want to
+reduce the number of loaded rules to improve performance you can instead
+use one or more of the following variables:
+
+* ``RULE_PATHS_COMMON``: rules not specific to any data type.
+* ``RULE_PATHS_ARTICLE``: rules for article websites.
+* ``RULE_PATHS_PRODUCT``: rules for e-commerce websites.
+
+As all of them are lists, you can combine them (e.g.
+``RULE_PATHS_COMMON + RULE_PATHS_PRODUCT``).

duplicate_url_discarder_rules/__init__.py

@@ -2,8 +2,27 @@
 from pathlib import Path
 from typing import List, Optional
 
+__all__ = [
+    "RULE_PATHS",
+    "RULE_PATHS_COMMON",
+    "RULE_PATHS_ARTICLE",
+    "RULE_PATHS_PRODUCT",
+]
+
 _current_path = Path(__file__).parent.resolve()
 RULE_PATHS: Optional[List[str]] = glob(str(_current_path / "**/*.json"), recursive=True)
 
+RULE_PATHS_COMMON: List[str] = []
+RULE_PATHS_ARTICLE: List[str] = []
+RULE_PATHS_PRODUCT: List[str] = []
+
+for path in RULE_PATHS:
+    filename = Path(path).name
+    if filename == "article.json":
+        RULE_PATHS_ARTICLE.append(path)
+    elif filename == "product.json":
+        RULE_PATHS_PRODUCT.append(path)
+    else:
+        RULE_PATHS_COMMON.append(path)
 
 __version__ = "2024.10.10"

tests/queryRemovalExcept/test_query_removal_except_main.py → tests/queryRemovalExcept/test_query_removal_except.py

@@ -4,9 +4,9 @@
 from duplicate_url_discarder_rules import RULE_PATHS
 
 
-def test_query_removal_except_main_rules():
+def test_query_removal_except_product_rules():
     rule_path = [
-        path for path in RULE_PATHS if path.endswith("queryRemovalExcept/main.json")
+        path for path in RULE_PATHS if path.endswith("queryRemovalExcept/product.json")
     ]
     assert len(rule_path) == 1
 

tests/subpathRemoval/test_subpath_removal_main.py → tests/subpathRemoval/test_subpath_removal.py

@@ -4,9 +4,9 @@
 from duplicate_url_discarder_rules import RULE_PATHS
 
 
-def test_subpath_removal_main_rules():
+def test_subpath_removal_product_rules():
     rule_path = [
-        path for path in RULE_PATHS if path.endswith("subpathRemoval/main.json")
+        path for path in RULE_PATHS if path.endswith("subpathRemoval/product.json")
     ]
     assert len(rule_path) == 1
 

tests/test_rules.py

@@ -2,7 +2,7 @@
 
 from url_matcher import Patterns
 
-from duplicate_url_discarder_rules import RULE_PATHS
+from duplicate_url_discarder_rules import RULE_PATHS, RULE_PATHS_ARTICLE, RULE_PATHS_PRODUCT, RULE_PATHS_COMMON
 
 
 def test_rule_validity():
@@ -37,3 +37,11 @@ def test_rule_validity():
         assert (
             json.dumps(data, indent=2, sort_keys=True) == raw_text.rstrip()
         ), f"Check {path} for formatting issue"
+
+
+def test_rules_concat():
+    all_rules = RULE_PATHS_COMMON + RULE_PATHS_ARTICLE + RULE_PATHS_PRODUCT
+    assert isinstance(all_rules, list)
+    for path in all_rules:
+        assert isinstance(path, str)
+    assert set(all_rules) == set(RULE_PATHS)