Merge pull request #13 from zytedata/modernize

Add Python 3.13, drop Python 3.8, add mypy and pre-commit

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/test.yml

@@ -16,12 +16,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.8"
         - python-version: "3.9"
         - python-version: "3.10"
         - python-version: "3.11"
         - python-version: "3.12"
-        - python-version: "3.12"
+        - python-version: "3.13"
 
     steps:
     - uses: actions/checkout@v4
@@ -42,8 +41,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12"]
-        tox-job: ["twine-check"]
+        python-version: ["3.13"]
+        tox-job: ["linters", "mypy", "twine-check"]
 
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+    - id: isort
+-   repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+    - id: black
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+    - id: flake8
+      additional_dependencies:
+        - flake8-docstrings
+        - flake8-print
+- repo: https://github.com/adamchainz/blacken-docs
+  rev: 1.19.1
+  hooks:
+    - id: blacken-docs
+      additional_dependencies:
+        - black==24.10.0

duplicate_url_discarder_rules/__init__.py

@@ -16,7 +16,7 @@
 RULE_PATHS_ARTICLE: List[str] = []
 RULE_PATHS_PRODUCT: List[str] = []
 
-for path in RULE_PATHS:
+for path in RULE_PATHS or []:
     filename = Path(path).name
     if filename == "article.json":
         RULE_PATHS_ARTICLE.append(path)

pyproject.toml

@@ -15,14 +15,14 @@ classifiers = [
     "Natural Language :: English",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: Implementation :: CPython",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dynamic = ["version"]
 
 [tool.setuptools.dynamic]
@@ -33,3 +33,22 @@ Source = "https://github.com/zytedata/duplicate-url-discarder-rules"
 
 [tool.setuptools.package-data]
 'duplicate_url_discarder_rules' = ['**/*.json']
+
+[tool.black]
+target-version = ["py39", "py310", "py311", "py312", "py313"]
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+
+[tool.mypy]
+strict = true
+implicit_reexport = true
+
+[[tool.mypy.overrides]]
+module = "tests.*.*"
+disallow_untyped_defs = false
+
+[[tool.mypy.overrides]]
+module = "url_matcher"
+ignore_missing_imports = true

setup.cfg

@@ -0,0 +1,4 @@
+[flake8]
+ignore = E203, E266, E501, W503, C901
+max-line-length = 88
+select = B,C,E,F,W,T4

tests/normalizer/test_normalizer.py

@@ -1,10 +1,11 @@
-from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 from duplicate_url_discarder.processors import NormalizerProcessor
+from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 
 from duplicate_url_discarder_rules import RULE_PATHS
 
 
 def test_normalizer_main_rules():
+    assert RULE_PATHS is not None
     rule_path = [path for path in RULE_PATHS if path.endswith("normalizer/main.json")]
     assert len(rule_path) == 1
 

tests/queryRemovalExcept/test_query_removal_except.py

@@ -1,10 +1,11 @@
-from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 from duplicate_url_discarder.processors import QueryRemovalExceptProcessor
+from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 
 from duplicate_url_discarder_rules import RULE_PATHS
 
 
 def test_query_removal_except_product_rules():
+    assert RULE_PATHS is not None
     rule_path = [
         path for path in RULE_PATHS if path.endswith("queryRemovalExcept/product.json")
     ]

tests/subpathRemoval/test_subpath_removal.py

@@ -1,10 +1,11 @@
-from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 from duplicate_url_discarder.processors import SubpathRemovalProcessor
+from duplicate_url_discarder.url_canonicalizer import UrlCanonicalizer
 
 from duplicate_url_discarder_rules import RULE_PATHS
 
 
 def test_subpath_removal_product_rules():
+    assert RULE_PATHS is not None
     rule_path = [
         path for path in RULE_PATHS if path.endswith("subpathRemoval/product.json")
     ]

tests/test_rules.py

@@ -2,10 +2,16 @@
 
 from url_matcher import Patterns
 
-from duplicate_url_discarder_rules import RULE_PATHS, RULE_PATHS_ARTICLE, RULE_PATHS_PRODUCT, RULE_PATHS_COMMON
+from duplicate_url_discarder_rules import (
+    RULE_PATHS,
+    RULE_PATHS_ARTICLE,
+    RULE_PATHS_COMMON,
+    RULE_PATHS_PRODUCT,
+)
 
 
 def test_rule_validity():
+    assert RULE_PATHS is not None
     for path in RULE_PATHS:
         try:
             with open(path, "r") as f:
@@ -40,6 +46,7 @@ def test_rule_validity():
 
 
 def test_rules_concat():
+    assert RULE_PATHS is not None
     all_rules = RULE_PATHS_COMMON + RULE_PATHS_ARTICLE + RULE_PATHS_PRODUCT
     assert isinstance(all_rules, list)
     for path in all_rules:

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38,py39,py310,py311,py312,twine-check
+envlist = py39,py310,py311,py312,py313,linters,mypy,twine-check
 
 [testenv]
 deps =
@@ -11,6 +11,16 @@ commands =
     py.test \
         {posargs:duplicate_url_discarder_rules tests}
 
+[testenv:linters]
+deps = pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
+
+[testenv:mypy]
+deps =
+    mypy==1.13.0
+    duplicate-url-discarder
+commands = mypy duplicate_url_discarder_rules tests
+
 [testenv:twine-check]
 deps =
     build