mypy: Cleans up most overrides. Adds some missing stubs

pyproject.toml

@@ -45,44 +45,17 @@ mypy_path = "$MYPY_CONFIG_FILE_DIR/src:$MYPY_CONFIG_FILE_DIR/stubs"
 [[tool.mypy.overrides]]
 # ignore missing imports for packages that have no stubs available
 module = [
-    "bjoern.*",
-    "cssutils.*",
-    "elasticsearch_dsl.*",
-    "docxtpl.*",
-    "dill.*",
-    "dukpy.*",
-    "genshi.*",
     # FIXME: Do we really need this? We have this implemented ourselves
     #        in OCQMS, so we could just copy that code over... plus the
     #        standard lib technically supports some ISO-8601 formats as well
     "isodate.*",
-    "kerberos.*",
     # TODO: Consider switching to fasttext-langdetect
     #       langdetect is really, really slow by comparison...
     "langdetect.*",
-    "lazy_object_proxy.*",
-    "mistletoe.*",
-    "msal.*",
-    "pdfdocument.*",
-    # FIXME: We only use this to convert a PDF to an XObject and then
-    #        embed it as a reportlab Flowable. I think we could implement
-    #        this using just pypdf, having multiple PDF parsers just for
-    #        this one use-case seems bad
-    "pdfrw.*",
-    "pdftotext.*",
-    "pglast.*",
-    "purl.*",
-    "svglib.*",
-    "stdnum.*",
-    "ua_parser.*",
+    # FIXME: Remove after https://github.com/lipoja/URLExtract/issues/164
     "urlextract.*",
-    "webassets.*",
-    "webtest.*",
-    "xlrd.*",
     # FIXME: Do we actually need this or can we just only use openpyxl?
     "xlsxwriter.*",
-    "yubico_client.*",
-    "zope.*"
 ]
 ignore_missing_imports = true
 

setup.cfg

@@ -140,6 +140,8 @@ install_requires =
     rjsmin
     sedate
     sentry_sdk
+    # FIXME: 71 breaks something with vendored importlib
+    setuptools<71
     sortedcontainers
     sqlalchemy<1.4.0
     sqlalchemy-utils
@@ -222,7 +224,7 @@ mypy =
     types-redis
     types-reportlab
     types-requests
-    types-setuptools
+    types-setuptools<71
     types-stripe
     types-toposort
     types-translationstring

src/onegov/agency/pdf.py

@@ -205,6 +205,8 @@ def page_fn_footer(canvas: 'Canvas', doc: 'Template') -> None:
 
         """
 
+        assert doc.title is not None
+        assert hasattr(doc, 'created')
         canvas.saveState()
         canvas.setFont('Helvetica', 9)
         canvas.drawString(
@@ -254,6 +256,7 @@ def page_fn_footer(canvas: 'Canvas', doc: 'Template') -> None:
 
         """
 
+        assert hasattr(doc, 'created')
         canvas.saveState()
         canvas.setFont('Helvetica', 9)
         canvas.drawString(
@@ -280,6 +283,7 @@ def page_fn_header_logo_and_footer(
         height = .81 * cm
         width = height * 5.72
 
+        assert hasattr(doc, 'logo')
         canvas.saveState()
         canvas.drawImage(
             doc.logo,
@@ -357,6 +361,7 @@ def page_fn_header(canvas: 'Canvas', doc: 'Template') -> None:
         numbers.
 
         """
+        assert hasattr(doc, 'logo')
         height = 1.85 * cm
         width = height * 2.77
 
@@ -374,7 +379,7 @@ def page_fn_header(canvas: 'Canvas', doc: 'Template') -> None:
 
     @staticmethod
     def page_fn_footer(canvas: 'Canvas', doc: 'Template') -> None:
-
+        assert hasattr(doc, 'created')
         canvas.saveState()
         canvas.setFont('Helvetica', 9)
         canvas.drawString(

src/onegov/core/cache.py

@@ -34,7 +34,7 @@ def my_view():
 
 """
 
-import dill
+import dill  # type:ignore[import-untyped]
 
 from dogpile.cache import CacheRegion
 from dogpile.cache.api import NO_VALUE

src/onegov/core/cli/core.py

@@ -198,7 +198,7 @@ def my_click_command():
 from sqlalchemy.pool import NullPool
 from sqlalchemy import create_engine
 from uuid import uuid4
-from webtest import TestApp as Client
+from webtest import TestApp as Client  # type:ignore[import-untyped]
 
 
 from typing import Any, NoReturn, TYPE_CHECKING

src/onegov/core/csv.py

@@ -6,7 +6,7 @@
 import re
 import sys
 import tempfile
-import xlrd
+import xlrd  # type:ignore[import-untyped]
 
 from collections import namedtuple, OrderedDict
 from csv import DictWriter, Sniffer, QUOTE_ALL

src/onegov/core/filters.py

@@ -2,10 +2,11 @@
 import os
 import rcssmin  # type:ignore[import-untyped]
 
-from webassets.filter import Filter, register_filter
-from webassets.filter.datauri import CSSDataUri, CSSUrlRewriter
-from dukpy.webassets import BabelJSX
-from dukpy import jsx_compile
+from webassets.filter import Filter, register_filter  # type:ignore
+from webassets.filter.datauri import (  # type:ignore[import-untyped]
+    CSSDataUri, CSSUrlRewriter)
+from dukpy.webassets import BabelJSX  # type:ignore[import-untyped]
+from dukpy import jsx_compile  # type:ignore[import-untyped]
 
 
 from typing import Any, IO

src/onegov/core/framework.py

@@ -731,9 +731,6 @@ def object_by_path(
 
         """
 
-        # strip host and scheme
-        path = URL(path).path()
-
         request = self.request_class(environ={
             'PATH_INFO': URL(path).path(),
             'SERVER_NAME': '',

src/onegov/core/html_diff.py

@@ -25,8 +25,8 @@
 from itertools import chain, zip_longest
 
 import html5lib
-from genshi.core import Stream, QName, Attrs, START, END, TEXT
-from genshi.input import ET
+from genshi.core import Stream, QName, Attrs, START, END, TEXT  # type:ignore
+from genshi.input import ET  # type:ignore[import-untyped]
 from markupsafe import Markup
 
 

src/onegov/core/markdown.py

@@ -1,13 +1,13 @@
 import html
 
-from mistletoe import Document, HtmlRenderer
+from mistletoe import Document, HtmlRenderer  # type:ignore[import-untyped]
 from onegov.core.html import sanitize_html
 
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from markupsafe import Markup
-    from mistletoe.span_token import HTMLBlock, HTMLSpan
+    from mistletoe.span_token import HTMLBlock, HTMLSpan  # type:ignore
 
 
 RENDERER_INSTANCES = {}

src/onegov/core/orm/abstract/adjacency_list.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from itertools import chain
-from lazy_object_proxy import Proxy
+from lazy_object_proxy import Proxy  # type:ignore[import-untyped]
 from onegov.core.orm import Base, observes
 from onegov.core.utils import normalize_for_url, increment_name, is_sorted
 from sqlalchemy import Column, ForeignKey, Integer, Text

src/onegov/core/orm/sql.py

@@ -1,5 +1,5 @@
 import os
-import pglast
+import pglast  # type:ignore[import-untyped]
 import re
 
 from onegov.core.cache import lru_cache
@@ -12,7 +12,7 @@
 from typing import Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Iterator
-    from pglast.ast import RawStmt
+    from pglast.ast import RawStmt  # type:ignore[import-untyped]
     from sqlalchemy.sql.selectable import Alias
     from sqlalchemy.types import TypeEngine
 

src/onegov/core/request.py

@@ -17,7 +17,7 @@
 from morepath.request import SAME_APP
 from onegov.core import utils
 from onegov.core.crypto import random_token
-from ua_parser import user_agent_parser
+from ua_parser import user_agent_parser  # type:ignore[import-untyped]
 from webob.exc import HTTPForbidden
 from wtforms.csrf.session import SessionCSRF
 

src/onegov/core/utils.py

@@ -37,9 +37,9 @@
 from unidecode import unidecode
 from uuid import UUID, uuid4
 from webob import static
-from yubico_client import Yubico
-from yubico_client.yubico_exceptions import SignatureVerificationError
-from yubico_client.yubico_exceptions import StatusCodeError
+from yubico_client import Yubico  # type:ignore[import-untyped]
+from yubico_client.yubico_exceptions import (  # type:ignore[import-untyped]
+    SignatureVerificationError, StatusCodeError)
 
 
 from typing import overload, Any, TypeVar, TYPE_CHECKING

src/onegov/election_day/pdf.py

@@ -11,7 +11,7 @@
     from datetime import date
     from datetime import datetime
     from gettext import GNUTranslations
-    from reportlab.lib.styles import ParagraphStyle
+    from reportlab.lib.styles import PropertySet
 
 
 class Pdf(PdfBase):
@@ -86,25 +86,25 @@ def translate(self, text: str) -> str:
                 translated = translator.gettext(text)
         return text.interpolate(translated)
 
-    def h1(self, title: str, style: 'ParagraphStyle | None' = None) -> None:
+    def h1(self, title: str, style: 'PropertySet | None' = None) -> None:
         """ Translated H1. """
 
         super().h1(self.translate(title), style=style)
 
-    def h2(self, title: str, style: 'ParagraphStyle | None' = None) -> None:
+    def h2(self, title: str, style: 'PropertySet | None' = None) -> None:
         """ Translated H2. """
 
         super().h2(self.translate(title), style=style)
 
-    def h3(self, title: str, style: 'ParagraphStyle | None' = None) -> None:
+    def h3(self, title: str, style: 'PropertySet | None' = None) -> None:
         """ Translated H3. """
 
         super().h3(self.translate(title), style=style)
 
     def figcaption(
         self,
         text: str,
-        style: 'ParagraphStyle | None' = None
+        style: 'PropertySet | None' = None
     ) -> None:
         """ Translated Figcaption. """
 

src/onegov/election_day/utils/pdf_generator.py

@@ -41,6 +41,7 @@
     from onegov.election_day.models import Municipality
     from onegov.election_day.models.vote.ballot import ResultsByDistrictRow
     from onegov.election_day.request import ElectionDayRequest
+    from reportlab.platypus import Paragraph
 
 
 class PdfGenerator:
@@ -276,7 +277,7 @@ def format_name(item: 'ElectionResult') -> str:
 
             connections = get_connection_results(election, self.session)
             spacers = []
-            table = [[
+            table: list[list[Paragraph | str]] = [[
                 '{} / {} / {}'.format(
                     pdf.translate(_('List connection')),
                     pdf.translate(_('Sublist connection')),

src/onegov/feriennet/qrbill.py

@@ -3,7 +3,7 @@
 from onegov.feriennet import log
 from onegov.feriennet.utils import NAME_SEPARATOR
 from qrbill.bill import QRBill, IBAN_ALLOWED_COUNTRIES, QR_IID
-from stdnum import iban
+from stdnum import iban  # type:ignore[import-untyped]
 
 
 from typing import Literal, TYPE_CHECKING

src/onegov/feriennet/views/booking.py

@@ -664,19 +664,17 @@ def signup_url(attendee: Attendee | None = None) -> str:
 
         # preselect the attendee when booking the occasion, and join this group
         if attendee:
-            url = (URL(url)
-                   .query_param('attendee_id', attendee.id)
-                   .as_string())
+            url_obj = URL(url).query_param('attendee_id', attendee.id.hex)
         else:
-            url = (URL(url)
-                   .query_param('attendee_id', 'other')
-                   .as_string())
+            url_obj = URL(url).query_param('attendee_id', 'other')
 
         # preselect the group code and the username
-        url = (URL(url)
-               .query_param('group_code', self.group_code)
-               .query_param('username', self.username)
-               .as_string())
+        url_obj = url_obj.query_param('group_code', self.group_code)
+
+        if self.username is not None:
+            url_obj = url_obj.query_param('username', self.username)
+
+        url = url_obj.as_string()
 
         # return to the current URL
         url = request.return_here(url)
@@ -710,7 +708,7 @@ def group_action(
             booking = actionable_bookings[attendee.id]
 
             url = (URL(request.link(self, action))
-                   .query_param('booking_id', booking.id)
+                   .query_param('booking_id', booking.id.hex)
                    .as_string())
         else:
             url = signup_url(attendee)

src/onegov/feriennet/views/invoice.py

@@ -17,7 +17,7 @@
 from sqlalchemy.orm import contains_eager
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.sql.expression import case
-from stdnum import iban
+from stdnum import iban  # type:ignore[import-untyped]
 from uuid import UUID
 
 

src/onegov/feriennet/views/settings.py

@@ -14,7 +14,7 @@
 from onegov.org.forms.fields import HtmlField
 from onegov.org.models import Organisation
 from onegov.org.views.settings import handle_generic_settings
-from stdnum import iban
+from stdnum import iban   # type:ignore[import-untyped]
 from wtforms.fields import BooleanField
 from wtforms.fields import RadioField
 from wtforms.fields import StringField

src/onegov/file/attachments.py

@@ -1,4 +1,4 @@
-import pdftotext
+import pdftotext  # type:ignore
 
 from depot.fields.upload import UploadedFile
 from depot.io import utils

src/onegov/form/fields.py

@@ -2,7 +2,7 @@
 import phonenumbers
 import sedate
 
-from cssutils.css import CSSStyleSheet
+from cssutils.css import CSSStyleSheet  # type:ignore[import-untyped]
 from itertools import zip_longest
 from email_validator import validate_email, EmailNotValidError
 from markupsafe import escape, Markup

src/onegov/form/validators.py

@@ -15,7 +15,8 @@
 from onegov.form.errors import FieldCompileError
 from onegov.form.errors import InvalidFormSyntax
 from onegov.form.errors import MixedTypeError
-from stdnum.exceptions import ValidationError as StdnumValidationError
+from stdnum.exceptions import (  # type:ignore[import-untyped]
+    ValidationError as StdnumValidationError)
 from wtforms import DateField, DateTimeLocalField, RadioField, TimeField
 from wtforms.fields import SelectField
 from wtforms.validators import DataRequired

src/onegov/gazette/models/issue.py

@@ -197,7 +197,8 @@ def publish(self, request: 'GazetteRequest') -> None:
             notice.publish(request)
 
         from onegov.gazette.pdf import IssuePdf  # circular
-        self.pdf = IssuePdf.from_issue(
+        # FIXME: asymmetric property
+        self.pdf = IssuePdf.from_issue(  # type:ignore[assignment]
             issue=self,
             request=request,
             first_publication_number=self.first_publication_number,