pythongh-104504: Run mypy on cases_generator

Author: corona10

.github/workflows/mypy.yml

@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - "Tools/clinic/**"
+      - "Tools/cases_generator/**"
       - ".github/workflows/mypy.yml"
   workflow_dispatch:
 
@@ -24,7 +25,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  mypy:
+  mypy-clinic:
     name: Run mypy on Tools/clinic/
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -37,3 +38,17 @@ jobs:
           cache-dependency-path: Tools/clinic/requirements-dev.txt
       - run: pip install -r Tools/clinic/requirements-dev.txt
       - run: mypy --config-file Tools/clinic/mypy.ini
+
+  mypy-cases-generator:
+    name: Run mypy on Tools/cases_generator/
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+          cache: pip
+          cache-dependency-path: Tools/cases_generator/requirements-dev.txt
+      - run: pip install -r Tools/cases_generator/requirements-dev.txt
+      - run: mypy --config-file Tools/cases_generator/mypy.ini

Tools/cases_generator/analysis.py

@@ -171,8 +171,6 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None:
                 case parsing.Pseudo(name):
                     self.pseudos[name] = thing
                     self.everything.append(thing)
-                case _:
-                    typing.assert_never(thing)
         if not psr.eof():
             raise psr.make_syntax_error(f"Extra stuff at the end of {filename}")
 
@@ -408,6 +406,4 @@ def check_macro_components(
                     components.append(self.instrs[name])
                 case parsing.CacheEffect():
                     components.append(uop)
-                case _:
-                    typing.assert_never(uop)
         return components

Tools/cases_generator/flags.py

@@ -16,11 +16,11 @@ class InstructionFlags:
     HAS_FREE_FLAG: bool
     HAS_LOCAL_FLAG: bool
 
-    def __post_init__(self):
+    def __post_init__(self) -> None:
         self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())}
 
     @staticmethod
-    def fromInstruction(instr: parsing.Node):
+    def fromInstruction(instr: parsing.Node) -> 'InstructionFlags':
 
         has_free = (
             variable_used(instr, "PyCell_New")
@@ -41,15 +41,15 @@ def fromInstruction(instr: parsing.Node):
         )
 
     @staticmethod
-    def newEmpty():
+    def newEmpty() -> 'InstructionFlags':
         return InstructionFlags(False, False, False, False, False, False)
 
     def add(self, other: "InstructionFlags") -> None:
         for name, value in dataclasses.asdict(other).items():
             if value:
                 setattr(self, name, value)
 
-    def names(self, value=None) -> list[str]:
+    def names(self, value: bool|None = None) -> list[str]:
         if value is None:
             return list(dataclasses.asdict(self).keys())
         return [n for n, v in dataclasses.asdict(self).items() if v == value]
@@ -62,7 +62,7 @@ def bitmap(self) -> int:
         return flags
 
     @classmethod
-    def emit_macros(cls, out: Formatter):
+    def emit_macros(cls, out: Formatter) -> None:
         flags = cls.newEmpty()
         for name, value in flags.bitmask.items():
             out.emit(f"#define {name} ({value})")

Tools/cases_generator/formatting.py

@@ -58,13 +58,13 @@ def reset_lineno(self) -> None:
             self.set_lineno(self.lineno + 1, self.filename)
 
     @contextlib.contextmanager
-    def indent(self):
+    def indent(self) -> typing.Iterator[None]:
         self.prefix += "    "
         yield
         self.prefix = self.prefix[:-4]
 
     @contextlib.contextmanager
-    def block(self, head: str, tail: str = ""):
+    def block(self, head: str, tail: str = "") -> typing.Iterator[None]:
         if head:
             self.emit(head + " {")
         else:
@@ -77,7 +77,7 @@ def stack_adjust(
         self,
         input_effects: list[StackEffect],
         output_effects: list[StackEffect],
-    ):
+    ) -> None:
         shrink, isym = list_effect_size(input_effects)
         grow, osym = list_effect_size(output_effects)
         diff = grow - shrink
@@ -90,7 +90,7 @@ def stack_adjust(
         if osym and osym != isym:
             self.emit(f"STACK_GROW({osym});")
 
-    def declare(self, dst: StackEffect, src: StackEffect | None):
+    def declare(self, dst: StackEffect, src: StackEffect | None) -> None:
         if dst.name == UNUSED or dst.cond == "0":
             return
         typ = f"{dst.type}" if dst.type else "PyObject *"
@@ -107,7 +107,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None):
         sepa = "" if typ.endswith("*") else " "
         self.emit(f"{typ}{sepa}{dst.name}{init};")
 
-    def assign(self, dst: StackEffect, src: StackEffect):
+    def assign(self, dst: StackEffect, src: StackEffect) -> None:
         if src.name == UNUSED or dst.name == UNUSED:
             return
         cast = self.cast(dst, src)

Tools/cases_generator/generate_cases.py

@@ -23,7 +23,6 @@
     MacroInstruction,
     MacroParts,
     PseudoInstruction,
-    StackEffect,
     OverriddenInstructionPlaceHolder,
     TIER_ONE,
     TIER_TWO,
@@ -183,17 +182,11 @@ def effect_str(effects: list[StackEffect]) -> str:
                     assert target_instr
                     target_popped = effect_str(target_instr.input_effects)
                     target_pushed = effect_str(target_instr.output_effects)
-                    if popped is None and pushed is None:
-                        popped, pushed = target_popped, target_pushed
-                    else:
-                        assert popped == target_popped
-                        assert pushed == target_pushed
-            case _:
-                typing.assert_never(thing)
+                    popped, pushed = target_popped, target_pushed
         return instr, popped, pushed
 
     @contextlib.contextmanager
-    def metadata_item(self, signature, open, close):
+    def metadata_item(self, signature: str, open: str, close: str) -> typing.Iterator[None]:
         self.out.emit("")
         self.out.emit(f"extern {signature};")
         self.out.emit("#ifdef NEED_OPCODE_METADATA")
@@ -243,21 +236,21 @@ def from_source_files(self) -> str:
         paths = f"\n{self.out.comment}   ".join(filenames)
         return f"{self.out.comment} from:\n{self.out.comment}   {paths}\n"
 
-    def write_provenance_header(self):
+    def write_provenance_header(self) -> None:
         self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n")
         self.out.write_raw(self.from_source_files())
         self.out.write_raw(f"{self.out.comment} Do not edit!\n")
 
-    def assign_opcode_ids(self):
+    def assign_opcode_ids(self) -> None:
         """Assign IDs to opcodes"""
 
-        ops: list[(bool, str)] = []  # (has_arg, name) for each opcode
+        ops: list[tuple[bool, str]] = []  # (has_arg, name) for each opcode
         instrumented_ops: list[str] = []
 
         for instr in itertools.chain(
             [instr for instr in self.instrs.values() if instr.kind != "op"],
             self.macro_instrs.values()):
-
+            assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction))
             name = instr.name
             if name.startswith('INSTRUMENTED_'):
                 instrumented_ops.append(name)
@@ -274,11 +267,11 @@ def assign_opcode_ids(self):
         assert len(set(ops)) == len(ops)
         assert len(set(instrumented_ops)) == len(instrumented_ops)
 
-        opname: list[str or None] = [None] * 512
-        opmap: dict = {}
-        markers: dict = {}
+        opname: list[str|None] = [None] * 512
+        opmap: dict[str, int] = {}
+        markers: dict[str, int] = {}
 
-        def map_op(op, name):
+        def map_op(op: int, name: str) -> None:
             assert op < len(opname)
             assert opname[op] is None
             assert name not in opmap
@@ -320,11 +313,11 @@ def map_op(op, name):
         for i, op in enumerate(sorted(self.pseudos)):
             map_op(256 + i, op)
 
-        assert 255 not in opmap  # 255 is reserved
+        assert 255 not in opmap.values()  # 255 is reserved
         self.opmap = opmap
         self.markers = markers
 
-    def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename):
+    def write_opcode_ids(self, opcode_ids_h_filename: str, opcode_targets_filename: str) -> None:
         """Write header file that defined the opcode IDs"""
 
         with open(opcode_ids_h_filename, "w") as f:
@@ -342,10 +335,10 @@ def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename):
             self.out.emit("")
             self.out.emit("/* Instruction opcodes for compiled code */")
 
-            def define(name, opcode):
+            def define(name: str, opcode: int) -> None:
                 self.out.emit(f"#define {name:<38} {opcode:>3}")
 
-            all_pairs = []
+            all_pairs: list[tuple[int, int, str]] = []
             # the second item in the tuple sorts the markers before the ops
             all_pairs.extend((i, 1, name) for (name, i) in self.markers.items())
             all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items())
@@ -392,11 +385,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No
                         assert target_instr
                         if format is None:
                             format = target_instr.instr_fmt
-                        else:
-                            assert format == target_instr.instr_fmt
                     assert format is not None
-                case _:
-                    typing.assert_never(thing)
             all_formats.add(format)
 
         # Turn it into a sorted list of enum values.
@@ -483,8 +472,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No
                             self.write_metadata_for_macro(self.macro_instrs[thing.name])
                         case parsing.Pseudo():
                             self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name])
-                        case _:
-                            typing.assert_never(thing)
 
             with self.metadata_item(
                 "const struct opcode_macro_expansion "
@@ -520,8 +507,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No
                             )
                         case parsing.Pseudo():
                             pass
-                        case _:
-                            typing.assert_never(thing)
 
             with self.metadata_item(
                 "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";"
@@ -770,8 +755,6 @@ def write_instructions(
                         # self.write_macro(self.macro_instrs[thing.name])
                     case parsing.Pseudo():
                         pass
-                    case _:
-                        typing.assert_never(thing)
 
         print(
             f"Wrote {n_instrs} instructions and {n_macros} macros "
@@ -814,8 +797,6 @@ def write_executor_instructions(
                         pass
                     case parsing.Pseudo():
                         pass
-                    case _:
-                        typing.assert_never(thing)
         print(
             f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}",
             file=sys.stderr,
@@ -843,8 +824,6 @@ def write_abstract_interpreter_instructions(
                         pass
                     case parsing.Pseudo():
                         pass
-                    case _:
-                        typing.assert_never(thing)
         print(
             f"Wrote some stuff to {abstract_interpreter_filename}",
             file=sys.stderr,
@@ -878,7 +857,7 @@ def write_instr(self, instr: Instruction) -> None:
                 self.out.emit(f"DISPATCH();")
 
 
-def main():
+def main() -> None:
     """Parse command line, parse input, analyze, write output."""
     args = arg_parser.parse_args()  # Prints message and sys.exit(2) on error
     if len(args.input) == 0:

Tools/cases_generator/lexer.py

@@ -4,8 +4,10 @@
 
 import re
 from dataclasses import dataclass
+from collections.abc import Iterator
 
-def choice(*opts):
+
+def choice(*opts: str) -> str:
     return "|".join("(%s)" % opt for opt in opts)
 
 # Regexes
@@ -123,13 +125,19 @@ def choice(*opts):
     'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID',
     'VOLATILE', 'WHILE'
 )
+
+# For mypy
+REGISTER = 'REGISTER'
+OVERRIDE = 'OVERRIDE'
+IF = 'IF'
+
 for name in kwds:
     globals()[name] = name
 keywords = { name.lower() : name for name in kwds }
 
 
 def make_syntax_error(
-    message: str, filename: str, line: int, column: int, line_text: str,
+    message: str, filename: str | None, line: int, column: int, line_text: str,
 ) -> SyntaxError:
     return SyntaxError(message, (filename, line, column, line_text))
 
@@ -142,30 +150,30 @@ class Token:
     end: tuple[int, int]
 
     @property
-    def line(self):
+    def line(self) -> int:
         return self.begin[0]
 
     @property
-    def column(self):
+    def column(self) -> int:
         return self.begin[1]
 
     @property
-    def end_line(self):
+    def end_line(self) -> int:
         return self.end[0]
 
     @property
-    def end_column(self):
+    def end_column(self) -> int:
         return self.end[1]
 
     @property
-    def width(self):
+    def width(self) -> int:
         return self.end[1] - self.begin[1]
 
-    def replaceText(self, txt):
+    def replaceText(self, txt: str) -> 'Token':
         assert isinstance(txt, str)
         return Token(self.kind, txt, self.begin, self.end)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         b0, b1 = self.begin
         e0, e1 = self.end
         if b0 == e0:
@@ -174,7 +182,7 @@ def __repr__(self):
             return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})"
 
 
-def tokenize(src, line=1, filename=None):
+def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]:
     linestart = -1
     for m in matcher.finditer(src):
         start, end = m.span()

Tools/cases_generator/parsing.py

@@ -32,7 +32,7 @@ class Context(NamedTuple):
     end: int
     owner: PLexer
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"<{self.owner.filename}: {self.begin}-{self.end}>"
 
 
@@ -75,7 +75,7 @@ class StackEffect(Node):
     size: str = ""  # Optional `[size]`
     # Note: size cannot be combined with type or cond
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         items = [self.name, self.type, self.cond, self.size]
         while items and items[-1] == "":
             del items[-1]

Tools/cases_generator/requirements-dev.txt

@@ -0,0 +1,2 @@
+# Requirements file for external linters and checks we run on Tools/clinic/ in CI
+mypy==1.4.1