feat: Compute and show some stats

src/griffe/cli.py

@@ -18,6 +18,7 @@
 import logging
 import os
 import sys
+from datetime import datetime
 from pathlib import Path
 from typing import Any, Sequence
 
@@ -40,6 +41,66 @@ def _print_data(data: str, output_file: str):
             print(data, file=fd)
 
 
+def _stats(stats):
+    lines = []
+    packages = stats["packages"]
+    modules = stats["modules"]
+    classes = stats["classes"]
+    functions = stats["functions"]
+    attributes = stats["attributes"]
+    objects = sum((modules, classes, functions, attributes))
+    lines.append("Statistics")
+    lines.append("---------------------")
+    lines.append("Number of loaded objects")
+    lines.append(f"  Modules: {modules}")
+    lines.append(f"  Classes: {classes}")
+    lines.append(f"  Functions: {functions}")
+    lines.append(f"  Attributes: {attributes}")
+    lines.append(f"  Total: {objects} across {packages} packages")
+    per_ext = stats["modules_by_extension"]
+    builtin = per_ext[""]
+    regular = per_ext[".py"]
+    compiled = modules - builtin - regular
+    lines.append("")
+    lines.append(f"Total number of lines: {stats['lines']}")
+    lines.append("")
+    lines.append("Modules")
+    lines.append(f"  Builtin: {builtin}")
+    lines.append(f"  Compiled: {compiled}")
+    lines.append(f"  Regular: {regular}")
+    lines.append("  Per extension:")
+    for ext, number in sorted(per_ext.items()):
+        if ext:
+            lines.append(f"    {ext}: {number}")
+    visit_time = stats["time_spent_visiting"] / 1000
+    inspect_time = stats["time_spent_inspecting"] / 1000
+    total_time = visit_time + inspect_time
+    visit_percent = visit_time / total_time * 100
+    inspect_percent = inspect_time / total_time * 100
+    try:
+        visit_time_per_module = visit_time / regular
+    except ZeroDivisionError:
+        visit_time_per_module = 0
+    inspected_modules = builtin + compiled
+    try:
+        inspect_time_per_module = visit_time / inspected_modules
+    except ZeroDivisionError:
+        inspect_time_per_module = 0
+    lines.append("")
+    lines.append(
+        f"Time spent visiting modules ({regular}): "
+        f"{visit_time}ms, {visit_time_per_module:.02f}ms/module ({visit_percent:.02f}%)"
+    )
+    lines.append(
+        f"Time spent inspecting modules ({inspected_modules}): "
+        f"{inspect_time}ms, {inspect_time_per_module:.02f}ms/module ({inspect_percent:.02f}%)"
+    )
+    serialize_time = stats["time_spent_serializing"] / 1000
+    serialize_time_per_module = serialize_time / modules
+    lines.append(f"Time spent serializing: " f"{serialize_time}ms, {serialize_time_per_module:.02f}ms/module")
+    return "\n".join(lines)
+
+
 def _load_packages(
     packages: Sequence[str],
     extensions: Extensions | None,
@@ -172,6 +233,12 @@ def get_parser() -> argparse.ArgumentParser:
         type=Path,
         help="Paths to search packages into.",
     )
+    parser.add_argument(
+        "-S",
+        "--stats",
+        action="store_true",
+        help="Show statistics at the end.",
+    )
 
     parser.add_argument("packages", metavar="PACKAGE", nargs="+", help="Packages to find and parse.")
     return parser
@@ -231,12 +298,17 @@ def main(args: list[str] | None = None) -> int:  # noqa: WPS231
     )
     packages = loader.modules_collection.members
 
+    started = datetime.now()
     if per_package_output:
         for package_name, data in packages.items():
             serialized = json.dumps(data, cls=Encoder, indent=2, full=opts.full)
             _print_data(serialized, output.format(package=package_name))
     else:
         serialized = json.dumps(packages, cls=Encoder, indent=2, full=opts.full)
         _print_data(serialized, output)
+    elapsed = datetime.now() - started
+
+    if opts.stats:
+        logger.info(_stats({"time_spent_serializing": elapsed.microseconds, **loader.stats()}))
 
     return 0 if len(packages) == len(opts.packages) else 1

src/griffe/dataclasses.py

@@ -339,6 +339,9 @@ def __init__(
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}({self.name!r}, {self.lineno!r}, {self.endlineno!r})>"
 
+    def __len__(self) -> int:
+        return len(self.members) + sum(len(member) for member in self.members.values())
+
     @property
     def has_docstring(self) -> bool:
         """Tell if this object has a non-empty docstring."""
@@ -739,6 +742,9 @@ def __setitem__(self, key, value):
         # not handled by __getattr__
         self.target[key] = value
 
+    def __len__(self) -> int:
+        return 1
+
     @property
     def kind(self) -> Kind:
         """Return the target's kind, or Kind.ALIAS if the target cannot be resolved.

src/griffe/loader.py

@@ -14,8 +14,8 @@
 
 import os
 import sys
-import traceback
 from contextlib import suppress
+from datetime import datetime
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Iterator, Sequence, Tuple
@@ -28,6 +28,7 @@
 from griffe.docstrings.parsers import Parser
 from griffe.exceptions import AliasResolutionError, CyclicAliasError, UnhandledPthFileError, UnimportableModuleError
 from griffe.logger import get_logger
+from griffe.stats import stats
 
 NamePartsType = Tuple[str, ...]
 NamePartsAndPathType = Tuple[NamePartsType, Path]
@@ -94,6 +95,10 @@ def __init__(
         self.docstring_options: dict[str, Any] = docstring_options or {}
         self.lines_collection: LinesCollection = lines_collection or LinesCollection()
         self.modules_collection: ModulesCollection = modules_collection or ModulesCollection()
+        self._time_stats: dict = {
+            "time_spent_visiting": 0,
+            "time_spent_inspecting": 0,
+        }
         patch_ast()
 
     def load_module(
@@ -261,6 +266,14 @@ def resolve_module_aliases(  # noqa: WPS231
 
         return resolved, unresolved
 
+    def stats(self) -> dict:
+        """Compute some statistics.
+
+        Returns:
+            Some statistics.
+        """
+        return {**stats(self), **self._time_stats}
+
     def _load_module_path(
         self,
         module_name: str,
@@ -313,7 +326,8 @@ def _create_module(self, module_name: str, module_path: Path) -> Module:
 
     def _visit_module(self, code: str, module_name: str, module_path: Path, parent: Module | None = None) -> Module:
         self.lines_collection[module_path] = code.splitlines(keepends=False)
-        return visit(
+        start = datetime.now()
+        module = visit(
             module_name,
             filepath=module_path,
             code=code,
@@ -324,13 +338,17 @@ def _visit_module(self, code: str, module_name: str, module_path: Path, parent:
             lines_collection=self.lines_collection,
             modules_collection=self.modules_collection,
         )
+        elapsed = datetime.now() - start
+        self._time_stats["time_spent_visiting"] += elapsed.microseconds
+        return module
 
     def _inspect_module(self, module_name: str, filepath: Path | None = None, parent: Module | None = None) -> Module:
         for prefix in _ignored_modules:
             if module_name.startswith(prefix):
                 raise ImportError(f"Ignored module '{module_name}'")
+        start = datetime.now()
         try:
-            return inspect(
+            module = inspect(
                 module_name,
                 filepath=filepath,
                 extensions=self.extensions,
@@ -341,6 +359,9 @@ def _inspect_module(self, module_name: str, filepath: Path | None = None, parent
             )
         except SystemExit as error:
             raise ImportError(f"Importing '{module_name}' raised a system exit") from error
+        elapsed = datetime.now() - start
+        self._time_stats["time_spent_inspecting"] += elapsed.microseconds
+        return module
 
     def _member_parent(self, module: Module, subparts: NamePartsType, subpath: Path) -> Module:
         parent_parts = subparts[:-1]

src/griffe/stats.py

@@ -0,0 +1,95 @@
+"""This module contains utilities to compute loading statistics."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+from griffe.exceptions import BuiltinModuleError
+
+if TYPE_CHECKING:
+    from griffe.loader import GriffeLoader
+
+
+def _direct(objects):
+    return [obj for obj in objects if not obj.is_alias]
+
+
+def _n_modules(module):
+    submodules = _direct(module.modules.values())
+    return len(submodules) + sum(_n_modules(mod) for mod in submodules)
+
+
+def _n_classes(module_or_class):
+    submodules = _direct(module_or_class.modules.values())
+    subclasses = _direct(module_or_class.classes.values())
+    mods_or_classes = [mc for mc in (*submodules, *subclasses) if not mc.is_alias]
+    return len(subclasses) + sum(_n_classes(mod_or_class) for mod_or_class in mods_or_classes)
+
+
+def _n_functions(module_or_class):
+    submodules = _direct(module_or_class.modules.values())
+    subclasses = _direct(module_or_class.classes.values())
+    functions = _direct(module_or_class.functions.values())
+    mods_or_classes = [*submodules, *subclasses]
+    return len(functions) + sum(_n_functions(mod_or_class) for mod_or_class in mods_or_classes)
+
+
+def _n_attributes(module_or_class):
+    submodules = _direct(module_or_class.modules.values())
+    subclasses = _direct(module_or_class.classes.values())
+    attributes = _direct(module_or_class.attributes.values())
+    mods_or_classes = [*submodules, *subclasses]
+    return len(attributes) + sum(_n_attributes(mod_or_class) for mod_or_class in mods_or_classes)
+
+
+def _merge_exts(exts1, exts2):
+    for ext, value in exts2.items():
+        exts1[ext] += value
+    return exts1
+
+
+def _sum_extensions(exts, module):
+    current_exts = defaultdict(int)
+    try:
+        suffix = module.filepath.suffix
+    except BuiltinModuleError:
+        current_exts[""] = 1
+    else:
+        if suffix:
+            current_exts[suffix] = 1
+        for submodule in _direct(module.modules.values()):
+            _sum_extensions(current_exts, submodule)
+    _merge_exts(exts, current_exts)
+
+
+def stats(loader: GriffeLoader) -> dict:
+    """Return some loading statistics.
+
+    Parameters:
+        loader: The loader to compute stats from.
+
+    Returns:
+        Some statistics.
+    """
+    modules_by_extension = {
+        "": 0,
+        ".py": 0,
+        ".pyc": 0,
+        ".pyo": 0,
+        ".pyd": 0,
+        ".so": 0,
+    }
+    top_modules = loader.modules_collection.members.values()
+    for module in top_modules:
+        _sum_extensions(modules_by_extension, module)
+    n_lines = sum(len(lines) for lines in loader.lines_collection.values())
+    return {
+        "packages": len(top_modules),
+        "modules": len(top_modules) + sum(_n_modules(mod) for mod in top_modules),
+        "classes": sum(_n_classes(mod) for mod in top_modules),
+        "functions": sum(_n_functions(mod) for mod in top_modules),
+        "attributes": sum(_n_attributes(mod) for mod in top_modules),
+        "modules_by_extension": modules_by_extension,
+        "lines": n_lines,
+    }