Restore the legacy implementation of autodoc for compatibility (#14091)

Author: AA-Turner

CHANGES.rst

@@ -14,6 +14,12 @@ Incompatible changes
 * #13639: :py:meth:`!SphinxComponentRegistry.create_source_parser` no longer
   has an *app* parameter, instead taking *config* and *env*.
   Patch by Adam Turner.
+* #13751, #14089: :mod:`sphinx.ext.autodoc` has been substantially rewritten,
+  and there may be some incompatible changes in edge cases, especially when
+  extensions interact with autodoc internals.
+  The :confval:`autodoc_use_legacy_class_based` option has been added to
+  use the legacy (pre-8.3) implementation of autodoc.
+  Patches by Adam Turner.
 
 Deprecated
 ----------

doc/development/tutorials/examples/autodoc_intenum.py

@@ -3,23 +3,21 @@
 from enum import IntEnum
 from typing import TYPE_CHECKING
 
-from sphinx.ext.autodoc import ClassDocumenter, bool_option
-from sphinx.ext.autodoc._generate import _docstring_source_name
+from sphinx.ext.autodoc import ClassDocumenter, Documenter, bool_option
 
 if TYPE_CHECKING:
     from typing import Any
 
     from docutils.statemachine import StringList
 
     from sphinx.application import Sphinx
-    from sphinx.ext.autodoc import Documenter
     from sphinx.util.typing import ExtensionMetadata
 
 
 class IntEnumDocumenter(ClassDocumenter):
     objtype = 'intenum'
     directivetype = ClassDocumenter.objtype
-    priority = 25
+    priority = 10 + ClassDocumenter.priority
     option_spec = dict(ClassDocumenter.option_spec)
     option_spec['hex'] = bool_option
 
@@ -32,33 +30,28 @@ def can_document_member(
         except TypeError:
             return False
 
-    def add_line(self, line: str, source: str = '', *lineno: int, indent: str) -> None:
-        """Append one line of generated reST to the output."""
-        analyzer_source = '' if self.analyzer is None else self.analyzer.srcname
-        source_name = _docstring_source_name(props=self.props, source=analyzer_source)
-        if line.strip():  # not a blank line
-            self.result.append(indent + line, source_name, *lineno)
-        else:
-            self.result.append('', source_name, *lineno)
+    def add_directive_header(self, sig: str) -> None:
+        super().add_directive_header(sig)
+        self.add_line('   :final:', self.get_sourcename())
 
-    def add_directive_header(self, *, indent: str) -> None:
-        super().add_directive_header(indent=indent)
-        self.add_line('   :final:', indent=indent)
+    def add_content(
+        self,
+        more_content: StringList | None,
+    ) -> None:
+        super().add_content(more_content)
 
-    def add_content(self, more_content: StringList | None, *, indent: str) -> None:
-        super().add_content(more_content, indent=indent)
-
-        enum_object: IntEnum = self.props._obj
+        source_name = self.get_sourcename()
+        enum_object: IntEnum = self.object
         use_hex = self.options.hex
-        self.add_line('', indent=indent)
+        self.add_line('', source_name)
 
         for the_member_name, enum_member in enum_object.__members__.items():  # type: ignore[attr-defined]
             the_member_value = enum_member.value
             if use_hex:
                 the_member_value = hex(the_member_value)
 
-            self.add_line(f'**{the_member_name}**: {the_member_value}', indent=indent)
-            self.add_line('', indent=indent)
+            self.add_line(f'**{the_member_name}**: {the_member_value}', source_name)
+            self.add_line('', source_name)
 
 
 def setup(app: Sphinx) -> ExtensionMetadata:

doc/usage/extensions/autodoc.rst

@@ -1003,6 +1003,26 @@ Configuration
 
 There are also config values that you can set:
 
+.. confval:: autodoc_use_legacy_class_based
+   :type: :code-py:`bool`
+   :default: :code-py:`False`
+
+   If true, autodoc will use the legacy class-based implementation.
+   This is the behaviour prior to Sphinx 8.3.
+   It is based on the ``Documenter`` class hierarchy.
+
+   This setting is provided for backwards compatibility if your documentation
+   or an extension you use uses or monkeypatches the legacy class-based API
+   in Python code.
+   If this is the case, set ``autodoc_use_legacy_class_based = True``
+   in your :file:`conf.py`.
+   Please also add a comment to `the tracking issue on GitHub
+   <https://github.com/sphinx-doc/sphinx/issues/14089>`__ so that the maintainers
+   are aware of your use case, for possible future improvements.
+
+   .. note:: The legacy class-based implementation does not support
+             PEP 695 type aliases.
+
 .. confval:: autoclass_content
    :type: :code-py:`str`
    :default: :code-py:`'class'`

pyproject.toml

@@ -173,7 +173,7 @@ exclude = [
 [tool.mypy]
 files = [
     "doc/conf.py",
-#    "doc/development/tutorials/examples/autodoc_intenum.py",
+    "doc/development/tutorials/examples/autodoc_intenum.py",
     "doc/development/tutorials/examples/helloworld.py",
     "sphinx",
     "tests",

sphinx/application.py

@@ -52,12 +52,12 @@
     from sphinx.config import ENUM, _ConfigRebuild
     from sphinx.domains import Domain, Index
     from sphinx.environment.collectors import EnvironmentCollector
-    from sphinx.ext.autodoc._documenters import Documenter
     from sphinx.ext.autodoc._event_listeners import (
         _AutodocProcessDocstringListener,
         _AutodocProcessSignatureListener,
         _AutodocSkipMemberListener,
     )
+    from sphinx.ext.autodoc._legacy_class_based._documenters import Documenter
     from sphinx.ext.todo import todo_node
     from sphinx.extension import Extension
     from sphinx.registry import (
@@ -1627,9 +1627,9 @@ def add_autodocumenter(self, cls: type[Documenter], override: bool = False) -> N
            Add *override* keyword.
         """
         logger.debug('[app] adding autodocumenter: %r', cls)
-        from sphinx.ext.autodoc._directive import AutodocDirective
+        from sphinx.ext.autodoc.directive import AutodocDirective
 
-        objtype = cls.objtype  # type: ignore[attr-defined]
+        objtype = cls.objtype
         self.registry.add_documenter(objtype, cls)
         self.add_directive('auto' + objtype, AutodocDirective, override=override)
 

sphinx/ext/autodoc/__init__.py

@@ -12,7 +12,9 @@
 import sphinx
 from sphinx.config import ENUM
 from sphinx.ext.autodoc._directive import AutodocDirective
-from sphinx.ext.autodoc._directive_options import (
+from sphinx.ext.autodoc._event_listeners import between, cut_lines
+from sphinx.ext.autodoc._legacy_class_based._directive_options import (
+    Options,
     annotation_option,
     bool_option,
     class_doc_from_option,
@@ -23,27 +25,67 @@
     members_option,
     merge_members_option,
 )
-from sphinx.ext.autodoc._dynamic._member_finder import ObjectMember, special_member_re
-from sphinx.ext.autodoc._event_listeners import between, cut_lines
-from sphinx.ext.autodoc._names import py_ext_sig_re
-from sphinx.ext.autodoc._sentinels import ALL, EMPTY, SUPPRESS, UNINITIALIZED_ATTR
-from sphinx.ext.autodoc._sentinels import INSTANCE_ATTR as INSTANCEATTR
-from sphinx.ext.autodoc._sentinels import SLOTS_ATTR as SLOTSATTR
+from sphinx.ext.autodoc._legacy_class_based._documenters import (
+    AttributeDocumenter,
+    ClassDocumenter,
+    ClassLevelDocumenter,
+    DataDocumenter,
+    DataDocumenterMixinBase,
+    DecoratorDocumenter,
+    DocstringSignatureMixin,
+    DocstringStripSignatureMixin,
+    Documenter,
+    ExceptionDocumenter,
+    FunctionDocumenter,
+    GenericAliasMixin,
+    MethodDocumenter,
+    ModuleDocumenter,
+    ModuleLevelDocumenter,
+    NonDataDescriptorMixin,
+    ObjectMember,
+    PropertyDocumenter,
+    RuntimeInstanceAttributeMixin,
+    SlotsMixin,
+    UninitializedGlobalVariableMixin,
+    UninitializedInstanceAttributeMixin,
+    autodoc_attrgetter,
+    py_ext_sig_re,
+    special_member_re,
+)
+from sphinx.ext.autodoc._legacy_class_based._sentinels import (
+    ALL,
+    EMPTY,
+    INSTANCEATTR,
+    SLOTSATTR,
+    SUPPRESS,
+    UNINITIALIZED_ATTR,
+)
 from sphinx.ext.autodoc.typehints import _merge_typehints
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
+    from sphinx.config import Config
     from sphinx.ext.autodoc._property_types import _AutodocObjType
     from sphinx.util.typing import ExtensionMetadata
 
 __all__ = (
     # Useful event listener factories for autodoc-process-docstring
     'cut_lines',
     'between',
+    # Documenters
+    'AttributeDocumenter',
+    'ClassDocumenter',
+    'DataDocumenter',
+    'DecoratorDocumenter',
+    'ExceptionDocumenter',
+    'FunctionDocumenter',
+    'MethodDocumenter',
+    'ModuleDocumenter',
+    'PropertyDocumenter',
     # This class is only used in ``sphinx.ext.autodoc.directive``,
     # but we export it here for compatibility.
     # See: https://github.com/sphinx-doc/sphinx/issues/4538
-    # 'Options',
+    'Options',
     # Option spec functions.
     # Exported for compatibility.
     'annotation_option',
@@ -68,26 +110,23 @@
     'ObjectMember',
     'py_ext_sig_re',
     'special_member_re',
+    'ModuleLevelDocumenter',
+    'ClassLevelDocumenter',
+    'DocstringSignatureMixin',
+    'DocstringStripSignatureMixin',
+    'DataDocumenterMixinBase',
+    'GenericAliasMixin',
+    'UninitializedGlobalVariableMixin',
+    'NonDataDescriptorMixin',
+    'SlotsMixin',
+    'RuntimeInstanceAttributeMixin',
+    'UninitializedInstanceAttributeMixin',
+    'autodoc_attrgetter',
+    'Documenter',
 )
 
 
 def setup(app: Sphinx) -> ExtensionMetadata:
-    obj_type: _AutodocObjType
-    for obj_type in (
-        'module',
-        'class',
-        'exception',
-        'function',
-        'decorator',
-        'method',
-        'property',
-        'attribute',
-        'data',
-        'type',
-    ):
-        # register the automodule, autoclass, etc. directives
-        app.add_directive(f'auto{obj_type}', AutodocDirective)
-
     app.add_config_value(
         'autoclass_content',
         'class',
@@ -142,6 +181,9 @@ def setup(app: Sphinx) -> ExtensionMetadata:
     app.add_config_value(
         'autodoc_use_type_comments', True, 'env', types=frozenset({bool})
     )
+    app.add_config_value(
+        'autodoc_use_legacy_class_based', False, 'env', types=frozenset({bool})
+    )
 
     app.add_event('autodoc-before-process-signature')
     app.add_event('autodoc-process-docstring')
@@ -151,7 +193,50 @@ def setup(app: Sphinx) -> ExtensionMetadata:
 
     app.connect('object-description-transform', _merge_typehints)
 
+    app.connect('config-inited', _register_directives)
+
     return {
         'version': sphinx.__display_version__,
         'parallel_read_safe': True,
     }
+
+
+def _register_directives(app: Sphinx, config: Config) -> None:
+    if not config.autodoc_use_legacy_class_based:
+        obj_type: _AutodocObjType
+        for obj_type in (
+            'module',
+            'class',
+            'exception',
+            'function',
+            'decorator',
+            'method',
+            'property',
+            'attribute',
+            'data',
+            'type',
+        ):
+            # register the automodule, autoclass, etc. directives
+            app.add_directive(f'auto{obj_type}', AutodocDirective)
+    else:
+        from sphinx.ext.autodoc.preserve_defaults import update_defvalue
+        from sphinx.ext.autodoc.type_comment import (
+            update_annotations_using_type_comments,
+        )
+        from sphinx.ext.autodoc.typehints import record_typehints
+
+        app.add_autodocumenter(ModuleDocumenter)
+        app.add_autodocumenter(ClassDocumenter)
+        app.add_autodocumenter(ExceptionDocumenter)
+        app.add_autodocumenter(DataDocumenter)
+        app.add_autodocumenter(FunctionDocumenter)
+        app.add_autodocumenter(DecoratorDocumenter)
+        app.add_autodocumenter(MethodDocumenter)
+        app.add_autodocumenter(AttributeDocumenter)
+        app.add_autodocumenter(PropertyDocumenter)
+
+        app.connect('autodoc-before-process-signature', update_defvalue)
+        app.connect(
+            'autodoc-before-process-signature', update_annotations_using_type_comments
+        )
+        app.connect('autodoc-process-signature', record_typehints)

sphinx/ext/autodoc/_documenters.py

@@ -20,7 +20,6 @@
 if TYPE_CHECKING:
     from typing import Any
 
-
 logger = logging.getLogger(__name__)
 _LAMBDA_NAME = (lambda: None).__name__
 

sphinx/ext/autodoc/_dynamic/_preserve_defaults.py

@@ -0,0 +1 @@
+"""The legacy (class-based) implementation of autodoc."""