https://github.com/jackdewinter/pymarkdown/issues/946

pymarkdown/file_scan_helper.py

@@ -14,6 +14,7 @@
 
 from pymarkdown.application_file_scanner import ApplicationFileScanner
 from pymarkdown.extensions.pragma_token import PragmaToken
+from pymarkdown.general.bad_tokenization_error import BadTokenizationError
 from pymarkdown.general.main_presentation import MainPresentation
 from pymarkdown.general.parser_logger import ParserLogger
 from pymarkdown.general.source_providers import FileSourceProvider
@@ -48,7 +49,7 @@ def __init__(
         plugins: PluginManager,
         presentation: MainPresentation,
         show_stack_trace: bool,
-        handle_error: Callable[[str, Exception], None],
+        handle_error: Callable[[str, Optional[Exception], bool, str], None],
     ):
         self.__tokenizer = tokenizer
         self.__plugins = plugins
@@ -69,8 +70,11 @@ def process_files_to_scan(
         Process the specified files to scan, or the string to scan.
         """
 
+        self.__continue_on_error = args.continue_on_error
+
         # sourcery skip: raise-specific-error
         did_fix_any_file = False
+        did_fail_any_file = False
         if use_standard_in:
             POGGER.debug("Scanning from: (stdin)")
             self.__scan_from_stdin(args, string_to_scan)
@@ -80,18 +84,20 @@ def process_files_to_scan(
             POGGER.debug("Scanning from: $", files_to_scan)
             for next_file in files_to_scan:
                 if args.x_fix:
-                    if self.__fix_specific_file(
+                    did_fix_file, did_succeed = self.__fix_specific_file(
                         next_file,
                         next_file,
                         args.x_fix_debug,
                         args.x_fix_file_debug,
-                        args.x_fix_no_rescan_log,
-                    ):
+                        args.x_fix_no_rescan_log)
+                    if did_fix_file:
                         self.__presentation.print_fix_message(next_file)
                         did_fix_any_file = True
                 else:
-                    self.__scan_specific_file(next_file, next_file)
-        return did_fix_any_file
+                    did_succeed = self.__scan_specific_file(next_file, next_file)
+                if not did_succeed:
+                    did_fail_any_file = True
+        return did_fix_any_file, did_fail_any_file
 
     def __scan_from_stdin(
         self, args: argparse.Namespace, string_to_scan: Optional[str]
@@ -128,12 +134,18 @@ def __scan_from_stdin(
             except IOError as this_exception:
                 self.__handle_scan_error(scan_id, this_exception)
 
-    def __scan_specific_file(self, next_file: str, next_file_name: str) -> None:
+    def __scan_specific_file(self, next_file: str, next_file_name: str) -> bool:
         try:
             source_provider = FileSourceProvider(next_file)
             self.__scan_file(source_provider, next_file_name)
+            return True
         except BadPluginError as this_exception:
-            self.__handle_scan_error(next_file, this_exception)
+            self.__handle_scan_error(next_file, this_exception, allow_shortcut=True)
+        except BadTokenizationError as this_exception:
+            if not self.__continue_on_error:
+                raise
+            self.__handle_scan_error(next_file, this_exception, allow_shortcut=True)
+        return False
 
     def __scan_file(
         self, source_provider: FileSourceProvider, next_file_name: str
@@ -198,6 +210,7 @@ def __fix_specific_file(
         fix_nolog_rescan: bool,
     ) -> bool:
         did_fix_file = False
+        did_succeed = False
         try:
             try:
                 POGGER.info("Starting file to fix '$'.", next_file_name)
@@ -211,25 +224,46 @@ def __fix_specific_file(
                 )
 
                 POGGER.info("Ending file to fix '$'.", next_file_name)
+                did_succeed = True
             except Exception:
                 POGGER.info("Ending file to fix '$' with exception.", next_file_name)
                 raise
         except (BadPluginError, BadPluginFixError) as this_exception:
-            self.__handle_scan_error(next_file, this_exception)
-        return did_fix_file
+            self.__handle_scan_error(next_file, this_exception, allow_shortcut=True)
+        except BadTokenizationError as this_exception:
+            if not self.__continue_on_error:
+                raise
+            self.__handle_scan_error(next_file, this_exception, allow_shortcut=True)
+        return did_fix_file, did_succeed
 
     # pylint: enable=too-many-arguments
 
-    def __handle_scan_error(self, next_file: str, this_exception: Exception) -> None:
+    def __handle_scan_error(self, next_file: str, this_exception: Exception, allow_shortcut:bool = False) -> None:
+
+        if not self.__continue_on_error:
+            allow_shortcut =False
+
+        if allow_shortcut:
+            show_extended_information = False
+            print_prefix = ""
+        else:
+            show_extended_information = self.__show_stack_trace
+            print_prefix = "\n\n"
+
         if formatted_error := self.__presentation.format_scan_error(
-            next_file, this_exception, self.__show_stack_trace
+            next_file, this_exception, show_extended_information, allow_shortcut
         ):
-            self.__handle_error(formatted_error, this_exception)
+            self.__handle_error(formatted_error, this_exception,not allow_shortcut, print_prefix)
 
         # If the `format_scan_error` call above returned None, it meant that
         # it handled any required output.  However, the application still needs
         # to terminate to respect that the error was called.
-        ReturnCodeHelper.exit_application(ApplicationResult.SYSTEM_ERROR)
+        #
+        # update: the allow_shortcut variable allows for a shorter error to be
+        #   logged to allow processing to continue.  When that happens, the
+        #   application is not exitted. 
+        if not allow_shortcut:
+            ReturnCodeHelper.exit_application(ApplicationResult.SYSTEM_ERROR)
 
     def __process_file_fix_rescan(
         self, fix_debug: bool, fix_nolog_rescan: bool, next_file_two: str

pymarkdown/general/main_presentation.py

@@ -29,19 +29,23 @@ def format_scan_error(
         next_file: str,
         this_exception: Exception,
         show_extended_information: bool = False,
+        allow_shortcut:bool = False
     ) -> Optional[str]:
         """
         Format a scan error for display.  Returning a value of None means that
         the function has handled any required output.
         """
-        formatted_error = f"{type(this_exception).__name__} encountered while scanning '{next_file}':\n{this_exception}"
-        if show_extended_information:
-            current_cause = this_exception.__cause__
-            while current_cause:
-                formatted_error += (
-                    f"\nCaused by: {type(current_cause).__name__}:\n   {current_cause}"
-                )
-                current_cause = current_cause.__cause__
+        if allow_shortcut:
+            formatted_error = f"{next_file}:0:0: {this_exception}"
+        else:
+            formatted_error = f"{type(this_exception).__name__} encountered while scanning '{next_file}':\n{this_exception}"
+            if show_extended_information:
+                current_cause = this_exception.__cause__
+                while current_cause:
+                    formatted_error += (
+                        f"\nCaused by: {type(current_cause).__name__}:\n   {current_cause}"
+                    )
+                    current_cause = current_cause.__cause__
         return formatted_error
 
     def print_pragma_failure(

pymarkdown/main.py

@@ -180,6 +180,13 @@ def __parse_arguments(self, direct_args: Optional[List[str]]) -> argparse.Namesp
             default=False,
             help="if an error occurs, print out the stack trace for debug purposes",
         )
+        parser.add_argument(
+            "--continue-on-error",
+            dest="continue_on_error",
+            action="store_true",
+            default=False,
+            help="if a tokenization or plugin error occurs, allow processing to continue",
+        )
         ApplicationLogging.add_default_command_line_arguments(parser)
         ReturnCodeHelper.add_command_line_arguments(parser)
 
@@ -314,6 +321,7 @@ def __handle_error(
         formatted_error: str,
         thrown_error: Optional[Exception],
         exit_on_error: bool = True,
+        print_prefix:str = "\n\n"
     ) -> None:
         LOGGER.warning(formatted_error, exc_info=thrown_error)
 
@@ -324,7 +332,7 @@ def __handle_error(
             and not isinstance(thrown_error, ValueError)
             else ""
         )
-        self.__presentation.print_system_error(f"\n\n{formatted_error}{stack_trace}")
+        self.__presentation.print_system_error(f"{print_prefix}{formatted_error}{stack_trace}")
         if exit_on_error:
             ReturnCodeHelper.exit_application(ApplicationResult.SYSTEM_ERROR)
 
@@ -359,10 +367,12 @@ def __scan_files_if_no_errors(
                 self.__show_stack_trace,
                 self.__handle_error,
             )
-            did_fix_any_files = fsh.process_files_to_scan(
+            did_fix_any_files, did_fail_any_file = fsh.process_files_to_scan(
                 args, use_standard_in, files_to_scan, self.__string_to_scan
             )
-            if did_fix_any_files:
+            if did_fail_any_file:
+                scan_result = ApplicationResult.SYSTEM_ERROR
+            elif did_fix_any_files:
                 scan_result = ApplicationResult.FIXED_AT_LEAST_ONE_FILE
             elif self.__plugins.number_of_scan_failures:
                 scan_result = ApplicationResult.SCAN_TRIGGERED_AT_LEAST_ONCE

test/markdown_scanner.py

@@ -43,6 +43,7 @@ def format_scan_error(
         next_file: str,
         this_exception: Exception,
         show_extended_information: bool = False,
+        allow_shortcut:bool = False
     ) -> Optional[str]:
         """
         Format a scan error for display.  Returning a value of None means that

test/resources/plugins/bad/bad_next_line_with_fix_trigger.py

@@ -0,0 +1,32 @@
+"""
+Module to implement a sample plugin that has a bad next_line function.
+"""
+from pymarkdown.plugin_manager.plugin_details import PluginDetailsV2
+from pymarkdown.plugin_manager.rule_plugin import RulePlugin
+
+
+class BadNextLineWithFixTrigger(RulePlugin):
+    """
+    Class to implement a sample plugin that has a bad starting_new_file function.
+    """
+
+    def get_details(self):
+        """
+        Get the details for the plugin.
+        """
+        return PluginDetailsV2(
+            plugin_name="bad-next-line-with-trigger",
+            plugin_id="MDE008",
+            plugin_enabled_by_default=True,
+            plugin_description="Plugin that has a bad next_line function.",
+            plugin_version="0.0.0",
+            plugin_fix_level=0,
+            plugin_supports_fix=True
+        )
+
+    def next_line(self, context, line):
+        """
+        Event that a new line is being processed.
+        """
+        if line == "throw_exception" and context.in_fix_mode:
+            raise Exception("bad next_line")

test/resources/plugins/bad/bad_next_line_with_scan_trigger.py

@@ -0,0 +1,30 @@
+"""
+Module to implement a sample plugin that has a bad next_line function.
+"""
+from pymarkdown.plugin_manager.plugin_details import PluginDetailsV2
+from pymarkdown.plugin_manager.rule_plugin import RulePlugin
+
+
+class BadNextLineWithScanTrigger(RulePlugin):
+    """
+    Class to implement a sample plugin that has a bad starting_new_file function.
+    """
+
+    def get_details(self):
+        """
+        Get the details for the plugin.
+        """
+        return PluginDetailsV2(
+            plugin_name="bad-next-line-with-trigger",
+            plugin_id="MDE008",
+            plugin_enabled_by_default=True,
+            plugin_description="Plugin that has a bad next_line function.",
+            plugin_version="0.0.0",
+        )
+
+    def next_line(self, context, line):
+        """
+        Event that a new line is being processed.
+        """
+        if line == "throw_exception":
+            raise Exception("bad next_line")