Add: Improve UpdateHeader to remove old header lines, that are not required anymore (#912)

Author: y0urself

pontos/updateheader/updateheader.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2019-2022 Greenbone AG
+# SPDX-FileCopyrightText: 2019-2023 Greenbone AG
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
@@ -26,7 +26,7 @@
 from datetime import datetime
 from pathlib import Path
 from subprocess import CalledProcessError, run
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from pontos.terminal import Terminal
 from pontos.terminal.null import NullTerminal
@@ -53,6 +53,29 @@
     "GPL-2.0-or-later",
     "GPL-3.0-or-later",
 ]
+OLD_LINES = [
+    "# \-\*\- coding: utf\-8 \-\*\-",
+    "This program is free software: you can redistribute it and/or modify",
+    "it under the terms of the GNU Affero General Public License as",
+    "published by the Free Software Foundation, either version 3 of the",
+    "License, or \(at your option\) any later version.",
+    "This program is free software; you can redistribute it and/or",
+    "modify it under the terms of the GNU General Public License",
+    "version 2 as published by the Free Software Foundation.",
+    "This program is free software: you can redistribute it and/or modify",
+    "it under the terms of the GNU General Public License as published by",
+    "the Free Software Foundation, either version 3 of the License, or",
+    "\(at your option\) any later version.",
+    "This program is distributed in the hope that it will be useful,",
+    "but WITHOUT ANY WARRANTY; without even the implied warranty of",
+    "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the",
+    "GNU Affero General Public License for more details.",
+    "GNU General Public License for more details.",
+    "You should have received a copy of the GNU Affero General Public License",
+    "along with this program.  If not, see <http://www.gnu.org/licenses/>.",
+    "along with this program; if not, write to the Free Software",
+    "Foundation, Inc\., 51 Franklin St, Fifth Floor, Boston, MA 02110\-1301 USA\.",  # noqa: E501
+]
 
 
 def _get_modified_year(f: Path) -> str:
@@ -112,11 +135,32 @@ def _add_header(
         raise ValueError
 
 
+def _remove_outdated(
+    content: str, cleanup_regexes: List[re.Pattern]
+) -> Optional[str]:
+    """Remove lines that contain outdated copyright header ..."""
+    changed = False
+    splitted_lines = content.splitlines()
+    i = 0
+    for line in splitted_lines[:20]:
+        for regex in cleanup_regexes:
+            if regex.match(line):
+                changed = True
+                splitted_lines.pop(i)
+                i = i - 1
+                break
+        i = i + 1
+    if changed:
+        return "\n".join(splitted_lines)
+    return None
+
+
 def _update_file(
     file: Path,
     regex: re.Pattern,
     parsed_args: Namespace,
     term: Terminal,
+    cleanup_regexes: Optional[List[re.Pattern]] = None,
 ) -> int:
     """Function to update the given file.
     Checks if header exists. If not it adds an
@@ -173,6 +217,15 @@ def _update_file(
                         "is not existing."
                     )
                 return 1
+            # old header existing - cleanup?
+            if cleanup_regexes:
+                old_content = file.read_text(encoding="utf-8")
+                new_content = _remove_outdated(
+                    content=old_content, cleanup_regexes=cleanup_regexes
+                )
+                if new_content:
+                    file.write_text(new_content, encoding="utf-8")
+                    print(f"{file}: Cleaned up!")
             # replace found header and write it to file
             if copyright_match and (
                 not copyright_match["modification_year"]
@@ -201,10 +254,9 @@ def _update_file(
                     f"{parsed_args.year}"
                 )
 
-                return 0
             else:
                 print(f"{file}: License Header is ok.")
-                return 0
+            return 0
     except FileNotFoundError as e:
         print(f"{file}: File is not existing.")
         raise e
@@ -335,9 +387,24 @@ def _parse_args(args=None):
         type=FileType("r"),
     )
 
+    parser.add_argument(
+        "--cleanup",
+        action="store_true",
+        default=False,
+        help="Do a cleanup: Remove lines from outdated header format",
+    )
+
     return parser.parse_args(args)
 
 
+def _compile_outdated_regex() -> List[re.Pattern]:
+    """prepare regex patterns to remove old copyright lines"""
+    regexes: List[re.Pattern] = []
+    for line in OLD_LINES:
+        regexes.append(re.compile(rf"^(([#*]|//) ?)?{line}"))
+    return regexes
+
+
 def main() -> None:
     parsed_args = _parse_args()
     exclude_list = []
@@ -376,18 +443,25 @@ def main() -> None:
         term.error("Specify files to update!")
         sys.exit(1)
 
-    regex = re.compile(
+    regex: re.Pattern = re.compile(
         "(SPDX-FileCopyrightText:|[Cc]opyright).*?(19[0-9]{2}|20[0-9]{2}) "
         f"?-? ?(19[0-9]{{2}}|20[0-9]{{2}})? ({parsed_args.company})"
     )
+    cleanup_regexes: Optional[List[re.Pattern]] = None
+    if parsed_args.cleanup:
+        cleanup_regexes = _compile_outdated_regex()
 
     for file in files:
         try:
             if file.absolute() in exclude_list:
                 term.warning(f"{file}: Ignoring file from exclusion list.")
             else:
                 _update_file(
-                    file=file, regex=regex, parsed_args=parsed_args, term=term
+                    file=file,
+                    regex=regex,
+                    parsed_args=parsed_args,
+                    term=term,
+                    cleanup_regexes=cleanup_regexes,
                 )
         except (FileNotFoundError, UnicodeDecodeError, ValueError):
             continue

tests/updateheader/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2022 Greenbone AG
+# SPDX-FileCopyrightText: 2020-2023 Greenbone AG
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 #

tests/updateheader/test_header.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2022 Greenbone AG
+# SPDX-FileCopyrightText: 2020-2023 Greenbone AG
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
@@ -28,6 +28,9 @@
 
 from pontos.terminal.terminal import ConsoleTerminal
 from pontos.updateheader.updateheader import _add_header as add_header
+from pontos.updateheader.updateheader import (
+    _compile_outdated_regex as compile_outdated_regex,
+)
 from pontos.updateheader.updateheader import _find_copyright as find_copyright
 from pontos.updateheader.updateheader import (
     _get_exclude_list as get_exclude_list,
@@ -36,6 +39,7 @@
     _get_modified_year as get_modified_year,
 )
 from pontos.updateheader.updateheader import _parse_args as parse_args
+from pontos.updateheader.updateheader import _remove_outdated as remove_outdated
 from pontos.updateheader.updateheader import _update_file as update_file
 from pontos.updateheader.updateheader import main
 
@@ -467,6 +471,7 @@ def test_main(self, argparser_mock):
         self.args.verbose = 0
         self.args.log_file = None
         self.args.quiet = False
+        self.args.cleanup = False
 
         argparser_mock.return_value = self.args
 
@@ -487,6 +492,7 @@ def test_main_never_happen(self, argparser_mock, mock_stdout):
         self.args.verbose = 0
         self.args.log_file = None
         self.args.quiet = False
+        self.args.cleanup = False
 
         argparser_mock.return_value = self.args
 
@@ -499,3 +505,41 @@ def test_main_never_happen(self, argparser_mock, mock_stdout):
             "Specify files to update!",
             ret,
         )
+
+    def test_remove_outdated(self):
+        test_content = """* This program is free software: you can redistribute it and/or modify
+*it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+//License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#but WITHOUT ANY WARRANTY; without even the implied warranty of
+# modify it under the terms of the GNU General Public License
+# This program is free software; you can redistribute it and/or
+# version 2 as published by the Free Software Foundation.
+This program is free software: you can redistribute it and/or modify"""  # noqa: E501
+
+        compiled_regexes = compile_outdated_regex()
+
+        new_content = remove_outdated(
+            content=test_content, cleanup_regexes=compiled_regexes
+        )
+        self.assertEqual(new_content, "")
+
+    def test_remove_outdated2(self):
+        test_content = """the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+* GNU General Public License for more details.
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -*- coding: utf-8 -*-
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA."""  # noqa: E501
+
+        compiled_regexes = compile_outdated_regex()
+
+        new_content = remove_outdated(
+            content=test_content, cleanup_regexes=compiled_regexes
+        )
+        self.assertEqual(new_content, "")