Add slfo-packagelist-uploader and a adjusted check_source

.github/workflows/ci-test.yml

@@ -83,7 +83,7 @@ jobs:
         run: |
           for f in $(find . -maxdepth 1 -type f -executable -print); do
               # skip completely broken scripts or those without --help
-              [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists " =~ "$f" ]] || "$f" --help
+              [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists ./slfo-packagelist-uploader.py" =~ "$f" ]] || "$f" --help
           done
 
   linters:

check_source.py

@@ -23,6 +23,8 @@
 from osclib.core import package_kind
 from osclib.core import create_add_role_request
 from osclib.core import package_role_expand
+from osclib.core import source_file_load
+from osclib.core import project_pseudometa_package
 from osc.core import show_package_meta, show_project_meta
 from osc.core import get_request_list
 from urllib.error import HTTPError
@@ -61,6 +63,7 @@ def target_project_config(self, project: str) -> None:
         self.devel_baseproject: str = config.get('check-source-devel-baseproject', '')
         self.allow_source_in_sle = str2bool(config.get('check-source-allow-source-in-sle', 'True'))
         self.sle_project_to_check = config.get('check-source-sle-project', '')
+        self.slfo_packagelist_to_check = config.get('check-source-slfo-packagelist-file', '')
         self.allow_valid_source_origin = str2bool(config.get('check-source-allow-valid-source-origin', 'False'))
         self.valid_source_origins: Set[str] = set(config.get('check-source-valid-source-origins', '').split(' '))
         self.add_devel_project_review = str2bool(config.get('check-source-add-devel-project-review', 'False'))
@@ -147,11 +150,23 @@ def check_source_submission(
             self.review_messages['declined'] = f'May not modify a non-source package of type {kind}'
             return False
 
-        if not self.allow_source_in_sle and self.sle_project_to_check:
-            if entity_exists(self.apiurl, self.sle_project_to_check, target_package):
+        if not self.allow_source_in_sle:
+            if self.sle_project_to_check and entity_exists(self.apiurl, self.sle_project_to_check, target_package):
                 self.review_messages['declined'] = ("SLE-base package, please submit to the corresponding SLE project."
                                                     "Or let us know the reason why needs to rebuild SLE-base package.")
                 return False
+            if self.slfo_packagelist_to_check:
+                pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, target_project)
+                if pseudometa_project and pseudometa_package:
+                    metafile = ET.fromstring(source_file_load(self.apiurl, pseudometa_project, pseudometa_package,
+                                                              self.slfo_packagelist_to_check))
+                    slfo_pkglist = [package.attrib['name'] for package in metafile.findall('package')]
+                    if target_package in slfo_pkglist:
+                        self.review_messages['declined'] = ("Please create a new feature request "
+                                                            f"https://code.opensuse.org/leap/features/issues for updating {target_package} "
+                                                            "from SLFO (SLES, SL Micro). Alternatively, please provide "
+                                                            "a reason for forking and rebuilding an existing SLFO package in Leap.")
+                        return False
 
         if self.ensure_source_exist_in_baseproject and self.devel_baseproject:
             if not entity_exists(self.apiurl, self.devel_baseproject, target_package) and source_project not in self.valid_source_origins:

dist/ci/testenv-tumbleweed/Dockerfile

@@ -10,7 +10,7 @@ RUN zypper in -y osc python3-pytest python3-httpretty python3-pyxdg python3-PyYA
   python3-influxdb python3-pytest-cov libxml2-tools curl python3-flake8 python3-requests \
   shadow vim vim-data strace git sudo patch unzip which cpio gawk openSUSE-release openSUSE-release-ftp \
   perl-Net-SSLeay perl-Text-Diff perl-XML-Simple perl-XML-Parser build \
-  obs-service-download_files obs-service-format_spec_file obs-scm-bridge
+  obs-service-download_files obs-service-format_spec_file obs-scm-bridge python3-GitPython
 RUN useradd tester -d /code/tests/home
 
 COPY run_as_tester /usr/bin

dist/package/openSUSE-release-tools.spec

@@ -238,6 +238,10 @@ Requires:       osclib = %{version}
 Requires:       python3-requests
 Requires:       python3-solv
 Requires:       zstd
+%if 0%{?suse_version} > 1500
+# slfo-packagelist-uploader.py
+Requires:       python3-GitPython
+%endif
 # we use the same user as repo-checker
 PreReq:         openSUSE-release-tools-repo-checker
 BuildArch:      noarch
@@ -316,6 +320,12 @@ OSC plugin for the staging workflow, see `osc staging --help`.
 
 %prep
 %setup -q
+# slfo-packagelist-uploader requires python-GitPython but 15.6 providing
+# python311-GitPython only, therefore do not ship slfo-packagelist-uploader
+# in 15.6 since python3-GitPython is not available
+%if 0%{?suse_version} <= 1500
+rm slfo-packagelist-uploader.py
+%endif
 
 %build
 %make_build
@@ -428,6 +438,7 @@ exit 0
 %exclude %{_datadir}/%{source_dir}/project-installcheck.py
 %exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py
 %exclude %{_datadir}/%{source_dir}/skippkg-finder.py
+%exclude %{_datadir}/%{source_dir}/slfo-packagelist-uploader.py
 %exclude %{_datadir}/%{source_dir}/osclib
 %exclude %{_datadir}/%{source_dir}/osc-cycle.py
 %exclude %{_datadir}/%{source_dir}/osc-origin.py
@@ -456,6 +467,10 @@ exit 0
 
 %files check-source
 %{_bindir}/osrt-check_source
+%if 0%{?suse_version} > 1500
+%{_bindir}/osrt-slfo-packagelist-uploader
+%{_datadir}/%{source_dir}/slfo-packagelist-uploader.py
+%endif
 %{_datadir}/%{source_dir}/check_source.py
 
 %files docker-publisher

gocd/checkers.opensuse.gocd.yaml

@@ -177,4 +177,23 @@ pipelines:
         tasks:
         - script: ./devel-project.py reviews --min-age 3 --remind
         - script: ./devel-project.py requests --min-age 3 --remind
-
+  openSUSE.SLFO.Packagelist.Uploader.Leap_16_0:
+    group: openSUSE.Checkers
+    lock_behavior: unlockWhenFinished
+    timer:
+      spec: 0 0 4 ? * *
+    environment_variables:
+      OSC_CONFIG: /home/go/config/oscrc-factory-maintainer
+    materials:
+      git:
+        git: https://github.com/openSUSE/openSUSE-release-tools.git
+    stages:
+    - Run:
+        approval:
+          type: manual
+        jobs:
+          Run:
+            resources:
+            - repo-checker
+            tasks:
+              - script: ./slfo-packagelist-uploader.py --project openSUSE:Leap:16.0

slfo-packagelist-uploader.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+import argparse
+import logging
+import sys
+import os
+import shutil
+import git
+
+from lxml import etree as ET
+
+import osc.conf
+from osclib.core import source_file_ensure
+from osclib.core import project_pseudometa_package
+
+META_FILE = 'SLFO_Packagelist.group'
+
+
+class PackagelistUploader(object):
+    def __init__(self, project, print_only, verbose):
+        self.project = project
+        self.print_only = print_only
+        self.verbose = verbose
+        self.apiurl = osc.conf.config['apiurl']
+        self.debug = osc.conf.config['debug']
+
+    def create_packagelist(self, packages=[]):
+        packagelist = ET.Element('packagelist')
+        for pkg in sorted(packages):
+            if not self.print_only and self.verbose:
+                print(pkg)
+            attr = {'name': pkg}
+            ET.SubElement(packagelist, 'package', attr)
+
+        return ET.tostring(packagelist, pretty_print=True, encoding='unicode')
+
+    def crawl(self):
+        """Main method"""
+        cwd = os.getcwd()
+        filepath = os.path.join(cwd, 'SLFO_main')
+        if os.path.isdir(filepath):
+            shutil.rmtree(filepath)
+        gitpath = 'https://src.opensuse.org/products/SLFO_main.git'
+        git.Repo.clone_from(gitpath, filepath, branch='main')
+        packages = []
+        files = os.listdir(filepath)
+        for f in files:
+            if f.startswith("."):
+                continue
+            fullpath = os.path.join(filepath, f)
+            if os.path.isdir(fullpath):
+                packages.append(f)
+
+        slfo_packagelist = self.create_packagelist(packages)
+        pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, self.project)
+        if not self.print_only:
+            source_file_ensure(self.apiurl, pseudometa_project, pseudometa_package, META_FILE,
+                               slfo_packagelist, 'Update SLFO packagelist')
+        else:
+            print(slfo_packagelist)
+
+
+def main(args):
+    osc.conf.get_config(override_apiurl=args.apiurl)
+    osc.conf.config['debug'] = args.debug
+
+    if args.project is None:
+        print("Please pass --project argument. See usage with --help.")
+        quit()
+
+    uc = PackagelistUploader(args.project, args.print_only, args.verbose)
+    uc.crawl()
+
+
+if __name__ == '__main__':
+    description = 'Upload SLFO packagelist from src.opensuse.org to staging meta package.'
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
+    parser.add_argument('-d', '--debug', action='store_true',
+                        help='print info useful for debuging')
+    parser.add_argument('-p', '--project', dest='project', metavar='PROJECT',
+                        help='Target project on buildservice')
+    parser.add_argument('-o', '--print-only', action='store_true',
+                        help='show the result instead of the uploading')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='show the diff')
+
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.debug
+                        else logging.INFO)
+
+    sys.exit(main(args))