From 729cf7d08ec56ee5e5b6c5e8389d2da04b81aad3 Mon Sep 17 00:00:00 2001 From: Max Lin Date: Tue, 10 Sep 2024 16:01:54 +0800 Subject: [PATCH 1/3] check_source: decline the request if target package exist in SLFO project --- check_source.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/check_source.py b/check_source.py index cc3489e78..a1137727e 100755 --- a/check_source.py +++ b/check_source.py @@ -23,6 +23,8 @@ from osclib.core import package_kind from osclib.core import create_add_role_request from osclib.core import package_role_expand +from osclib.core import source_file_load +from osclib.core import project_pseudometa_package from osc.core import show_package_meta, show_project_meta from osc.core import get_request_list from urllib.error import HTTPError @@ -61,6 +63,7 @@ def target_project_config(self, project: str) -> None: self.devel_baseproject: str = config.get('check-source-devel-baseproject', '') self.allow_source_in_sle = str2bool(config.get('check-source-allow-source-in-sle', 'True')) self.sle_project_to_check = config.get('check-source-sle-project', '') + self.slfo_packagelist_to_check = config.get('check-source-slfo-packagelist-file', '') self.allow_valid_source_origin = str2bool(config.get('check-source-allow-valid-source-origin', 'False')) self.valid_source_origins: Set[str] = set(config.get('check-source-valid-source-origins', '').split(' ')) self.add_devel_project_review = str2bool(config.get('check-source-add-devel-project-review', 'False')) @@ -147,11 +150,23 @@ def check_source_submission( self.review_messages['declined'] = f'May not modify a non-source package of type {kind}' return False - if not self.allow_source_in_sle and self.sle_project_to_check: - if entity_exists(self.apiurl, self.sle_project_to_check, target_package): + if not self.allow_source_in_sle: + if self.sle_project_to_check and entity_exists(self.apiurl, self.sle_project_to_check, target_package): self.review_messages['declined'] = ("SLE-base package, please submit to the corresponding SLE project." "Or let us know the reason why needs to rebuild SLE-base package.") return False + if self.slfo_packagelist_to_check: + pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, target_project) + if pseudometa_project and pseudometa_package: + metafile = ET.fromstring(source_file_load(self.apiurl, pseudometa_project, pseudometa_package, + self.slfo_packagelist_to_check)) + slfo_pkglist = [package.attrib['name'] for package in metafile.findall('package')] + if target_package in slfo_pkglist: + self.review_messages['declined'] = ("Please create a new feature request " + f"https://code.opensuse.org/leap/features/issues for updating {target_package} " + "from SLFO (SLES, SL Micro). Alternatively, please provide " + "a reason for forking and rebuilding an existing SLFO package in Leap.") + return False if self.ensure_source_exist_in_baseproject and self.devel_baseproject: if not entity_exists(self.apiurl, self.devel_baseproject, target_package) and source_project not in self.valid_source_origins: From a6cd8cb3453d3463c76d6cec808b1766ead939e9 Mon Sep 17 00:00:00 2001 From: Max Lin Date: Tue, 10 Sep 2024 15:57:34 +0800 Subject: [PATCH 2/3] Add slfo-packagelist-uploader.py for uploading SLFO packagelist to meta package slfo-packagelist-uploader.py is be able to checkout pacakgelist from src.opensue.org/products/SLFO_main and uplaoding the packagelist to project's pseudometa package. --- .github/workflows/ci-test.yml | 2 +- dist/ci/testenv-tumbleweed/Dockerfile | 2 +- dist/package/openSUSE-release-tools.spec | 15 ++++ slfo-packagelist-uploader.py | 93 ++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100755 slfo-packagelist-uploader.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b4471a9c3..0db95a8af 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -83,7 +83,7 @@ jobs: run: | for f in $(find . -maxdepth 1 -type f -executable -print); do # skip completely broken scripts or those without --help - [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists " =~ "$f" ]] || "$f" --help + [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists ./slfo-packagelist-uploader.py" =~ "$f" ]] || "$f" --help done linters: diff --git a/dist/ci/testenv-tumbleweed/Dockerfile b/dist/ci/testenv-tumbleweed/Dockerfile index 88c896963..4348f754e 100644 --- a/dist/ci/testenv-tumbleweed/Dockerfile +++ b/dist/ci/testenv-tumbleweed/Dockerfile @@ -10,7 +10,7 @@ RUN zypper in -y osc python3-pytest python3-httpretty python3-pyxdg python3-PyYA python3-influxdb python3-pytest-cov libxml2-tools curl python3-flake8 python3-requests \ shadow vim vim-data strace git sudo patch unzip which cpio gawk openSUSE-release openSUSE-release-ftp \ perl-Net-SSLeay perl-Text-Diff perl-XML-Simple perl-XML-Parser build \ - obs-service-download_files obs-service-format_spec_file obs-scm-bridge + obs-service-download_files obs-service-format_spec_file obs-scm-bridge python3-GitPython RUN useradd tester -d /code/tests/home COPY run_as_tester /usr/bin diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 0de5f9cef..7eb220082 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -238,6 +238,10 @@ Requires: osclib = %{version} Requires: python3-requests Requires: python3-solv Requires: zstd +%if 0%{?suse_version} > 1500 +# slfo-packagelist-uploader.py +Requires: python3-GitPython +%endif # we use the same user as repo-checker PreReq: openSUSE-release-tools-repo-checker BuildArch: noarch @@ -316,6 +320,12 @@ OSC plugin for the staging workflow, see `osc staging --help`. %prep %setup -q +# slfo-packagelist-uploader requires python-GitPython but 15.6 providing +# python311-GitPython only, therefore do not ship slfo-packagelist-uploader +# in 15.6 since python3-GitPython is not available +%if 0%{?suse_version} <= 1500 +rm slfo-packagelist-uploader.py +%endif %build %make_build @@ -428,6 +438,7 @@ exit 0 %exclude %{_datadir}/%{source_dir}/project-installcheck.py %exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py %exclude %{_datadir}/%{source_dir}/skippkg-finder.py +%exclude %{_datadir}/%{source_dir}/slfo-packagelist-uploader.py %exclude %{_datadir}/%{source_dir}/osclib %exclude %{_datadir}/%{source_dir}/osc-cycle.py %exclude %{_datadir}/%{source_dir}/osc-origin.py @@ -456,6 +467,10 @@ exit 0 %files check-source %{_bindir}/osrt-check_source +%if 0%{?suse_version} > 1500 +%{_bindir}/osrt-slfo-packagelist-uploader +%{_datadir}/%{source_dir}/slfo-packagelist-uploader.py +%endif %{_datadir}/%{source_dir}/check_source.py %files docker-publisher diff --git a/slfo-packagelist-uploader.py b/slfo-packagelist-uploader.py new file mode 100755 index 000000000..a1cfd1b34 --- /dev/null +++ b/slfo-packagelist-uploader.py @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +import argparse +import logging +import sys +import os +import shutil +import git + +from lxml import etree as ET + +import osc.conf +from osclib.core import source_file_ensure +from osclib.core import project_pseudometa_package + +META_FILE = 'SLFO_Packagelist.group' + + +class PackagelistUploader(object): + def __init__(self, project, print_only, verbose): + self.project = project + self.print_only = print_only + self.verbose = verbose + self.apiurl = osc.conf.config['apiurl'] + self.debug = osc.conf.config['debug'] + + def create_packagelist(self, packages=[]): + packagelist = ET.Element('packagelist') + for pkg in sorted(packages): + if not self.print_only and self.verbose: + print(pkg) + attr = {'name': pkg} + ET.SubElement(packagelist, 'package', attr) + + return ET.tostring(packagelist, pretty_print=True, encoding='unicode') + + def crawl(self): + """Main method""" + cwd = os.getcwd() + filepath = os.path.join(cwd, 'SLFO_main') + if os.path.isdir(filepath): + shutil.rmtree(filepath) + gitpath = 'https://src.opensuse.org/products/SLFO_main.git' + git.Repo.clone_from(gitpath, filepath, branch='main') + packages = [] + files = os.listdir(filepath) + for f in files: + if f.startswith("."): + continue + fullpath = os.path.join(filepath, f) + if os.path.isdir(fullpath): + packages.append(f) + + slfo_packagelist = self.create_packagelist(packages) + pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, self.project) + if not self.print_only: + source_file_ensure(self.apiurl, pseudometa_project, pseudometa_package, META_FILE, + slfo_packagelist, 'Update SLFO packagelist') + else: + print(slfo_packagelist) + + +def main(args): + osc.conf.get_config(override_apiurl=args.apiurl) + osc.conf.config['debug'] = args.debug + + if args.project is None: + print("Please pass --project argument. See usage with --help.") + quit() + + uc = PackagelistUploader(args.project, args.print_only, args.verbose) + uc.crawl() + + +if __name__ == '__main__': + description = 'Upload SLFO packagelist from src.opensuse.org to staging meta package.' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL') + parser.add_argument('-d', '--debug', action='store_true', + help='print info useful for debuging') + parser.add_argument('-p', '--project', dest='project', metavar='PROJECT', + help='Target project on buildservice') + parser.add_argument('-o', '--print-only', action='store_true', + help='show the result instead of the uploading') + parser.add_argument('-v', '--verbose', action='store_true', + help='show the diff') + + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug + else logging.INFO) + + sys.exit(main(args)) From 262f0f1b11a79ff2a8ccc4c74b85378a9780323a Mon Sep 17 00:00:00 2001 From: Max Lin Date: Wed, 11 Sep 2024 15:13:12 +0800 Subject: [PATCH 3/3] gocd: run slfo-packagelist-uploader.py once a day at 4AM --- gocd/checkers.opensuse.gocd.yaml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/gocd/checkers.opensuse.gocd.yaml b/gocd/checkers.opensuse.gocd.yaml index 8c613036a..fb0865ffe 100644 --- a/gocd/checkers.opensuse.gocd.yaml +++ b/gocd/checkers.opensuse.gocd.yaml @@ -177,4 +177,23 @@ pipelines: tasks: - script: ./devel-project.py reviews --min-age 3 --remind - script: ./devel-project.py requests --min-age 3 --remind - + openSUSE.SLFO.Packagelist.Uploader.Leap_16_0: + group: openSUSE.Checkers + lock_behavior: unlockWhenFinished + timer: + spec: 0 0 4 ? * * + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-factory-maintainer + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + stages: + - Run: + approval: + type: manual + jobs: + Run: + resources: + - repo-checker + tasks: + - script: ./slfo-packagelist-uploader.py --project openSUSE:Leap:16.0