Skip to content

Commit

Permalink
Merge pull request #3158 from nilxam/slfo_check_source
Browse files Browse the repository at this point in the history
Add slfo-packagelist-uploader and a adjusted check_source
  • Loading branch information
nilxam committed Sep 11, 2024
2 parents 9054fef + 262f0f1 commit 44779bb
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 5 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
run: |
for f in $(find . -maxdepth 1 -type f -executable -print); do
# skip completely broken scripts or those without --help
[[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists " =~ "$f" ]] || "$f" --help
[[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists ./slfo-packagelist-uploader.py" =~ "$f" ]] || "$f" --help
done
linters:
Expand Down
19 changes: 17 additions & 2 deletions check_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
from osclib.core import package_kind
from osclib.core import create_add_role_request
from osclib.core import package_role_expand
from osclib.core import source_file_load
from osclib.core import project_pseudometa_package
from osc.core import show_package_meta, show_project_meta
from osc.core import get_request_list
from urllib.error import HTTPError
Expand Down Expand Up @@ -61,6 +63,7 @@ def target_project_config(self, project: str) -> None:
self.devel_baseproject: str = config.get('check-source-devel-baseproject', '')
self.allow_source_in_sle = str2bool(config.get('check-source-allow-source-in-sle', 'True'))
self.sle_project_to_check = config.get('check-source-sle-project', '')
self.slfo_packagelist_to_check = config.get('check-source-slfo-packagelist-file', '')
self.allow_valid_source_origin = str2bool(config.get('check-source-allow-valid-source-origin', 'False'))
self.valid_source_origins: Set[str] = set(config.get('check-source-valid-source-origins', '').split(' '))
self.add_devel_project_review = str2bool(config.get('check-source-add-devel-project-review', 'False'))
Expand Down Expand Up @@ -147,11 +150,23 @@ def check_source_submission(
self.review_messages['declined'] = f'May not modify a non-source package of type {kind}'
return False

if not self.allow_source_in_sle and self.sle_project_to_check:
if entity_exists(self.apiurl, self.sle_project_to_check, target_package):
if not self.allow_source_in_sle:
if self.sle_project_to_check and entity_exists(self.apiurl, self.sle_project_to_check, target_package):
self.review_messages['declined'] = ("SLE-base package, please submit to the corresponding SLE project."
"Or let us know the reason why needs to rebuild SLE-base package.")
return False
if self.slfo_packagelist_to_check:
pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, target_project)
if pseudometa_project and pseudometa_package:
metafile = ET.fromstring(source_file_load(self.apiurl, pseudometa_project, pseudometa_package,
self.slfo_packagelist_to_check))
slfo_pkglist = [package.attrib['name'] for package in metafile.findall('package')]
if target_package in slfo_pkglist:
self.review_messages['declined'] = ("Please create a new feature request "
f"https://code.opensuse.org/leap/features/issues for updating {target_package} "
"from SLFO (SLES, SL Micro). Alternatively, please provide "
"a reason for forking and rebuilding an existing SLFO package in Leap.")
return False

if self.ensure_source_exist_in_baseproject and self.devel_baseproject:
if not entity_exists(self.apiurl, self.devel_baseproject, target_package) and source_project not in self.valid_source_origins:
Expand Down
2 changes: 1 addition & 1 deletion dist/ci/testenv-tumbleweed/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ RUN zypper in -y osc python3-pytest python3-httpretty python3-pyxdg python3-PyYA
python3-influxdb python3-pytest-cov libxml2-tools curl python3-flake8 python3-requests \
shadow vim vim-data strace git sudo patch unzip which cpio gawk openSUSE-release openSUSE-release-ftp \
perl-Net-SSLeay perl-Text-Diff perl-XML-Simple perl-XML-Parser build \
obs-service-download_files obs-service-format_spec_file obs-scm-bridge
obs-service-download_files obs-service-format_spec_file obs-scm-bridge python3-GitPython
RUN useradd tester -d /code/tests/home

COPY run_as_tester /usr/bin
Expand Down
15 changes: 15 additions & 0 deletions dist/package/openSUSE-release-tools.spec
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,10 @@ Requires: osclib = %{version}
Requires: python3-requests
Requires: python3-solv
Requires: zstd
%if 0%{?suse_version} > 1500
# slfo-packagelist-uploader.py
Requires: python3-GitPython
%endif
# we use the same user as repo-checker
PreReq: openSUSE-release-tools-repo-checker
BuildArch: noarch
Expand Down Expand Up @@ -316,6 +320,12 @@ OSC plugin for the staging workflow, see `osc staging --help`.

%prep
%setup -q
# slfo-packagelist-uploader requires python-GitPython but 15.6 providing
# python311-GitPython only, therefore do not ship slfo-packagelist-uploader
# in 15.6 since python3-GitPython is not available
%if 0%{?suse_version} <= 1500
rm slfo-packagelist-uploader.py
%endif

%build
%make_build
Expand Down Expand Up @@ -428,6 +438,7 @@ exit 0
%exclude %{_datadir}/%{source_dir}/project-installcheck.py
%exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py
%exclude %{_datadir}/%{source_dir}/skippkg-finder.py
%exclude %{_datadir}/%{source_dir}/slfo-packagelist-uploader.py
%exclude %{_datadir}/%{source_dir}/osclib
%exclude %{_datadir}/%{source_dir}/osc-cycle.py
%exclude %{_datadir}/%{source_dir}/osc-origin.py
Expand Down Expand Up @@ -456,6 +467,10 @@ exit 0

%files check-source
%{_bindir}/osrt-check_source
%if 0%{?suse_version} > 1500
%{_bindir}/osrt-slfo-packagelist-uploader
%{_datadir}/%{source_dir}/slfo-packagelist-uploader.py
%endif
%{_datadir}/%{source_dir}/check_source.py

%files docker-publisher
Expand Down
21 changes: 20 additions & 1 deletion gocd/checkers.opensuse.gocd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -177,4 +177,23 @@ pipelines:
tasks:
- script: ./devel-project.py reviews --min-age 3 --remind
- script: ./devel-project.py requests --min-age 3 --remind

openSUSE.SLFO.Packagelist.Uploader.Leap_16_0:
group: openSUSE.Checkers
lock_behavior: unlockWhenFinished
timer:
spec: 0 0 4 ? * *
environment_variables:
OSC_CONFIG: /home/go/config/oscrc-factory-maintainer
materials:
git:
git: https://github.com/openSUSE/openSUSE-release-tools.git
stages:
- Run:
approval:
type: manual
jobs:
Run:
resources:
- repo-checker
tasks:
- script: ./slfo-packagelist-uploader.py --project openSUSE:Leap:16.0
93 changes: 93 additions & 0 deletions slfo-packagelist-uploader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/usr/bin/python3

import argparse
import logging
import sys
import os
import shutil
import git

from lxml import etree as ET

import osc.conf
from osclib.core import source_file_ensure
from osclib.core import project_pseudometa_package

META_FILE = 'SLFO_Packagelist.group'


class PackagelistUploader(object):
def __init__(self, project, print_only, verbose):
self.project = project
self.print_only = print_only
self.verbose = verbose
self.apiurl = osc.conf.config['apiurl']
self.debug = osc.conf.config['debug']

def create_packagelist(self, packages=[]):
packagelist = ET.Element('packagelist')
for pkg in sorted(packages):
if not self.print_only and self.verbose:
print(pkg)
attr = {'name': pkg}
ET.SubElement(packagelist, 'package', attr)

return ET.tostring(packagelist, pretty_print=True, encoding='unicode')

def crawl(self):
"""Main method"""
cwd = os.getcwd()
filepath = os.path.join(cwd, 'SLFO_main')
if os.path.isdir(filepath):
shutil.rmtree(filepath)
gitpath = 'https://src.opensuse.org/products/SLFO_main.git'
git.Repo.clone_from(gitpath, filepath, branch='main')
packages = []
files = os.listdir(filepath)
for f in files:
if f.startswith("."):
continue
fullpath = os.path.join(filepath, f)
if os.path.isdir(fullpath):
packages.append(f)

slfo_packagelist = self.create_packagelist(packages)
pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, self.project)
if not self.print_only:
source_file_ensure(self.apiurl, pseudometa_project, pseudometa_package, META_FILE,
slfo_packagelist, 'Update SLFO packagelist')
else:
print(slfo_packagelist)


def main(args):
osc.conf.get_config(override_apiurl=args.apiurl)
osc.conf.config['debug'] = args.debug

if args.project is None:
print("Please pass --project argument. See usage with --help.")
quit()

uc = PackagelistUploader(args.project, args.print_only, args.verbose)
uc.crawl()


if __name__ == '__main__':
description = 'Upload SLFO packagelist from src.opensuse.org to staging meta package.'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
parser.add_argument('-d', '--debug', action='store_true',
help='print info useful for debuging')
parser.add_argument('-p', '--project', dest='project', metavar='PROJECT',
help='Target project on buildservice')
parser.add_argument('-o', '--print-only', action='store_true',
help='show the result instead of the uploading')
parser.add_argument('-v', '--verbose', action='store_true',
help='show the diff')

args = parser.parse_args()

logging.basicConfig(level=logging.DEBUG if args.debug
else logging.INFO)

sys.exit(main(args))

0 comments on commit 44779bb

Please sign in to comment.