From a5f602a3ff247f16f1d8ab7c2b87164430c2e9af Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Thu, 13 Oct 2022 15:25:41 +0200 Subject: [PATCH 01/13] Fixes for osc-1.x --- tests/OBSLocal.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/OBSLocal.py b/tests/OBSLocal.py index 3d838ec4a..a211cfae9 100644 --- a/tests/OBSLocal.py +++ b/tests/OBSLocal.py @@ -71,14 +71,18 @@ def oscrc(userid): with open(OSCRC, 'w+') as f: f.write('\n'.join([ '[general]', - 'apiurl = http://api:3000', + 'apiurl = {}'.format(APIURL), 'http_debug = false', 'debug = false', 'cookiejar = {}'.format(OSCCOOKIEJAR), - '[http://api:3000]', + '[{}]'.format(APIURL), 'user = {}'.format(userid), 'pass = opensuse', 'email = {}@example.com'.format(userid), + # allow plain http even if it is insecure; we're testing after all + 'allow_http = 1', + # disable cert checking to allow self-signed certs + 'sslcertck = 0', '', ])) @@ -87,8 +91,7 @@ def osc_user(self, userid): self.users.append(userid) self.oscrc(userid) - # Rather than modify userid and email, just re-parse entire config and - # reset authentication by clearing opener to avoid edge-cases. + # Rather than modify userid and email, just re-parse entire config self.oscParse() def osc_user_pop(self): @@ -96,16 +99,12 @@ def osc_user_pop(self): self.osc_user(self.users.pop()) def oscParse(self): - # Otherwise, will stick to first user for a given apiurl. - conf._build_opener.last_opener = (None, None) - # Otherwise, will not re-parse same config file. if 'cp' in conf.get_configParser.__dict__: del conf.get_configParser.cp conf.get_config(override_conffile=OSCRC, - override_no_keyring=True, - override_no_gnome_keyring=True) + override_no_keyring=True) os.environ['OSC_CONFIG'] = OSCRC os.environ['OSRT_DISABLE_CACHE'] = 'true' @@ -310,8 +309,7 @@ def __init__(self, project=PROJECT): memoize_session_reset() osc.core.conf.get_config(override_conffile=oscrc, - override_no_keyring=True, - override_no_gnome_keyring=True) + override_no_keyring=True) os.environ['OSC_CONFIG'] = oscrc if os.environ.get('OSC_DEBUG'): From 08240ab45c89dd072badffeafa2a0b230f2eaeec Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Mon, 10 Oct 2022 09:37:23 +0200 Subject: [PATCH 02/13] osclib: Add type annotations --- osclib/core.py | 16 ++++++++++++---- osclib/pkglistgen_comments.py | 27 ++++++++++++++------------- osclib/stagingapi.py | 5 +++-- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/osclib/core.py b/osclib/core.py index fcf72c767..f9ea1db3c 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -6,6 +6,7 @@ import logging from lxml import etree as ET from urllib.error import HTTPError +from typing import Optional from osc.core import create_submit_request from osc.core import get_binarylist @@ -413,7 +414,7 @@ def package_list_kind_filtered(apiurl, project, kinds_allowed=['source']): yield package -def attribute_value_load(apiurl, project, name, namespace='OSRT', package=None): +def attribute_value_load(apiurl: str, project: str, name: str, namespace='OSRT', package: Optional[str] = None): path = list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name])) url = makeurl(apiurl, path) @@ -444,7 +445,14 @@ def attribute_value_load(apiurl, project, name, namespace='OSRT', package=None): # Remember to create for both OBS and IBS as necessary. -def attribute_value_save(apiurl, project, name, value, namespace='OSRT', package=None): +def attribute_value_save( + apiurl: str, + project: str, + name: str, + value: str, + namespace='OSRT', + package: Optional[str] = None +): root = ET.Element('attributes') attribute = ET.SubElement(root, 'attribute') @@ -463,13 +471,13 @@ def attribute_value_save(apiurl, project, name, value, namespace='OSRT', package raise e -def attribute_value_delete(apiurl, project, name, namespace='OSRT', package=None): +def attribute_value_delete(apiurl: str, project: str, name: str, namespace='OSRT', package: Optional[str] = None): http_DELETE(makeurl( apiurl, list(filter(None, ['source', project, package, '_attribute', namespace + ':' + name])))) @memoize(session=True) -def repository_path_expand(apiurl, project, repo, visited_repos=None): +def repository_path_expand(apiurl: str, project: str, repo: str, visited_repos: Optional[set] = None): """Recursively list underlying projects.""" if visited_repos is None: visited_repos = set() diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py index 217c5c39d..bb27a23ce 100644 --- a/osclib/pkglistgen_comments.py +++ b/osclib/pkglistgen_comments.py @@ -5,6 +5,7 @@ import logging import os import sys +from typing import Dict, List, Union, Optional from lxml import etree as ET from osclib.comments import CommentAPI @@ -21,7 +22,7 @@ def __init__(self, apiurl): self.apiurl = apiurl self.comment = CommentAPI(apiurl) - def read_summary_file(self, file): + def read_summary_file(self, file: str) -> Dict[str, List[str]]: ret = dict() with open(file, 'r') as f: for line in f: @@ -30,7 +31,7 @@ def read_summary_file(self, file): ret[pkg].append(group) return ret - def write_summary_file(self, file, content): + def write_summary_file(self, file: str, content: dict): output = [] for pkg in sorted(content): for group in sorted(content[pkg]): @@ -40,7 +41,7 @@ def write_summary_file(self, file, content): for line in sorted(output): f.write(line + '\n') - def calculcate_package_diff(self, old_file, new_file): + def calculcate_package_diff(self, old_file: str, new_file: str): old_file = self.read_summary_file(old_file) new_file = self.read_summary_file(new_file) @@ -102,7 +103,7 @@ def calculcate_package_diff(self, old_file, new_file): return report.strip() - def handle_package_diff(self, project, old_file, new_file): + def handle_package_diff(self, project: str, old_file: str, new_file: str): comments = self.comment.get_comments(project_name=project) comment, _ = self.comment.comment_find(comments, MARKER) @@ -134,7 +135,7 @@ def handle_package_diff(self, project, old_file, new_file): return 1 - def is_approved(self, comment, comments): + def is_approved(self, comment, comments: dict) -> str | None: if not comment: return None @@ -145,7 +146,7 @@ def is_approved(self, comment, comments): return c['who'] return None - def parse_title(self, line): + def parse_title(self, line: str) -> Optional[Dict[str, Union[str, List[str]]]]: m = re.match(r'\*\*Add to (.*)\*\*', line) if m: return {'cmd': 'add', 'to': m.group(1).split(','), 'pkgs': []} @@ -157,7 +158,7 @@ def parse_title(self, line): return {'cmd': 'remove', 'from': m.group(1).split(','), 'pkgs': []} return None - def parse_sections(self, comment): + def parse_sections(self, comment: str) -> List[Dict[str, Union[str, List[str]]]]: current_section = None sections = [] in_quote = False @@ -179,7 +180,7 @@ def parse_sections(self, comment): sections.append(current_section) return sections - def apply_move(self, content, section): + def apply_move(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): for pkg in section['pkgs']: pkg_content = content[pkg] for group in section['from']: @@ -192,12 +193,12 @@ def apply_move(self, content, section): pkg_content.append(group) content[pkg] = pkg_content - def apply_add(self, content, section): + def apply_add(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): for pkg in section['pkgs']: content.setdefault(pkg, []) content[pkg] += section['to'] - def apply_remove(self, content, section): + def apply_remove(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): for pkg in section['pkgs']: pkg_content = content[pkg] for group in section['from']: @@ -208,7 +209,7 @@ def apply_remove(self, content, section): sys.exit(1) content[pkg] = pkg_content - def apply_commands(self, filename, sections): + def apply_commands(self, filename: str, sections: List[Dict[str, Union[str, List[str]]]]): content = self.read_summary_file(filename) for section in sections: if section['cmd'] == 'move': @@ -239,7 +240,7 @@ def format_remove(self, section): text = f" * Remove from {gfrom}:\n" return text + self.format_pkgs(section['pkgs']) - def apply_changes(self, filename, sections, approver): + def apply_changes(self, filename: str, sections: List[Dict[str, Union[str, List[str]]]], approver: str): text = "-------------------------------------------------------------------\n" now = datetime.datetime.utcnow() date = now.strftime("%a %b %d %H:%M:%S UTC %Y") @@ -262,7 +263,7 @@ def apply_changes(self, filename, sections, approver): writer.write(line) os.rename(filename + '.new', filename) - def check_staging_accept(self, project, target): + def check_staging_accept(self, project: str, target: str): comments = self.comment.get_comments(project_name=project) comment, _ = self.comment.comment_find(comments, MARKER) approver = self.is_approved(comment, comments) diff --git a/osclib/stagingapi.py b/osclib/stagingapi.py index e3ded69f8..62abb9df8 100644 --- a/osclib/stagingapi.py +++ b/osclib/stagingapi.py @@ -1,5 +1,6 @@ from io import StringIO from datetime import datetime +from typing import List import dateutil.parser import logging import textwrap @@ -52,7 +53,7 @@ class StagingAPI(object): Class containing various api calls to work with staging projects. """ - def __init__(self, apiurl, project): + def __init__(self, apiurl: str, project: str): """Initialize instance variables.""" self.apiurl = apiurl @@ -149,7 +150,7 @@ def project_has_repo(self, repo_name): xpath = f'repository[@name="{repo_name}"]' return len(meta.xpath(xpath)) > 0 - def makeurl(self, paths, query=None): + def makeurl(self, paths: List[str], query=None) -> str: """ Wrapper around osc's makeurl passing our apiurl :return url made for l and query From d5531d65b2beba7d40554eb13b6ae5563f612b0d Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Tue, 11 Oct 2022 14:59:13 +0200 Subject: [PATCH 03/13] pkglistgen: Add type annotations --- pkglistgen/cli.py | 2 +- pkglistgen/tool.py | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/pkglistgen/cli.py b/pkglistgen/cli.py index 5d82a5232..5ad9218aa 100755 --- a/pkglistgen/cli.py +++ b/pkglistgen/cli.py @@ -86,7 +86,7 @@ def do_update_and_solve(self, subcmd, opts): if apiurl.find('opensuse.org') > 0: os.environ['OBS_NAME'] = 'build.opensuse.org' - def solve_project(project, scope): + def solve_project(project, scope: str): try: self.tool.reset() self.tool.dry_run = self.options.dry diff --git a/pkglistgen/tool.py b/pkglistgen/tool.py index 82cacd39b..d7c6b190a 100644 --- a/pkglistgen/tool.py +++ b/pkglistgen/tool.py @@ -8,6 +8,8 @@ import subprocess import yaml +from typing import Any, Mapping, Optional + from lxml import etree as ET from osc.core import checkout_package @@ -175,7 +177,7 @@ def solve_module(self, groupname, includes, excludes, use_recommends): if package[0] not in g.solved_packages['*']: self.logger.error(f'Missing {package[0]} in {groupname} for {arch}') - def expand_repos(self, project, repo='standard'): + def expand_repos(self, project: str, repo='standard'): return repository_path_expand(self.apiurl, project, repo) def _check_supplements(self): @@ -506,7 +508,13 @@ def create_weakremovers(self, target, target_config, directory, output): print('%endif', file=output) output.flush() - def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None): + def solve_project( + self, + ignore_unresolvable=False, + ignore_recommended=False, + locale: Optional[str] = None, + locales_from: Optional[str] = None + ): self.load_all_groups() if not self.output: self.logger.error('OUTPUT not defined') @@ -601,9 +609,19 @@ def replace_product_version(self, product_file, product_version): new_lines.append(line.replace('', product_version)) open(product_file, 'w').write(''.join(new_lines)) - def update_and_solve_target(self, api, target_project, target_config, main_repo, - project, scope, force, no_checkout, - only_release_packages, stop_after_solve): + def update_and_solve_target( + self, + api, + target_project: str, + target_config: Mapping[str, Any], + main_repo: str, + project: str, + scope: str, + force: bool, + no_checkout: bool, + only_release_packages: bool, + stop_after_solve: bool + ): self.all_architectures = target_config.get('pkglistgen-archs').split(' ') self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False')) self.repos = self.expand_repos(project, main_repo) @@ -682,11 +700,12 @@ def update_and_solve_target(self, api, target_project, target_config, main_repo, self.load_all_groups() self.write_group_stubs() else: - summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')), - ignore_recommended=str2bool( - target_config.get('pkglistgen-ignore-recommended')), - locale=target_config.get('pkglistgen-locale'), - locales_from=target_config.get('pkglistgen-locales-from')) + summary = self.solve_project( + ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')), + ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')), + locale=target_config.get('pkglistgen-locale'), + locales_from=target_config.get('pkglistgen-locales-from') + ) if stop_after_solve: return From f187e9f4870af66b127ff51f87463702c96dec35 Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Tue, 11 Oct 2022 14:50:57 +0200 Subject: [PATCH 04/13] Docs: Fix typos --- docs/pkglistgen.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/pkglistgen.md b/docs/pkglistgen.md index 59736d777..058566579 100644 --- a/docs/pkglistgen.md +++ b/docs/pkglistgen.md @@ -1,25 +1,25 @@ # Package List Generator -pkglistgen.py is a self contained script to generate and update OBS products for openSUSE and SLE. +pkglistgen.py is a self contained script to generate and update OBS products for openSUSE and SLE. It works on the products and its staging projects and ports. The main input is a package named 000package-groups and it will update the content of other packages from that. For that it will read [YAML](https://en.wikipedia.org/wiki/YAML) input from e.g. 000package-groups/groups.yml and generate .group files into 000product. The rest of 000package-groups is copied into 000product as well and it runs the OBS product converter service (See [OBS Documentation](https://en.opensuse.org/openSUSE:Build_Service_product_definition) for details) -The generated release spec files are split into 000release-packages to avoid needless rebuilds. +The generated release spec files are split into 000release-packages to avoid needless rebuilds. ## Input -The package list generator reads several files. The most important are group*.yml (tradionally only groups.yml) within 000package-groups. +The package list generator reads several files. The most important are group*.yml (traditionally only groups.yml) within 000package-groups. ### supportstatus.txt The file lists the packages and their support level. It's only necessary to list packages here that have a different level than the default level specificied in the groups. The format is plain text: - the level is handed over 1:1 to KIWI file. Currently used values are: unsupported, l2 and l3 - + ### group*.yml The file is a list of package lists and the special hash 'OUTPUT'. OUTPUT contains an entry for every group file that needs to be written out. The group name of it needs to exist as package list as well. OUTPUT also contains flags for the groups. We currently support: * default-support - Sets the support level in case there is no explicitly entry in [supportstatus.txt](#supportstatus.txt), defaults to 'unsupported' + Sets the support level in case there is no explicitly entry in [supportstatus.txt](#supportstatustxt), defaults to 'unsupported' * recommends If the solver should take recommends into account when solving the package list, defaults to false. * includes @@ -31,7 +31,7 @@ We currently support: Be aware that group names must not contain a '-'. -You can also adapt the solving on a package level by putting a hash into the package list. Normally the package name is a string, in case it's a hash the key needs to be the package name and the value is a list of following modifiers: +You can also adapt the solving on a package level by putting a hash into the package list. Normally the package name is a string, in case it's a hash the key needs to be the package name and the value is a list of following modifiers: * recommended Evaluate also 'Recommends' in package to determine dependencies. Otherwise only 'required' are considered. Used mainly for patterns in SLE. It can not be combined with platforms, For architecture specific recommends, use patterns. @@ -46,7 +46,7 @@ You can also adapt the solving on a package level by putting a hash into the pac * required If the package is missing or is uninstallable, don't leave a comment but put the error as package entry for OBS to create unresolvable error to avoid building a DVD -Note that you can write yaml lists in 2 ways. You can put the modifier lists as multiple lines starting with -, but it's recommended to put them as [M1,M2] behind the package name. See the difference between pkg4 and pkg5 in the example. +Note that you can write yaml lists in 2 ways. You can put the modifier lists as multiple lines starting with -, but it's recommended to put them as [M1,M2] behind the package name. See the difference between pkg4 and pkg5 in the example. #### Example: @@ -68,32 +68,32 @@ OUTPUT: - group3: includes: - list2 - + group1: - pkg1 - + group2: - pkg2: [locked] - pkg3 - + group3: - pkg4: [x86_64] - + list1: - pkg5: - x86_64 - + list2: - pkg6: [recommended] -``` +``` -## Overlap calculcation - TODO +## Overlap calculation + TODO ## Handling in staging workflow -If 000package-groups contains a file named summary-staging.txt, the bot will trigger a diff mode on staging projects. +If 000package-groups contains a file named summary-staging.txt, the bot will trigger a diff mode on staging projects. It will create an equal summary-staging.txt in 000product and create a comment with a human readable diff in the staging project. This comment can be replied to. If the reply starts with 'approve', staging accept will apply the diff to this txt file. If the reply starts with 'ignore', the bot will continue with the pipeline and do nothing on staging accept. -This way simple changes to the summary can be accepted without a submit request (of which there can only be one at a time). \ No newline at end of file +This way simple changes to the summary can be accepted without a submit request (of which there can only be one at a time). From 2cffb3d749ac20cf393b6f9a5051fdb0a66ec46b Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Thu, 13 Oct 2022 14:09:59 +0200 Subject: [PATCH 05/13] pkglistgen: Convert sections to class --- osclib/pkglistgen_comments.py | 125 ++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py index bb27a23ce..a090a7f97 100644 --- a/osclib/pkglistgen_comments.py +++ b/osclib/pkglistgen_comments.py @@ -1,24 +1,54 @@ import datetime -import textwrap -import re -import tempfile +import enum import logging import os +import re import sys -from typing import Dict, List, Union, Optional +import tempfile +import textwrap +from typing import Dict, List, Optional + from lxml import etree as ET +from osc.core import Package, checkout_package, http_GET, makeurl from osclib.comments import CommentAPI -from osc.core import checkout_package, http_GET, makeurl -from osc.core import Package MARKER = 'PackageListDiff' -class PkglistComments(object): +class PkglistSectionCommend(enum.Enum): + ADD = "add" + REMOVE = "remove" + MOVE = "move" + + +class PkglistSection: + def __init__( + self, + command: PkglistSectionCommend, + pkgs: Optional[List[str]] = None, + to_module: Optional[List[str]] = None, + from_module: Optional[List[str]] = None + ): + self.command = command + if pkgs is None: + self.pkgs = [] + else: + self.pkgs = pkgs + if pkgs is None: + self.to_module = [] + else: + self.to_module = to_module + if pkgs is None: + self.from_module = [] + else: + self.from_module = from_module + + +class PkglistComments: """Handling staging comments of diffs""" - def __init__(self, apiurl): + def __init__(self, apiurl: str): self.apiurl = apiurl self.comment = CommentAPI(apiurl) @@ -146,19 +176,28 @@ def is_approved(self, comment, comments: dict) -> str | None: return c['who'] return None - def parse_title(self, line: str) -> Optional[Dict[str, Union[str, List[str]]]]: + def parse_title(self, line: str) -> Optional[PkglistSection]: m = re.match(r'\*\*Add to (.*)\*\*', line) if m: - return {'cmd': 'add', 'to': m.group(1).split(','), 'pkgs': []} + return PkglistSection(PkglistSectionCommend.ADD, pkgs=[], to_module=m.group(1).split(',')) m = re.match(r'\*\*Move from (.*) to (.*)\*\*', line) if m: - return {'cmd': 'move', 'from': m.group(1).split(','), 'to': m.group(2).split(','), 'pkgs': []} + return PkglistSection( + PkglistSectionCommend.MOVE, + pkgs=[], + from_module=m.group(1).split(','), + to_module=m.group(2).split(','), + ) m = re.match(r'\*\*Remove from (.*)\*\*', line) if m: - return {'cmd': 'remove', 'from': m.group(1).split(','), 'pkgs': []} + return PkglistSection( + PkglistSectionCommend.REMOVE, + pkgs=[], + from_module=m.group(1).split(','), + ) return None - def parse_sections(self, comment: str) -> List[Dict[str, Union[str, List[str]]]]: + def parse_sections(self, comment: str) -> List[PkglistSection]: current_section = None sections = [] in_quote = False @@ -175,33 +214,33 @@ def parse_sections(self, comment: str) -> List[Dict[str, Union[str, List[str]]]] for pkg in line.split(','): pkg = pkg.strip() if pkg: - current_section['pkgs'].append(pkg) + current_section.pkgs.append(pkg) if current_section: sections.append(current_section) return sections - def apply_move(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): - for pkg in section['pkgs']: + def apply_move(self, content: Dict[str, List[str]], section: PkglistSection): + for pkg in section.pkgs: pkg_content = content[pkg] - for group in section['from']: + for group in section.from_module: try: pkg_content.remove(group) except ValueError: logging.error(f"Can't remove {pkg} from {group}, not there. Mismatch.") sys.exit(1) - for group in section['to']: + for group in section.to_module: pkg_content.append(group) content[pkg] = pkg_content - def apply_add(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): - for pkg in section['pkgs']: + def apply_add(self, content: Dict[str, List[str]], section: PkglistSection): + for pkg in section.pkgs: content.setdefault(pkg, []) - content[pkg] += section['to'] + content[pkg] += section.to_module - def apply_remove(self, content: Dict[str, List[str]], section: Dict[str, Union[str, List[str]]]): - for pkg in section['pkgs']: + def apply_remove(self, content: Dict[str, List[str]], section: PkglistSection): + for pkg in section.pkgs: pkg_content = content[pkg] - for group in section['from']: + for group in section.from_module: try: pkg_content.remove(group) except ValueError: @@ -209,38 +248,38 @@ def apply_remove(self, content: Dict[str, List[str]], section: Dict[str, Union[s sys.exit(1) content[pkg] = pkg_content - def apply_commands(self, filename: str, sections: List[Dict[str, Union[str, List[str]]]]): + def apply_commands(self, filename: str, sections: List[PkglistSection]): content = self.read_summary_file(filename) for section in sections: - if section['cmd'] == 'move': + if section.command == PkglistSectionCommend.MOVE: self.apply_move(content, section) - elif section['cmd'] == 'add': + elif section.command == PkglistSectionCommend.ADD: self.apply_add(content, section) - elif section['cmd'] == 'remove': + elif section.command == PkglistSectionCommend.REMOVE: self.apply_remove(content, section) self.write_summary_file(filename, content) - def format_pkgs(self, pkgs): + def format_pkgs(self, pkgs: List[str]): text = ', '.join(pkgs) return " " + "\n ".join(textwrap.wrap(text, width=68, break_long_words=False, break_on_hyphens=False)) + "\n\n" - def format_move(self, section): - gfrom = ','.join(section['from']) - gto = ','.join(section['to']) + def format_move(self, section: PkglistSection): + gfrom = ','.join(section.from_module) + gto = ','.join(section.to_module) text = f" * Move from {gfrom} to {gto}:\n" - return text + self.format_pkgs(section['pkgs']) + return text + self.format_pkgs(section.pkgs) - def format_add(self, section): - gto = ','.join(section['to']) + def format_add(self, section: PkglistSection): + gto = ','.join(section.to_module) text = f" * Add to {gto}:\n" - return text + self.format_pkgs(section['pkgs']) + return text + self.format_pkgs(section.pkgs) - def format_remove(self, section): - gfrom = ','.join(section['from']) + def format_remove(self, section: PkglistSection): + gfrom = ','.join(section.from_module) text = f" * Remove from {gfrom}:\n" - return text + self.format_pkgs(section['pkgs']) + return text + self.format_pkgs(section.pkgs) - def apply_changes(self, filename: str, sections: List[Dict[str, Union[str, List[str]]]], approver: str): + def apply_changes(self, filename: str, sections: List[PkglistSection], approver: str): text = "-------------------------------------------------------------------\n" now = datetime.datetime.utcnow() date = now.strftime("%a %b %d %H:%M:%S UTC %Y") @@ -250,11 +289,11 @@ def apply_changes(self, filename: str, sections: List[Dict[str, Union[str, List[ email = root.find('email').text text += f"{date} - {realname} <{email}>\n\n- Approved changes to summary-staging.txt\n" for section in sections: - if section['cmd'] == 'move': + if section.command == PkglistSectionCommend.MOVE: text += self.format_move(section) - elif section['cmd'] == 'add': + elif section.command == PkglistSectionCommend.ADD: text += self.format_add(section) - elif section['cmd'] == 'remove': + elif section.command == PkglistSectionCommend.REMOVE: text += self.format_remove(section) with open(filename + '.new', 'w') as writer: writer.write(text) From 16caed8aed69ce2fc6664477ecb61a1227a3964a Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Mon, 10 Oct 2022 09:37:44 +0200 Subject: [PATCH 06/13] ignore: venv, vscode and IDEA config directories --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index a5edb1def..82a30ee9f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ .docker-tmp/ .bash_history osc +.venv +.vscode +.idea From c1eb784c7fd54fe3bb235d1b4cb88fc057a94e6b Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Thu, 13 Oct 2022 15:12:54 +0200 Subject: [PATCH 07/13] osclib: Add typehints --- osclib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osclib/core.py b/osclib/core.py index f9ea1db3c..c41219ca2 100644 --- a/osclib/core.py +++ b/osclib/core.py @@ -229,7 +229,7 @@ def binary_src_debug(binary): @memoize(session=True) -def devel_project_get(apiurl, target_project, target_package): +def devel_project_get(apiurl: str, target_project: str, target_package: str): try: meta = ET.fromstringlist(show_package_meta(apiurl, target_project, target_package)) node = meta.find('devel') From 10554d5c31642ad4399c2613ded4cd3a0d89afb0 Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Thu, 13 Oct 2022 15:13:16 +0200 Subject: [PATCH 08/13] osclib: Submit summary-staging.txt to the devel project --- osclib/pkglistgen_comments.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py index a090a7f97..8321b7956 100644 --- a/osclib/pkglistgen_comments.py +++ b/osclib/pkglistgen_comments.py @@ -12,6 +12,7 @@ from osc.core import Package, checkout_package, http_GET, makeurl from osclib.comments import CommentAPI +from osclib.core import devel_project_get MARKER = 'PackageListDiff' @@ -309,8 +310,11 @@ def check_staging_accept(self, project: str, target: str): if not approver: return sections = self.parse_sections(comment['comment']) + project, package = devel_project_get(self.apiurl, target, '000package-groups') + if project is None or package is None: + raise ValueError('Could not determine devel project or package for the "000package-groups"!') with tempfile.TemporaryDirectory() as tmpdirname: - checkout_package(self.apiurl, target, '000package-groups', expand_link=True, outdir=tmpdirname) + checkout_package(self.apiurl, project, package, expand_link=True, outdir=tmpdirname) self.apply_commands(tmpdirname + '/summary-staging.txt', sections) self.apply_changes(tmpdirname + '/package-groups.changes', sections, approver) package = Package(tmpdirname) From d3c4563e9ae22245924bff63e7286768c47cc4e6 Mon Sep 17 00:00:00 2001 From: Enno Gotthold Date: Fri, 14 Oct 2022 15:33:39 +0200 Subject: [PATCH 09/13] osclib: Correct syntax error in typing --- osclib/pkglistgen_comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py index 8321b7956..3e2818f50 100644 --- a/osclib/pkglistgen_comments.py +++ b/osclib/pkglistgen_comments.py @@ -166,7 +166,7 @@ def handle_package_diff(self, project: str, old_file: str, new_file: str): return 1 - def is_approved(self, comment, comments: dict) -> str | None: + def is_approved(self, comment, comments: dict) -> Optional[str]: if not comment: return None From bc832a185c8409f6307d33cc37a16175e3e3848a Mon Sep 17 00:00:00 2001 From: Max Lin Date: Fri, 21 Oct 2022 15:57:22 +0800 Subject: [PATCH 10/13] gocd: add openSUSE:Leap:15.4:Images to pkglistgen pipeline for the Leap 15.4 CR --- gocd/pkglistgen.opensuse.gocd.yaml | 44 +++++++++++++++++++++++++- gocd/pkglistgen.opensuse.gocd.yaml.erb | 34 ++++++++++++++++++-- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/gocd/pkglistgen.opensuse.gocd.yaml b/gocd/pkglistgen.opensuse.gocd.yaml index d9a125e41..b98546d13 100644 --- a/gocd/pkglistgen.opensuse.gocd.yaml +++ b/gocd/pkglistgen.opensuse.gocd.yaml @@ -145,7 +145,28 @@ pipelines: tasks: - script: python3 ./skippkg-finder.py -A https://api.opensuse.org -o openSUSE:Leap:15.3:Update:Respin -t openSUSE:Leap:15.3:Update -s SUSE:SLE-15-SP3:Update - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p openSUSE:Leap:15.3:Update:Respin -s target - + Pkglistgen.openSUSE_Leap_15.4_CR: + group: Leap + lock_behavior: unlockWhenFinished + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + timer: + spec: 0 40 * ? * * + only_on_changes: false + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + stages: + - pkglistgen: + approval: + type: manual + jobs: + openSUSE_Leap_15.4_Images_target: + resources: + - repo-checker + tasks: + - script: python3 ./skippkg-finder.py -A https://api.opensuse.org -o openSUSE:Leap:15.4:Images -t openSUSE:Leap:15.4:Update -s SUSE:SLE-15-SP4:Update + - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p openSUSE:Leap:15.4:Images -s target Update.Repos.Leap.openSUSE_Leap_15.5: group: Leap lock_behavior: unlockWhenFinished @@ -167,6 +188,27 @@ pipelines: - repo-checker tasks: - script: python3 ./pkglistgen.py --apiurl https://api.opensuse.org handle_update_repos openSUSE:Leap:15.5 + Update.Repos.Leap.openSUSE_Leap_15.4_Images: + group: Leap + lock_behavior: unlockWhenFinished + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + timer: + spec: 0 0 0 */3 * ? + only_on_changes: false + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + stages: + - Update: + approval: + type: manual + jobs: + openSUSE_Leap_15.4_Images: + resources: + - repo-checker + tasks: + - script: python3 ./pkglistgen.py --apiurl https://api.opensuse.org handle_update_repos openSUSE:Leap:15.4:Images Update.Repos.Leap.openSUSE_Leap_15.3_Update_Respin: group: Leap lock_behavior: unlockWhenFinished diff --git a/gocd/pkglistgen.opensuse.gocd.yaml.erb b/gocd/pkglistgen.opensuse.gocd.yaml.erb index d2072b5dd..8048191c6 100644 --- a/gocd/pkglistgen.opensuse.gocd.yaml.erb +++ b/gocd/pkglistgen.opensuse.gocd.yaml.erb @@ -119,8 +119,38 @@ pipelines: - script: python3 ./skippkg-finder.py -A https://api.opensuse.org -o openSUSE:Leap:15.3:Update:Respin -t openSUSE:Leap:15.3:Update -s SUSE:SLE-15-SP3:Update - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p <%= project[0] %><%= options %> <% end -%> - -<% %w(openSUSE:Leap:15.5 openSUSE:Leap:15.3:Update:Respin).each do |project| -%> + Pkglistgen.openSUSE_Leap_15.4_CR: + group: Leap + lock_behavior: unlockWhenFinished + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + timer: + spec: 0 40 * ? * * + only_on_changes: false + materials: + git: + git: https://github.com/openSUSE/openSUSE-release-tools.git + stages: + - pkglistgen: + approval: + type: manual + jobs: +<% ['openSUSE:Leap:15.4:Images/target'].each do |project| + project=project.split('/') + name=project[0].gsub(':', '_') + if project.size > 1 + options=" -s #{project[1]}" + name = name + "_#{project[1]}" + end + -%> + <%= name %>: + resources: + - repo-checker + tasks: + - script: python3 ./skippkg-finder.py -A https://api.opensuse.org -o openSUSE:Leap:15.4:Images -t openSUSE:Leap:15.4:Update -s SUSE:SLE-15-SP4:Update + - script: python3 ./pkglistgen.py -d -A https://api.opensuse.org update_and_solve -p <%= project[0] %><%= options %> +<% end -%> +<% %w(openSUSE:Leap:15.5 openSUSE:Leap:15.4:Images openSUSE:Leap:15.3:Update:Respin).each do |project| -%> Update.Repos.Leap.<%= project.gsub(':', '_') %>: group: Leap lock_behavior: unlockWhenFinished From 523d46cc1399f63f7dd7471a780a72d44d375bc3 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 25 Oct 2022 12:19:54 +0200 Subject: [PATCH 11/13] Don't hard code bash as interpreter for source_validator scripts --- check_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_source.py b/check_source.py index b6883cef3..3e3ed2304 100755 --- a/check_source.py +++ b/check_source.py @@ -585,7 +585,7 @@ def run_source_validator(self, old, directory): for script in scripts: if os.path.isdir(script): continue - res = subprocess.run(['/bin/bash', script, '--batchmode', directory, old], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + res = subprocess.run([script, '--batchmode', directory, old], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if res.returncode: text = "Source validator failed. Try \"osc service runall source_validator\"\n" text += res.stdout.decode('utf-8') From 3a7f30144b8a2e948c417d2a3b816199cbbb07cf Mon Sep 17 00:00:00 2001 From: Frederic Crozat Date: Tue, 25 Oct 2022 09:59:45 +0200 Subject: [PATCH 12/13] add configuration for ALP staging --- gocd/alp-stagings.gocd.yaml | 142 ++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 gocd/alp-stagings.gocd.yaml diff --git a/gocd/alp-stagings.gocd.yaml b/gocd/alp-stagings.gocd.yaml new file mode 100644 index 000000000..8b263ffce --- /dev/null +++ b/gocd/alp-stagings.gocd.yaml @@ -0,0 +1,142 @@ +--- +format_version: 3 +pipelines: + ALP.Stagings.RelPkgs: + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + group: LEO + lock_behavior: unlockWhenFinished + timer: + spec: 0 0 * ? * * + only_on_changes: false + materials: + scripts: + git: https://github.com/openSUSE/openSUSE-release-tools.git + stages: + - Generate.Release.Package: + approval: manual + jobs: + ALP.Staging.A: + resources: + - repo-checker + tasks: + - script: ./pkglistgen.py -A https://api.opensuse.org update_and_solve --staging SUSE:ALP:Staging:A --only-release-packages --force + ALP.Staging.B: + resources: + - repo-checker + tasks: + - script: ./pkglistgen.py -A https://api.opensuse.org update_and_solve --staging SUSE:ALP:Staging:B --only-release-packages --force + + ALP.Staging.A: + environment_variables: + STAGING_PROJECT: SUSE:ALP:Staging:A + STAGING_API: https://api.opensuse.org + OSC_CONFIG: /home/go/config/oscrc-staging-bot + group: LEO + lock_behavior: unlockWhenFinished + materials: + scripts: + auto_update: true + git: https://github.com/openSUSE/openSUSE-release-tools.git + whitelist: + - DO_NOT_TRIGGER + destination: scripts + stages: + - Checks: + jobs: + Check.Build.Succeeds: + resources: + - staging-bot + tasks: + - script: |- + export PYTHONPATH=$PWD/scripts + cd scripts/gocd + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s pending + ./verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r standard + Repo.Checker: + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + resources: + - repo-checker + tasks: + - script: |- + ./scripts/staging-installcheck.py -A $STAGING_API -p SUSE:ALP -s $STAGING_PROJECT + + - Update.000product: + resources: + - repo-checker + tasks: + - script: |- + export PYTHONPATH=$PWD/scripts + cd scripts/gocd + + if ../pkglistgen.py --debug -A $STAGING_API update_and_solve --staging $STAGING_PROJECT --force; then + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s success + else + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s failure + exit 1 + fi + + - Enable.images.repo: + resources: + - staging-bot + tasks: + - script: |- + osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=images&flag=build" + + ALP.Staging.B: + environment_variables: + STAGING_PROJECT: SUSE:ALP:Staging:B + STAGING_API: https://api.opensuse.org + OSC_CONFIG: /home/go/config/oscrc-staging-bot + group: LEO + lock_behavior: unlockWhenFinished + materials: + scripts: + auto_update: true + git: https://github.com/openSUSE/openSUSE-release-tools.git + whitelist: + - DO_NOT_TRIGGER + destination: scripts + stages: + - Checks: + jobs: + Check.Build.Succeeds: + resources: + - staging-bot + tasks: + - script: |- + export PYTHONPATH=$PWD/scripts + cd scripts/gocd + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s pending + ./verify-repo-built-successful.py -A $STAGING_API -p $STAGING_PROJECT -r standard + Repo.Checker: + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-staging-bot + resources: + - repo-checker + tasks: + - script: |- + ./scripts/staging-installcheck.py -A $STAGING_API -p SUSE:ALP -s $STAGING_PROJECT + + - Update.000product: + resources: + - repo-checker + tasks: + - script: |- + export PYTHONPATH=$PWD/scripts + cd scripts/gocd + + if ../pkglistgen.py --debug -A $STAGING_API update_and_solve --staging $STAGING_PROJECT --force; then + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s success + else + ./report-status.py -A $STAGING_API -p $STAGING_PROJECT -n packagelists -r standard -s failure + exit 1 + fi + + - Enable.images.repo: + resources: + - staging-bot + tasks: + - script: |- + osc -A $STAGING_API api -X POST "/source/$STAGING_PROJECT?cmd=remove_flag&repository=images&flag=build" From f32c097f6620ffa88e04c570dd72324efddcc0db Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Tue, 25 Oct 2022 14:35:06 +0200 Subject: [PATCH 13/13] Use assertEqual instead of assertTrue a == b --- tests/comment_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/comment_tests.py b/tests/comment_tests.py index 862b5f0b5..f36dbe9b6 100644 --- a/tests/comment_tests.py +++ b/tests/comment_tests.py @@ -166,7 +166,7 @@ def test_delete_batch(self): self.assertTrue(comment_count >= len(users)) self.api.delete_from_where_user(users[0], project_name=PROJECT) - self.assertTrue(len(self.api.get_comments(project_name=PROJECT)) == comment_count - 1) + self.assertEqual(len(self.api.get_comments(project_name=PROJECT)), comment_count - 1) self.api.delete_from(project_name=PROJECT) self.assertFalse(len(self.api.get_comments(project_name=PROJECT)))