-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2993 from Vogtinator/bcirepopublisher
Add BCI repo publishing bot
- Loading branch information
Showing
3 changed files
with
275 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,199 @@ | ||
| #!/usr/bin/python3 | ||
| # (c) 2023 [email protected] | ||
| # GPL-2.0-or-later | ||
|
|
||
| # This is a "mini ttm" for the BCI repo. Differences: | ||
| # * No :ToTest staging area | ||
| # * Only the repo is built, so it needs BCI specific openQA queries | ||
| # * The publishing location is arch specific | ||
| # * Uses a token for releasing to the publishing project | ||
|
|
||
|
|
||
| import cmdln | ||
| import logging | ||
| import ToolBase | ||
| import requests | ||
| import sys | ||
| import time | ||
| import re | ||
| from lxml import etree as ET | ||
| from openqa_client.client import OpenQA_Client | ||
| from osc.core import http_GET, makeurl | ||
|
|
||
|
|
||
| class BCIRepoPublisher(ToolBase.ToolBase): | ||
| def __init__(self): | ||
| ToolBase.ToolBase.__init__(self) | ||
| self.logger = logging.getLogger(__name__) | ||
| self.openqa = OpenQA_Client(server='https://openqa.suse.de') | ||
|
|
||
| def version_of_product(self, project, package, repo, arch): | ||
| """Get the build version of the given product build, based on the binary name.""" | ||
| url = makeurl(self.apiurl, ['build', project, repo, arch, package]) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
| for binary in root.findall('binary'): | ||
| result = re.match(r'.*-Build(.*)-Media1.report', binary.get('filename')) | ||
| if result: | ||
| return result.group(1) | ||
|
|
||
| raise RuntimeError(f"Failed to get version of {project}/{package}") | ||
|
|
||
| def mtime_of_product(self, project, package, repo, arch): | ||
| """Get the build time stamp of the given product, based on _buildenv.""" | ||
| url = makeurl(self.apiurl, ['build', project, repo, arch, package]) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
| mtime = root.xpath('/binarylist/binary[@filename = "_buildenv"]/@mtime') | ||
| return mtime[0] | ||
|
|
||
| def openqa_jobs_for_product(self, arch, version, build): | ||
| """Query openQA for all relevant jobs""" | ||
| values = { | ||
| 'group': 'BCI repo', | ||
| 'flavor': 'BCI-Repo-Updates', | ||
| 'arch': arch, | ||
| 'version': version, | ||
| 'build': build, | ||
| 'scope': 'current', | ||
| 'latest': '1', | ||
| } | ||
| return self.openqa.openqa_request('GET', 'jobs', values)['jobs'] | ||
|
|
||
| def is_repo_published(self, project, repo): | ||
| """Validates that the given prj/repo is fully published and all builds | ||
| have succeeded.""" | ||
| url = makeurl(self.apiurl, ['build', project, '_result'], | ||
| {'view': 'summary', 'repository': repo}) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
| for result in root.findall('result'): | ||
| if result.get('dirty', 'false') != 'false': | ||
| return False | ||
| if result.get('code') != 'published' or result.get('state') != 'published': | ||
| return False | ||
|
|
||
| for statuscount in root.findall('statuscount'): | ||
| if statuscount.get('code') not in ('succeeded', 'disabled', 'excluded'): | ||
| return False | ||
|
|
||
| return True | ||
|
|
||
| def run(self, version, token=None): | ||
| build_prj = f'SUSE:SLE-{version}:Update:BCI' | ||
|
|
||
| if not self.is_repo_published(build_prj, 'images'): | ||
| self.logger.info(f'{build_prj}/images not successfully built') | ||
| return | ||
|
|
||
| # Build the list of packages with metainfo | ||
| packages = [] | ||
| # As long as it's the same everywhere, hardcoding this list here | ||
| # is easier and safer than trying to derive it from the package list. | ||
| for arch in ('aarch64', 'ppc64le', 's390x', 'x86_64'): | ||
| packages.append({ | ||
| 'arch': arch, | ||
| 'name': f'000product:SLE_BCI-ftp-POOL-{arch}', | ||
| 'build_prj': build_prj, | ||
| 'publish_prj': f'SUSE:Products:SLE-BCI:{version}:{arch}' | ||
| }) | ||
|
|
||
| # Fetch the build numbers of built products. | ||
| # After release, the BuildXXX part vanishes, so the mtime has to be | ||
| # used instead for comparing built and published binaries. | ||
| for pkg in packages: | ||
| pkg['built_version'] = self.version_of_product(pkg['build_prj'], pkg['name'], | ||
| 'images', 'local') | ||
| pkg['built_mtime'] = self.mtime_of_product(pkg['build_prj'], pkg['name'], | ||
| 'images', 'local') | ||
| pkg['published_mtime'] = self.mtime_of_product(pkg['publish_prj'], pkg['name'], | ||
| 'images', 'local') | ||
|
|
||
| # Verify that the builds for all archs are in sync | ||
| built_versions = {pkg['built_version'] for pkg in packages} | ||
| if len(built_versions) != 1: | ||
| # This should not be the case if everything is built and idle | ||
| self.logger.warning(f'Different builds found - not releasing: {packages}') | ||
| return | ||
|
|
||
| # Compare versions | ||
| newer_version_available = [int(pkg['built_mtime']) > int(pkg['published_mtime']) | ||
| for pkg in packages] | ||
| if not any(newer_version_available): | ||
| self.logger.info('Current build already published, nothing to do.') | ||
| return | ||
|
|
||
| # Check openQA results | ||
| openqa_passed = True | ||
| for pkg in packages: | ||
| passed = 0 | ||
| pending = 0 | ||
| failed = 0 | ||
| for job in self.openqa_jobs_for_product(arch=pkg['arch'], version=version, | ||
| build=pkg['built_version']): | ||
| if job['result'] in ('passed', 'softfailed'): | ||
| passed += 1 | ||
| elif job['result'] == 'none': | ||
| self.logger.info(f'https://openqa.suse.de/tests/{job["id"]} pending') | ||
| pending += 1 | ||
| else: | ||
| self.logger.warning(f'https://openqa.suse.de/tests/{job["id"]} failed') | ||
| failed += 1 | ||
|
|
||
| if passed == 0 or pending > 0 or failed > 0: | ||
| openqa_passed = False | ||
|
|
||
| if not openqa_passed: | ||
| self.logger.info('No positive result from openQA (yet)') | ||
| return | ||
|
|
||
| # Trigger publishing | ||
| if token is None: | ||
| self.logger.warning('Would publish now, but no token specified') | ||
| return | ||
|
|
||
| for pkg in packages: | ||
| self.logger.info(f'Releasing {pkg["name"]}...') | ||
| params = { | ||
| 'project': pkg['build_prj'], 'package': pkg['name'], | ||
| 'filter_source_repository': 'images', | ||
| 'targetproject': pkg['publish_prj'], 'targetrepository': 'images' | ||
| } | ||
| url = makeurl(self.apiurl, ['trigger', 'release'], params) | ||
| # No bindings for using tokens yet, so do the request manually | ||
| req = requests.post(url, headers={'Authorization': f'Token {token}'}) | ||
| if req.status_code != 200: | ||
| raise RuntimeError(f'Releasing failed: {req.text}') | ||
|
|
||
| self.logger.info('Waiting for publishing to finish') | ||
| for pkg in packages: | ||
| while not self.is_repo_published(pkg['publish_prj'], 'images'): | ||
| self.logger.debug(f'Waiting for {pkg["publish_prj"]}') | ||
| time.sleep(20) | ||
|
|
||
|
|
||
| class CommandLineInterface(ToolBase.CommandLineInterface): | ||
| def __init__(self, *args, **kwargs): | ||
| ToolBase.CommandLineInterface.__init__(self, args, kwargs) | ||
|
|
||
| def setup_tool(self): | ||
| tool = BCIRepoPublisher() | ||
| if self.options.debug: | ||
| logging.basicConfig(level=logging.DEBUG) | ||
| elif self.options.verbose: | ||
| logging.basicConfig(level=logging.INFO) | ||
|
|
||
| return tool | ||
|
|
||
| @cmdln.option('--token', help='The token for publishing. Does a dry run if not given.') | ||
| def do_run(self, subcmd, opts, project): | ||
| """${cmd_name}: run the BCI repo publisher for the specified project, | ||
| e.g. 15-SP3 | ||
| ${cmd_usage} | ||
| ${cmd_option_list} | ||
| """ | ||
|
|
||
| self.tool.run(project, token=opts.token) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| cli = CommandLineInterface() | ||
| sys.exit(cli.main()) |