BCI repo publisher: also trigger a rebuild of devel:BCI:$VERSION #3023
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,8 @@ | |
| # * Uses a token for releasing to the publishing project | ||
|
|
||
|
|
||
| from typing import List, Optional | ||
| from typing_extensions import Literal, TypedDict | ||
| import cmdln | ||
| import logging | ||
| import ToolBase | ||
|
|
@@ -18,17 +20,20 @@ | |
| import re | ||
| from lxml import etree as ET | ||
| from openqa_client.client import OpenQA_Client | ||
| from osc.core import makeurl | ||
| from osc.connection import http_GET | ||
| from random import randint | ||
|
|
||
| _SLE_VERSION_T = Literal['15-SP3', '15-SP4', '15-SP5', '15-SP6'] | ||
|
|
||
|
|
||
| class BCIRepoPublisher(ToolBase.ToolBase): | ||
| def __init__(self): | ||
| ToolBase.ToolBase.__init__(self) | ||
| self.logger = logging.getLogger(__name__) | ||
| self.openqa = OpenQA_Client(server='https://openqa.suse.de') | ||
|
|
||
| def version_of_product(self, project: str, package: str, repo: str, arch: str) -> str: | ||
| """Get the build version of the given product build, based on the binary name.""" | ||
| url = makeurl(self.apiurl, ['build', project, repo, arch, package]) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
|
|
@@ -39,12 +44,14 @@ def version_of_product(self, project, package, repo, arch): | |
|
|
||
| raise RuntimeError(f"Failed to get version of {project}/{package}") | ||
|
|
||
| def mtime_of_product(self, project: str, package: str, repo: str, arch: str) -> int: | ||
| """Get the build time stamp of the given product, based on _buildenv.""" | ||
| url = makeurl(self.apiurl, ['build', project, repo, arch, package]) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
| mtime = root.xpath('/binarylist/binary[@filename = "_buildenv"]/@mtime') | ||
| if not mtime: | ||
| raise RuntimeError(f"Failed to get mtime of {project}/{package}") | ||
| return int(mtime[0]) | ||
|
|
||
| def openqa_jobs_for_product(self, arch, version, build): | ||
| """Query openQA for all relevant jobs""" | ||
|
|
@@ -59,9 +66,12 @@ def openqa_jobs_for_product(self, arch, version, build): | |
| } | ||
| return self.openqa.openqa_request('GET', 'jobs', values)['jobs'] | ||
|
|
||
| def is_repo_published(self, project: str, repo: str, arch: Optional[str] = None) -> bool: | ||
| """Validate that the given prj/repo is fully published and all builds | ||
| have succeeded. If an architecture is provided, then only that | ||
| architecture is considered for checking the publishing & build state. | ||
|
|
||
| """ | ||
| result_filter = {'view': 'summary', 'repository': repo} | ||
| if arch: | ||
| result_filter['arch'] = arch | ||
|
|
@@ -79,38 +89,53 @@ def is_repo_published(self, project, repo, arch=None): | |
|
|
||
| return True | ||
|
|
||
| def fetch_package_names_in_project(self, apiurl: str, prj_name: str) -> List[str]: | ||
| url = makeurl(apiurl, ['source', prj_name]) | ||
| root = ET.parse(http_GET(url)).getroot() | ||
| return [elem.get("name") for elem in root] | ||
|
|
||
| def run(self, version: _SLE_VERSION_T, publish_token: Optional[str] = None, | ||
| rebuild_token: Optional[str] = None) -> None: | ||
|
|
||
| class Package(TypedDict): | ||
| arch: str | ||
| name: str | ||
| build_prj: str | ||
| publish_prj: str | ||
| built_version: str | ||
| built_mtime: int | ||
| published_mtime: int | ||
|
|
||
| build_prj = f'SUSE:SLE-{version}:Update:BCI' | ||
| devel_prj = f'devel:BCI:SLE-{version}' | ||
| _DEVEL_PRJ_APIURL = "https://api.opensuse.org" | ||
|
|
||
| if not self.is_repo_published(build_prj, 'images', 'local'): | ||
| self.logger.info(f'{build_prj}/images not successfully built') | ||
| return | ||
|
|
||
| # Build the list of packages with metainfo | ||
| packages: List[Package] = [] | ||
| # List of packages that have passed openQA | ||
| openqa_passed_packages: List[Package] = [] | ||
| # As long as it's the same everywhere, hardcoding this list here | ||
| # is easier and safer than trying to derive it from the package list. | ||
| for arch in ('aarch64', 'ppc64le', 's390x', 'x86_64'): | ||
| name = f'000product:SLE_BCI-ftp-POOL-{arch}' | ||
| publish_prj = f'SUSE:Products:SLE-BCI:{version}:{arch}' | ||
| packages.append({ | ||
| 'arch': arch, | ||
| 'name': name, | ||
| 'build_prj': build_prj, | ||
| 'publish_prj': publish_prj, | ||
| # Fetch the build numbers of built products. | ||
| # After release, the BuildXXX part vanishes, so the mtime has to be | ||
| # used instead for comparing built and published binaries. | ||
| 'built_version': self.version_of_product(build_prj, name, 'images', 'local'), | ||
| 'built_mtime': self.mtime_of_product(build_prj, name, 'images', 'local'), | ||
| 'published_mtime': self.mtime_of_product(publish_prj, name, 'images', 'local') | ||
| }) | ||
|
|
||
| # Verify that the builds for all archs are in sync | ||
| built_versions = {pkg['built_version'] for pkg in packages} | ||
| if len(built_versions) != 1: | ||
|
|
@@ -157,8 +182,16 @@ def run(self, version, token=None): | |
| return | ||
| openqa_passed_packages.append(pkg) | ||
|
|
||
| if rebuild_token: | ||
| for pkg in self.fetch_package_names_in_project(_DEVEL_PRJ_APIURL, devel_prj): | ||
|
There was a problem hiding this comment. jftr OBS team will fix this that we can use unscoped tokens to trigger project wide rebuilds There was a problem hiding this comment. So shall we wait for them to implement this before moving this forward? There was a problem hiding this comment. implementation is ready at openSUSE/open-build-service#15119 There was a problem hiding this comment.
no, I think we can do this optimisation to use only one project wide trigger after merging this. however I'd really like to get one repo published first before adding more risk of regressions.. |
||
| url = makeurl(_DEVEL_PRJ_APIURL, ['trigger', 'rebuild'], {'project': devel_prj, 'package': pkg}) | ||
|
There was a problem hiding this comment. this could now be changed based on openSUSE/open-build-service#15261 (which is deployed) |
||
| req = requests.post(url, headers={'Authorization': f'Token {rebuild_token}'}) | ||
| req.raise_for_status() | ||
| else: | ||
| self.logger.warning('Would rebuild %s now, but no token specified', devel_prj) | ||
|
|
||
| # Trigger publishing | ||
| if publish_token is None: | ||
| self.logger.warning(f'Would publish {[pkg["name"] for pkg in openqa_passed_packages]}, but no token specified') | ||
| return | ||
|
|
||
|
|
@@ -171,9 +204,8 @@ def run(self, version, token=None): | |
| } | ||
| url = makeurl(self.apiurl, ['trigger', 'release'], params) | ||
| # No bindings for using tokens yet, so do the request manually | ||
| req = requests.post(url, headers={'Authorization': f'Token {publish_token}'}) | ||
| req.raise_for_status() | ||
|
|
||
| self.logger.info('Waiting for publishing to finish') | ||
| for pkg in openqa_passed_packages: | ||
|
|
@@ -195,15 +227,16 @@ def setup_tool(self): | |
|
|
||
| return tool | ||
|
|
||
| @cmdln.option('--publish-token', help='The token for publishing. Does a dry run if not given.') | ||
| @cmdln.option('--rebuild-token', help='The token for rebuilding devel:BCI, no rebuild is triggered if omitted.') | ||
| def do_run(self, subcmd, opts, project): | ||
| """${cmd_name}: run BCI repo publisher for the project, e.g. 15-SP5. | ||
|
|
||
| ${cmd_usage} | ||
| ${cmd_option_list} | ||
| """ | ||
|
|
||
| self.tool.run(project, publish_token=opts.publish_token, rebuild_token=opts.rebuild_token) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this really need to be typed?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No it doesn't have to, but I found it quite useful, because I was wondering what the values of this variable are. Initially I thought this was a package version only to realize later that it's actually
15-SP$N.