From 72b299ce778d38117bb04abbb93992adca50f452 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Sun, 7 Jul 2024 21:29:19 +0200 Subject: [PATCH 1/7] feat(bake): new --build-arg cli parameter overrides conf.py properties --- CHANGELOG.md | 7 ++++ README.md | 40 +++++++++++++++++++---- src/image_tools/args.py | 71 +++++++++++++++++++++++++++++------------ src/image_tools/bake.py | 6 ++-- 4 files changed, 93 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cefb28..cd697af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- New `bake` command line argument `--build-arg` to override conf.py build arguments ([#38]) + +[#38]: https://github.com/stackabletech/image-tools/pull/38 + + ## [0.0.11] ### Added diff --git a/README.md b/README.md index 72db64f..794372a 100644 --- a/README.md +++ b/README.md @@ -22,19 +22,32 @@ Below are some common usage examples: ```shell # Build images of the hello-world containers -bake -p hello-world -i 0.0.0-dev +bake --product hello-world # Build only one version [0.37.2] of OPA -bake -p opa=0.37.2 -i 0.0.0-dev +bake --product opa=0.37.2 + +# Dry run. Do not buikd anything. Print the the generated Bakefile. +bake --product hello-world --dry + +# Build all OPA images and set the organisation to "sandbox" +bake --product opa=0.37.2 --organization sandbox + +# Build all OPA images and set the image version to a release 24.7.0 +bake --product opa --image-version 24.7.0 # Enable distributed docker cache (requires credentials to access the cache registry) -bake -p opa --cache +bake --product opa --cache + +# Build the HBase images but use Java 21 instead of the values in conf.py +# for the java-base and java-devel images. +bake --product hbase --build-arg 'java-base=21' --build-arg 'java-devel=21' # Build half of all versions defined for OPA -bake -p opa -i 0.0.0-dev --shard-count 2 --shard-index 0 +bake --product opa --shard-count 2 --shard-index 0 # Build the other half of all versions defined for OPA -bake -p opa -i 0.0.0-dev --shard-count 2 --shard-index 1 +bake --product opa --shard-count 2 --shard-index 1 ``` ## Installation @@ -63,8 +76,8 @@ let image-tools = pkgs.callPackage (pkgs.fetchFromGitHub { owner = "stackabletech"; repo = "image-tools"; - rev = "caa4d993bcbb8b884097c89a54ee246f975e2ec6"; - hash = "sha256-gjTCroHw4iJhXPW+s3mHBzIH8seIKH1tPb82lUb8+a0="; # comment out to find new hashes when upgrading + rev = "caa4d993bcbb8b884097c89a54ee246f975e2ec6"; # pragma: allowlist secret + hash = "sha256-gjTCroHw4iJhXPW+s3mHBzIH8seIKH1tPb82lUb8+a0="; # pragma: allowlist secret ; comment out to find new hashes when upgrading } + "/image-tools.nix") {}; in { @@ -97,6 +110,19 @@ pipx install --editable . With the activated virtual environment, you can now run the tools from the `docker-images` repository and any local changes are immediately in effect. +We also recommend installing the `pre-commit` hooks in the activated virtual environment. + +```shell +pip install pre-commit +pre-commit install +``` + +To run the hooks, stage the changes you want to commit and run: + +```shell +pre-commit run +``` + ## Release a new version 1. Create a release PR where you: diff --git a/src/image_tools/args.py b/src/image_tools/args.py index befd400..169dc13 100644 --- a/src/image_tools/args.py +++ b/src/image_tools/args.py @@ -4,6 +4,8 @@ import importlib.util import sys import os +from types import ModuleType +from typing import List, Tuple from .version import version @@ -20,14 +22,13 @@ def bake_args() -> Namespace: ) parser.add_argument("-v", "--version", help="Display version", action="store_true") - ( - parser.add_argument( - "-c", - "--configuration", - help="Configuration file. Default: './conf.py'.", - default="./conf.py", - ), + parser.add_argument( + "-c", + "--configuration", + help="Configuration file. Default: './conf.py'.", + default="./conf.py", ) + parser.add_argument( "-i", "--image-version", @@ -75,13 +76,19 @@ def bake_args() -> Namespace: help="Image registry to publish to. Default: docker.stackable.tech.", default="docker.stackable.tech", ) - ( - parser.add_argument( - "--export-tags-file", - help="Write target image tags to a text file. Useful for signing or other follow-up CI steps.", - ), + parser.add_argument( + "--export-tags-file", + help="Write target image tags to a text file. Useful for signing or other follow-up CI steps.", + ) + + parser.add_argument("--cache", help="Enable distributed build cache", action="store_true") + + parser.add_argument( + "--build-arg", + help="Override build arguments. Expecting an ARGV=VALUE format.", + nargs="*", + type=check_build_arg, ) - (parser.add_argument("--cache", help="Enable distributed build cache", action="store_true"),) result = parser.parse_args() @@ -133,6 +140,13 @@ def check_image_version_format(image_version) -> str: raise ValueError(f"Invalid image version: {image_version}") +def check_build_arg(build_args: str) -> Tuple[str, str]: + kv = build_args.split("=") + if len(kv) != 2: + raise ValueError + return kv[0], kv[1] + + def preflight_args() -> Namespace: parser = ArgumentParser( description="Run OpenShift certification checks and submit results to RedHat Partner Connect portal" @@ -180,13 +194,11 @@ def preflight_args() -> Namespace: help="Name of the preflight program. Default: preflight", default="preflight", ) - ( - parser.add_argument( - "-c", - "--configuration", - help="Configuration file.", - default="./conf.py", - ), + parser.add_argument( + "-c", + "--configuration", + help="Configuration file.", + default="./conf.py", ) result = parser.parse_args() @@ -206,7 +218,11 @@ def check_architecture_input(architecture: str) -> str: return architecture -def load_configuration(conf_file_name: str): +def load_configuration(conf_file_name: str, user_build_args: List[Tuple[str, str]] = []) -> ModuleType: + """Load the configuration module conf.py and potentially override build arguments + with values provided by the user with the --build-arg flag. + The build arguments are key, value pairs from the "conf.products..versions." dictionary. + """ module_name = "conf" sys.path.append(str(os.getcwd())) spec = importlib.util.spec_from_file_location(module_name, conf_file_name) @@ -215,5 +231,18 @@ def load_configuration(conf_file_name: str): sys.modules[module_name] = module if spec.loader: spec.loader.exec_module(module) + override_build_args(module, user_build_args) return module raise ImportError(name=module_name, path=conf_file_name) + + +def override_build_args(conf: ModuleType, user_build_args: List[Tuple[str, str]] = []) -> None: + if not user_build_args: + return + # convert user_build_args to a dictionary for easier lookup + user_build_args_dict = {kv[0]: kv[1] for kv in user_build_args} + for product in conf.products: + for conf_build_args in product["versions"]: + for arg in conf_build_args: + if arg in user_build_args_dict: + conf_build_args[arg] = user_build_args_dict[arg] diff --git a/src/image_tools/bake.py b/src/image_tools/bake.py index 1804712..d7f560d 100644 --- a/src/image_tools/bake.py +++ b/src/image_tools/bake.py @@ -22,7 +22,7 @@ from .version import version -def build_image_args(version: Dict[str, str], release_version: str): +def build_image_args(conf_build_args: Dict[str, str], release_version: str): """ Returns a list of --build-arg command line arguments that are used by the docker build command. @@ -33,7 +33,7 @@ def build_image_args(version: Dict[str, str], release_version: str): """ result = {} - for k, v in version.items(): + for k, v in conf_build_args.items(): result[k.upper()] = v result["RELEASE"] = release_version @@ -213,7 +213,7 @@ def main() -> int: print(version()) return 0 - conf = load_configuration(args.configuration) + conf = load_configuration(args.configuration, args.build_arg) bakefile = generate_bakefile(args, conf) From f5b9632a69bc26104f229b7f56b2d4e0f1763869 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 04:24:14 -0400 Subject: [PATCH 2/7] Update README.md Co-authored-by: Lars Francke --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 794372a..ec933fc 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ bake --product hello-world # Build only one version [0.37.2] of OPA bake --product opa=0.37.2 -# Dry run. Do not buikd anything. Print the the generated Bakefile. +# Dry run. Do not build anything. Print the the generated Bakefile. bake --product hello-world --dry # Build all OPA images and set the organisation to "sandbox" From eb5f658faf1610d743bdb265726cb6cba569e594 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:07:38 +0200 Subject: [PATCH 3/7] fix example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec933fc..33fd7de 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ bake --product opa=0.37.2 bake --product hello-world --dry # Build all OPA images and set the organisation to "sandbox" -bake --product opa=0.37.2 --organization sandbox +bake --product opa --organization sandbox # Build all OPA images and set the image version to a release 24.7.0 bake --product opa --image-version 24.7.0 From f7063c5027b88c3c07385320d9123f7e4594bec0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:24:34 +0200 Subject: [PATCH 4/7] docs: build cache --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 33fd7de..890c1aa 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,35 @@ The `bake` command provides the following features: * Use one or more distributed docker cache servers * Publish images +## Docker Build Cache + +Docker's `buildx` plugin supports different types of build cache back ends. Since Stackable product images are built by distributed GitHub actions, the distributed back ends are relevant. + +To use the build cache, you have to configure one or more back ends and enable them by calling `bake` with the `--cache` flag. + +To configure one or more cache back ends, add the relevant properties to the `cache` property of the configuration module. + +Here an example with the `registry` backend: + +```python +cache = [ + { + "type": "registry", + "ref_prefix": "build-repo.stackable.tech:8083/sandbox/cache", + "mode": "max", + "compression": "zstd", + "ignore-error": "true", + }, +] +``` + +Here `ref_prefix` is used to build the unique `ref` property for each target. + +NOTE: it's your responsibility to ensure that `bake` can read/write to the cache registry by performing a `docker login` before running `bake`. + + +For more information about the cache back ends, see the [Docker documentation](https://docs.docker.com/build/cache/backends/). + ## Usage examples Run either `bake` or `check-container` with `--help` to get an overview of the accepted flags and their functionality. From e15a3f6640a558d70770b7aa84e3093e82fe4511 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:26:14 +0200 Subject: [PATCH 5/7] docs: mention build arg name case insensitivity --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 890c1aa..2cb8ea0 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ bake --product opa --cache # Build the HBase images but use Java 21 instead of the values in conf.py # for the java-base and java-devel images. +# It doesn't matter if you use lower or upper case for the build argument names. bake --product hbase --build-arg 'java-base=21' --build-arg 'java-devel=21' # Build half of all versions defined for OPA From c24672c8157a33bf8f421b1ecf3d2d16ec425268 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:28:48 +0200 Subject: [PATCH 6/7] fix: update help output --- src/image_tools/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image_tools/args.py b/src/image_tools/args.py index 169dc13..b295b78 100644 --- a/src/image_tools/args.py +++ b/src/image_tools/args.py @@ -85,7 +85,7 @@ def bake_args() -> Namespace: parser.add_argument( "--build-arg", - help="Override build arguments. Expecting an ARGV=VALUE format.", + help="Override build arguments. Expecting an KEY=VALUE format. The key is case insensitive.", nargs="*", type=check_build_arg, ) From 6c8feb11356ae5232e66c41327e43503e97b516b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 8 Jul 2024 05:37:31 -0400 Subject: [PATCH 7/7] Update README.md Co-authored-by: Lars Francke --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cb8ea0..bb24fd0 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,8 @@ bake --product opa --cache # Build the HBase images but use Java 21 instead of the values in conf.py # for the java-base and java-devel images. -# It doesn't matter if you use lower or upper case for the build argument names. +# It doesn't matter if you use lower or upper case for the build argument names, +# bake will normalize all of them to upper case. bake --product hbase --build-arg 'java-base=21' --build-arg 'java-devel=21' # Build half of all versions defined for OPA