From da44de6fbc4489e78588143388649dd13a6eb57d Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Fri, 20 Feb 2026 11:37:17 +0200 Subject: [PATCH 1/2] dedup task scripts Signed-off-by: Benny Zlotnik --- internal/common/tasks/scripts.go | 16 +- .../common/tasks/scripts/build_builder.sh | 142 +------ internal/common/tasks/scripts/build_image.sh | 372 ++++++------------ internal/common/tasks/scripts/common.sh | 274 +++++++++++++ internal/common/tasks/tasks.go | 9 + 5 files changed, 423 insertions(+), 390 deletions(-) create mode 100644 internal/common/tasks/scripts/common.sh diff --git a/internal/common/tasks/scripts.go b/internal/common/tasks/scripts.go index 16ab848e..ac6d8d12 100644 --- a/internal/common/tasks/scripts.go +++ b/internal/common/tasks/scripts.go @@ -5,15 +5,20 @@ import ( _ "embed" ) +//go:embed scripts/common.sh +var commonScript string + //go:embed scripts/find_manifest.sh // FindManifestScript contains the embedded shell script for finding build manifests. var FindManifestScript string //go:embed scripts/build_image.sh +var buildImageScript string // BuildImageScript contains the embedded shell script for building images. -var BuildImageScript string +// It is the concatenation of common.sh and build_image.sh. +var BuildImageScript = "" //go:embed scripts/push_artifact.sh @@ -21,11 +26,18 @@ var BuildImageScript string var PushArtifactScript string //go:embed scripts/build_builder.sh +var buildBuilderScript string // BuildBuilderScript contains the embedded shell script for building the builder image. -var BuildBuilderScript string +// It is the concatenation of common.sh and build_builder.sh. +var BuildBuilderScript = "" //go:embed scripts/flash_image.sh // FlashImageScript contains the embedded shell script for flashing images via Jumpstarter. var FlashImageScript string + +func init() { + BuildImageScript = commonScript + "\n" + buildImageScript + BuildBuilderScript = commonScript + "\n" + buildBuilderScript +} diff --git a/internal/common/tasks/scripts/build_builder.sh b/internal/common/tasks/scripts/build_builder.sh index 29d4fdea..d69bcf24 100644 --- a/internal/common/tasks/scripts/build_builder.sh +++ b/internal/common/tasks/scripts/build_builder.sh @@ -1,25 +1,4 @@ -#!/bin/bash -set -e - -validate_arg() { - local arg="$1" - local name="$2" - # Block shell metacharacters that could be used for injection - if [[ "$arg" =~ [\;\|\&\$\`\(\)\{\}\<\>\!\\] ]]; then - echo "ERROR: Invalid characters in $name: $arg" - exit 1 - fi -} - -validate_custom_def() { - local def="$1" - # Custom defs should be KEY=VALUE format only - if [[ ! "$def" =~ ^[a-zA-Z_][a-zA-Z0-9_]*=.*$ ]]; then - echo "ERROR: Invalid custom definition format: $def (expected KEY=VALUE)" - exit 1 - fi - validate_arg "$def" "custom definition" -} +# NOTE: common.sh is prepended to this script at embed time. echo "Prepare builder for distro: $DISTRO, arch: $TARGET_ARCH" @@ -30,16 +9,11 @@ if [ -n "$BUILDER_IMAGE" ]; then exit 0 fi -# Set up cluster registry details -TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - +# Determine registry and set up authentication if [ -n "$CLUSTER_REGISTRY_ROUTE" ]; then echo "Using external registry route: $CLUSTER_REGISTRY_ROUTE" - REGISTRY="$CLUSTER_REGISTRY_ROUTE" -else - REGISTRY="image-registry.openshift-image-registry.svc:5000" fi +setup_cluster_auth "${CLUSTER_REGISTRY_ROUTE:-}" # Include a short hash of the AIB image in the registry tag so that different # AIB versions cache their builder images separately and don't overwrite each other. @@ -47,45 +21,13 @@ AIB_HASH=$(echo -n "$AIB_IMAGE" | sha256sum | cut -c1-8) TARGET_IMAGE="${REGISTRY}/${NAMESPACE}/aib-build:${DISTRO}-${TARGET_ARCH}-${AIB_HASH}" echo "AIB image: $AIB_IMAGE (hash: $AIB_HASH)" -mkdir -p $HOME/.config -cat > $HOME/.authjson < /etc/containers/registries.conf << EOF -[registries.insecure] -registries = ['image-registry.openshift-image-registry.svc:5000'] -EOF +# Local target name for pushing to registry +LOCAL_TARGET="localhost/aib-build:${DISTRO}-${TARGET_ARCH}-${AIB_HASH}" -echo "Configuring kernel overlay storage driver" -cat > /etc/containers/storage.conf << EOF -[storage] -driver = "overlay" -runroot = "/run/containers/storage" -graphroot = "/var/lib/containers/storage" -EOF - -if ! mountpoint -q /var/tmp; then - VAR_TMP_SIZE="${VAR_TMP_SIZE:-20G}" - echo "Creating loopback ext4 filesystem for /var/tmp (${VAR_TMP_SIZE} sparse)" - truncate -s "$VAR_TMP_SIZE" /tmp/var-tmp.img - mkfs.ext4 -q /tmp/var-tmp.img - mount -o loop /tmp/var-tmp.img /var/tmp -fi - -# Local image name (what we'll actually use - nested containers can access this) -LOCAL_IMAGE="localhost/aib-build:${DISTRO}-${TARGET_ARCH}" - -# Check if image already exists in cluster registry (skip if rebuild requested) +# Check if image already exists in cluster registry if [ "$REBUILD_BUILDER" = "true" ]; then echo "Rebuild requested, skipping cache check" else @@ -99,72 +41,18 @@ fi echo "Builder image not found, building..." -# Install custom CA certificates if available -if [ -d /etc/pki/ca-trust/custom ] && ls /etc/pki/ca-trust/custom/*.pem >/dev/null 2>&1; then - echo "Installing custom CA certificates..." - cp /etc/pki/ca-trust/custom/*.pem /etc/pki/ca-trust/source/anchors/ 2>/dev/null || true - update-ca-trust extract 2>/dev/null || true -fi - -# Set up SELinux contexts for osbuild -osbuildPath="/usr/bin/osbuild" -storePath="/_build" -runTmp="/run/osbuild/" - -mkdir -p "$storePath" -mkdir -p "$runTmp" - -rootType="system_u:object_r:root_t:s0" -chcon "$rootType" "$storePath" || true - -installType="system_u:object_r:install_exec_t:s0" -if ! mountpoint -q "$runTmp"; then - mount -t tmpfs tmpfs "$runTmp" -fi - -destPath="$runTmp/osbuild" -cp -p "$osbuildPath" "$destPath" -chcon "$installType" "$destPath" || true +install_custom_ca_certs +setup_osbuild -mount --bind "$destPath" "$osbuildPath" +load_custom_definitions "$(workspaces.manifest-config-workspace.path)/custom-definitions.env" -# Load custom definitions (e.g., distro_baseurl) from manifest config workspace -declare -a CUSTOM_DEFS_ARGS=() -CUSTOM_DEFS_FILE="/workspace/manifest-config/custom-definitions.env" -if [ -f "$CUSTOM_DEFS_FILE" ]; then - while IFS= read -r line || [[ -n "$line" ]]; do - # Skip empty lines and comments - [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - validate_custom_def "$line" - CUSTOM_DEFS_ARGS+=("--define" "$line") - echo " Custom definition: $line" - done < "$CUSTOM_DEFS_FILE" -fi - -echo "Running: aib build-builder --distro $DISTRO ${CUSTOM_DEFS_ARGS[*]}" -aib --verbose build-builder --distro "$DISTRO" "${CUSTOM_DEFS_ARGS[@]}" - -# Find what image was actually created by aib (it might use distro-target naming) -echo "Checking for created builder images..." -ACTUAL_IMAGE="" -for img in $(podman images --format "{{.Repository}}:{{.Tag}}" | grep "localhost/aib-build:"); do - echo "Found image: $img" - if [[ "$img" == *"$DISTRO"* ]]; then - ACTUAL_IMAGE="$img" - echo "Using image: $ACTUAL_IMAGE" - break - fi -done - -if [ -z "$ACTUAL_IMAGE" ]; then - echo "ERROR: Could not find builder image containing '$DISTRO'" - podman images | grep aib-build || echo "No aib-build images found" - exit 1 -fi +echo "Running: aib build-builder --distro $DISTRO ${CUSTOM_DEFS_ARGS[*]} $LOCAL_TARGET" +aib --verbose build-builder --distro "$DISTRO" "${CUSTOM_DEFS_ARGS[@]}" "$LOCAL_TARGET" +echo "Built local image: $LOCAL_TARGET" echo "Pushing to cluster registry: $TARGET_IMAGE" skopeo copy --authfile="$REGISTRY_AUTH_FILE" \ - "containers-storage:$ACTUAL_IMAGE" \ + "containers-storage:$LOCAL_TARGET" \ "docker://$TARGET_IMAGE" echo "Builder image ready: $TARGET_IMAGE" diff --git a/internal/common/tasks/scripts/build_image.sh b/internal/common/tasks/scripts/build_image.sh index 455c8ec5..fc8b83dc 100644 --- a/internal/common/tasks/scripts/build_image.sh +++ b/internal/common/tasks/scripts/build_image.sh @@ -1,137 +1,40 @@ -#!/bin/bash -set -e - -validate_arg() { - local arg="$1" - local name="$2" - # Block shell metacharacters that could be used for injection - if [[ "$arg" =~ [\;\|\&\$\`\(\)\{\}\<\>\!\\] ]]; then - echo "ERROR: Invalid characters in $name: $arg" - exit 1 - fi -} - -validate_custom_def() { - local def="$1" - # Custom defs should be KEY=VALUE format only - if [[ ! "$def" =~ ^[a-zA-Z_][a-zA-Z0-9_]*=.*$ ]]; then - echo "ERROR: Invalid custom definition format: $def (expected KEY=VALUE)" - exit 1 - fi - validate_arg "$def" "custom definition" -} +# NOTE: common.sh is prepended to this script at embed time. +# Initialize optimizations +echo "DEBUG: Starting build script" +WORKSPACE_PATH="$(workspaces.shared-workspace.path)" +echo "DEBUG: WORKSPACE_PATH=$WORKSPACE_PATH" +detect_stat_command +echo "DEBUG: Stat command detected" # Make the internal registry trusted # TODO think about whether this is really the right approach -mkdir -p /etc/containers -cat > /etc/containers/registries.conf << EOF -[registries.insecure] -registries = ['image-registry.openshift-image-registry.svc:5000'] -EOF - -echo "Configuring kernel overlay storage driver" -cat > /etc/containers/storage.conf << EOF -[storage] -driver = "overlay" -runroot = "/run/containers/storage" -graphroot = "/var/lib/containers/storage" -EOF - -if ! mountpoint -q /var/tmp; then - VAR_TMP_SIZE="${VAR_TMP_SIZE:-20G}" - echo "Creating loopback ext4 filesystem for /var/tmp (${VAR_TMP_SIZE} sparse)" - truncate -s "$VAR_TMP_SIZE" /tmp/var-tmp.img - mkfs.ext4 -q /tmp/var-tmp.img - mount -o loop /tmp/var-tmp.img /var/tmp -fi +setup_container_config +echo "DEBUG: Container config set up" +setup_var_tmp +echo "DEBUG: /var/tmp set up" umask 0077 +echo "DEBUG: umask set" -TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -REGISTRY="image-registry.openshift-image-registry.svc:5000" -NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - -mkdir -p $HOME/.config -cat > $HOME/.authjson < $HOME/.custom_authjson - export REGISTRY_AUTH_FILE=$HOME/.custom_authjson -elif [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ] && [ -n "$REGISTRY_URL" ]; then - echo "Creating registry auth from username/password for $REGISTRY_URL" - mkdir -p $HOME/.config - AUTH_STRING=$(echo -n "$REGISTRY_USERNAME:$REGISTRY_PASSWORD" | base64 -w0) - cat > $HOME/.custom_authjson < $HOME/.custom_authjson </dev/null 2>&1; then - echo "Installing custom CA certificates..." - cp /etc/pki/ca-trust/custom/*.pem /etc/pki/ca-trust/source/anchors/ 2>/dev/null || true - update-ca-trust extract 2>/dev/null || true -fi - -rootType="system_u:object_r:root_t:s0" -chcon "$rootType" "$storePath" - -installType="system_u:object_r:install_exec_t:s0" -if ! mountpoint -q "$runTmp"; then - mount -t tmpfs tmpfs "$runTmp" -fi - -destPath="$runTmp/osbuild" -cp -p "$osbuildPath" "$destPath" -chcon "$installType" "$destPath" +install_custom_ca_certs +setup_osbuild -mount --bind "$destPath" "$osbuildPath" - -cd $(workspaces.shared-workspace.path) +cd "$WORKSPACE_PATH" EXPORT_FORMAT="$(params.export-format)" # If format is empty, AIB defaults to raw @@ -216,7 +101,7 @@ load_args_from_file() { while IFS= read -r line || [[ -n "$line" ]]; do # Skip empty lines and comments [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue - $validator "$line" "$description" + [ -n "$validator" ] && $validator "$line" "$description" result_array+=("$line") done < "$file" echo "Loaded ${#result_array[@]} items for $description" @@ -224,19 +109,13 @@ load_args_from_file() { } # Load custom definitions -declare -a CUSTOM_DEFS_ARGS=() -CUSTOM_DEFS_FILE="$(workspaces.manifest-config-workspace.path)/custom-definitions.env" -if load_args_from_file "$CUSTOM_DEFS_FILE" "custom definitions" validate_custom_def temp_defs; then - for def in "${temp_defs[@]}"; do - CUSTOM_DEFS_ARGS+=("--define" "$def") - done -fi +load_custom_definitions "$(workspaces.manifest-config-workspace.path)/custom-definitions.env" # Load AIB extra arguments declare -a AIB_EXTRA_ARGS=() AIB_EXTRA_ARGS_FILE="$(workspaces.manifest-config-workspace.path)/aib-extra-args.txt" -if load_args_from_file "$AIB_EXTRA_ARGS_FILE" "AIB extra args" validate_arg AIB_EXTRA_ARGS; then +if load_args_from_file "$AIB_EXTRA_ARGS_FILE" "AIB extra args" "" AIB_EXTRA_ARGS; then : # Extra args loaded successfully else echo "No AIB extra args file found" @@ -252,7 +131,6 @@ case "$arch" in ;; esac - CONTAINER_PUSH="$(params.container-push)" BUILD_DISK_IMAGE="$(params.build-disk-image)" EXPORT_OCI="$(params.export-oci)" @@ -267,20 +145,17 @@ echo "BUILD_DISK_IMAGE: $BUILD_DISK_IMAGE" echo "EXPORT_OCI: ${EXPORT_OCI:-}" echo "===========================" -if [ -n "$CONTAINER_PUSH" ]; then - BOOTC_CONTAINER_NAME="$CONTAINER_PUSH" -else - BOOTC_CONTAINER_NAME="localhost/aib-build:$(params.distro)-$(params.target)" -fi +# Use parameter expansion for cleaner default value assignment +BOOTC_CONTAINER_NAME="${CONTAINER_PUSH:-localhost/aib-build:$(params.distro)-$(params.target)}" -BUILD_CONTAINER_ARG="" -LOCAL_BUILDER_IMAGE="localhost/aib-build:$(params.distro)-$TARGET_ARCH" +# Calculate AIB hash for consistent naming with build_builder.sh +AIB_HASH=$(echo -n "$(params.automotive-image-builder)" | sha256sum | cut -c1-8) +# Local builder image name (matches what build_builder.sh creates with --out) +LOCAL_BUILDER_IMAGE="localhost/aib-build:$(params.distro)-${TARGET_ARCH}-${AIB_HASH}" # For bootc/disk builds, if no builder-image is provided but cluster-registry-route is set, # use the image that prepare-builder cached in the cluster registry if [ -z "$BUILDER_IMAGE" ] && { [ "$BUILD_MODE" = "bootc" ] || [ "$BUILD_MODE" = "disk" ]; } && [ -n "$CLUSTER_REGISTRY_ROUTE" ]; then - NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - AIB_HASH=$(echo -n "$(params.automotive-image-builder)" | sha256sum | cut -c1-8) BUILDER_IMAGE="${CLUSTER_REGISTRY_ROUTE}/${NAMESPACE}/aib-build:$(params.distro)-${TARGET_ARCH}-${AIB_HASH}" echo "Using builder image from cluster registry: $BUILDER_IMAGE" fi @@ -288,21 +163,15 @@ fi # Record the effective builder image used for annotation echo -n "${BUILDER_IMAGE:-}" > /tekton/results/builder-image +# Set up builder image if needed (consolidated logic) +declare -a BUILD_CONTAINER_ARGS=() if [ -n "$BUILDER_IMAGE" ] && { [ "$BUILD_MODE" = "bootc" ] || [ "$BUILD_MODE" = "disk" ]; }; then echo "Pulling builder image to local storage: $BUILDER_IMAGE" TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null || echo "") if [ -n "$TOKEN" ]; then REGISTRY_HOST=$(echo "$BUILDER_IMAGE" | cut -d'/' -f1) - cat > /tmp/builder-auth.json < "$dest" -} - -compress_file_lz4() { - src="$1"; dest="$2" - lz4 -z -f -q "$src" "$dest" -} - -tar_dir_gzip() { - dir="$1"; out="$2" - tar -C $(workspaces.shared-workspace.path) -czf "$out" "$dir" -} - -tar_dir_lz4() { - dir="$1"; out="$2" - tar -C $(workspaces.shared-workspace.path) -cf - "$dir" | lz4 -z -f -q > "$out" -} - +# Simplified compression functions - no unnecessary dispatching compress_file() { - src="$1"; dest="$2" + local src="$1" dest="$2" case "$COMPRESSION" in - lz4) compress_file_lz4 "$src" "$dest" ;; - gzip|*) compress_file_gzip "$src" "$dest" ;; + lz4) lz4 -z -f -q "$src" "$dest" ;; + gzip|*) gzip -c "$src" > "$dest" ;; esac } tar_dir() { - dir="$1"; out="$2" + local dir="$1" out="$2" case "$COMPRESSION" in - lz4) tar_dir_lz4 "$dir" "$out" ;; - gzip|*) tar_dir_gzip "$dir" "$out" ;; + lz4) tar -C "$WORKSPACE_PATH" -cf - "$dir" | lz4 -z -f -q > "$out" ;; + gzip|*) tar -C "$WORKSPACE_PATH" -czf "$out" "$dir" ;; esac } @@ -574,19 +431,19 @@ final_name="" if [ "$DISK_IMAGE_EXISTS" = "false" ] && [ -n "$CONTAINER_PUSH" ]; then echo "Container-only build completed. Container pushed to: $CONTAINER_PUSH" final_name="container:$CONTAINER_PUSH" -elif [ -d "$(workspaces.shared-workspace.path)/${exportFile}" ]; then +elif [ -d "$WORKSPACE_PATH/${exportFile}" ]; then echo "Preparing compressed parts for directory ${exportFile}..." final_compressed_name="${exportFile}${EXT_DIR}" - parts_dir="$(workspaces.shared-workspace.path)/${final_compressed_name}-parts" + parts_dir="$WORKSPACE_PATH/${final_compressed_name}-parts" mkdir -p "$parts_dir" ( - cd "$(workspaces.shared-workspace.path)" + cd "$WORKSPACE_PATH" for item in "${exportFile}"/*; do [ -e "$item" ] || continue base=$(basename "$item") if [ -f "$item" ]; then # Record uncompressed size before compression (for OCI layer annotations) - uncompressed_size=$(stat -c%s "$item" 2>/dev/null || stat -f%z "$item" 2>/dev/null || echo "") + uncompressed_size=$($GET_SIZE_CMD "$item" 2>/dev/null || echo "") echo "Creating $parts_dir/${base}${EXT_FILE} (uncompressed: ${uncompressed_size:-unknown} bytes)" compress_file "$item" "$parts_dir/${base}${EXT_FILE}" || echo "Failed to create $parts_dir/${base}${EXT_FILE}" # Store uncompressed size in sidecar file for push_artifact.sh @@ -600,28 +457,28 @@ elif [ -d "$(workspaces.shared-workspace.path)/${exportFile}" ]; then done ) echo "Creating compressed archive ${final_compressed_name} in shared workspace..." - tar_dir "${exportFile}" "$(workspaces.shared-workspace.path)/${final_compressed_name}" || echo "Failed to create ${final_compressed_name}" - echo "Compressed archive size:" && ls -lah $(workspaces.shared-workspace.path)/${final_compressed_name} || true - if [ -f "$(workspaces.shared-workspace.path)/${final_compressed_name}" ]; then + tar_dir "${exportFile}" "$WORKSPACE_PATH/${final_compressed_name}" || echo "Failed to create ${final_compressed_name}" + echo "Compressed archive size:" && ls -lah "$WORKSPACE_PATH/${final_compressed_name}" || true + if [ -f "$WORKSPACE_PATH/${final_compressed_name}" ]; then echo "Removing uncompressed directory ${exportFile} (keeping parts directory)" - rm -rf "$(workspaces.shared-workspace.path)/${exportFile}" - pushd $(workspaces.shared-workspace.path) + rm -rf "$WORKSPACE_PATH/${exportFile}" + pushd "$WORKSPACE_PATH" ln -sf ${final_compressed_name} disk.img final_name="${final_compressed_name}" popd echo "Available artifacts:" - ls -la $(workspaces.shared-workspace.path)/ || true - if [ -d "$(workspaces.shared-workspace.path)/${final_compressed_name}-parts" ]; then + ls -la "$WORKSPACE_PATH/" || true + if [ -d "$WORKSPACE_PATH/${final_compressed_name}-parts" ]; then echo "Individual compressed parts in ${final_compressed_name}-parts/:" - ls -la "$(workspaces.shared-workspace.path)/${final_compressed_name}-parts/" || true + ls -la "$WORKSPACE_PATH/${final_compressed_name}-parts/" || true fi fi -elif [ -f "$(workspaces.shared-workspace.path)/${exportFile}" ]; then +elif [ -f "$WORKSPACE_PATH/${exportFile}" ]; then echo "Creating compressed file ${exportFile}${EXT_FILE} in shared workspace..." - compress_file "$(workspaces.shared-workspace.path)/${exportFile}" "$(workspaces.shared-workspace.path)/${exportFile}${EXT_FILE}" || echo "Failed to create ${exportFile}${EXT_FILE}" - echo "Compressed file size:" && ls -lah $(workspaces.shared-workspace.path)/${exportFile}${EXT_FILE} || true - if [ -f "$(workspaces.shared-workspace.path)/${exportFile}${EXT_FILE}" ]; then - pushd $(workspaces.shared-workspace.path) + compress_file "$WORKSPACE_PATH/${exportFile}" "$WORKSPACE_PATH/${exportFile}${EXT_FILE}" || echo "Failed to create ${exportFile}${EXT_FILE}" + echo "Compressed file size:" && ls -lah "$WORKSPACE_PATH/${exportFile}${EXT_FILE}" || true + if [ -f "$WORKSPACE_PATH/${exportFile}${EXT_FILE}" ]; then + pushd "$WORKSPACE_PATH" ln -sf ${exportFile}${EXT_FILE} disk.img final_name="${exportFile}${EXT_FILE}" popd @@ -629,8 +486,6 @@ elif [ -f "$(workspaces.shared-workspace.path)/${exportFile}" ]; then fi if [ -z "$final_name" ]; then - workspace_path=$(workspaces.shared-workspace.path) - # Try to find artifact with priority: compressed file > compressed dir > any file # This ensures we prefer compressed artifacts when compression is enabled patterns_to_try=( @@ -644,15 +499,11 @@ if [ -z "$final_name" ]; then patterns_to_try=("${cleanName}*") fi - for pattern in "${patterns_to_try[@]}"; do - guess=$(ls -1 "${workspace_path}/${pattern}" 2>/dev/null | head -n1) - if [ -n "$guess" ]; then - final_name=$(basename "$guess") - echo "Fallback: using found artifact: $final_name" - break - fi - done + if final_name=$(find_artifact "$WORKSPACE_PATH" "${patterns_to_try[@]}"); then + echo "Fallback: using found artifact: $final_name" + fi fi + if [ -n "$final_name" ]; then echo "Writing artifact filename to Tekton result: $final_name" echo "$final_name" > /tekton/results/artifact-filename || echo "Failed to write Tekton result" @@ -662,7 +513,6 @@ else echo "Warning: final_name is empty, no artifact filename will be recorded" fi - echo "Syncing filesystem to ensure all artifacts are written..." sync -echo "Filesystem sync completed" +echo "Filesystem sync completed" \ No newline at end of file diff --git a/internal/common/tasks/scripts/common.sh b/internal/common/tasks/scripts/common.sh new file mode 100644 index 00000000..d5f3db11 --- /dev/null +++ b/internal/common/tasks/scripts/common.sh @@ -0,0 +1,274 @@ +#!/bin/bash +set -e + +# Common constants and functions shared between build scripts. +# This file is prepended to build_image.sh and build_builder.sh at embed time. + +INTERNAL_REGISTRY="image-registry.openshift-image-registry.svc:5000" +OSBUILD_PATH="/usr/bin/osbuild" +OSBUILD_STORE="/_build" +OSBUILD_RUN="/run/osbuild/" + +# --- Validation --- + +validate_container_ref() { + local ref="$1" + # Container image references may only contain alphanumerics and . / : - _ @ + if [[ ! "$ref" =~ ^[a-zA-Z0-9][a-zA-Z0-9./_:@-]*$ ]]; then + echo "ERROR: Invalid container reference: $ref" + exit 1 + fi +} + +validate_custom_def() { + local def="$1" + # Custom defs should be KEY=VALUE format only + if [[ ! "$def" =~ ^[a-zA-Z_][a-zA-Z0-9_]*=.*$ ]]; then + echo "ERROR: Invalid custom definition format: $def (expected KEY=VALUE)" + exit 1 + fi +} + +# --- Setup functions --- + +# Configure container registries (insecure internal registry) and overlay storage driver. +setup_container_config() { + mkdir -p /etc/containers + cat > /etc/containers/registries.conf << EOF +[registries.insecure] +registries = ['$INTERNAL_REGISTRY'] +EOF + + echo "Configuring kernel overlay storage driver" + cat > /etc/containers/storage.conf << EOF +[storage] +driver = "overlay" +runroot = "/run/containers/storage" +graphroot = "/var/lib/containers/storage" +EOF + + export CONTAINERS_REGISTRIES_CONF="/etc/containers/registries.conf" +} + +# Create a filesystem for /var/tmp if not already mounted. +# Uses tmpfs (RAM) when USE_MEMORY_VOLUMES=true, otherwise loopback ext4 for SELinux isolation. +setup_var_tmp() { + if ! mountpoint -q /var/tmp; then + if [ "$USE_MEMORY_VOLUMES" = "true" ]; then + if [ -n "$VAR_TMP_SIZE" ]; then + echo "Creating tmpfs filesystem for /var/tmp (${VAR_TMP_SIZE} memory)" + mount -t tmpfs -o size="$VAR_TMP_SIZE" tmpfs /var/tmp + else + echo "Creating tmpfs filesystem for /var/tmp (default size)" + mount -t tmpfs tmpfs /var/tmp + fi + else + # Larger default for sparse loopback (doesn't use real disk space initially) + VAR_TMP_SIZE="${VAR_TMP_SIZE:-20G}" + echo "Creating loopback ext4 filesystem for /var/tmp (${VAR_TMP_SIZE} sparse)" + truncate -s "$VAR_TMP_SIZE" /tmp/var-tmp.img + mkfs.ext4 -q /tmp/var-tmp.img + mount -o loop /tmp/var-tmp.img /var/tmp + fi + fi +} + +# Set up Kubernetes service account authentication for a container registry. +# Sets globals: TOKEN, NAMESPACE, REGISTRY +# Exports: REGISTRY_AUTH_FILE +# Args: $1 - registry URL (defaults to INTERNAL_REGISTRY) +setup_cluster_auth() { + echo "DEBUG: Reading service account token" + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + echo "DEBUG: Reading service account namespace" + NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + REGISTRY="${1:-$INTERNAL_REGISTRY}" + echo "DEBUG: Using registry: $REGISTRY" + + mkdir -p "$HOME/.config" + echo "DEBUG: Creating auth JSON" + (umask 0177; cat > "$HOME/.authjson" </dev/null 2>&1; then + echo "Installing custom CA certificates..." + cp /etc/pki/ca-trust/custom/*.pem /etc/pki/ca-trust/source/anchors/ 2>/dev/null || true + update-ca-trust extract 2>/dev/null || true + fi +} + +# Set up SELinux contexts and bind-mount osbuild for privileged execution. +# Creates OSBUILD_STORE and OSBUILD_RUN directories. +setup_osbuild() { + mkdir -p "$OSBUILD_STORE" + mkdir -p "$OSBUILD_RUN" + + chcon "system_u:object_r:root_t:s0" "$OSBUILD_STORE" || true + + if ! mountpoint -q "$OSBUILD_RUN"; then + mount -t tmpfs tmpfs "$OSBUILD_RUN" + fi + + local destPath="$OSBUILD_RUN/osbuild" + cp -p "$OSBUILD_PATH" "$destPath" + chcon "system_u:object_r:install_exec_t:s0" "$destPath" || true + + mount --bind "$destPath" "$OSBUILD_PATH" +} + +# Load custom definitions (KEY=VALUE) from a file into CUSTOM_DEFS_ARGS array. +# Sets global: CUSTOM_DEFS_ARGS (array of --define KEY=VALUE pairs) +# Args: $1 - path to custom definitions file +load_custom_definitions() { + local defs_file="$1" + declare -g -a CUSTOM_DEFS_ARGS=() + + if [ ! -f "$defs_file" ]; then + return + fi + + echo "Loading custom definitions from $defs_file" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue + validate_custom_def "$line" + CUSTOM_DEFS_ARGS+=("--define" "$line") + echo " Custom definition: $line" + done < "$defs_file" + echo "Loaded $((${#CUSTOM_DEFS_ARGS[@]} / 2)) custom definitions" +} + +# Create service account authentication JSON for container registries. +# Args: $1 - registry URL, $2 - output file path, $3 - optional token (defaults to SA token) +create_service_account_auth() { + local registry="$1" + local output_file="$2" + local token="${3:-$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null)}" + + cat > "$output_file" < "$auth_file" + elif [ -n "$REGISTRY_USERNAME" ] && [ -n "$REGISTRY_PASSWORD" ] && [ -n "$REGISTRY_URL" ]; then + echo "Creating registry auth from username/password for $REGISTRY_URL" + create_auth_json "$auth_file" "$REGISTRY_URL" "$(echo -n "$REGISTRY_USERNAME:$REGISTRY_PASSWORD" | base64 -w0)" + elif [ -n "$REGISTRY_TOKEN" ] && [ -n "$REGISTRY_URL" ]; then + echo "Creating registry auth from token for $REGISTRY_URL" + echo "DEBUG: Creating dual registry auth JSON" + # Create auth JSON with both custom registry and cluster registry + cat > "$auth_file" < "$file" </dev/null 2>&1; then + declare -g GET_SIZE_CMD="stat -c%s" + echo "DEBUG: Using GNU stat" + elif stat -f%z /dev/null >/dev/null 2>&1; then + declare -g GET_SIZE_CMD="stat -f%z" + echo "DEBUG: Using BSD stat" + else + declare -g GET_SIZE_CMD="echo ''" + echo "DEBUG: No working stat command found" + fi + echo "DEBUG: detect_stat_command completed" +} + +# Find artifact file using bash globbing instead of ls. +# Returns first matching file basename, or empty string if none found. +# Args: $1 - workspace path, $2+ - glob patterns to try +find_artifact() { + local workspace="$1" + shift + local patterns=("$@") + + for pattern in "${patterns[@]}"; do + for file in "$workspace"/$pattern; do + [ -e "$file" ] && { basename "$file"; return 0; } + done + done + return 1 +} diff --git a/internal/common/tasks/tasks.go b/internal/common/tasks/tasks.go index e192dd1a..88935e6d 100644 --- a/internal/common/tasks/tasks.go +++ b/internal/common/tasks/tasks.go @@ -2,6 +2,7 @@ package tasks import ( _ "embed" // Required for go:embed directives + "fmt" "time" automotivev1alpha1 "github.com/centos-automotive-suite/automotive-dev-operator/api/v1alpha1" @@ -329,6 +330,10 @@ func GenerateBuildAutomotiveImageTask(namespace string, buildConfig *BuildConfig Name: "TARGET_ARCH", Value: "$(params.target-architecture)", }, + { + Name: "USE_MEMORY_VOLUMES", + Value: fmt.Sprintf("%t", buildConfig != nil && buildConfig.UseMemoryVolumes), + }, }, VolumeMounts: []corev1.VolumeMount{ { @@ -1251,6 +1256,10 @@ func GeneratePrepareBuilderTask(namespace string, buildConfig *BuildConfig) *tek Name: "AIB_IMAGE", Value: "$(params.automotive-image-builder)", }, + { + Name: "USE_MEMORY_VOLUMES", + Value: fmt.Sprintf("%t", buildConfig != nil && buildConfig.UseMemoryVolumes), + }, }, Script: BuildBuilderScript, VolumeMounts: []corev1.VolumeMount{ From 2eb799830dd05ca5b43320b20c4b8ee13e08dc8c Mon Sep 17 00:00:00 2001 From: Benny Zlotnik Date: Fri, 20 Feb 2026 20:14:01 +0200 Subject: [PATCH 2/2] use pigz instead gzip Signed-off-by: Benny Zlotnik --- internal/common/tasks/scripts/build_image.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/common/tasks/scripts/build_image.sh b/internal/common/tasks/scripts/build_image.sh index fc8b83dc..03ab5f3d 100644 --- a/internal/common/tasks/scripts/build_image.sh +++ b/internal/common/tasks/scripts/build_image.sh @@ -373,6 +373,7 @@ ls -la "$WORKSPACE_PATH/" COMPRESSION="$(params.compression)" echo "Requested compression: $COMPRESSION" +GZIP_COMPRESSOR="gzip" ensure_lz4() { if ! command -v lz4 >/dev/null 2>&1; then @@ -393,8 +394,19 @@ ensure_lz4() { fi } +setup_gzip_compressor() { + if command -v pigz >/dev/null 2>&1; then + GZIP_COMPRESSOR="pigz" + echo "Using pigz for gzip compression" + else + echo "pigz not found; using gzip for compression" + fi +} + if [ "$COMPRESSION" = "lz4" ]; then ensure_lz4 +elif [ "$COMPRESSION" = "gzip" ]; then + setup_gzip_compressor fi # Simplified compression functions - no unnecessary dispatching @@ -402,7 +414,7 @@ compress_file() { local src="$1" dest="$2" case "$COMPRESSION" in lz4) lz4 -z -f -q "$src" "$dest" ;; - gzip|*) gzip -c "$src" > "$dest" ;; + gzip|*) "$GZIP_COMPRESSOR" -c "$src" > "$dest" ;; esac } @@ -410,7 +422,7 @@ tar_dir() { local dir="$1" out="$2" case "$COMPRESSION" in lz4) tar -C "$WORKSPACE_PATH" -cf - "$dir" | lz4 -z -f -q > "$out" ;; - gzip|*) tar -C "$WORKSPACE_PATH" -czf "$out" "$dir" ;; + gzip|*) tar -C "$WORKSPACE_PATH" -cf - "$dir" | "$GZIP_COMPRESSOR" -c > "$out" ;; esac }