diff --git a/.github/scripts/create-values-diff.sh b/.github/scripts/create-values-diff.sh index 76c22eafda..1bc3502f84 100755 --- a/.github/scripts/create-values-diff.sh +++ b/.github/scripts/create-values-diff.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 set -eu set -o pipefail @@ -38,46 +39,11 @@ GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" GITHUB_DEFAULT_BRANCH="${GITHUB_DEFAULT_BRANCH:-main}" GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}" GITHUB_REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" -SCRIPTS="$GITHUB_WORKSPACE/.github/scripts/" TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT cd "$GITHUB_WORKSPACE" -function splitYamlIntoDir() { - local yaml="${1?}" - local dir="${2?}" - local IFS=$'\n' - local selector - - for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do - local resourceName - local kind - local namespace - local name - kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')" - namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')" - name="$(jq --argjson selector "$selector" -n -r '$selector.name')" - - resourceName="$dir/$namespace/$kind/$name.yaml" - if [[ -f "$resourceName" ]]; then - echo "'$resourceName' shouldn't already exist" >/dev/stderr - return 1 - fi - mkdir -p "$(dirname "$resourceName")" - # shellcheck disable=SC2016 - yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName" - if [[ "$kind" == "HelmRelease" ]]; then - ( - "$SCRIPTS/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated" - splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")" - rm "${resourceName}_templated" - ) & - fi - done - wait -} - function generateComment() { local chart="charts/${1?}" local -A diffs @@ -93,13 +59,13 @@ function generateComment() { mkdir "$originalResourcesDir" "$newResourcesDir" ( - "$SCRIPTS/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml" - splitYamlIntoDir "$originalResourcesDir.yaml" "$originalResourcesDir" + "$(dirname "$0")/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml" + "$(dirname "$0")/splitYamlIntoDir" "$originalResourcesDir.yaml" "$originalResourcesDir" ) & ( - "$SCRIPTS/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml" - splitYamlIntoDir "$newResourcesDir.yaml" "$newResourcesDir" + "$(dirname "$0")/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml" + "$(dirname "$0")/splitYamlIntoDir" "$newResourcesDir.yaml" "$newResourcesDir" ) & wait diff --git a/.github/scripts/enforce-trusted-registries.sh b/.github/scripts/enforce-trusted-registries.sh index 4c41d1fe85..6f27b33999 100755 --- a/.github/scripts/enforce-trusted-registries.sh +++ b/.github/scripts/enforce-trusted-registries.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 set -eu set -o pipefail @@ -11,9 +12,10 @@ function getUntrustedImages() { trustedImagesRegex="$(yq -r -f .github/scripts/trusted_images_regex.jq <.github/trusted_registries.yaml)" - yq -r '.annotations["artifacthub.io/images"]' <"$chart/Chart.yaml" | + yq -r '.annotations["artifacthub.io/images"]' "$chart/Chart.yaml" | yq -r '.[] | .image' | - grep -v -E "$trustedImagesRegex" + grep -v -E "$trustedImagesRegex" | + sort -u } function enforceTrustedImages() { @@ -21,10 +23,13 @@ function enforceTrustedImages() { local untrustedImages=() mapfile -t untrustedImages < <(getUntrustedImages "$chart") - if ((${#untrustedImages[@]} > 0)); then + if [[ "${#untrustedImages[@]}" -gt 0 ]]; then echo "found ${#untrustedImages[@]} untrusted images in '$chart', please fix;" >/dev/stderr for untrustedImage in "${untrustedImages[@]}"; do - echo " $untrustedImage" >/dev/stderr + echo " > $untrustedImage, found in the following resources:" >/dev/stderr + # shellcheck disable=SC2016 + yq --arg image "$untrustedImage" -r '.annotations["artifacthub.io/images"] | split("\n")[] | select(contains($image))' "$chart/Chart.yaml" | + awk '{print " - " $NF}' >/dev/stderr done return 1 fi diff --git a/.github/scripts/extract-artifacthub-images.sh b/.github/scripts/extract-artifacthub-images.sh index 7f0d8bc071..34ae16fee4 100755 --- a/.github/scripts/extract-artifacthub-images.sh +++ b/.github/scripts/extract-artifacthub-images.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 set -eu set -o pipefail @@ -8,14 +9,6 @@ set -o pipefail TMP_DIR=$(mktemp -d) trap 'rm -rf "$TMP_DIR"' EXIT -function templateRemoteHelmChart() { - "$(dirname "$0")/templateRemoteHelmChart" -} - -function templateLocalHelmChart() { - "$(dirname "$0")/templateLocalHelmChart" "$@" -} - function templateHelmChart() { local chart="$1" local yaml @@ -28,20 +21,33 @@ function templateHelmChart() { if [[ "$numberOfHelmReleases" -gt 0 ]]; then for index in $(seq 0 $((numberOfHelmReleases - 1))); do echo --- - yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateRemoteHelmChart + yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | "$(dirname "$0")/templateRemoteHelmChart" done fi } function getImages() { local chart="$1" - templateLocalHelmChart "$chart" | - grep -E '\s+image: \S+' | - grep -v 'artifacthub-ignore' | - awk '{print $NF}' | - tr -d '"' | - sort -u | - jq -Rn '[[inputs][] | {image: .}]' | yq -y + local tmpDir + tmpDir="$(mktemp -d -p "$TMP_DIR")" + "$(dirname "$0")/templateLocalHelmChart" -1 "$chart" >"$tmpDir/helmRelease.yaml" + "$(dirname "$0")/splitYamlIntoDir" "$tmpDir/helmRelease.yaml" "$tmpDir/helmRelease" + + ( + cd "$tmpDir/helmRelease" + rm -- */HelmRelease/*.yaml + grep -Er '\s+image: \S+' | + grep -v 'artifacthub-ignore' | + awk '{print $3 " # " $1}' | + tr -d '"' | + sed 's#:$##' | + sort -k1 -k2 | + uniq | + column -t | + jq -Rn '[[inputs][] | {image: .}]' | + yq -y | + tr -d "'" + ) } function updateChartYaml() { @@ -53,13 +59,13 @@ function updateChartYaml() { ( echo "artifacthub.io/images: |" getImages "$chart" | awk '{print " " $0}' - ) | tee "$tmpDir/images.yaml" + ) | tee "$tmpDir/images.yaml" >/dev/stderr if yq -e .annotations "$chart/Chart.yaml" >/dev/null; then echo "Existing annotations:" >/dev/stderr - yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml" + yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml" >/dev/stderr echo "Cleaned Chart.yaml:" >/dev/stderr - yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml") + yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml") >/dev/stderr else touch "$tmpDir/annotations.yaml" fi diff --git a/.github/scripts/prepare-values.sh b/.github/scripts/prepare-values.sh index 0aeaa6db89..aa0c177646 100755 --- a/.github/scripts/prepare-values.sh +++ b/.github/scripts/prepare-values.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +[[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 + function mergeYaml() { local valuesFile="${1?}" local overrideJson="${2?}" diff --git a/.github/scripts/splitYamlIntoDir b/.github/scripts/splitYamlIntoDir new file mode 100755 index 0000000000..e6379f725b --- /dev/null +++ b/.github/scripts/splitYamlIntoDir @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +[[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 + +set -eu +set -o pipefail + +function splitYamlIntoDir() { + local yaml="${1?}" + local dir="${2?}" + local IFS=$'\n' + local selector + + for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do + local resourceName + local kind + local namespace + local name + kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')" + namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')" + name="$(jq --argjson selector "$selector" -n -r '$selector.name')" + + resourceName="$dir/$namespace/$kind/$name.yaml" + if [[ -f "$resourceName" ]]; then + echo "'$resourceName' shouldn't already exist" >/dev/stderr + return 1 + fi + mkdir -p "$(dirname "$resourceName")" + # shellcheck disable=SC2016 + yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName" + if [[ "$kind" == "HelmRelease" ]]; then + ( + "$(dirname "$0")/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated" + splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")" + rm "${resourceName}_templated" + ) & + fi + done + wait +} + +splitYamlIntoDir "$@" diff --git a/.github/scripts/sync-codeowners.sh b/.github/scripts/sync-codeowners.sh index 12813e578b..ca7ad5e267 100755 --- a/.github/scripts/sync-codeowners.sh +++ b/.github/scripts/sync-codeowners.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 echo "* @teutonet/k8s" diff --git a/.github/scripts/templateHelmChart.sh b/.github/scripts/templateHelmChart.sh index fdfe4a3a2e..6064a30351 100755 --- a/.github/scripts/templateHelmChart.sh +++ b/.github/scripts/templateHelmChart.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 set -eu set -o pipefail @@ -115,11 +116,9 @@ function templateSubHelmCharts() { yaml=$(cat -) numberOfHelmReleases=$(yq <<<"$yaml" -ers '[.[] | select(.kind == "HelmRelease")] | length') echo "$yaml" - if [[ "$numberOfHelmReleases" -gt 0 ]]; then - for index in $(seq 0 $((numberOfHelmReleases - 1))); do - yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" & - done - fi + for index in $(seq 0 $((numberOfHelmReleases - 1))); do + yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" & + done wait for index in $(seq 0 $((numberOfHelmReleases - 1))); do echo --- diff --git a/.github/scripts/trusted_images_regex.jq b/.github/scripts/trusted_images_regex.jq index 2e15441ef0..0e13708bf3 100644 --- a/.github/scripts/trusted_images_regex.jq +++ b/.github/scripts/trusted_images_regex.jq @@ -8,6 +8,6 @@ else "\(.[0:-1] | join("/")):\($type)" end | - "^\(.)$" + "^\(.)(\\s|$)" ] | join("|") diff --git a/.github/scripts/validate-pullrequest.sh b/.github/scripts/validate-pullrequest.sh index 7e1005c782..ddc003435c 100755 --- a/.github/scripts/validate-pullrequest.sh +++ b/.github/scripts/validate-pullrequest.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash [[ "$RUNNER_DEBUG" == 1 ]] && set -x +[[ $- == *x* ]] && export RUNNER_DEBUG=1 set -eu set -o pipefail