Skip to content

feat(ci/artifacthub-images): improve collection and display of images #809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cwrau merged 1 commit into from
Mar 5, 2024
Merged

feat(ci/artifacthub-images): improve collection and display of images #809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 5 additions & 39 deletions .github/scripts/create-values-diff.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail
Expand Down Expand Up @@ -38,46 +39,11 @@ GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}"
GITHUB_DEFAULT_BRANCH="${GITHUB_DEFAULT_BRANCH:-main}"
GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}"
GITHUB_REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
SCRIPTS="$GITHUB_WORKSPACE/.github/scripts/"
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT

cd "$GITHUB_WORKSPACE"

function splitYamlIntoDir() {
local yaml="${1?}"
local dir="${2?}"
local IFS=$'\n'
local selector

for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do
local resourceName
local kind
local namespace
local name
kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')"
namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')"
name="$(jq --argjson selector "$selector" -n -r '$selector.name')"

resourceName="$dir/$namespace/$kind/$name.yaml"
if [[ -f "$resourceName" ]]; then
echo "'$resourceName' shouldn't already exist" >/dev/stderr
return 1
fi
mkdir -p "$(dirname "$resourceName")"
# shellcheck disable=SC2016
yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName"
if [[ "$kind" == "HelmRelease" ]]; then
(
"$SCRIPTS/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated"
splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")"
rm "${resourceName}_templated"
) &
fi
done
wait
}

function generateComment() {
local chart="charts/${1?}"
local -A diffs
Expand All @@ -93,13 +59,13 @@ function generateComment() {
mkdir "$originalResourcesDir" "$newResourcesDir"

(
"$SCRIPTS/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml"
splitYamlIntoDir "$originalResourcesDir.yaml" "$originalResourcesDir"
"$(dirname "$0")/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml"
"$(dirname "$0")/splitYamlIntoDir" "$originalResourcesDir.yaml" "$originalResourcesDir"
) &

(
"$SCRIPTS/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml"
splitYamlIntoDir "$newResourcesDir.yaml" "$newResourcesDir"
"$(dirname "$0")/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml"
"$(dirname "$0")/splitYamlIntoDir" "$newResourcesDir.yaml" "$newResourcesDir"
) &

wait
Expand Down
13 changes: 9 additions & 4 deletions .github/scripts/enforce-trusted-registries.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail
Expand All @@ -11,20 +12,24 @@ function getUntrustedImages() {

trustedImagesRegex="$(yq -r -f .github/scripts/trusted_images_regex.jq <.github/trusted_registries.yaml)"

yq -r '.annotations["artifacthub.io/images"]' <"$chart/Chart.yaml" |
yq -r '.annotations["artifacthub.io/images"]' "$chart/Chart.yaml" |
yq -r '.[] | .image' |
grep -v -E "$trustedImagesRegex"
grep -v -E "$trustedImagesRegex" |
sort -u
}

function enforceTrustedImages() {
local chart="${1?}"
local untrustedImages=()

mapfile -t untrustedImages < <(getUntrustedImages "$chart")
if ((${#untrustedImages[@]} > 0)); then
if [[ "${#untrustedImages[@]}" -gt 0 ]]; then
echo "found ${#untrustedImages[@]} untrusted images in '$chart', please fix;" >/dev/stderr
for untrustedImage in "${untrustedImages[@]}"; do
echo " $untrustedImage" >/dev/stderr
echo " > $untrustedImage, found in the following resources:" >/dev/stderr
# shellcheck disable=SC2016
yq --arg image "$untrustedImage" -r '.annotations["artifacthub.io/images"] | split("\n")[] | select(contains($image))' "$chart/Chart.yaml" |
awk '{print " - " $NF}' >/dev/stderr
done
return 1
fi
Expand Down
44 changes: 25 additions & 19 deletions .github/scripts/extract-artifacthub-images.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail

TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT

function templateRemoteHelmChart() {
"$(dirname "$0")/templateRemoteHelmChart"
}

function templateLocalHelmChart() {
"$(dirname "$0")/templateLocalHelmChart" "$@"
}

function templateHelmChart() {
local chart="$1"
local yaml
Expand All @@ -28,20 +21,33 @@ function templateHelmChart() {
if [[ "$numberOfHelmReleases" -gt 0 ]]; then
for index in $(seq 0 $((numberOfHelmReleases - 1))); do
echo ---
yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateRemoteHelmChart
yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | "$(dirname "$0")/templateRemoteHelmChart"
done
fi
}

function getImages() {
local chart="$1"
templateLocalHelmChart "$chart" |
grep -E '\s+image: \S+' |
grep -v 'artifacthub-ignore' |
awk '{print $NF}' |
tr -d '"' |
sort -u |
jq -Rn '[[inputs][] | {image: .}]' | yq -y
local tmpDir
tmpDir="$(mktemp -d -p "$TMP_DIR")"
"$(dirname "$0")/templateLocalHelmChart" -1 "$chart" >"$tmpDir/helmRelease.yaml"
"$(dirname "$0")/splitYamlIntoDir" "$tmpDir/helmRelease.yaml" "$tmpDir/helmRelease"

(
cd "$tmpDir/helmRelease"
rm -- */HelmRelease/*.yaml
grep -Er '\s+image: \S+' |
grep -v 'artifacthub-ignore' |
awk '{print $3 " # " $1}' |
tr -d '"' |
sed 's#:$##' |
sort -k1 -k2 |
uniq |
column -t |
jq -Rn '[[inputs][] | {image: .}]' |
yq -y |
tr -d "'"
)
}

function updateChartYaml() {
Expand All @@ -53,13 +59,13 @@ function updateChartYaml() {
(
echo "artifacthub.io/images: |"
getImages "$chart" | awk '{print " " $0}'
) | tee "$tmpDir/images.yaml"
) | tee "$tmpDir/images.yaml" >/dev/stderr

if yq -e .annotations "$chart/Chart.yaml" >/dev/null; then
echo "Existing annotations:" >/dev/stderr
yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml"
yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml" >/dev/stderr
echo "Cleaned Chart.yaml:" >/dev/stderr
yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml")
yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml") >/dev/stderr
else
touch "$tmpDir/annotations.yaml"
fi
Expand Down
3 changes: 3 additions & 0 deletions .github/scripts/prepare-values.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

function mergeYaml() {
local valuesFile="${1?}"
local overrideJson="${2?}"
Expand Down
43 changes: 43 additions & 0 deletions .github/scripts/splitYamlIntoDir
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail

function splitYamlIntoDir() {
local yaml="${1?}"
local dir="${2?}"
local IFS=$'\n'
local selector

for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do
local resourceName
local kind
local namespace
local name
kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')"
namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')"
name="$(jq --argjson selector "$selector" -n -r '$selector.name')"

resourceName="$dir/$namespace/$kind/$name.yaml"
if [[ -f "$resourceName" ]]; then
echo "'$resourceName' shouldn't already exist" >/dev/stderr
return 1
fi
mkdir -p "$(dirname "$resourceName")"
# shellcheck disable=SC2016
yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName"
if [[ "$kind" == "HelmRelease" ]]; then
(
"$(dirname "$0")/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated"
splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")"
rm "${resourceName}_templated"
) &
fi
done
wait
}

splitYamlIntoDir "$@"
1 change: 1 addition & 0 deletions .github/scripts/sync-codeowners.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

echo "* @teutonet/k8s"

Expand Down
9 changes: 4 additions & 5 deletions .github/scripts/templateHelmChart.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail
Expand Down Expand Up @@ -115,11 +116,9 @@ function templateSubHelmCharts() {
yaml=$(cat -)
numberOfHelmReleases=$(yq <<<"$yaml" -ers '[.[] | select(.kind == "HelmRelease")] | length')
echo "$yaml"
if [[ "$numberOfHelmReleases" -gt 0 ]]; then
for index in $(seq 0 $((numberOfHelmReleases - 1))); do
yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" &
done
fi
for index in $(seq 0 $((numberOfHelmReleases - 1))); do
yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" &
done
wait
for index in $(seq 0 $((numberOfHelmReleases - 1))); do
echo ---
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/trusted_images_regex.jq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@
else
"\(.[0:-1] | join("/")):\($type)"
end |
"^\(.)$"
"^\(.)(\\s|$)"
] |
join("|")
1 change: 1 addition & 0 deletions .github/scripts/validate-pullrequest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash

[[ "$RUNNER_DEBUG" == 1 ]] && set -x
[[ $- == *x* ]] && export RUNNER_DEBUG=1

set -eu
set -o pipefail
Expand Down