Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 87 additions & 8 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ on:
required: false
type: boolean
default: false
helm-version:
description: "Optional exact Helm chart version override for stable Helm-only releases."
required: false
type: string
default: ""
update-ngc-metadata:
description: "Synchronize NGC metadata from the this branch."
required: false
Expand Down Expand Up @@ -143,11 +148,13 @@ jobs:
release_label: ${{ steps.plan.outputs.release_label }}
nightly_timestamp: ${{ steps.plan.outputs.nightly_timestamp }}
wheel_version: ${{ steps.wheel-version.outputs.wheel_version }}
helm_version: ${{ steps.helm-version.outputs.helm_version }}
wheel_ids: ${{ steps.plan.outputs.wheel_ids }}
container_ids: ${{ steps.plan.outputs.container_ids }}
has_wheels: ${{ steps.plan.outputs.has_wheels }}
has_containers: ${{ steps.plan.outputs.has_containers }}
include_helm: ${{ steps.plan.outputs.include_helm }}
helm_version_override: ${{ steps.plan.outputs.helm_version_override }}
Comment thread
crookedstorm marked this conversation as resolved.
update_ngc_metadata: ${{ steps.plan.outputs.update_ngc_metadata }}
send_notifications: ${{ steps.plan.outputs.send_notifications }}
dry_run: ${{ steps.plan.outputs.dry_run }}
Expand All @@ -166,13 +173,22 @@ jobs:
const allContainers = JSON.parse(process.env.RELEASE_CONTAINERS_JSON);
const allWheelIds = allWheels.map((wheel) => wheel.id);
const allContainerIds = allContainers.map((container) => container.id);
const semverCorePattern = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
// ECMA-compatible SemVer 2.0.0 pattern from https://semver.org/.
const semverPattern = new RegExp(
"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"
+ "(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)"
+ "(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
+ "(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$",
);
const inputs = context.payload.inputs ?? {};
const isManual = context.eventName === "workflow_dispatch";
const releaseType = isManual ? inputs["release-type"] : "nightly";
const releaseScope = isManual ? (inputs["release-scope"] || "all") : "all";
const updateNgcMetadata = isManual && inputs["update-ngc-metadata"] === "true";
const sendNotifications = inputs["send-notifications"] !== "false";
const dryRun = isManual && inputs["dry-run"] === "true";
const helmVersionOverride = isManual ? (inputs["helm-version"] ?? "").trim() : "";

let sourceSha = isManual ? (inputs["source-sha"] ?? "").trim() : context.sha;
const version = releaseType === "stable" ? (inputs.version ?? "").trim() : "";
Expand All @@ -181,7 +197,7 @@ jobs:
if (!/^[0-9a-f]{40}$/i.test(sourceSha)) {
throw new Error("Stable releases require an exact 40-character source SHA.");
}
if (!/^\d+\.\d+\.\d+$/.test(version)) {
if (!semverCorePattern.test(version)) {
throw new Error("Stable releases require a MAJOR.MINOR.PATCH version.");
}
} else {
Expand Down Expand Up @@ -271,6 +287,20 @@ jobs:
const wheelIds = wheels.map((wheel) => wheel.id);
const containerIds = containers.map((container) => container.id);

if (helmVersionOverride) {
if (
releaseType !== "stable"
|| !includeHelm
|| wheelIds.length > 0
|| containerIds.length > 0
) {
throw new Error("helm-version can only be used for stable Helm-only releases.");
}
if (!semverPattern.test(helmVersionOverride)) {
throw new Error("helm-version must be a SemVer chart version.");
}
}

const nightlyTimestamp = releaseType === "nightly"
? new Date().toISOString().replace(/\D/g, "").slice(0, 14)
: "";
Expand All @@ -291,6 +321,7 @@ jobs:
core.setOutput("has_wheels", String(wheelIds.length > 0));
core.setOutput("has_containers", String(containerIds.length > 0));
core.setOutput("include_helm", String(includeHelm));
core.setOutput("helm_version_override", helmVersionOverride);
core.setOutput("update_ngc_metadata", String(updateNgcMetadata));
core.setOutput("send_notifications", String(sendNotifications));
core.setOutput("dry_run", String(dryRun));
Expand Down Expand Up @@ -447,6 +478,58 @@ jobs:
wheel_version="$(python3 .github/scripts/stamp_sdk_version.py "${args[@]}")"
echo "wheel_version=${wheel_version}" >> "${GITHUB_OUTPUT}"

- name: Resolve Helm version
id: helm-version
if: steps.plan.outputs.include_helm == 'true'
shell: bash
env:
HELM_CHART: ${{ env.RELEASE_HELM_PATH }}
RELEASE_TYPE: ${{ steps.plan.outputs.release_type }}
RELEASE_VERSION: ${{ steps.plan.outputs.version }}
NIGHTLY_TIMESTAMP: ${{ steps.plan.outputs.nightly_timestamp }}
HELM_VERSION_OVERRIDE: ${{ steps.plan.outputs.helm_version_override }}
run: |
set -euo pipefail

if [[ -n "${HELM_VERSION_OVERRIDE}" ]]; then
helm_version="${HELM_VERSION_OVERRIDE}"
elif [[ "${RELEASE_TYPE}" == "nightly" ]]; then
# Nightlies align to the release core from the latest release or
# RC tag. Build-metadata chart fixes use helm-version instead.
base_version="$(
git tag --merged HEAD --list \
| awk '
/^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$/ {
sub(/-rc[0-9]+$/, "")
print
}
' \
| sort -V -u \
| tail -n 1
)"
if [[ -z "${base_version}" ]]; then
base_version="$(
awk '
$1 == "version:" {
version = $2
gsub(/^["'\''"]|["'\''"]$/, "", version)
print version
exit
}
' "${HELM_CHART}/Chart.yaml"
)"
fi
if [[ -z "${base_version}" ]]; then
echo "Unable to resolve Helm chart base version for nightly release." >&2
exit 1
fi
helm_version="${base_version}-nightly-${NIGHTLY_TIMESTAMP}"
else
helm_version="${RELEASE_VERSION}"
fi

echo "helm_version=${helm_version}" >> "${GITHUB_OUTPUT}"

notify-start:
# Start alerts intentionally run for dry runs so the webhook can be tested.
name: Notify release start
Expand Down Expand Up @@ -666,9 +749,8 @@ jobs:
NIGHTLY_IMAGE_REGISTRY: ${{ env.RELEASE_NIGHTLY_CONTAINER_REGISTRY }}
STABLE_IMAGE_REGISTRY: ${{ env.RELEASE_HELM_REGISTRY }}
RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
RELEASE_VERSION: ${{ needs.plan-release.outputs.version }}
RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
NIGHTLY_TIMESTAMP: ${{ needs.plan-release.outputs.nightly_timestamp }}
HELM_VERSION: ${{ needs.plan-release.outputs.helm_version }}
run: |
set -euo pipefail

Expand All @@ -680,17 +762,14 @@ jobs:
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm dependency build "${chart_dir}"

chart_version="$(yq -r '.version' "${chart_dir}/Chart.yaml")"
if [[ "${RELEASE_TYPE}" == "nightly" ]]; then
chart_version="${chart_version}-nightly-${NIGHTLY_TIMESTAMP}"
release_registry="${NIGHTLY_IMAGE_REGISTRY}"
else
# TODO: Decide whether stable chart versions should track the platform
# release version or remain independently managed in Chart.yaml.
chart_version="${RELEASE_VERSION}"
release_registry="${STABLE_IMAGE_REGISTRY}"
fi

chart_version="${HELM_VERSION}"

yq -i ".platformConfig.platform.image_registry = \"${release_registry}\"" "${chart_dir}/values.yaml"
yq -i ".api.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml"
yq -i ".core.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml"
Expand Down
19 changes: 11 additions & 8 deletions RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ and select **Run workflow**. The form shows the allowed custom artifact IDs.
| `release-scope` | `all` by default. Select `wheels`, `containers`, `helm`, or `custom` for a subset. |
| `wheel-ids`, `container-ids` | Comma-separated IDs used only with `release-scope: custom`. Each ID must be in the catalog above; duplicates and empty entries fail validation. |
| `include-helm` | Includes the Helm chart in a custom release. |
| `helm-version` | Optional exact SemVer Helm chart version for stable Helm-only releases. The stable release label still comes from `version`. |
| `update-ngc-metadata` | Also runs the reusable NGC metadata workflow for `nemo-platform` and `nemo-platform-dev`. It checks out the workflow ref, normally `main`. |
| `send-notifications` | Sends Slack start and final-status notifications. Defaults to `true`. |
| `dry-run` | Validates the selected source and packages the selected Helm chart, but does not publish, dispatch external work, poll, create a GitHub release, or signal deployment. The start notification intentionally still runs when notifications are enabled. |
Expand All @@ -68,26 +69,28 @@ Examples:
| Stable full release | `release-type: stable`, `source-sha: <40-character SHA>`, `version: <MAJOR.MINOR.PATCH>`, `release-scope: all`. |
| One container | `release-scope: custom`, `container-ids: nmp-automodel-tasks`. |
| Helm-only validation | `release-scope: helm`, `dry-run: true`. |
| Stable Helm-only chart override | `release-type: stable`, `source-sha: <40-character SHA>`, `version: 0.1.0`, `release-scope: helm`, `helm-version: 0.1.0+helmfix1`. |

Nightlies also run automatically Monday through Friday at 8:00 PM
America/Los_Angeles.

## What the workflow does

1. Resolves the source, release label, selected artifacts, and wheel version.
Stable versions use the supplied release version. Nightly labels use
`nightly-<UTC timestamp>` and the wheel version is resolved by
`.github/scripts/stamp_sdk_version.py`.
1. Resolves the source, release label, selected artifacts, wheel version, and
Helm chart version. Stable versions use the supplied release version.
Nightly labels use `nightly-<UTC timestamp>` and the wheel version is
resolved by `.github/scripts/stamp_sdk_version.py`.
2. Checks out the selected source and validates the selected wheel paths,
Docker Bake targets, and NGC overview files.
3. Optionally synchronizes NGC metadata, when requested on a non-dry-run.
4. Dispatches wheel, container, and stable-release registration work to the
configured internal release repository. The selected source SHA, release
type, version, and selected IDs are passed with the dispatch.
5. Packages the Helm chart. A nightly chart uses the `Chart.yaml` version with
`-nightly-<UTC timestamp>` appended. A stable chart currently uses the
stable release version. Whether stable chart versions should instead remain
independently managed in `Chart.yaml` is an open policy decision.
5. Packages the Helm chart with the planned chart version. A nightly chart uses
the latest release or RC Git tag core reachable from the selected source with
`-nightly-<UTC timestamp>` appended, falling back to the `Chart.yaml`
version if no such tag exists. A stable chart uses the stable release version
unless `helm-version` is set for a stable Helm-only release.
6. Waits for every selected final artifact to become public before continuing.
The polling job times out after four hours and sends a Slack alert after two
hours if it is still waiting.
Expand Down