diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53574fdea22..a238f2c9999 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,16 @@ .merge_train_rule: &merge_train_rule - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" + INTEGRATION_TEST: 'no' INTEGRATION_TEST_SCOPE: mr - FUNCTIONAL_TEST: "yes" + FUNCTIONAL_TEST: 'yes' FUNCTIONAL_TEST_SCOPE: mr-slim FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' workflow: rules: @@ -35,30 +35,30 @@ workflow: # For push to main - if: $CI_PIPELINE_SOURCE == 'push' && ($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH =~ /^core_/) variables: - UNIT_TEST: "no" - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "yes" + UNIT_TEST: 'no' + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'yes' FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 5 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' FUNCTIONAL_TEST_TIME_LIMIT: 3600 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' auto_cancel: on_new_commit: interruptible # For merge-trains that need to be fast-tracked - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' && $CI_MERGE_REQUEST_LABELS =~ /fast-track/ variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "no" - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'no' + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' # For normal merge-trains - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merge_train' @@ -67,75 +67,75 @@ workflow: # For MRs with integration suite - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run tests/ variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "yes" + INTEGRATION_TEST: 'yes' INTEGRATION_TEST_SCOPE: mr - FUNCTIONAL_TEST: "no" + FUNCTIONAL_TEST: 'no' FUNCTIONAL_TEST_SCOPE: mr-slim FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' # For MRs with nightly - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run nightly/ variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "yes" + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'yes' FUNCTIONAL_TEST_SCOPE: nightly FUNCTIONAL_TEST_REPEAT: 5 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' # For MRs with weekly - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run weekly/ variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "yes" + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'yes' FUNCTIONAL_TEST_SCOPE: weekly FUNCTIONAL_TEST_REPEAT: 1 - FUNCTIONAL_TEST_RECORD_CHECKPOINTS: "no" + FUNCTIONAL_TEST_RECORD_CHECKPOINTS: 'no' FUNCTIONAL_TEST_TIME_LIMIT: 9000 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' # For MRs with heavy suite - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' && $CI_MERGE_REQUEST_LABELS =~ /Run functional tests/ variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "yes" + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'yes' FUNCTIONAL_TEST_SCOPE: mr FUNCTIONAL_TEST_REPEAT: 1 FUNCTIONAL_TEST_TIME_LIMIT: 2700 - CLUSTER_A100: "" - CLUSTER_H100: "" - PUBLISH: "no" + CLUSTER_A100: '' + CLUSTER_H100: '' + PUBLISH: 'no' # Default MRs - if: $CI_MERGE_REQUEST_EVENT_TYPE == 'merged_result' variables: - UNIT_TEST: "yes" + UNIT_TEST: 'yes' UNIT_TEST_REPEAT: 1 UNIT_TEST_TIMEOUT: 30 - INTEGRATION_TEST: "no" - FUNCTIONAL_TEST: "no" - PUBLISH: "no" + INTEGRATION_TEST: 'no' + FUNCTIONAL_TEST: 'no' + PUBLISH: 'no' - when: never @@ -157,104 +157,109 @@ default: variables: BUILD: - value: "yes" + value: 'yes' UNIT_TEST: - value: "yes" + value: 'yes' options: - - "yes" - - "no" + - 'yes' + - 'no' description: To run the funtional test suite UNIT_TEST_REPEAT: - value: "1" - description: "Number of repetitions" + value: '1' + description: 'Number of repetitions' UNIT_TEST_TIMEOUT: - value: "30" + value: '30' description: Timeout (minutes) for Unit tests (all repeats) INTEGRATION_TEST: - value: "yes" + value: 'yes' options: - - "yes" - - "no" + - 'yes' + - 'no' description: To run the integration test suite INTEGRATION_TEST_SCOPE: - value: "mr" + value: 'mr' options: - - "mr" - - "nightly" - - "weekly" - - "pre-release" - - "release" - description: "Testsuite to run (only for INTEGRATION_TEST=yes)" + - 'mr' + - 'nightly' + - 'weekly' + - 'pre-release' + - 'release' + description: 'Testsuite to run (only for INTEGRATION_TEST=yes)' INTEGRATION_TEST_TIME_LIMIT: - value: "900" - description: "Timeout in seconds per test" + value: '900' + description: 'Timeout in seconds per test' INTEGRATION_TEST_CASES: - value: "all" + value: 'all' description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite." FUNCTIONAL_TEST: - value: "yes" + value: 'yes' options: - - "yes" - - "no" + - 'yes' + - 'no' description: To run the funtional test suite FUNCTIONAL_TEST_SCOPE: - value: "mr" + value: 'mr' options: - - "mr" - - "nightly" - - "weekly" - - "pre-release" - - "release" - description: "Testsuite to run (only for FUNCTIONAL_TEST=yes)" + - 'mr' + - 'nightly' + - 'weekly' + - 'pre-release' + - 'release' + description: 'Testsuite to run (only for FUNCTIONAL_TEST=yes)' FUNCTIONAL_TEST_REPEAT: - value: "5" - description: "Number of repetitions per test" + value: '5' + description: 'Number of repetitions per test' FUNCTIONAL_TEST_TIME_LIMIT: - value: "2700" - description: "Timeout in seconds per test" + value: '2700' + description: 'Timeout in seconds per test' FUNCTIONAL_TEST_CASES: - value: "all" + value: 'all' description: "Comma-separated list of test_cases to run. Use 'all' to run the full suite." FUNCTIONAL_TEST_NAME: - description: "Name of functional test run (only for pre-release and release)" - value: "$$CI_COMMIT_SHA" + description: 'Name of functional test run (only for pre-release and release)' + value: '$$CI_COMMIT_SHA' FUNCTIONAL_TEST_RECORD_CHECKPOINTS: - value: "no" - description: "Record golden checkpoints" + value: 'no' + description: 'Record golden checkpoints' options: - - "yes" - - "no" + - 'yes' + - 'no' CLUSTER_A100: - value: "dgxa100_dracooci" + value: 'dgxa100_dracooci' options: - - "dgxa100_dracooci" - - "dgxa100_dracooci-ord" - description: "Cluster for A100 workloads" + - 'dgxa100_dracooci' + - 'dgxa100_dracooci-ord' + description: 'Cluster for A100 workloads' CLUSTER_H100: - value: "dgxh100_coreweave" + value: 'dgxh100_coreweave' options: - - "dgxh100_coreweave" - - "dgxh100_eos" - description: "Cluster for H100 workloads" + - 'dgxh100_coreweave' + - 'dgxh100_eos' + description: 'Cluster for H100 workloads' + CLUSTER_GB200: + value: 'dgxgb200_oci-hsg' + options: + - 'dgxgb200_oci-hsg' + description: 'Cluster for H100 workloads' PUBLISH: - value: "no" + value: 'no' options: - - "yes" - - "no" + - 'yes' + - 'no' description: Build and publish a wheel to PyPi PUBLISH_COMMIT: - value: "$$CI_COMMIT_SHA" + value: '$$CI_COMMIT_SHA' description: Which commit to publish PUBLISH_VERSION_BUMP_BRANCH: - value: "$$CI_COMMIT_BRANCH" + value: '$$CI_COMMIT_BRANCH' description: Which branch to target for version bump PUBLISH_SCOPE: - value: "code-freeze" + value: 'code-freeze' options: - - "code-freeze" - - "release" - - "review-reminder" - - "upgrade-dependencies" + - 'code-freeze' + - 'release' + - 'review-reminder' + - 'upgrade-dependencies' description: Type of publish (freeze or final release) # CI wide variables @@ -262,7 +267,7 @@ variables: CI_MCORE_DEV_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci_dev CI_NEMO_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/nemo_ci UTILITY_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_utility - TE_GIT_REF: "" + TE_GIT_REF: '' include: - .gitlab/stages/00.pre.yml diff --git a/.gitlab/scripts/build.sh b/.gitlab/scripts/build.sh index e64434e834d..8359731e3d7 100644 --- a/.gitlab/scripts/build.sh +++ b/.gitlab/scripts/build.sh @@ -22,15 +22,21 @@ ADDITIONAL_PARAMS=() if [[ "$CI_COMMIT_BRANCH" == "ci-rebuild-mcore-nemo-image" || "$CI_COMMIT_BRANCH" == "main" || "$CI_COMMIT_BRANCH" == "dev" ]]; then ADDITIONAL_PARAMS+=("--pull") - ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:main,mode=max") - ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_COMMIT_BRANCH}") -elif [[ -n "$CI_MERGE_REQUEST_IID" ]]; then - ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID},mode=max") - ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_MERGE_REQUEST_IID}") +fi + +CI_COMMIT_BRANCH=$(echo "$CI_COMMIT_BRANCH" | tr '/' '-' | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g') +ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_COMMIT_BRANCH}-${PLATFORM},mode=max") +ADDITIONAL_PARAMS+=("--cache-from type=registry,ref=${IMAGE}-buildcache:${CI_COMMIT_BRANCH}-${PLATFORM}") +ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_COMMIT_BRANCH}-${PLATFORM}") + +if [[ -n "$CI_MERGE_REQUEST_IID" ]]; then + ADDITIONAL_PARAMS+=("--cache-to type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID}-${PLATFORM},mode=max") + ADDITIONAL_PARAMS+=("--cache-from type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID}-${PLATFORM}") + ADDITIONAL_PARAMS+=("-t ${IMAGE}:${CI_MERGE_REQUEST_IID}-${PLATFORM}") fi if [[ "$CI_COMMIT_BRANCH" == "ci-nightly" ]]; then - ADDITIONAL_PARAMS+=("-t ${IMAGE}:nightly") + ADDITIONAL_PARAMS+=("-t ${IMAGE}:nightly-${PLATFORM}") fi if [[ -n "$TE_GIT_REF" ]]; then @@ -46,13 +52,11 @@ DOCKER_BUILDKIT=1 docker build \ --secret id=LOGGER_INDEX_URL \ --target $STAGE \ -f docker/$FILE \ - -t ${IMAGE}:${CI_PIPELINE_ID} \ + -t ${IMAGE}:${CI_PIPELINE_ID}-${PLATFORM} \ --builder=container \ --build-arg JET_API_VERSION=$JET_API_VERSION \ - --cache-from type=registry,ref=${IMAGE}-buildcache:${CI_MERGE_REQUEST_IID} \ - --cache-from type=registry,ref=${IMAGE}-buildcache:dev \ - --cache-from type=registry,ref=${IMAGE}-buildcache:main \ --build-arg FROM_IMAGE_NAME=$BASE_IMAGE \ + --provenance=false \ --push \ --progress plain \ ${ADDITIONAL_PARAMS[@]} . diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index b3ab8cc5bd5..20252e7d045 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -9,21 +9,20 @@ extends: [.build_rules, .dind_rules] stage: build tags: - - arch/amd64 + - arch/${PLATFORM} - origin/jet-fleet - env/prod - - ${TAG} + - purpose/builder-large services: - name: docker:24.0.5-dind variables: - HEALTHCHECK_TCP_PORT: '2376' + HEALTHCHECK_TCP_PORT: "2376" timeout: 180m variables: DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: '/certs' + DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: '$DOCKER_TLS_CERTDIR/client' - TAG: purpose/builder-large + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" STAGE: jet MCORE_BACKWARDS_REF: core_r0.14.0 KUBERNETES_SERVICE_MEMORY_REQUEST: 90Gi @@ -48,7 +47,7 @@ reports: dotenv: build.env -test:build_image: +test:pre_build_image: extends: [.build_image] parallel: matrix: @@ -56,13 +55,30 @@ test:build_image: FILE: Dockerfile.ci.dev IMAGE_TYPE: lts BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 + PLATFORM: amd64 + - IMAGE: CI_MCORE_LTS_IMAGE + FILE: Dockerfile.ci.dev + IMAGE_TYPE: lts + BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 + PLATFORM: arm64 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev IMAGE_TYPE: dev BASE_IMAGE: nvcr.io/nvidia/pytorch:25.11-py3 + PLATFORM: amd64 + - IMAGE: CI_MCORE_DEV_IMAGE + FILE: Dockerfile.ci.dev + IMAGE_TYPE: dev + BASE_IMAGE: nvcr.io/nvidia/pytorch:25.11-py3 + PLATFORM: arm64 + - IMAGE: UTILITY_IMAGE + FILE: Dockerfile.linting + BASE_IMAGE: python:3.10 + PLATFORM: amd64 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting BASE_IMAGE: python:3.10 + PLATFORM: arm64 test:build_nemo_image: extends: [.build_image] @@ -70,6 +86,57 @@ test:build_nemo_image: IMAGE: CI_NEMO_IMAGE FILE: Dockerfile.ci.nemo BASE_IMAGE: nvcr.io/nvidian/nemo:nightly + PLATFORM: amd64 rules: - if: $FUNCTIONAL_TEST == "yes" || $INTEGRATION_TEST == "yes" || $CI_COMMIT_BRANCH == "ci-rebuild-mcore-nemo-image" when: on_success + +test:build_image: + needs: [test:pre_build_image] + extends: [.build_rules, .dind_rules] + parallel: + matrix: + - IMAGE: CI_MCORE_LTS_IMAGE + - IMAGE: CI_MCORE_DEV_IMAGE + - IMAGE: UTILITY_IMAGE + stage: build + tags: + - arch/amd64 + - origin/jet-fleet + - env/prod + - purpose/builder-large + services: + - name: docker:24.0.5-dind + variables: + HEALTHCHECK_TCP_PORT: "2376" + timeout: 180m + variables: + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + STAGE: jet + MCORE_BACKWARDS_REF: core_r0.14.0 + KUBERNETES_SERVICE_MEMORY_REQUEST: 90Gi + KUBERNETES_SERVICE_MEMORY_LIMIT: 90Gi + SHARED_PATH: /builds/$CI_PROJECT_PATH/shared + script: + - | + set -x + + env + eval "IMAGE=\$$IMAGE" + + docker manifest create ${IMAGE}:${CI_PIPELINE_ID} \ + ${IMAGE}:${CI_PIPELINE_ID}-amd64 \ + ${IMAGE}:${CI_PIPELINE_ID}-arm64 + + docker manifest push ${IMAGE}:${CI_PIPELINE_ID} + - echo "MCORE_MR_COMMIT=$CI_COMMIT_SHA" | tee -a build.env + - echo "MCORE_BACKWARDS_COMMIT=$MCORE_BACKWARDS_COMMIT" | tee -a build.env + - cat build.env + retry: + max: 2 + artifacts: + reports: + dotenv: build.env diff --git a/.gitlab/stages/03.integration-tests.yml b/.gitlab/stages/03.integration-tests.yml index 824721b9fb1..d28ecd8e137 100644 --- a/.gitlab/stages/03.integration-tests.yml +++ b/.gitlab/stages/03.integration-tests.yml @@ -43,6 +43,7 @@ integration:configure: - | A100_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo $CLUSTER_A100 || echo $DEFAULT_A100_CLUSTER) H100_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo $CLUSTER_H100 || echo $DEFAULT_H100_CLUSTER) + GB200_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo $CLUSTER_GB200 || echo $DEFAULT_GB200_CLUSTER) - | ARGS=( "--scope $INTEGRATION_TEST_SCOPE" @@ -88,12 +89,30 @@ integration:configure: --platform dgx_h100 \ --cluster $H100_CLUSTER \ --output-path "functional-test-job-lts-H100.yaml" + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${ARGS[@]} \ + --environment lts \ + --platform dgx_gb2100 \ + --cluster $GB200_CLUSTER \ + --output-path "functional-test-job-lts-GB200.yaml" + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${ARGS[@]} \ + --environment lts \ + --platform dgx_gb200 \ + --cluster $GB200_CLUSTER \ + --output-path "functional-test-job-lts-GB200.yaml" artifacts: paths: - functional-test-job-lts-A100.yaml - functional-test-job-lts-H100.yaml - functional-test-job-dev-H100.yaml - functional-test-job-dev-A100.yaml + - functional-test-job-lts-GB200.yaml + - functional-test-job-dev-GB200.yaml - tests/test_utils/local_recipes .integration_run: @@ -132,6 +151,12 @@ integration:run_lts_dgx_h100: ENVIRONMENT: lts CLUSTER: H100 +integration:run_lts_dgx_gb200: + extends: [.integration_run] + variables: + ENVIRONMENT: lts + CLUSTER: GB200 + integration:run_dev_dgx_a100: extends: [.integration_run] variables: @@ -143,3 +168,9 @@ integration:run_dev_dgx_h100: variables: ENVIRONMENT: dev CLUSTER: H100 + +integration:run_dev_dgx_gb200: + extends: [.integration_run] + variables: + ENVIRONMENT: dev + CLUSTER: GB200 diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index eee5a9b80fe..d32ff86a344 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -50,6 +50,7 @@ functional:configure: - | A100_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo $CLUSTER_A100 || echo $DEFAULT_A100_CLUSTER) H100_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo $CLUSTER_H100 || echo $DEFAULT_H100_CLUSTER) + GB200_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo $CLUSTER_GB200 || echo $DEFAULT_GB200_CLUSTER) - | RECORD_CHECKPOINTS=$([[ "$CI_MERGE_REQUEST_LABELS" == *"Record checkpoints"* || "$FUNCTIONAL_TEST_RECORD_CHECKPOINTS" == "yes" ]] && echo "true" || echo "false") - | @@ -113,12 +114,32 @@ functional:configure: --cluster $H100_CLUSTER \ --output-path "functional-test-job-lts-H100.yaml" \ ${RELEASE_ARGS[@]} + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${ARGS[@]} \ + --environment dev \ + --platform dgx_gb200 \ + --cluster $GB200_CLUSTER \ + --output-path "functional-test-job-dev-GB200.yaml" \ + ${RELEASE_ARGS[@]} + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${ARGS[@]} \ + --environment lts \ + --platform dgx_gb200 \ + --cluster $GB200_CLUSTER \ + --output-path "functional-test-job-lts-GB200.yaml" \ + ${RELEASE_ARGS[@]} artifacts: paths: - functional-test-job-lts-A100.yaml - functional-test-job-lts-H100.yaml - functional-test-job-dev-A100.yaml - functional-test-job-dev-H100.yaml + - functional-test-job-lts-GB200.yaml + - functional-test-job-dev-GB200.yaml - tests/test_utils/local_recipes .functional_run: @@ -157,6 +178,12 @@ functional:run_lts_dgx_h100: ENVIRONMENT: lts CLUSTER: H100 +functional:run_lts_dgx_gb200: + extends: [.functional_run] + variables: + ENVIRONMENT: lts + CLUSTER: GB200 + functional:run_dev_dgx_a100: extends: [.functional_run] variables: @@ -169,6 +196,12 @@ functional:run_dev_dgx_h100: ENVIRONMENT: dev CLUSTER: H100 +functional:run_dev_dgx_gb200: + extends: [.functional_run] + variables: + ENVIRONMENT: dev + CLUSTER: GB200 + functional:run_nemo: extends: [.functional_tests_rules] trigger: diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index d8c1dd33942..4e1a4de55e8 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -17,10 +17,17 @@ ENV UV_LINK_MODE=copy RUN bash -ex <<"EOF" apt-get update - apt-get install -y --no-install-recommends gettext python3-venv psmisc + apt-get install -y --no-install-recommends gettext python3-venv psmisc uuid-runtime apt-get clean python -m venv /opt/jet - wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 -O /usr/local/bin/yq + ARCH=$(uname -m) + case "${ARCH}" in \ + "x86_64") YQ_ARCH=amd64 ;; \ + "aarch64") YQ_ARCH=arm64 ;; \ + "armv7l") YQ_ARCH=arm ;; \ + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; \ + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/local/bin/yq chmod a+x /usr/local/bin/yq curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh EOF diff --git a/megatron/core/datasets/Makefile b/megatron/core/datasets/Makefile index e745f52399b..16f251bf903 100644 --- a/megatron/core/datasets/Makefile +++ b/megatron/core/datasets/Makefile @@ -1,4 +1,4 @@ -CXXFLAGS += -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color +CXXFLAGS += -O3 -Wall -shared -std=c++17 -fPIC -fdiagnostics-color CPPFLAGS += $(shell python3 -m pybind11 --includes) LIBNAME = helpers_cpp diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh index 1d0e77a3477..72fd187d19d 100644 --- a/tests/functional_tests/shell_test_utils/_run_training.sh +++ b/tests/functional_tests/shell_test_utils/_run_training.sh @@ -159,7 +159,7 @@ MASTER_PORT=${MASTER_PORT:-6000} NUM_NODES=${NUM_NODES:-${SLURM_NNODES:-1}} GPUS_PER_NODE=${GPUS_PER_NODE:-8} NODE_RANK=${SLURM_NODEID:-${SLURM_NODEID:-0}} -LAST_RANK=7 +LAST_RANK=$((GPUS_PER_NODE - 1)) export LOG_DIR=$OUTPUT_PATH/logs/$REPEAT mkdir -p $LOG_DIR @@ -170,7 +170,7 @@ DISTRIBUTED_ARGS=( --master_port $MASTER_PORT --node_rank $NODE_RANK --log-dir $LOG_DIR - --tee "0:3,7:3" + --tee "0:3,$LAST_RANK:3" --redirects "3" ) diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..f023ed07c99 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82558, + "2": 10.83322, + "3": 10.82737, + "4": 10.79588, + "5": 10.85708, + "6": 10.86392, + "7": 10.8269, + "8": 10.82588, + "9": 10.83699, + "10": 10.79719, + "11": 10.87851, + "12": 10.85797, + "13": 10.85368, + "14": 10.87548, + "15": 10.79177, + "16": 10.80301, + "17": 10.7745, + "18": 10.80399, + "19": 10.79365, + "20": 10.69588, + "21": 10.6855, + "22": 10.53152, + "23": 10.70658, + "24": 10.57319, + "25": 10.51545, + "26": 10.59076, + "27": 10.60738, + "28": 10.57025, + "29": 10.58904, + "30": 10.34674, + "31": 10.07736, + "32": 10.46317, + "33": 10.45705, + "34": 10.19923, + "35": 10.25593, + "36": 10.21246, + "37": 10.34689, + "38": 10.18008, + "39": 10.40796, + "40": 10.07602, + "41": 10.12935, + "42": 10.21132, + "43": 9.81692, + "44": 9.94027, + "45": 9.817, + "46": 9.80608, + "47": 10.12473, + "48": 9.84047, + "49": 9.50975, + "50": 9.88932 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1691.0, + "2": 1553.0, + "3": 1673.0, + "4": 1760.0, + "5": 1852.0, + "6": 1861.0, + "7": 1852.0, + "8": 1755.0, + "9": 1952.0, + "10": 1427.0, + "11": 1857.0, + "12": 1820.0, + "13": 1948.0, + "14": 1828.0, + "15": 1913.0, + "16": 1881.0, + "17": 1770.0, + "18": 1683.0, + "19": 1784.0, + "20": 1714.0, + "21": 1969.0, + "22": 1701.0, + "23": 1972.0, + "24": 1545.0, + "25": 1537.0, + "26": 1650.0, + "27": 1770.0, + "28": 1889.0, + "29": 1946.0, + "30": 2031.0, + "31": 1511.0, + "32": 1848.0, + "33": 2009.0, + "34": 1749.0, + "35": 1978.0, + "36": 1926.0, + "37": 2358.0, + "38": 2036.0, + "39": 2202.0, + "40": 2015.0, + "41": 2184.0, + "42": 2304.0, + "43": 2079.0, + "44": 2042.0, + "45": 2082.0, + "46": 2206.0, + "47": 2417.0, + "48": 2284.0, + "49": 2231.0, + "50": 2430.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 552193536.0, + "2": 552193536.0, + "3": 552193536.0, + "4": 553242112.0, + "5": 552193536.0, + "6": 553242112.0, + "7": 553242112.0, + "8": 552193536.0, + "9": 552193536.0, + "10": 552193536.0, + "11": 553242112.0, + "12": 552193536.0, + "13": 552193536.0, + "14": 552193536.0, + "15": 552193536.0, + "16": 553242112.0, + "17": 553242112.0, + "18": 552193536.0, + "19": 553242112.0, + "20": 552193536.0, + "21": 552193536.0, + "22": 552193536.0, + "23": 552193536.0, + "24": 552193536.0, + "25": 552193536.0, + "26": 552193536.0, + "27": 552193536.0, + "28": 552193536.0, + "29": 552193536.0, + "30": 552193536.0, + "31": 552193536.0, + "32": 552193536.0, + "33": 552193536.0, + "34": 552193536.0, + "35": 552193536.0, + "36": 552193536.0, + "37": 552193536.0, + "38": 552193536.0, + "39": 552193536.0, + "40": 552193536.0, + "41": 552193536.0, + "42": 552193536.0, + "43": 552193536.0, + "44": 552193536.0, + "45": 553242112.0, + "46": 552193536.0, + "47": 552193536.0, + "48": 552193536.0, + "49": 552193536.0, + "50": 552193536.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3798208000.0, + "2": 3942086144.0, + "3": 3942086144.0, + "4": 3942086144.0, + "5": 3942086144.0, + "6": 3942086144.0, + "7": 3942086144.0, + "8": 3942086144.0, + "9": 3942086144.0, + "10": 3942086144.0, + "11": 3942086144.0, + "12": 3942086144.0, + "13": 3942086144.0, + "14": 3942086144.0, + "15": 3942086144.0, + "16": 3942086144.0, + "17": 3942086144.0, + "18": 3942086144.0, + "19": 3942086144.0, + "20": 3942086144.0, + "21": 3942086144.0, + "22": 3942086144.0, + "23": 3942086144.0, + "24": 3942086144.0, + "25": 3942086144.0, + "26": 3942086144.0, + "27": 3942086144.0, + "28": 3942086144.0, + "29": 3942086144.0, + "30": 3942086144.0, + "31": 3942086144.0, + "32": 3942086144.0, + "33": 3942086144.0, + "34": 3942086144.0, + "35": 3942086144.0, + "36": 3942086144.0, + "37": 3942086144.0, + "38": 3942086144.0, + "39": 3942086144.0, + "40": 3942086144.0, + "41": 3942086144.0, + "42": 3942086144.0, + "43": 3942086144.0, + "44": 3942086144.0, + "45": 3942086144.0, + "46": 3942086144.0, + "47": 3942086144.0, + "48": 3942086144.0, + "49": 3942086144.0, + "50": 3942086144.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.06303, + "2": 0.15398, + "3": 0.27325, + "4": 0.13945, + "5": 0.25021, + "6": 0.16329, + "7": 0.27717, + "8": 0.18718, + "9": 0.12007, + "10": 0.21402, + "11": 0.2385, + "12": 0.61603, + "13": 0.24413, + "14": 0.18837, + "15": 0.14999, + "16": 0.12555, + "17": 0.24832, + "18": 0.1361, + "19": 0.13136, + "20": 0.27497, + "21": 0.22444, + "22": 0.11923, + "23": 0.11996, + "24": 0.25718, + "25": 0.20275, + "26": 0.35028, + "27": 0.11968, + "28": 0.23901, + "29": 0.12079, + "30": 0.12184, + "31": 0.21733, + "32": 0.28054, + "33": 0.11829, + "34": 0.17717, + "35": 0.1215, + "36": 0.27112, + "37": 0.22357, + "38": 0.12158, + "39": 0.12105, + "40": 0.12099, + "41": 0.21658, + "42": 0.22641, + "43": 0.12146, + "44": 0.1201, + "45": 0.253, + "46": 0.12142, + "47": 0.23268, + "48": 0.13569, + "49": 0.1302, + "50": 0.24153 + } + } +} \ No newline at end of file diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py index 6ecd98a06c1..7f60ceb12d6 100644 --- a/tests/test_utils/python_scripts/launch_jet_workload.py +++ b/tests/test_utils/python_scripts/launch_jet_workload.py @@ -8,6 +8,7 @@ import signal import sys import time +import uuid import zipfile from typing import Dict, List, Optional @@ -111,15 +112,12 @@ def launch_and_wait_for_completion( "HF_HUB_CACHE": "/lustre/fsw/coreai_dlalgo_mcore/hf_hub", "TRANSFORMERS_OFFLINE": "1", "CLUSTER": cluster, + "RUN_ID": str(uuid.uuid4()), } } } } }, - "outputs": { - "enabled": True, - "artifacts_storages": [recipe_parser.resolve_artifact_config(cluster)], - }, }, wait_for_validation=True, max_wait_time=(60 * 60), diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py index b866fbbf5c2..c6e7c5517e8 100644 --- a/tests/test_utils/python_scripts/recipe_parser.py +++ b/tests/test_utils/python_scripts/recipe_parser.py @@ -24,6 +24,8 @@ class dotdict(dict): def resolve_cluster_config(cluster: str) -> str: if cluster == "dgxh100_eos": return "eos" + if cluster == "dgxgb200_oci-hsg": + return "oci-hsg" if cluster == "dgxa100_dracooci": return "draco-oci-iad" if cluster == "dgxa100_dracooci-ord": @@ -35,18 +37,6 @@ def resolve_cluster_config(cluster: str) -> str: raise ValueError(f"Unknown cluster {cluster} provided.") -def resolve_artifact_config(cluster: str) -> str: - if cluster == "dgxh100_eos": - return "eos_lustre" - if cluster == "dgxa100_dracooci": - return "draco-oci_lustre" - if cluster == "dgxa100_dracooci-ord": - return "draco-oci-ord_lustre" - if cluster == "dgxh100_coreweave": - return "coreweave_lustre" - raise ValueError(f"Unknown cluster {cluster} provided.") - - def flatten_products(workload_manifest: dotdict) -> dotdict: """Flattens a nested dict of products""" expanded_products = [] diff --git a/tests/test_utils/recipes/_build-mcore-dev.yaml b/tests/test_utils/recipes/_build-mcore-dev.yaml index 123250d7469..d82417ea5e3 100644 --- a/tests/test_utils/recipes/_build-mcore-dev.yaml +++ b/tests/test_utils/recipes/_build-mcore-dev.yaml @@ -3,7 +3,7 @@ format_version: 1 maintainers: [maanug] spec: name: mcore-pyt-dev - platforms: [linux/amd64] + platforms: [linux/amd64,linux/arm64] source: # The image tag will be added via `jet-tests.yaml` # Tags are one of {buildcache, $CI_PIPELINE_ID} diff --git a/tests/test_utils/recipes/_build-mcore-lts.yaml b/tests/test_utils/recipes/_build-mcore-lts.yaml index d017b71c101..8efa6faa1e5 100644 --- a/tests/test_utils/recipes/_build-mcore-lts.yaml +++ b/tests/test_utils/recipes/_build-mcore-lts.yaml @@ -3,7 +3,7 @@ format_version: 1 maintainers: [maanug] spec: name: mcore-pyt-lts - platforms: [linux/amd64] + platforms: [linux/amd64,linux/arm64] source: # The image tag will be added via `jet-tests.yaml` # Tags are one of {buildcache, $CI_PIPELINE_ID} diff --git a/tests/test_utils/recipes/gpt-gb200.yaml b/tests/test_utils/recipes/gpt-gb200.yaml new file mode 100644 index 00000000000..c32d141bbf4 --- /dev/null +++ b/tests/test_utils/recipes/gpt-gb200.yaml @@ -0,0 +1,73 @@ +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: "{test_case}_{environment}_{platforms}" + model: gpt + build: mcore-pyt-{environment} + nodes: 2 + gpus: 4 + n_repeat: 5 + platforms: dgx_a100 + script_setup: | + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + # Checkout latest + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + + # Checkout backwards-ref + cd /opt + rm -rf /opt/megatron-lm-legacy; mkdir megatron-lm-legacy; cd megatron-lm-legacy + git init + git remote add origin $MCORE_REPO + git fetch origin $MCORE_BACKWARDS_COMMIT + git checkout $MCORE_BACKWARDS_COMMIT + git rev-parse HEAD + rm -rf megatron; cp -a /opt/megatron-lm/megatron ./ + script: |- + ls + cd /opt/megatron-lm + + NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g') + export GPUS_PER_NODE={gpus} + + ARGUMENTS=( + "DATA_PATH=/mnt/artifacts" + "DATA_CACHE_PATH=/lustre/fsw/coreai_dlalgo_mcore/mcore_ci/data/$RUN_ID/cache/" + "OUTPUT_PATH={assets_dir}" + "TENSORBOARD_PATH={assets_dir}/tensorboard" + "CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts/" + "TRAINING_SCRIPT_PATH=pretrain_gpt.py" + "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" + "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json" + "N_REPEAT={n_repeat}" + "ENABLE_LIGHTWEIGHT_MODE=${{ENABLE_LIGHTWEIGHT_MODE}}" + "RECORD_CHECKPOINTS=${{RECORD_CHECKPOINTS}}" + ) + + set +x + bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}} + exit_code=$? + echo "Exit code: $exit_code" + rm -rf /lustre/fsw/coreai_dlalgo_mcore/mcore_ci/data/$RUN_ID || true + set -x + exit $exit_code + +products: + - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] + products: + - environment: [lts] + scope: [mr] + - environment: [dev] + scope: [mr, mr-github, mr-github-slim] + platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index f403ac20e3f..eab62026381 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -3,7 +3,7 @@ format_version: 1 maintainers: [mcore] loggers: [stdout] spec: - name: '{test_case}_{environment}_{platforms}' + name: "{test_case}_{environment}_{platforms}" model: gpt build: mcore-pyt-{environment} nodes: 1