diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index d69985bff40..469f102f748 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -345,7 +345,7 @@ jobs: - name: Parse unit tests id: parse-unit-tests run: | - cat tests/test_utils/recipes/unit-tests.yaml | yq -o json '[.products[].test_case[] | { "bucket": .}] | sort_by(.model, .test_case)' | jq -c > unit-tests.json + cat tests/test_utils/recipes/h100/unit-tests.yaml | yq -o json '[.products[].test_case[] | { "bucket": .}] | sort_by(.model, .test_case)' | jq -c > unit-tests.json echo "unit-tests=$(cat unit-tests.json)" | tee -a $GITHUB_OUTPUT cicd-unit-tests-latest: diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh index 3f1500b502c..3d47e591749 100644 --- a/tests/functional_tests/shell_test_utils/run_ci_test.sh +++ b/tests/functional_tests/shell_test_utils/run_ci_test.sh @@ -132,6 +132,8 @@ SKIP_PYTEST=$(cat $TRAINING_PARAMS_PATH | export RECORD_CHECKPOINTS=${RECORD_CHECKPOINTS:-"false"} +NODE_RANK=${SLURM_NODEID:-${SLURM_NODEID:-0}} + for i in $(seq 1 $N_REPEAT); do # Move TB logs into a repeat-specific directory DIR=$(dirname "$_TENSORBOARD_PATH") diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json index 4770792474b..0174aaf4684 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp1_pp4_memory_speed/golden_values_dev_dgx_gb200.json @@ -133,29 +133,29 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.8109, + "2": 3.93489, "3": "nan", - "4": 0.8316, + "4": 0.83885, "5": "nan", - "6": 0.83072, + "6": 0.86101, "7": "nan", - "8": 0.82637, + "8": 0.82617, "9": "nan", - "10": 0.823, + "10": 0.8264, "11": "nan", - "12": 0.82386, + "12": 0.82456, "13": "nan", - "14": 0.82343, + "14": 0.82414, "15": "nan", - "16": 0.82487, + "16": 0.82604, "17": "nan", - "18": 0.82227, + "18": 0.83002, "19": "nan", - "20": 0.82121, + "20": 0.8234, "21": "nan", - "22": 0.82248, + "22": 0.82298, "23": "nan", - "24": 0.81939, + "24": 0.82311, "25": "nan" } } diff --git a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json index 5c13c9d624f..ca51cd1bcb3 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_7b_tp4_pp1_memory_speed/golden_values_dev_dgx_gb200.json @@ -133,29 +133,29 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.75605, + "2": 2.88247, "3": "nan", - "4": 1.05448, + "4": 0.98359, "5": "nan", - "6": 1.24087, + "6": 0.91373, "7": "nan", - "8": 0.89299, + "8": 1.07044, "9": "nan", - "10": 0.89376, + "10": 0.91309, "11": "nan", - "12": 0.8965, + "12": 0.91579, "13": "nan", - "14": 0.89831, + "14": 0.90609, "15": "nan", - "16": 0.89733, + "16": 0.90906, "17": "nan", - "18": 1.02538, + "18": 0.91134, "19": "nan", - "20": 0.89305, + "20": 0.90623, "21": "nan", - "22": 0.89255, + "22": 0.91236, "23": "nan", - "24": 0.91075, + "24": 0.9145, "25": "nan" } } diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json index 7650494228d..bac18297ae6 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5.51862, - "2": 0.11775, - "3": 0.11561, - "4": 0.1042, - "5": 0.10208, - "6": 0.09598, - "7": 0.09542, - "8": 0.095, - "9": 0.09404, - "10": 0.09596, - "11": 0.09825, - "12": 0.09507, - "13": 0.0943, - "14": 0.09595, - "15": 0.09454, - "16": 0.09354, - "17": 0.09423, - "18": 0.09638, - "19": 0.09698, - "20": 0.09656, - "21": 0.09629, - "22": 0.09731, - "23": 0.09913, - "24": 0.09535, - "25": 0.09314, - "26": 0.09324, - "27": 0.09374, - "28": 0.0992, - "29": 0.09647, - "30": 0.11416, - "31": 0.09524, - "32": 0.09418, - "33": 0.09544, - "34": 0.09428, - "35": 0.09432, - "36": 0.09584, - "37": 0.096, - "38": 0.09539, - "39": 0.09482, - "40": 0.09568, - "41": 0.09682, - "42": 0.0964, - "43": 0.09675, - "44": 0.09583, - "45": 0.09482, - "46": 0.09426, - "47": 0.09537, - "48": 0.09383, - "49": 0.09397, - "50": 0.09592 + "1": "nan", + "2": 2.46491, + "3": 0.11014, + "4": 0.09722, + "5": 0.09703, + "6": 0.09705, + "7": 0.09593, + "8": 0.09584, + "9": 0.09505, + "10": 0.0949, + "11": 0.09504, + "12": 0.09589, + "13": 0.09506, + "14": 0.09425, + "15": 0.09404, + "16": 0.09465, + "17": 0.09237, + "18": 0.09201, + "19": 0.09159, + "20": 0.09124, + "21": 0.09092, + "22": 0.09028, + "23": 0.08966, + "24": 0.08893, + "25": 0.09042, + "26": 0.09055, + "27": 0.08889, + "28": 0.08857, + "29": 0.0884, + "30": 0.08807, + "31": 0.08777, + "32": 0.08747, + "33": 0.0876, + "34": 0.08733, + "35": 0.0886, + "36": 0.08828, + "37": 0.08789, + "38": 0.08768, + "39": 0.08819, + "40": 0.08922, + "41": 0.08797, + "42": 0.0876, + "43": 0.0868, + "44": 0.08693, + "45": 0.08661, + "46": 0.08657, + "47": 0.08769, + "48": 0.08644, + "49": 0.08681, + "50": 0.08702 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json index 0405b9dc312..8bcd3aa91d5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5.87663, - "2": 0.11967, - "3": 0.10376, - "4": 0.09966, - "5": 0.0967, - "6": 0.09666, - "7": 0.09702, - "8": 0.09962, - "9": 0.10053, - "10": 0.10019, - "11": 0.09818, - "12": 0.37487, - "13": 0.10166, - "14": 0.10015, - "15": 0.10189, - "16": 0.09883, - "17": 0.10229, - "18": 0.09859, - "19": 0.09957, - "20": 0.09987, - "21": 0.09747, - "22": 0.09678, - "23": 0.09865, - "24": 0.09988, - "25": 0.11712, - "26": 0.11559, - "27": 0.11626, - "28": 0.11634, - "29": 0.11701, - "30": 0.13544, - "31": 0.13258, - "32": 0.12643, - "33": 0.12858, - "34": 0.18682, - "35": 0.12702, - "36": 0.09639, - "37": 0.09478, - "38": 0.09349, - "39": 0.09417, - "40": 0.09272, - "41": 0.09563, - "42": 0.09369, - "43": 0.09427, - "44": 0.09501, - "45": 0.09141, - "46": 0.09367, - "47": 0.0929, - "48": 0.09322, - "49": 0.09223, - "50": 0.0936 + "1": "nan", + "2": 2.34829, + "3": 0.10511, + "4": 0.09797, + "5": 0.09705, + "6": 0.09665, + "7": 0.09616, + "8": 0.09616, + "9": 0.0968, + "10": 0.09708, + "11": 0.09598, + "12": 0.09533, + "13": 0.09549, + "14": 0.09665, + "15": 0.09303, + "16": 0.0963, + "17": 0.10058, + "18": 0.09955, + "19": 0.10067, + "20": 0.10221, + "21": 0.09941, + "22": 0.09872, + "23": 0.09975, + "24": 0.10322, + "25": 0.09837, + "26": 0.09834, + "27": 0.09843, + "28": 0.09692, + "29": 0.09907, + "30": 0.09889, + "31": 0.10064, + "32": 0.09748, + "33": 0.09927, + "34": 0.09831, + "35": 0.09862, + "36": 0.09852, + "37": 0.09869, + "38": 0.09941, + "39": 0.09945, + "40": 0.10014, + "41": 0.09934, + "42": 0.10081, + "43": 0.10148, + "44": 0.09766, + "45": 0.09746, + "46": 0.09842, + "47": 0.09924, + "48": 0.09864, + "49": 0.09829, + "50": 0.09685 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index b280d123468..eb5a06ac1fc 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.16302, - "2": 0.10393, - "3": 0.10318, - "4": 0.08757, - "5": 0.08719, - "6": 0.08686, - "7": 0.08532, - "8": 0.0858, - "9": 0.08669, - "10": 0.08615, - "11": 0.08684, - "12": 0.08786, - "13": 0.15333, - "14": 0.08821, - "15": 0.18235, - "16": 0.08981, - "17": 0.08651, - "18": 0.0876, - "19": 0.08798, - "20": 0.08911, - "21": 0.08738, - "22": 0.08768, - "23": 0.08719, - "24": 0.087, - "25": 0.08861, - "26": 0.08768, - "27": 0.08826, - "28": 0.08976, - "29": 0.0886, - "30": 0.08951, - "31": 0.08933, - "32": 0.08963, - "33": 0.09543, - "34": 0.10061, - "35": 0.10664, - "36": 0.09906, - "37": 0.11365, - "38": 0.82081, - "39": 0.08864, - "40": 0.08743, - "41": 0.08722, - "42": 0.08656, - "43": 0.09145, - "44": 0.08801, - "45": 0.17031, - "46": 0.0894, - "47": 0.08943, - "48": 0.08707, - "49": 0.08683, - "50": 0.08738, - "51": 0.11089, - "52": 0.08833, - "53": 0.08713, - "54": 0.08847, - "55": 0.09031, - "56": 0.08636, - "57": 0.08753, - "58": 0.08716, - "59": 0.08699, - "60": 0.08807, - "61": 0.6943, - "62": 0.09219, - "63": 0.08631, - "64": 0.0882, - "65": 0.08874, - "66": 0.08909, - "67": 0.08792, - "68": 0.08836, - "69": 0.08825, - "70": 0.08851, - "71": 0.08764, - "72": 0.08728, - "73": 0.08806, - "74": 0.08749, - "75": 0.09031, - "76": 0.08768, - "77": 0.08844, - "78": 0.08914, - "79": 0.08957, - "80": 0.08909, - "81": 0.08925, - "82": 0.09031, - "83": 0.08817, - "84": 0.08786, - "85": 0.08912, - "86": 0.08785, - "87": 0.08907, - "88": 0.08837, - "89": 0.08812, - "90": 0.0872, - "91": 0.08931, - "92": 0.0876, - "93": 0.16836, - "94": 0.09054, - "95": 0.09081, - "96": 0.09078, - "97": 0.09068, - "98": 0.09042, - "99": 0.09008, - "100": 0.08863 + "1": "nan", + "2": 2.38769, + "3": 0.09884, + "4": 0.08514, + "5": 0.08435, + "6": 0.08412, + "7": 0.08558, + "8": 0.08425, + "9": 0.08436, + "10": 0.08457, + "11": 0.08469, + "12": 0.0848, + "13": 0.08487, + "14": 0.08571, + "15": 0.08487, + "16": 0.08529, + "17": 0.08559, + "18": 0.0898, + "19": 0.08482, + "20": 0.08509, + "21": 0.08527, + "22": 0.08597, + "23": 0.08592, + "24": 0.08654, + "25": 0.08608, + "26": 0.08574, + "27": 0.08542, + "28": 0.0856, + "29": 0.08581, + "30": 0.08539, + "31": 0.08675, + "32": 0.08679, + "33": 0.08699, + "34": 0.08599, + "35": 0.08568, + "36": 0.08528, + "37": 0.08527, + "38": 0.08526, + "39": 0.08614, + "40": 0.08507, + "41": 0.08552, + "42": 0.08586, + "43": 0.08568, + "44": 0.0866, + "45": 0.08692, + "46": 0.08614, + "47": 0.0859, + "48": 0.0863, + "49": 0.08723, + "50": 0.08703, + "51": 0.09195, + "52": 0.0775, + "53": 0.07822, + "54": 0.07813, + "55": 0.0784, + "56": 0.07871, + "57": 0.07816, + "58": 0.07787, + "59": 0.07958, + "60": 0.07893, + "61": 0.07873, + "62": 0.07887, + "63": 0.07945, + "64": 0.07879, + "65": 0.08059, + "66": 0.08041, + "67": 0.08127, + "68": 0.07996, + "69": 0.0799, + "70": 0.07821, + "71": 0.07712, + "72": 0.07745, + "73": 0.07774, + "74": 0.07859, + "75": 0.07741, + "76": 0.07753, + "77": 0.07725, + "78": 0.07676, + "79": 0.07838, + "80": 0.07786, + "81": 0.07743, + "82": 0.07732, + "83": 0.0773, + "84": 0.07664, + "85": 0.07753, + "86": 0.07826, + "87": 0.07764, + "88": 0.07681, + "89": 0.07911, + "90": 0.07799, + "91": 0.07796, + "92": 0.07853, + "93": 0.07736, + "94": 0.07777, + "95": 0.07791, + "96": 0.07723, + "97": 0.07753, + "98": 0.07789, + "99": 0.07782, + "100": 0.07733 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..5c26c56ee5a --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.83614, + "52": 9.72315, + "53": 10.05318, + "54": 9.93747, + "55": 9.87384, + "56": 9.60449, + "57": 9.4523, + "58": 9.8188, + "59": 9.5772, + "60": 9.48534, + "61": 9.68548, + "62": 9.97906, + "63": 9.36419, + "64": 9.76203, + "65": 8.94097, + "66": 9.69475, + "67": 9.36656, + "68": 9.77745, + "69": 9.79001, + "70": 9.72374, + "71": 9.62037, + "72": 9.57423, + "73": 9.48575, + "74": 8.92729, + "75": 9.41651, + "76": 9.07747, + "77": 10.05444, + "78": 9.71914, + "79": 9.37306, + "80": 9.40003, + "81": 9.47844, + "82": 9.69867, + "83": 9.31155, + "84": 9.41457, + "85": 9.61163, + "86": 9.07418, + "87": 9.5939, + "88": 9.74928, + "89": 9.5985, + "90": 9.82761, + "91": 9.33631, + "92": 9.35805, + "93": 9.08552, + "94": 8.82786, + "95": 9.5303, + "96": 9.52663, + "97": 9.30483, + "98": 9.67007, + "99": 8.89606, + "100": 9.40702 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2650.0, + "52": 2700.0, + "53": 2863.0, + "54": 2676.0, + "55": 2390.0, + "56": 2753.0, + "57": 2430.0, + "58": 2919.0, + "59": 2831.0, + "60": 2428.0, + "61": 2932.0, + "62": 2724.0, + "63": 2579.0, + "64": 2987.0, + "65": 2506.0, + "66": 2886.0, + "67": 2871.0, + "68": 2870.0, + "69": 3001.0, + "70": 3294.0, + "71": 3043.0, + "72": 2614.0, + "73": 3054.0, + "74": 2024.0, + "75": 2507.0, + "76": 3020.0, + "77": 3253.0, + "78": 3230.0, + "79": 3210.0, + "80": 3252.0, + "81": 3614.0, + "82": 3395.0, + "83": 2919.0, + "84": 3296.0, + "85": 3320.0, + "86": 2865.0, + "87": 3931.0, + "88": 3240.0, + "89": 3428.0, + "90": 3127.0, + "91": 2815.0, + "92": 3098.0, + "93": 2796.0, + "94": 3324.0, + "95": 3428.0, + "96": 3541.0, + "97": 3216.0, + "98": 3705.0, + "99": 3184.0, + "100": 3073.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 581489664.0, + "52": 581489664.0, + "53": 581489664.0, + "54": 581489664.0, + "55": 581489664.0, + "56": 581489664.0, + "57": 581489664.0, + "58": 581489664.0, + "59": 581489664.0, + "60": 581489664.0, + "61": 581489664.0, + "62": 581489664.0, + "63": 581489664.0, + "64": 581489664.0, + "65": 581489664.0, + "66": 581489664.0, + "67": 581489664.0, + "68": 581489664.0, + "69": 581489664.0, + "70": 581489664.0, + "71": 581489664.0, + "72": 581489664.0, + "73": 581489664.0, + "74": 581489664.0, + "75": 581489664.0, + "76": 581489664.0, + "77": 581489664.0, + "78": 581489664.0, + "79": 581489664.0, + "80": 581489664.0, + "81": 581489664.0, + "82": 581489664.0, + "83": 581489664.0, + "84": 581489664.0, + "85": 581489664.0, + "86": 581489664.0, + "87": 581489664.0, + "88": 581489664.0, + "89": 581489664.0, + "90": 581489664.0, + "91": 581489664.0, + "92": 581489664.0, + "93": 581489664.0, + "94": 581489664.0, + "95": 581489664.0, + "96": 581489664.0, + "97": 581489664.0, + "98": 581489664.0, + "99": 581489664.0, + "100": 581489664.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2708568576.0, + "52": 2742124032.0, + "53": 2742124032.0, + "54": 2742124032.0, + "55": 2742124032.0, + "56": 2742124032.0, + "57": 2742124032.0, + "58": 2742124032.0, + "59": 2742124032.0, + "60": 2742124032.0, + "61": 2742124032.0, + "62": 2742124032.0, + "63": 2742124032.0, + "64": 2742124032.0, + "65": 2742124032.0, + "66": 2742124032.0, + "67": 2742124032.0, + "68": 2742124032.0, + "69": 2742124032.0, + "70": 2742124032.0, + "71": 2742124032.0, + "72": 2742124032.0, + "73": 2742124032.0, + "74": 2742124032.0, + "75": 2742124032.0, + "76": 2742124032.0, + "77": 2742124032.0, + "78": 2742124032.0, + "79": 2742124032.0, + "80": 2742124032.0, + "81": 2742124032.0, + "82": 2742124032.0, + "83": 2742124032.0, + "84": 2742124032.0, + "85": 2742124032.0, + "86": 2742124032.0, + "87": 2742124032.0, + "88": 2742124032.0, + "89": 2742124032.0, + "90": 2742124032.0, + "91": 2742124032.0, + "92": 2742124032.0, + "93": 2742124032.0, + "94": 2742124032.0, + "95": 2742124032.0, + "96": 2742124032.0, + "97": 2742124032.0, + "98": 2742124032.0, + "99": 2742124032.0, + "100": 2742124032.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.1716, + "53": 0.09643, + "54": 0.08435, + "55": 0.08492, + "56": 0.08409, + "57": 0.08624, + "58": 0.08522, + "59": 0.08521, + "60": 0.08445, + "61": 0.08447, + "62": 0.08412, + "63": 0.08534, + "64": 0.08529, + "65": 0.08566, + "66": 0.08409, + "67": 0.08468, + "68": 0.08268, + "69": 0.08161, + "70": 0.08416, + "71": 0.08383, + "72": 0.08425, + "73": 0.08363, + "74": 0.08451, + "75": 0.08423, + "76": 0.08453, + "77": 0.08475, + "78": 0.08435, + "79": 0.0844, + "80": 0.08466, + "81": 0.08777, + "82": 0.08524, + "83": 0.08559, + "84": 0.08524, + "85": 0.08501, + "86": 0.08518, + "87": 0.08503, + "88": 0.08555, + "89": 0.0855, + "90": 0.08584, + "91": 0.08419, + "92": 0.08467, + "93": 0.08514, + "94": 0.08518, + "95": 0.08444, + "96": 0.08484, + "97": 0.08521, + "98": 0.08697, + "99": 0.08772, + "100": 0.08544 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json index ccf3054dcf0..a084bf35662 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 4.35869, - "2": 0.14764, - "3": 0.13356, - "4": 0.12382, - "5": 0.1223, - "6": 0.1198, - "7": 0.12014, - "8": 0.12507, - "9": 0.12529, - "10": 0.13316, - "11": 0.13102, - "12": 0.13889, - "13": 0.13638, - "14": 0.14898, - "15": 0.16074, - "16": 0.14966, - "17": 0.17711, - "18": 0.17201, - "19": 0.14817, - "20": 0.14956, - "21": 0.17491, - "22": 0.29045, - "23": 0.49855, - "24": 0.12704, - "25": 0.12527, - "26": 0.12833, - "27": 0.12762, - "28": 0.12497, - "29": 0.1258, - "30": 0.12747, - "31": 0.1272, - "32": 0.12749, - "33": 0.12753, - "34": 0.12763, - "35": 0.12697, - "36": 0.12734, - "37": 0.12802, - "38": 0.12925, - "39": 0.1278, - "40": 0.1273, - "41": 0.1284, - "42": 0.12646, - "43": 0.12669, - "44": 0.12781, - "45": 0.12751, - "46": 0.12772, - "47": 0.12712, - "48": 0.12664, - "49": 0.12679, - "50": 0.13142, - "51": 0.13902, - "52": 0.12241, - "53": 0.12407, - "54": 0.12462, - "55": 0.1225, - "56": 0.12498, - "57": 0.12564, - "58": 0.12627, - "59": 0.12399, - "60": 0.12468, - "61": 0.12629, - "62": 0.12645, - "63": 0.12377, - "64": 0.12505, - "65": 0.1271, - "66": 0.12603, - "67": 0.12556, - "68": 0.12634, - "69": 0.1332, - "70": 0.13504, - "71": 0.13164, - "72": 0.13511, - "73": 0.14002, - "74": 0.14488, - "75": 0.14064, - "76": 0.14236, - "77": 0.14155, - "78": 0.14042, - "79": 0.14188, - "80": 0.14414, - "81": 0.14147, - "82": 0.14264, - "83": 0.14126, - "84": 0.1423, - "85": 0.14311, - "86": 0.144, - "87": 0.1445, - "88": 0.14401, - "89": 0.14198, - "90": 0.14227, - "91": 0.14119, - "92": 0.14076, - "93": 0.14281, - "94": 0.14283, - "95": 0.1438, - "96": 0.14188, - "97": 0.14623, - "98": 0.14374, - "99": 0.14361, - "100": 0.14481 + "1": "nan", + "2": 2.02256, + "3": 0.13455, + "4": 0.12293, + "5": 0.12302, + "6": 0.1233, + "7": 0.12328, + "8": 0.12248, + "9": 0.12446, + "10": 0.12285, + "11": 0.12255, + "12": 0.12296, + "13": 0.12411, + "14": 0.12369, + "15": 0.12438, + "16": 0.12387, + "17": 0.12481, + "18": 0.12591, + "19": 0.12445, + "20": 0.12257, + "21": 0.12141, + "22": 0.12289, + "23": 0.12296, + "24": 0.12246, + "25": 0.12246, + "26": 0.12219, + "27": 0.12283, + "28": 0.12209, + "29": 0.12164, + "30": 0.12236, + "31": 0.1236, + "32": 0.12251, + "33": 0.12372, + "34": 0.12054, + "35": 0.12166, + "36": 0.12052, + "37": 0.12268, + "38": 0.12181, + "39": 0.12231, + "40": 0.1195, + "41": 0.12001, + "42": 0.12145, + "43": 0.12238, + "44": 0.12054, + "45": 0.11842, + "46": 0.11812, + "47": 0.11785, + "48": 0.11631, + "49": 0.11798, + "50": 0.11707, + "51": 0.12234, + "52": 0.11424, + "53": 0.11577, + "54": 0.11058, + "55": 0.11651, + "56": 0.12356, + "57": 0.12837, + "58": 0.1238, + "59": 0.13093, + "60": 0.14556, + "61": 0.1747, + "62": 0.14486, + "63": 0.15679, + "64": 0.14116, + "65": 0.13574, + "66": 0.16023, + "67": 0.14862, + "68": 0.14163, + "69": 0.14244, + "70": 0.13512, + "71": 0.1407, + "72": 0.14689, + "73": 0.13238, + "74": 0.1279, + "75": 0.12535, + "76": 0.12172, + "77": 0.12314, + "78": 0.12089, + "79": 0.11925, + "80": 0.11854, + "81": 0.11618, + "82": 0.11706, + "83": 0.11632, + "84": 0.11839, + "85": 0.11763, + "86": 0.11977, + "87": 0.12013, + "88": 0.11954, + "89": 0.11859, + "90": 0.11546, + "91": 0.11494, + "92": 0.1154, + "93": 0.11866, + "94": 0.25826, + "95": 0.11359, + "96": 0.11427, + "97": 0.11526, + "98": 0.11269, + "99": 0.11239, + "100": 0.11374 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..9853cec1655 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.83617, + "52": 9.72317, + "53": 10.05321, + "54": 9.93744, + "55": 9.87386, + "56": 9.60451, + "57": 9.45231, + "58": 9.81883, + "59": 9.57722, + "60": 9.48536, + "61": 9.68547, + "62": 9.97907, + "63": 9.36417, + "64": 9.76205, + "65": 8.94102, + "66": 9.69479, + "67": 9.36657, + "68": 9.77743, + "69": 9.78996, + "70": 9.72377, + "71": 9.62042, + "72": 9.57421, + "73": 9.48574, + "74": 8.92728, + "75": 9.41652, + "76": 9.07749, + "77": 10.05445, + "78": 9.71913, + "79": 9.37304, + "80": 9.40003, + "81": 9.47846, + "82": 9.69869, + "83": 9.31156, + "84": 9.41458, + "85": 9.61162, + "86": 9.07419, + "87": 9.59392, + "88": 9.74925, + "89": 9.59851, + "90": 9.82763, + "91": 9.33629, + "92": 9.35804, + "93": 9.08549, + "94": 8.8279, + "95": 9.53033, + "96": 9.52662, + "97": 9.30484, + "98": 9.67007, + "99": 8.89604, + "100": 9.407 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2708.0, + "52": 2707.0, + "53": 2812.0, + "54": 2620.0, + "55": 2399.0, + "56": 2790.0, + "57": 2301.0, + "58": 3008.0, + "59": 2863.0, + "60": 2465.0, + "61": 2808.0, + "62": 2607.0, + "63": 2442.0, + "64": 2977.0, + "65": 2646.0, + "66": 3061.0, + "67": 2818.0, + "68": 2891.0, + "69": 3036.0, + "70": 3160.0, + "71": 3064.0, + "72": 2618.0, + "73": 2978.0, + "74": 2000.0, + "75": 2580.0, + "76": 2967.0, + "77": 3281.0, + "78": 3131.0, + "79": 3108.0, + "80": 3217.0, + "81": 3614.0, + "82": 3411.0, + "83": 2834.0, + "84": 3191.0, + "85": 3306.0, + "86": 2806.0, + "87": 3808.0, + "88": 3237.0, + "89": 3425.0, + "90": 3202.0, + "91": 2829.0, + "92": 3105.0, + "93": 2882.0, + "94": 3303.0, + "95": 3310.0, + "96": 3499.0, + "97": 3211.0, + "98": 3741.0, + "99": 3167.0, + "100": 3049.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1261849088.0, + "52": 1261849088.0, + "53": 1261849088.0, + "54": 1261849088.0, + "55": 1261849088.0, + "56": 1261849088.0, + "57": 1261849088.0, + "58": 1261849088.0, + "59": 1261849088.0, + "60": 1261849088.0, + "61": 1261849088.0, + "62": 1261849088.0, + "63": 1261849088.0, + "64": 1261849088.0, + "65": 1261849088.0, + "66": 1261849088.0, + "67": 1261849088.0, + "68": 1261849088.0, + "69": 1261849088.0, + "70": 1261849088.0, + "71": 1261849088.0, + "72": 1261849088.0, + "73": 1261849088.0, + "74": 1261849088.0, + "75": 1261849088.0, + "76": 1261849088.0, + "77": 1261849088.0, + "78": 1261849088.0, + "79": 1261849088.0, + "80": 1261849088.0, + "81": 1261849088.0, + "82": 1261849088.0, + "83": 1261849088.0, + "84": 1261849088.0, + "85": 1261849088.0, + "86": 1261849088.0, + "87": 1261849088.0, + "88": 1261849088.0, + "89": 1261849088.0, + "90": 1261849088.0, + "91": 1261849088.0, + "92": 1261849088.0, + "93": 1261849088.0, + "94": 1261849088.0, + "95": 1261849088.0, + "96": 1261849088.0, + "97": 1261849088.0, + "98": 1261849088.0, + "99": 1261849088.0, + "100": 1261849088.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2530924544.0, + "52": 2564480000.0, + "53": 2564480000.0, + "54": 2564480000.0, + "55": 2564480000.0, + "56": 2564480000.0, + "57": 2564480000.0, + "58": 2564480000.0, + "59": 2564480000.0, + "60": 2564480000.0, + "61": 2564480000.0, + "62": 2564480000.0, + "63": 2564480000.0, + "64": 2564480000.0, + "65": 2564480000.0, + "66": 2564480000.0, + "67": 2564480000.0, + "68": 2564480000.0, + "69": 2564480000.0, + "70": 2564480000.0, + "71": 2564480000.0, + "72": 2564480000.0, + "73": 2564480000.0, + "74": 2564480000.0, + "75": 2564480000.0, + "76": 2564480000.0, + "77": 2564480000.0, + "78": 2564480000.0, + "79": 2564480000.0, + "80": 2564480000.0, + "81": 2564480000.0, + "82": 2564480000.0, + "83": 2564480000.0, + "84": 2564480000.0, + "85": 2564480000.0, + "86": 2564480000.0, + "87": 2564480000.0, + "88": 2564480000.0, + "89": 2564480000.0, + "90": 2564480000.0, + "91": 2564480000.0, + "92": 2564480000.0, + "93": 2564480000.0, + "94": 2564480000.0, + "95": 2564480000.0, + "96": 2564480000.0, + "97": 2564480000.0, + "98": 2564480000.0, + "99": 2564480000.0, + "100": 2564480000.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 1.73474, + "53": 0.13329, + "54": 0.12193, + "55": 0.12308, + "56": 0.12634, + "57": 0.12745, + "58": 0.12425, + "59": 0.12199, + "60": 0.12359, + "61": 0.11982, + "62": 0.12161, + "63": 0.11993, + "64": 0.12221, + "65": 0.12364, + "66": 0.12245, + "67": 0.12126, + "68": 0.12211, + "69": 0.11961, + "70": 0.12166, + "71": 0.11999, + "72": 0.12512, + "73": 0.12157, + "74": 0.11996, + "75": 0.12183, + "76": 0.11982, + "77": 0.1205, + "78": 0.12225, + "79": 0.12245, + "80": 0.12222, + "81": 0.12087, + "82": 0.11834, + "83": 0.11849, + "84": 0.11754, + "85": 0.1168, + "86": 0.11739, + "87": 0.11786, + "88": 0.1178, + "89": 0.11801, + "90": 0.11757, + "91": 0.11668, + "92": 0.11659, + "93": 0.11656, + "94": 0.11741, + "95": 0.11613, + "96": 0.11801, + "97": 0.11735, + "98": 0.1168, + "99": 0.11724, + "100": 0.1175 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json index 5470d60dcdb..684fd4831fe 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200.json @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 3.09607, - "3": 0.15089, - "4": 0.16387, - "5": 0.13417, - "6": 0.12738, - "7": 0.12788, - "8": 0.132, - "9": 0.28261, - "10": 0.12697, - "11": 0.13182, - "12": 0.13355, - "13": 0.13045, - "14": 0.13241, - "15": 0.1311, - "16": 0.13108, - "17": 0.13531, - "18": 0.13102, - "19": 0.13307, - "20": 0.13285, - "21": 0.13577, - "22": 0.13581, - "23": 0.13315, - "24": 0.13227, - "25": 0.13286, - "26": 0.13451, - "27": 0.13303, - "28": 0.13253, - "29": 0.29925, - "30": 0.13379, - "31": 0.13315, - "32": 0.13374, - "33": 0.13205, - "34": 0.13144, - "35": 0.13199, - "36": 0.13191, - "37": 0.13367, - "38": 0.13204, - "39": 0.13375, - "40": 0.1347, - "41": 0.13056, - "42": 0.13244, - "43": 0.13361, - "44": 0.13216, - "45": 0.13279, - "46": 0.12873, - "47": 0.13055, - "48": 0.13334, - "49": 0.1341, - "50": 0.13588, - "51": 0.1385, - "52": 0.12954, - "53": 0.1309, - "54": 0.12956, - "55": 0.12942, - "56": 0.12835, - "57": 0.13126, - "58": 0.13085, - "59": 0.17194, - "60": 0.12864, - "61": 0.13121, - "62": 0.13254, - "63": 0.17379, - "64": 0.1288, - "65": 0.13106, - "66": 0.13033, - "67": 0.13051, - "68": 0.12867, - "69": 0.13001, - "70": 0.12842, - "71": 0.13086, - "72": 0.13042, - "73": 0.13305, - "74": 0.13253, - "75": 0.13136, - "76": 0.13325, - "77": 0.13253, - "78": 0.13157, - "79": 0.13256, - "80": 0.13095, - "81": 0.13101, - "82": 0.13389, - "83": 0.13228, - "84": 0.13283, - "85": 0.13274, - "86": 0.13308, - "87": 0.13089, - "88": 0.13159, - "89": 0.13218, - "90": 0.13253, - "91": 0.13284, - "92": 0.13376, - "93": 0.13307, - "94": 0.13459, - "95": 0.13415, - "96": 0.13629, - "97": 0.13635, - "98": 0.1381, - "99": 0.13441, - "100": 0.1359 + "2": 2.22064, + "3": 0.13024, + "4": 0.11768, + "5": 0.11875, + "6": 0.11742, + "7": 0.11821, + "8": 0.11878, + "9": 0.11922, + "10": 0.11834, + "11": 0.11707, + "12": 0.1171, + "13": 0.11874, + "14": 0.12245, + "15": 0.11821, + "16": 0.1177, + "17": 0.11857, + "18": 0.11778, + "19": 0.1187, + "20": 0.11835, + "21": 0.12351, + "22": 0.11771, + "23": 0.11773, + "24": 0.11819, + "25": 0.11705, + "26": 0.12602, + "27": 0.12585, + "28": 0.12677, + "29": 0.12752, + "30": 0.12847, + "31": 0.12883, + "32": 0.12586, + "33": 0.12437, + "34": 0.12277, + "35": 0.12212, + "36": 0.12255, + "37": 0.12084, + "38": 0.12104, + "39": 0.12124, + "40": 0.12086, + "41": 0.12101, + "42": 0.11969, + "43": 0.1197, + "44": 0.11956, + "45": 0.11977, + "46": 0.11865, + "47": 0.11795, + "48": 0.11928, + "49": 0.11794, + "50": 0.11851, + "51": 0.12726, + "52": 0.11929, + "53": 0.11813, + "54": 0.11818, + "55": 0.11748, + "56": 0.11707, + "57": 0.11896, + "58": 0.11832, + "59": 0.11799, + "60": 0.11784, + "61": 0.11888, + "62": 0.11879, + "63": 0.11819, + "64": 0.1185, + "65": 0.11926, + "66": 0.11924, + "67": 0.11982, + "68": 0.11873, + "69": 0.11986, + "70": 0.11895, + "71": 0.11964, + "72": 0.11906, + "73": 0.12017, + "74": 0.11976, + "75": 0.11759, + "76": 0.11921, + "77": 0.11907, + "78": 0.11823, + "79": 0.11867, + "80": 0.11934, + "81": 0.11888, + "82": 0.11988, + "83": 0.1213, + "84": 0.11913, + "85": 0.12002, + "86": 0.12046, + "87": 0.11952, + "88": 0.11819, + "89": 0.11901, + "90": 0.11918, + "91": 0.11919, + "92": 0.11824, + "93": 0.12018, + "94": 0.11929, + "95": 0.11974, + "96": 0.11767, + "97": 0.11845, + "98": 0.11695, + "99": 0.11892, + "100": 0.11948 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..f92f1c4672e --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.76757, + "52": 9.64732, + "53": 9.995, + "54": 9.88603, + "55": 9.81763, + "56": 9.53914, + "57": 9.38192, + "58": 9.75896, + "59": 9.52106, + "60": 9.42443, + "61": 9.63665, + "62": 9.92974, + "63": 9.29595, + "64": 9.70631, + "65": 8.88066, + "66": 9.64072, + "67": 9.32146, + "68": 9.73692, + "69": 9.75346, + "70": 9.68289, + "71": 9.58117, + "72": 9.52491, + "73": 9.44094, + "74": 8.86077, + "75": 9.36671, + "76": 9.01691, + "77": 10.02224, + "78": 9.68354, + "79": 9.33325, + "80": 9.3582, + "81": 9.43786, + "82": 9.66102, + "83": 9.26223, + "84": 9.37189, + "85": 9.56652, + "86": 9.04493, + "87": 9.5575, + "88": 9.70541, + "89": 9.55092, + "90": 9.79196, + "91": 9.29173, + "92": 9.31225, + "93": 9.0433, + "94": 8.78683, + "95": 9.49525, + "96": 9.48391, + "97": 9.25966, + "98": 9.62611, + "99": 8.85031, + "100": 9.36043 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2613.0, + "52": 2647.0, + "53": 2908.0, + "54": 2580.0, + "55": 2486.0, + "56": 2687.0, + "57": 2577.0, + "58": 2824.0, + "59": 2720.0, + "60": 2410.0, + "61": 2744.0, + "62": 2536.0, + "63": 2652.0, + "64": 2918.0, + "65": 2742.0, + "66": 2927.0, + "67": 2920.0, + "68": 2652.0, + "69": 3019.0, + "70": 2996.0, + "71": 2835.0, + "72": 2664.0, + "73": 3211.0, + "74": 2311.0, + "75": 2658.0, + "76": 3155.0, + "77": 3051.0, + "78": 3073.0, + "79": 3116.0, + "80": 3191.0, + "81": 3237.0, + "82": 3218.0, + "83": 2689.0, + "84": 3294.0, + "85": 3209.0, + "86": 2558.0, + "87": 3462.0, + "88": 3287.0, + "89": 3201.0, + "90": 3331.0, + "91": 3183.0, + "92": 3201.0, + "93": 2942.0, + "94": 3274.0, + "95": 3132.0, + "96": 3200.0, + "97": 3054.0, + "98": 3544.0, + "99": 3387.0, + "100": 3192.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 917459968.0, + "52": 917459968.0, + "53": 917459968.0, + "54": 917459968.0, + "55": 917459968.0, + "56": 917459968.0, + "57": 917459968.0, + "58": 917459968.0, + "59": 917459968.0, + "60": 917459968.0, + "61": 917459968.0, + "62": 917459968.0, + "63": 917459968.0, + "64": 917459968.0, + "65": 917459968.0, + "66": 917459968.0, + "67": 917459968.0, + "68": 917459968.0, + "69": 917459968.0, + "70": 917459968.0, + "71": 917459968.0, + "72": 917459968.0, + "73": 917459968.0, + "74": 917459968.0, + "75": 917459968.0, + "76": 917459968.0, + "77": 917459968.0, + "78": 917459968.0, + "79": 917459968.0, + "80": 917459968.0, + "81": 917459968.0, + "82": 917459968.0, + "83": 917459968.0, + "84": 917459968.0, + "85": 917459968.0, + "86": 917459968.0, + "87": 917459968.0, + "88": 917459968.0, + "89": 917459968.0, + "90": 917459968.0, + "91": 917459968.0, + "92": 917459968.0, + "93": 917459968.0, + "94": 917459968.0, + "95": 917459968.0, + "96": 917459968.0, + "97": 917459968.0, + "98": 917459968.0, + "99": 917459968.0, + "100": 917459968.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2622354432.0, + "52": 2622355456.0, + "53": 2622355456.0, + "54": 2622355456.0, + "55": 2622355456.0, + "56": 2622355456.0, + "57": 2622355456.0, + "58": 2622355456.0, + "59": 2622355456.0, + "60": 2622355456.0, + "61": 2622355456.0, + "62": 2622355456.0, + "63": 2622355456.0, + "64": 2622355456.0, + "65": 2622355456.0, + "66": 2622355456.0, + "67": 2622355456.0, + "68": 2622355456.0, + "69": 2622355456.0, + "70": 2622355456.0, + "71": 2622355456.0, + "72": 2622355456.0, + "73": 2622355456.0, + "74": 2622355456.0, + "75": 2622355456.0, + "76": 2622355456.0, + "77": 2622355456.0, + "78": 2622355456.0, + "79": 2622355456.0, + "80": 2622355456.0, + "81": 2622355456.0, + "82": 2622355456.0, + "83": 2622355456.0, + "84": 2622355456.0, + "85": 2622355456.0, + "86": 2622355456.0, + "87": 2622355456.0, + "88": 2622355456.0, + "89": 2622355456.0, + "90": 2622355456.0, + "91": 2622355456.0, + "92": 2622355456.0, + "93": 2622355456.0, + "94": 2622355456.0, + "95": 2622355456.0, + "96": 2622355456.0, + "97": 2622355456.0, + "98": 2622355456.0, + "99": 2622355456.0, + "100": 2622355456.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.13245, + "53": 0.1297, + "54": 0.11767, + "55": 0.11927, + "56": 0.12061, + "57": 0.12305, + "58": 0.12043, + "59": 0.11822, + "60": 0.11725, + "61": 0.11813, + "62": 0.11746, + "63": 0.11736, + "64": 0.11897, + "65": 0.12036, + "66": 0.11746, + "67": 0.11937, + "68": 0.11862, + "69": 0.11914, + "70": 0.11949, + "71": 0.11638, + "72": 0.11794, + "73": 0.11866, + "74": 0.11751, + "75": 0.11637, + "76": 0.11834, + "77": 0.11768, + "78": 0.11854, + "79": 0.11727, + "80": 0.11732, + "81": 0.11811, + "82": 0.11878, + "83": 0.11805, + "84": 0.11921, + "85": 0.11932, + "86": 0.11908, + "87": 0.12476, + "88": 0.12628, + "89": 0.12876, + "90": 0.12617, + "91": 0.12743, + "92": 0.12783, + "93": 0.12812, + "94": 0.12493, + "95": 0.12584, + "96": 0.12791, + "97": 0.12455, + "98": 0.1269, + "99": 0.12715, + "100": 0.12714 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json index 13a8f84be7c..d7fe14d5d09 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 6.53475, - "2": 0.15984, - "3": 0.14377, - "4": 0.29674, - "5": 0.13063, - "6": 0.13043, - "7": 0.13235, - "8": 0.13474, - "9": 0.13363, - "10": 0.13339, - "11": 0.13581, - "12": 0.13012, - "13": 0.13019, - "14": 0.13252, - "15": 0.13313, - "16": 0.13357, - "17": 0.13327, - "18": 0.13417, - "19": 0.1331, - "20": 0.1329, - "21": 0.13223, - "22": 0.32857, - "23": 0.13492, - "24": 0.13386, - "25": 0.13206, - "26": 0.13477, - "27": 0.13149, - "28": 0.13502, - "29": 0.13417, - "30": 0.13168, - "31": 0.13588, - "32": 0.13436, - "33": 0.13143, - "34": 0.13205, - "35": 0.13068, - "36": 0.13116, - "37": 0.13337, - "38": 0.22586, - "39": 0.13222, - "40": 0.13032, - "41": 0.13333, - "42": 0.13093, - "43": 0.13146, - "44": 0.17904, - "45": 0.13291, - "46": 0.13299, - "47": 0.13217, - "48": 0.19742, - "49": 0.24879, - "50": 0.13041, - "51": 0.17217, - "52": 0.14728, - "53": 0.14883, - "54": 0.15217, - "55": 0.15333, - "56": 0.15162, - "57": 0.14349, - "58": 0.5576, - "59": 0.13842, - "60": 0.13366, - "61": 0.13505, - "62": 0.13481, - "63": 0.13475, - "64": 0.13594, - "65": 0.13184, - "66": 0.13558, - "67": 0.1672, - "68": 0.13268, - "69": 0.13176, - "70": 0.13495, - "71": 0.13816, - "72": 0.13681, - "73": 0.13679, - "74": 0.13748, - "75": 0.13564, - "76": 0.13376, - "77": 0.13018, - "78": 0.13137, - "79": 0.13475, - "80": 0.1358, - "81": 0.1337, - "82": 0.13153, - "83": 0.13119, - "84": 0.13428, - "85": 0.15135, - "86": 0.13542, - "87": 0.13379, - "88": 0.13317, - "89": 0.13159, - "90": 0.1344, - "91": 0.13415, - "92": 0.1338, - "93": 0.13311, - "94": 0.13567, - "95": 0.13426, - "96": 0.13525, - "97": 0.13575, - "98": 0.133, - "99": 0.13093, - "100": 0.13623 + "1": "nan", + "2": 2.44626, + "3": 0.14544, + "4": 0.14069, + "5": 0.13132, + "6": 0.13447, + "7": 0.13519, + "8": 0.13562, + "9": 0.13513, + "10": 0.13387, + "11": 0.13378, + "12": 0.13575, + "13": 0.13462, + "14": 0.13384, + "15": 0.13412, + "16": 0.13347, + "17": 0.13555, + "18": 0.13515, + "19": 0.13443, + "20": 0.14433, + "21": 0.14638, + "22": 0.14561, + "23": 0.13968, + "24": 0.13694, + "25": 0.14479, + "26": 0.14038, + "27": 0.1473, + "28": 0.14099, + "29": 0.13829, + "30": 0.13782, + "31": 0.13746, + "32": 0.13573, + "33": 0.13325, + "34": 0.14385, + "35": 0.14367, + "36": 0.14113, + "37": 0.1394, + "38": 0.136, + "39": 0.13678, + "40": 0.13539, + "41": 0.1364, + "42": 0.13593, + "43": 0.13738, + "44": 0.13238, + "45": 0.13667, + "46": 0.14472, + "47": 0.1358, + "48": 0.13697, + "49": 0.13391, + "50": 0.13536, + "51": 0.16637, + "52": 0.15213, + "53": 0.14685, + "54": 0.14134, + "55": 0.14007, + "56": 0.13524, + "57": 0.13779, + "58": 0.13841, + "59": 0.13821, + "60": 0.13687, + "61": 0.13663, + "62": 0.13401, + "63": 0.13389, + "64": 0.13289, + "65": 0.13362, + "66": 0.13754, + "67": 0.13473, + "68": 0.13402, + "69": 0.13491, + "70": 0.13536, + "71": 0.13258, + "72": 0.13482, + "73": 0.13371, + "74": 0.13507, + "75": 0.13595, + "76": 0.13613, + "77": 0.13395, + "78": 0.13252, + "79": 0.13394, + "80": 0.13329, + "81": 0.13388, + "82": 0.13407, + "83": 0.13522, + "84": 0.13579, + "85": 0.13452, + "86": 0.13422, + "87": 0.13388, + "88": 0.1343, + "89": 0.13546, + "90": 0.13522, + "91": 0.13458, + "92": 0.1341, + "93": 0.13519, + "94": 0.13534, + "95": 0.13521, + "96": 0.13886, + "97": 0.13832, + "98": 0.14048, + "99": 0.14022, + "100": 0.13732 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..13e04822669 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.76773, + "52": 9.64757, + "53": 9.99521, + "54": 9.88624, + "55": 9.81783, + "56": 9.53944, + "57": 9.38198, + "58": 9.75913, + "59": 9.52125, + "60": 9.42463, + "61": 9.63669, + "62": 9.93001, + "63": 9.29627, + "64": 9.70638, + "65": 8.88076, + "66": 9.64079, + "67": 9.32154, + "68": 9.737, + "69": 9.75369, + "70": 9.68294, + "71": 9.58129, + "72": 9.52492, + "73": 9.44113, + "74": 8.86077, + "75": 9.3667, + "76": 9.01682, + "77": 10.0224, + "78": 9.68369, + "79": 9.33323, + "80": 9.35819, + "81": 9.43805, + "82": 9.66108, + "83": 9.26227, + "84": 9.37195, + "85": 9.56661, + "86": 9.04515, + "87": 9.55767, + "88": 9.70545, + "89": 9.55104, + "90": 9.79186, + "91": 9.29174, + "92": 9.31247, + "93": 9.04313, + "94": 8.7869, + "95": 9.49543, + "96": 9.48418, + "97": 9.25973, + "98": 9.62635, + "99": 8.85054, + "100": 9.36076 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2647.0, + "52": 2648.0, + "53": 2878.0, + "54": 2654.0, + "55": 2580.0, + "56": 2658.0, + "57": 2547.0, + "58": 2739.0, + "59": 2779.0, + "60": 2349.0, + "61": 2741.0, + "62": 2617.0, + "63": 2512.0, + "64": 2800.0, + "65": 2697.0, + "66": 2966.0, + "67": 2952.0, + "68": 2833.0, + "69": 3029.0, + "70": 2977.0, + "71": 2813.0, + "72": 2664.0, + "73": 3085.0, + "74": 2292.0, + "75": 2810.0, + "76": 3025.0, + "77": 3025.0, + "78": 3037.0, + "79": 3181.0, + "80": 3234.0, + "81": 3273.0, + "82": 3294.0, + "83": 2707.0, + "84": 3332.0, + "85": 3336.0, + "86": 2585.0, + "87": 3448.0, + "88": 3239.0, + "89": 3137.0, + "90": 3341.0, + "91": 3188.0, + "92": 3246.0, + "93": 2823.0, + "94": 3358.0, + "95": 3202.0, + "96": 3118.0, + "97": 3163.0, + "98": 3645.0, + "99": 3345.0, + "100": 3201.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 917459968.0, + "52": 917459968.0, + "53": 917459968.0, + "54": 917459968.0, + "55": 917459968.0, + "56": 917459968.0, + "57": 917459968.0, + "58": 917459968.0, + "59": 917459968.0, + "60": 917459968.0, + "61": 917459968.0, + "62": 917459968.0, + "63": 917459968.0, + "64": 917459968.0, + "65": 917459968.0, + "66": 917459968.0, + "67": 917459968.0, + "68": 917459968.0, + "69": 917459968.0, + "70": 917459968.0, + "71": 917459968.0, + "72": 917459968.0, + "73": 917459968.0, + "74": 917459968.0, + "75": 917459968.0, + "76": 917459968.0, + "77": 917459968.0, + "78": 917459968.0, + "79": 917459968.0, + "80": 917459968.0, + "81": 917459968.0, + "82": 917459968.0, + "83": 917459968.0, + "84": 917459968.0, + "85": 917459968.0, + "86": 917459968.0, + "87": 917459968.0, + "88": 917459968.0, + "89": 917459968.0, + "90": 917459968.0, + "91": 917459968.0, + "92": 917459968.0, + "93": 917459968.0, + "94": 917459968.0, + "95": 917459968.0, + "96": 917459968.0, + "97": 917459968.0, + "98": 917459968.0, + "99": 917459968.0, + "100": 917459968.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2625500160.0, + "52": 2625501184.0, + "53": 2625501184.0, + "54": 2625501184.0, + "55": 2625501184.0, + "56": 2625501184.0, + "57": 2625501184.0, + "58": 2625501184.0, + "59": 2625501184.0, + "60": 2625501184.0, + "61": 2625501184.0, + "62": 2625501184.0, + "63": 2625501184.0, + "64": 2625501184.0, + "65": 2625501184.0, + "66": 2625501184.0, + "67": 2625501184.0, + "68": 2625501184.0, + "69": 2625501184.0, + "70": 2625501184.0, + "71": 2625501184.0, + "72": 2625501184.0, + "73": 2625501184.0, + "74": 2625501184.0, + "75": 2625501184.0, + "76": 2625501184.0, + "77": 2625501184.0, + "78": 2625501184.0, + "79": 2625501184.0, + "80": 2625501184.0, + "81": 2625501184.0, + "82": 2625501184.0, + "83": 2625501184.0, + "84": 2625501184.0, + "85": 2625501184.0, + "86": 2625501184.0, + "87": 2625501184.0, + "88": 2625501184.0, + "89": 2625501184.0, + "90": 2625501184.0, + "91": 2625501184.0, + "92": 2625501184.0, + "93": 2625501184.0, + "94": 2625501184.0, + "95": 2625501184.0, + "96": 2625501184.0, + "97": 2625501184.0, + "98": 2625501184.0, + "99": 2625501184.0, + "100": 2625501184.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.60445, + "53": 0.14686, + "54": 0.13325, + "55": 0.13174, + "56": 0.13234, + "57": 0.13268, + "58": 0.13337, + "59": 0.13324, + "60": 0.13107, + "61": 0.13206, + "62": 0.1329, + "63": 0.13379, + "64": 0.1348, + "65": 0.13602, + "66": 0.13298, + "67": 0.13465, + "68": 0.13495, + "69": 0.13454, + "70": 0.13536, + "71": 0.13494, + "72": 0.13541, + "73": 0.13997, + "74": 0.1423, + "75": 0.13785, + "76": 0.14664, + "77": 0.16548, + "78": 0.17704, + "79": 0.15011, + "80": 0.14471, + "81": 0.13952, + "82": 0.13892, + "83": 0.13568, + "84": 0.13463, + "85": 0.13878, + "86": 0.13867, + "87": 0.13899, + "88": 0.13819, + "89": 0.13945, + "90": 0.13964, + "91": 0.13862, + "92": 0.13655, + "93": 0.13587, + "94": 0.13572, + "95": 0.1357, + "96": 0.13598, + "97": 0.13642, + "98": 0.13742, + "99": 0.13474, + "100": 0.13647 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json index 4143efc2988..63c74381364 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.52284, - "2": 0.157, - "3": 0.14283, - "4": 0.12717, - "5": 0.23804, - "6": 0.12672, - "7": 0.23745, - "8": 0.12054, - "9": 0.21684, - "10": 0.11896, - "11": 0.13284, - "12": 0.11855, - "13": 0.11845, - "14": 0.11744, - "15": 0.11809, - "16": 0.11959, - "17": 0.11704, - "18": 0.22382, - "19": 0.30417, - "20": 0.13849, - "21": 0.11644, - "22": 0.24942, - "23": 0.11902, - "24": 0.11673, - "25": 0.11881, - "26": 0.11714, - "27": 0.26517, - "28": 0.11796, - "29": 0.11692, - "30": 0.1177, - "31": 0.1199, - "32": 0.11855, - "33": 0.20894, - "34": 0.1189, - "35": 0.11946, - "36": 0.11731, - "37": 0.11585, - "38": 0.22438, - "39": 0.11586, - "40": 0.31661, - "41": 0.27224, - "42": 0.11828, - "43": 0.11576, - "44": 0.31558, - "45": 0.11735, - "46": 0.11931, - "47": 0.2329, - "48": 0.20057, - "49": 0.11638, - "50": 0.14553, - "51": 0.15092, - "52": 0.12868, - "53": 0.29978, - "54": 0.13487, - "55": 0.1206, - "56": 0.117, - "57": 0.117, - "58": 0.11712, - "59": 0.11789, - "60": 0.11693, - "61": 0.11525, - "62": 0.24109, - "63": 0.11906, - "64": 0.12054, - "65": 0.11805, - "66": 0.11831, - "67": 0.11744, - "68": 0.11454, - "69": 0.39474, - "70": 0.11683, - "71": 0.117, - "72": 0.11875, - "73": 0.28446, - "74": 0.22373, - "75": 0.11573, - "76": 0.1177, - "77": 0.11707, - "78": 0.24184, - "79": 0.11755, - "80": 0.11784, - "81": 0.21803, - "82": 0.11787, - "83": 0.23349, - "84": 0.22596, - "85": 0.11587, - "86": 0.11507, - "87": 0.16522, - "88": 0.24306, - "89": 0.12003, - "90": 0.23071, - "91": 0.12051, - "92": 0.12072, - "93": 0.11991, - "94": 0.22186, - "95": 0.12105, - "96": 0.12128, - "97": 0.11916, - "98": 0.12303, - "99": 0.1197, - "100": 0.1207 + "1": "nan", + "2": 2.20838, + "3": 0.13042, + "4": 0.11826, + "5": 0.11718, + "6": 0.11797, + "7": 0.1177, + "8": 0.11717, + "9": 0.11846, + "10": 0.11778, + "11": 0.11712, + "12": 0.11866, + "13": 0.12004, + "14": 0.11788, + "15": 0.11787, + "16": 0.1181, + "17": 0.11903, + "18": 0.11843, + "19": 0.11754, + "20": 0.11834, + "21": 0.11897, + "22": 0.12726, + "23": 0.13834, + "24": 0.15039, + "25": 0.14107, + "26": 0.14586, + "27": 0.16343, + "28": 0.2297, + "29": 0.26681, + "30": 0.19748, + "31": 0.2586, + "32": 0.12068, + "33": 0.11944, + "34": 0.11896, + "35": 0.11984, + "36": 0.11823, + "37": 0.11997, + "38": 0.11949, + "39": 0.11877, + "40": 0.11898, + "41": 0.11996, + "42": 0.11893, + "43": 0.12547, + "44": 0.13195, + "45": 0.12144, + "46": 0.11997, + "47": 0.12005, + "48": 0.11855, + "49": 0.11944, + "50": 0.11842, + "51": 0.14635, + "52": 0.12016, + "53": 0.11762, + "54": 0.11802, + "55": 0.1184, + "56": 0.11774, + "57": 0.12181, + "58": 0.11784, + "59": 0.11936, + "60": 0.11831, + "61": 0.11819, + "62": 0.11807, + "63": 0.11828, + "64": 0.11663, + "65": 0.11901, + "66": 0.1168, + "67": 0.1167, + "68": 0.12002, + "69": 0.12016, + "70": 0.1186, + "71": 0.11772, + "72": 0.1189, + "73": 0.11915, + "74": 0.11908, + "75": 0.11898, + "76": 0.11863, + "77": 0.11869, + "78": 0.11971, + "79": 0.11843, + "80": 0.1198, + "81": 0.12003, + "82": 0.11885, + "83": 0.11905, + "84": 0.12002, + "85": 0.1192, + "86": 0.11872, + "87": 0.11777, + "88": 0.11801, + "89": 0.11864, + "90": 0.11769, + "91": 0.11692, + "92": 0.12015, + "93": 0.12072, + "94": 0.11802, + "95": 0.11798, + "96": 0.12278, + "97": 0.11941, + "98": 0.1174, + "99": 0.11816, + "100": 0.12102 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..12556b60c96 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.83807, + "52": 9.72058, + "53": 10.0568, + "54": 9.95032, + "55": 9.88328, + "56": 9.60431, + "57": 9.45518, + "58": 9.81927, + "59": 9.58262, + "60": 9.48844, + "61": 9.68577, + "62": 9.97779, + "63": 9.36765, + "64": 9.75913, + "65": 8.9376, + "66": 9.69257, + "67": 9.36621, + "68": 9.78303, + "69": 9.79318, + "70": 9.72699, + "71": 9.62875, + "72": 9.58004, + "73": 9.487, + "74": 8.92041, + "75": 9.41128, + "76": 9.07564, + "77": 10.05848, + "78": 9.72184, + "79": 9.3732, + "80": 9.40079, + "81": 9.4792, + "82": 9.69754, + "83": 9.31037, + "84": 9.41777, + "85": 9.61194, + "86": 9.07155, + "87": 9.59661, + "88": 9.74709, + "89": 9.59667, + "90": 9.82915, + "91": 9.33725, + "92": 9.3564, + "93": 9.08552, + "94": 8.82807, + "95": 9.52842, + "96": 9.52611, + "97": 9.30632, + "98": 9.66808, + "99": 8.89461, + "100": 9.40666 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2546.0, + "52": 2590.0, + "53": 2879.0, + "54": 2697.0, + "55": 2316.0, + "56": 2549.0, + "57": 2261.0, + "58": 2904.0, + "59": 2740.0, + "60": 2434.0, + "61": 2801.0, + "62": 2663.0, + "63": 2502.0, + "64": 2948.0, + "65": 2644.0, + "66": 2961.0, + "67": 2813.0, + "68": 2686.0, + "69": 2912.0, + "70": 3096.0, + "71": 2854.0, + "72": 2454.0, + "73": 3081.0, + "74": 1933.0, + "75": 2465.0, + "76": 3012.0, + "77": 3163.0, + "78": 2997.0, + "79": 3089.0, + "80": 3187.0, + "81": 3500.0, + "82": 3339.0, + "83": 2705.0, + "84": 3205.0, + "85": 3033.0, + "86": 2818.0, + "87": 3671.0, + "88": 3190.0, + "89": 3336.0, + "90": 3320.0, + "91": 2698.0, + "92": 3072.0, + "93": 2750.0, + "94": 3397.0, + "95": 3317.0, + "96": 3290.0, + "97": 3116.0, + "98": 3732.0, + "99": 3049.0, + "100": 2974.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 746195456.0, + "52": 746195456.0, + "53": 746195456.0, + "54": 746195456.0, + "55": 746195456.0, + "56": 746195456.0, + "57": 746195456.0, + "58": 746195456.0, + "59": 746195456.0, + "60": 746195456.0, + "61": 746195456.0, + "62": 746195456.0, + "63": 746195456.0, + "64": 746195456.0, + "65": 746195456.0, + "66": 746195456.0, + "67": 746195456.0, + "68": 746195456.0, + "69": 746195456.0, + "70": 746195456.0, + "71": 746195456.0, + "72": 746195456.0, + "73": 746195456.0, + "74": 746195456.0, + "75": 746195456.0, + "76": 746195456.0, + "77": 746195456.0, + "78": 746195456.0, + "79": 746195456.0, + "80": 746195456.0, + "81": 746195456.0, + "82": 746195456.0, + "83": 746195456.0, + "84": 746195456.0, + "85": 746195456.0, + "86": 746195456.0, + "87": 746195456.0, + "88": 746195456.0, + "89": 746195456.0, + "90": 746195456.0, + "91": 746195456.0, + "92": 746195456.0, + "93": 746195456.0, + "94": 746195456.0, + "95": 746195456.0, + "96": 746195456.0, + "97": 746195456.0, + "98": 746195456.0, + "99": 746195456.0, + "100": 746195456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2223482880.0, + "52": 2223483904.0, + "53": 2223483904.0, + "54": 2223483904.0, + "55": 2223483904.0, + "56": 2223483904.0, + "57": 2223483904.0, + "58": 2223483904.0, + "59": 2223483904.0, + "60": 2223483904.0, + "61": 2223483904.0, + "62": 2223483904.0, + "63": 2223483904.0, + "64": 2223483904.0, + "65": 2223483904.0, + "66": 2223483904.0, + "67": 2223483904.0, + "68": 2223483904.0, + "69": 2223483904.0, + "70": 2223483904.0, + "71": 2223483904.0, + "72": 2223483904.0, + "73": 2223483904.0, + "74": 2223483904.0, + "75": 2223483904.0, + "76": 2223483904.0, + "77": 2223483904.0, + "78": 2223483904.0, + "79": 2223483904.0, + "80": 2223483904.0, + "81": 2223483904.0, + "82": 2223483904.0, + "83": 2223483904.0, + "84": 2223483904.0, + "85": 2223483904.0, + "86": 2223483904.0, + "87": 2223483904.0, + "88": 2223483904.0, + "89": 2223483904.0, + "90": 2223483904.0, + "91": 2223483904.0, + "92": 2223483904.0, + "93": 2223483904.0, + "94": 2223483904.0, + "95": 2223483904.0, + "96": 2223483904.0, + "97": 2223483904.0, + "98": 2223483904.0, + "99": 2223483904.0, + "100": 2223483904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 1.94458, + "53": 0.13487, + "54": 0.12133, + "55": 0.12128, + "56": 0.12059, + "57": 0.11937, + "58": 0.11813, + "59": 0.11931, + "60": 0.12225, + "61": 0.1198, + "62": 0.1197, + "63": 0.12083, + "64": 0.12132, + "65": 0.12067, + "66": 0.12047, + "67": 0.12065, + "68": 0.12005, + "69": 0.12047, + "70": 0.11977, + "71": 0.1205, + "72": 0.11909, + "73": 0.11956, + "74": 0.12277, + "75": 0.11982, + "76": 0.12087, + "77": 0.12003, + "78": 0.12188, + "79": 0.12094, + "80": 0.12076, + "81": 0.12072, + "82": 0.12053, + "83": 0.11961, + "84": 0.12306, + "85": 0.12275, + "86": 0.11989, + "87": 0.11996, + "88": 0.1294, + "89": 0.12077, + "90": 0.1204, + "91": 0.12138, + "92": 0.11998, + "93": 0.12202, + "94": 0.12092, + "95": 0.11985, + "96": 0.11995, + "97": 0.12124, + "98": 0.12243, + "99": 0.12016, + "100": 0.12049 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json index 29e5fc62d41..a9a12874e97 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.46714, - "2": 0.15122, - "3": 0.25092, - "4": 0.27729, - "5": 0.24367, - "6": 0.26751, - "7": 0.2545, - "8": 0.12105, - "9": 0.24325, - "10": 0.12174, - "11": 0.12002, - "12": 0.12125, - "13": 0.25244, - "14": 0.11939, - "15": 0.24088, - "16": 0.11887, - "17": 0.44496, - "18": 0.16003, - "19": 0.25508, - "20": 0.21211, - "21": 0.3079, - "22": 0.12004, - "23": 0.43335, - "24": 0.12, - "25": 0.12101, - "26": 0.12096, - "27": 0.29192, - "28": 0.19864, - "29": 0.26692, - "30": 0.11884, - "31": 0.12045, - "32": 0.12079, - "33": 0.12032, - "34": 0.12022, - "35": 0.21894, - "36": 0.11918, - "37": 0.22006, - "38": 0.34871, - "39": 0.12088, - "40": 0.12089, - "41": 0.12159, - "42": 0.18229, - "43": 0.16394, - "44": 0.11984, - "45": 0.12064, - "46": 0.12128, - "47": 0.17743, - "48": 0.1593, - "49": 0.12034, - "50": 0.11831, - "51": 0.13446, - "52": 0.12243, - "53": 0.11866, - "54": 0.11939, - "55": 0.20902, - "56": 0.13705, - "57": 0.11709, - "58": 0.11749, - "59": 0.11871, - "60": 0.22163, - "61": 0.11825, - "62": 0.22086, - "63": 0.11702, - "64": 0.11919, - "65": 0.12009, - "66": 0.19788, - "67": 0.42941, - "68": 0.11868, - "69": 0.22718, - "70": 0.20618, - "71": 0.13003, - "72": 0.134, - "73": 0.13466, - "74": 0.14293, - "75": 0.22299, - "76": 0.12996, - "77": 0.13433, - "78": 0.13652, - "79": 0.1285, - "80": 0.13881, - "81": 0.13014, - "82": 0.12942, - "83": 0.22639, - "84": 0.1185, - "85": 0.22799, - "86": 0.23089, - "87": 0.11774, - "88": 0.22926, - "89": 0.12055, - "90": 0.11828, - "91": 0.25019, - "92": 0.11977, - "93": 0.1173, - "94": 0.11879, - "95": 0.1161, - "96": 0.34968, - "97": 0.11818, - "98": 0.21965, - "99": 0.12107, - "100": 0.11838 + "1": "nan", + "2": 2.22807, + "3": 0.13601, + "4": 0.12128, + "5": 0.1198, + "6": 0.1228, + "7": 0.12056, + "8": 0.11886, + "9": 0.11944, + "10": 0.11995, + "11": 0.11935, + "12": 0.11905, + "13": 0.11975, + "14": 0.12242, + "15": 0.12061, + "16": 0.12046, + "17": 0.1208, + "18": 0.12205, + "19": 0.12427, + "20": 0.12315, + "21": 0.11965, + "22": 0.12231, + "23": 0.12286, + "24": 0.12394, + "25": 0.12377, + "26": 0.12221, + "27": 0.11936, + "28": 0.11894, + "29": 0.11945, + "30": 0.12192, + "31": 0.12571, + "32": 0.12346, + "33": 0.12413, + "34": 0.12225, + "35": 0.12328, + "36": 0.12241, + "37": 0.12432, + "38": 0.12195, + "39": 0.12262, + "40": 0.12198, + "41": 0.12396, + "42": 0.12194, + "43": 0.12435, + "44": 0.12108, + "45": 0.12326, + "46": 0.1218, + "47": 0.12308, + "48": 0.12384, + "49": 0.12795, + "50": 0.12572, + "51": 0.13502, + "52": 0.13106, + "53": 0.14515, + "54": 0.12597, + "55": 0.1249, + "56": 0.12535, + "57": 0.12569, + "58": 0.12489, + "59": 0.12862, + "60": 0.12778, + "61": 0.12731, + "62": 0.12786, + "63": 0.13022, + "64": 0.12789, + "65": 0.12838, + "66": 0.12571, + "67": 0.12651, + "68": 0.12592, + "69": 0.12663, + "70": 0.12691, + "71": 0.12636, + "72": 0.12638, + "73": 0.12671, + "74": 0.12637, + "75": 0.12602, + "76": 0.12598, + "77": 0.12554, + "78": 0.12553, + "79": 0.12501, + "80": 0.13898, + "81": 0.14589, + "82": 0.14718, + "83": 0.14665, + "84": 0.16017, + "85": 0.14231, + "86": 0.15628, + "87": 0.14055, + "88": 0.13961, + "89": 0.14878, + "90": 0.14486, + "91": 0.1432, + "92": 0.14946, + "93": 0.14581, + "94": 0.1623, + "95": 0.15638, + "96": 0.12895, + "97": 0.12907, + "98": 0.12824, + "99": 0.12741, + "100": 0.12543 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..8cb40558f2c --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.83807, + "52": 9.72058, + "53": 10.0568, + "54": 9.95032, + "55": 9.88328, + "56": 9.60431, + "57": 9.45518, + "58": 9.81927, + "59": 9.58262, + "60": 9.48844, + "61": 9.68577, + "62": 9.97779, + "63": 9.36765, + "64": 9.75913, + "65": 8.9376, + "66": 9.69257, + "67": 9.36621, + "68": 9.78303, + "69": 9.79318, + "70": 9.72699, + "71": 9.62875, + "72": 9.58004, + "73": 9.487, + "74": 8.92041, + "75": 9.41128, + "76": 9.07564, + "77": 10.05848, + "78": 9.72184, + "79": 9.3732, + "80": 9.40079, + "81": 9.4792, + "82": 9.69754, + "83": 9.31037, + "84": 9.41777, + "85": 9.61194, + "86": 9.07155, + "87": 9.59661, + "88": 9.74709, + "89": 9.59667, + "90": 9.82915, + "91": 9.33725, + "92": 9.3564, + "93": 9.08552, + "94": 8.82807, + "95": 9.52842, + "96": 9.52611, + "97": 9.30632, + "98": 9.66808, + "99": 8.89461, + "100": 9.40666 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2546.0, + "52": 2590.0, + "53": 2879.0, + "54": 2697.0, + "55": 2316.0, + "56": 2549.0, + "57": 2261.0, + "58": 2904.0, + "59": 2740.0, + "60": 2434.0, + "61": 2801.0, + "62": 2663.0, + "63": 2502.0, + "64": 2948.0, + "65": 2644.0, + "66": 2961.0, + "67": 2813.0, + "68": 2686.0, + "69": 2912.0, + "70": 3096.0, + "71": 2854.0, + "72": 2454.0, + "73": 3081.0, + "74": 1933.0, + "75": 2465.0, + "76": 3012.0, + "77": 3163.0, + "78": 2997.0, + "79": 3089.0, + "80": 3187.0, + "81": 3500.0, + "82": 3339.0, + "83": 2705.0, + "84": 3205.0, + "85": 3033.0, + "86": 2818.0, + "87": 3671.0, + "88": 3190.0, + "89": 3336.0, + "90": 3320.0, + "91": 2698.0, + "92": 3072.0, + "93": 2750.0, + "94": 3397.0, + "95": 3317.0, + "96": 3290.0, + "97": 3116.0, + "98": 3732.0, + "99": 3049.0, + "100": 2974.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 746195456.0, + "52": 746195456.0, + "53": 746195456.0, + "54": 746195456.0, + "55": 746195456.0, + "56": 746195456.0, + "57": 746195456.0, + "58": 746195456.0, + "59": 746195456.0, + "60": 746195456.0, + "61": 746195456.0, + "62": 746195456.0, + "63": 746195456.0, + "64": 746195456.0, + "65": 746195456.0, + "66": 746195456.0, + "67": 746195456.0, + "68": 746195456.0, + "69": 746195456.0, + "70": 746195456.0, + "71": 746195456.0, + "72": 746195456.0, + "73": 746195456.0, + "74": 746195456.0, + "75": 746195456.0, + "76": 746195456.0, + "77": 746195456.0, + "78": 746195456.0, + "79": 746195456.0, + "80": 746195456.0, + "81": 746195456.0, + "82": 746195456.0, + "83": 746195456.0, + "84": 746195456.0, + "85": 746195456.0, + "86": 746195456.0, + "87": 746195456.0, + "88": 746195456.0, + "89": 746195456.0, + "90": 746195456.0, + "91": 746195456.0, + "92": 746195456.0, + "93": 746195456.0, + "94": 746195456.0, + "95": 746195456.0, + "96": 746195456.0, + "97": 746195456.0, + "98": 746195456.0, + "99": 746195456.0, + "100": 746195456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2223482880.0, + "52": 2223483904.0, + "53": 2223483904.0, + "54": 2223483904.0, + "55": 2223483904.0, + "56": 2223483904.0, + "57": 2223483904.0, + "58": 2223483904.0, + "59": 2223483904.0, + "60": 2223483904.0, + "61": 2223483904.0, + "62": 2223483904.0, + "63": 2223483904.0, + "64": 2223483904.0, + "65": 2223483904.0, + "66": 2223483904.0, + "67": 2223483904.0, + "68": 2223483904.0, + "69": 2223483904.0, + "70": 2223483904.0, + "71": 2223483904.0, + "72": 2223483904.0, + "73": 2223483904.0, + "74": 2223483904.0, + "75": 2223483904.0, + "76": 2223483904.0, + "77": 2223483904.0, + "78": 2223483904.0, + "79": 2223483904.0, + "80": 2223483904.0, + "81": 2223483904.0, + "82": 2223483904.0, + "83": 2223483904.0, + "84": 2223483904.0, + "85": 2223483904.0, + "86": 2223483904.0, + "87": 2223483904.0, + "88": 2223483904.0, + "89": 2223483904.0, + "90": 2223483904.0, + "91": 2223483904.0, + "92": 2223483904.0, + "93": 2223483904.0, + "94": 2223483904.0, + "95": 2223483904.0, + "96": 2223483904.0, + "97": 2223483904.0, + "98": 2223483904.0, + "99": 2223483904.0, + "100": 2223483904.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.08357, + "53": 0.13321, + "54": 0.11949, + "55": 0.11861, + "56": 0.11817, + "57": 0.12088, + "58": 0.11937, + "59": 0.11893, + "60": 0.11961, + "61": 0.11894, + "62": 0.11953, + "63": 0.11978, + "64": 0.11983, + "65": 0.12255, + "66": 0.12188, + "67": 0.12135, + "68": 0.11972, + "69": 0.11963, + "70": 0.11929, + "71": 0.11924, + "72": 0.12023, + "73": 0.12093, + "74": 0.12082, + "75": 0.11862, + "76": 0.11797, + "77": 0.11862, + "78": 0.12219, + "79": 0.12137, + "80": 0.11873, + "81": 0.11752, + "82": 0.1208, + "83": 0.11974, + "84": 0.1182, + "85": 0.11721, + "86": 0.11748, + "87": 0.11944, + "88": 0.11934, + "89": 0.11847, + "90": 0.11837, + "91": 0.11938, + "92": 0.11761, + "93": 0.11737, + "94": 0.12142, + "95": 0.12574, + "96": 0.12197, + "97": 0.12384, + "98": 0.12251, + "99": 0.13032, + "100": 0.12305 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json index 6d18d551f69..1fc5ef869c5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 9.70264, - "2": 0.16719, - "3": 0.1517, - "4": 0.13783, - "5": 0.26129, - "6": 0.13706, - "7": 0.13419, - "8": 0.23253, - "9": 0.27748, - "10": 0.13541, - "11": 0.2497, - "12": 0.16837, - "13": 0.18244, - "14": 0.25112, - "15": 0.13528, - "16": 0.13665, - "17": 0.1335, - "18": 0.24242, - "19": 0.13551, - "20": 0.1359, - "21": 0.23117, - "22": 0.23904, - "23": 0.14673, - "24": 0.21295, - "25": 0.13514, - "26": 0.13371, - "27": 0.27353, - "28": 0.13711, - "29": 0.13562, - "30": 0.14989, - "31": 0.13559, - "32": 0.25304, - "33": 0.13594, - "34": 0.23626, - "35": 0.21619, - "36": 0.13222, - "37": 0.22334, - "38": 0.17132, - "39": 0.13473, - "40": 0.13527, - "41": 0.13612, - "42": 0.13601, - "43": 0.13671, - "44": 0.13525, - "45": 0.13595, - "46": 0.13781, - "47": 0.13561, - "48": 0.21607, - "49": 0.13778, - "50": 0.13576, - "51": 0.15841, - "52": 0.19731, - "53": 0.13535, - "54": 0.13412, - "55": 0.13529, - "56": 0.20892, - "57": 0.136, - "58": 0.13447, - "59": 0.13492, - "60": 0.22138, - "61": 0.1371, - "62": 0.13221, - "63": 0.31035, - "64": 0.13635, - "65": 0.18383, - "66": 0.13523, - "67": 0.21619, - "68": 0.13406, - "69": 0.24552, - "70": 0.13459, - "71": 0.24237, - "72": 0.13438, - "73": 0.13314, - "74": 0.2234, - "75": 0.13466, - "76": 0.13379, - "77": 0.23131, - "78": 0.13685, - "79": 0.2198, - "80": 0.13574, - "81": 0.13541, - "82": 0.24005, - "83": 0.13618, - "84": 0.13532, - "85": 0.13462, - "86": 0.13568, - "87": 0.13402, - "88": 0.22458, - "89": 0.13468, - "90": 0.23352, - "91": 0.14917, - "92": 0.14938, - "93": 0.14799, - "94": 0.23609, - "95": 0.15009, - "96": 0.22721, - "97": 0.15604, - "98": 0.22921, - "99": 0.1552, - "100": 0.15308 + "1": "nan", + "2": 2.5568, + "3": 0.14788, + "4": 0.13602, + "5": 0.13596, + "6": 0.136, + "7": 0.13621, + "8": 0.13502, + "9": 0.13408, + "10": 0.23083, + "11": 0.14377, + "12": 0.14332, + "13": 0.15453, + "14": 0.15537, + "15": 0.15549, + "16": 0.15444, + "17": 0.15453, + "18": 0.15178, + "19": 0.21432, + "20": 0.15336, + "21": 0.1534, + "22": 0.15483, + "23": 0.15395, + "24": 0.15469, + "25": 0.15447, + "26": 0.15509, + "27": 0.1545, + "28": 0.15527, + "29": 0.15593, + "30": 0.15688, + "31": 0.15659, + "32": 0.15629, + "33": 0.15533, + "34": 0.155, + "35": 0.15519, + "36": 0.15784, + "37": 0.15943, + "38": 0.15552, + "39": 0.15486, + "40": 0.15539, + "41": 0.15618, + "42": 0.15569, + "43": 0.15725, + "44": 0.15522, + "45": 0.1553, + "46": 0.15719, + "47": 0.15571, + "48": 0.15568, + "49": 0.15362, + "50": 0.15495, + "51": 0.18287, + "52": 0.16115, + "53": 0.15739, + "54": 0.15665, + "55": 0.15684, + "56": 0.15658, + "57": 0.15631, + "58": 0.22153, + "59": 0.15604, + "60": 0.15313, + "61": 0.15485, + "62": 0.15518, + "63": 0.15719, + "64": 0.15757, + "65": 0.15904, + "66": 0.15846, + "67": 0.15846, + "68": 0.15754, + "69": 0.15779, + "70": 0.1589, + "71": 0.16037, + "72": 0.15778, + "73": 0.15771, + "74": 0.155, + "75": 0.15611, + "76": 0.15702, + "77": 0.15564, + "78": 0.15892, + "79": 0.15669, + "80": 0.15768, + "81": 0.15805, + "82": 0.15778, + "83": 0.15674, + "84": 0.15715, + "85": 0.15834, + "86": 0.15763, + "87": 0.15855, + "88": 0.15589, + "89": 0.15616, + "90": 0.15639, + "91": 0.15722, + "92": 0.15788, + "93": 0.15597, + "94": 0.15817, + "95": 0.15819, + "96": 0.15869, + "97": 0.15875, + "98": 0.15993, + "99": 0.16297, + "100": 0.16682 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..2cf6e26ff95 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.9812, + "52": 9.89198, + "53": 10.19208, + "54": 10.09574, + "55": 10.00506, + "56": 9.78714, + "57": 9.64607, + "58": 9.9862, + "59": 9.72684, + "60": 9.67172, + "61": 9.80984, + "62": 10.11126, + "63": 9.54877, + "64": 9.90929, + "65": 9.08735, + "66": 9.84659, + "67": 9.48264, + "68": 9.89439, + "69": 9.87695, + "70": 9.82469, + "71": 9.72751, + "72": 9.72911, + "73": 9.62051, + "74": 9.11601, + "75": 9.55057, + "76": 9.21504, + "77": 10.14893, + "78": 9.8138, + "79": 9.47515, + "80": 9.51582, + "81": 9.58685, + "82": 9.79026, + "83": 9.45587, + "84": 9.50503, + "85": 9.71387, + "86": 9.17463, + "87": 9.66601, + "88": 9.84354, + "89": 9.70734, + "90": 9.8955, + "91": 9.48652, + "92": 9.47023, + "93": 9.21481, + "94": 8.94327, + "95": 9.6154, + "96": 9.63634, + "97": 9.37644, + "98": 9.74975, + "99": 9.01753, + "100": 9.50515 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2676.0, + "52": 2581.0, + "53": 2898.0, + "54": 2849.0, + "55": 2548.0, + "56": 2661.0, + "57": 2510.0, + "58": 2758.0, + "59": 2650.0, + "60": 2242.0, + "61": 2628.0, + "62": 2899.0, + "63": 2605.0, + "64": 2939.0, + "65": 2572.0, + "66": 2896.0, + "67": 2640.0, + "68": 2709.0, + "69": 2889.0, + "70": 3012.0, + "71": 2978.0, + "72": 2536.0, + "73": 2964.0, + "74": 2163.0, + "75": 2603.0, + "76": 2974.0, + "77": 3007.0, + "78": 3138.0, + "79": 3197.0, + "80": 2984.0, + "81": 3280.0, + "82": 3341.0, + "83": 2757.0, + "84": 3399.0, + "85": 3320.0, + "86": 2882.0, + "87": 3407.0, + "88": 3278.0, + "89": 3336.0, + "90": 3322.0, + "91": 2472.0, + "92": 3061.0, + "93": 2911.0, + "94": 3005.0, + "95": 2984.0, + "96": 2991.0, + "97": 3178.0, + "98": 3343.0, + "99": 2929.0, + "100": 2588.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 744880640.0, + "52": 744880640.0, + "53": 744880640.0, + "54": 744880640.0, + "55": 744880640.0, + "56": 744880640.0, + "57": 744880640.0, + "58": 744880640.0, + "59": 744880640.0, + "60": 744880640.0, + "61": 744880640.0, + "62": 744880640.0, + "63": 744880640.0, + "64": 744880640.0, + "65": 744880640.0, + "66": 744880640.0, + "67": 744880640.0, + "68": 744880640.0, + "69": 744880640.0, + "70": 744880640.0, + "71": 744880640.0, + "72": 744880640.0, + "73": 744880640.0, + "74": 744880640.0, + "75": 744880640.0, + "76": 744880640.0, + "77": 744880640.0, + "78": 744880640.0, + "79": 744880640.0, + "80": 744880640.0, + "81": 744880640.0, + "82": 744880640.0, + "83": 744880640.0, + "84": 744880640.0, + "85": 744880640.0, + "86": 744880640.0, + "87": 744880640.0, + "88": 744880640.0, + "89": 744880640.0, + "90": 744880640.0, + "91": 744880640.0, + "92": 744880640.0, + "93": 744880640.0, + "94": 744880640.0, + "95": 744880640.0, + "96": 744880640.0, + "97": 744880640.0, + "98": 744880640.0, + "99": 744880640.0, + "100": 744880640.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2222430208.0, + "52": 2222431232.0, + "53": 2222431232.0, + "54": 2222431232.0, + "55": 2222431232.0, + "56": 2222431232.0, + "57": 2222431232.0, + "58": 2222431232.0, + "59": 2222431232.0, + "60": 2222431232.0, + "61": 2222431232.0, + "62": 2222431232.0, + "63": 2222431232.0, + "64": 2222431232.0, + "65": 2222431232.0, + "66": 2222431232.0, + "67": 2222431232.0, + "68": 2222431232.0, + "69": 2222431232.0, + "70": 2222431232.0, + "71": 2222431232.0, + "72": 2222431232.0, + "73": 2222431232.0, + "74": 2222431232.0, + "75": 2222431232.0, + "76": 2222431232.0, + "77": 2222431232.0, + "78": 2222431232.0, + "79": 2222431232.0, + "80": 2222431232.0, + "81": 2222431232.0, + "82": 2222431232.0, + "83": 2222431232.0, + "84": 2222431232.0, + "85": 2222431232.0, + "86": 2222431232.0, + "87": 2222431232.0, + "88": 2222431232.0, + "89": 2222431232.0, + "90": 2222431232.0, + "91": 2222431232.0, + "92": 2222431232.0, + "93": 2222431232.0, + "94": 2222431232.0, + "95": 2222431232.0, + "96": 2222431232.0, + "97": 2222431232.0, + "98": 2222431232.0, + "99": 2222431232.0, + "100": 2222431232.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.21538, + "53": 0.14615, + "54": 0.13599, + "55": 0.13518, + "56": 0.13401, + "57": 0.13944, + "58": 0.13509, + "59": 0.1377, + "60": 0.13698, + "61": 0.137, + "62": 0.13756, + "63": 0.14119, + "64": 0.13937, + "65": 0.13725, + "66": 0.13667, + "67": 0.13894, + "68": 0.13705, + "69": 0.1375, + "70": 0.13655, + "71": 0.13624, + "72": 0.13743, + "73": 0.13786, + "74": 0.13678, + "75": 0.13803, + "76": 0.13591, + "77": 0.13654, + "78": 0.13783, + "79": 0.13724, + "80": 0.13943, + "81": 0.13808, + "82": 0.13899, + "83": 0.13956, + "84": 0.14004, + "85": 0.14504, + "86": 0.14078, + "87": 0.14075, + "88": 0.14222, + "89": 0.14283, + "90": 0.14178, + "91": 0.14143, + "92": 0.14178, + "93": 0.14108, + "94": 0.14248, + "95": 0.14123, + "96": 0.14274, + "97": 0.14429, + "98": 0.14312, + "99": 0.14121, + "100": 0.14248 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json index 87eebe31670..34725e2965a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 8.41422, - "2": 0.18257, - "3": 0.66774, - "4": 0.24561, - "5": 0.26628, - "6": 0.28507, - "7": 0.15561, - "8": 0.31346, - "9": 0.1544, - "10": 0.23888, - "11": 0.24945, - "12": 0.15494, - "13": 0.20224, - "14": 0.15018, - "15": 0.21414, - "16": 0.15279, - "17": 0.15714, - "18": 0.16051, - "19": 0.23287, - "20": 0.17277, - "21": 0.18416, - "22": 0.18504, - "23": 0.22706, - "24": 0.17428, - "25": 0.15714, - "26": 0.24051, - "27": 0.16163, - "28": 0.15307, - "29": 0.15547, - "30": 0.15066, - "31": 0.18968, - "32": 0.20133, - "33": 0.15407, - "34": 0.15375, - "35": 0.22411, - "36": 0.1654, - "37": 0.23902, - "38": 0.15259, - "39": 0.15371, - "40": 0.15185, - "41": 0.21089, - "42": 0.15272, - "43": 0.21496, - "44": 0.15539, - "45": 0.15507, - "46": 0.1557, - "47": 0.15641, - "48": 0.15434, - "49": 0.15017, - "50": 0.23326, - "51": 0.17863, - "52": 0.15471, - "53": 0.1511, - "54": 0.1513, - "55": 0.14791, - "56": 0.23169, - "57": 0.15152, - "58": 0.27611, - "59": 0.15101, - "60": 0.15075, - "61": 0.15095, - "62": 0.15099, - "63": 0.40681, - "64": 0.15196, - "65": 0.4085, - "66": 0.15392, - "67": 0.15079, - "68": 0.18374, - "69": 0.16595, - "70": 0.17343, - "71": 0.2083, - "72": 0.23324, - "73": 0.17579, - "74": 0.2442, - "75": 0.15263, - "76": 0.15001, - "77": 0.14836, - "78": 0.22649, - "79": 0.15368, - "80": 0.15125, - "81": 0.15382, - "82": 0.15532, - "83": 0.1536, - "84": 0.15494, - "85": 0.1516, - "86": 0.2253, - "87": 0.1656, - "88": 0.16481, - "89": 0.16686, - "90": 0.19956, - "91": 0.15647, - "92": 0.15231, - "93": 0.15013, - "94": 0.22716, - "95": 0.15151, - "96": 0.15158, - "97": 0.21549, - "98": 0.15054, - "99": 0.16863, - "100": 0.15247 + "1": "nan", + "2": 2.33774, + "3": 0.14705, + "4": 0.13192, + "5": 0.13017, + "6": 0.1292, + "7": 0.13024, + "8": 0.13032, + "9": 0.12858, + "10": 0.12948, + "11": 0.12888, + "12": 0.12888, + "13": 0.12811, + "14": 0.12943, + "15": 0.12948, + "16": 0.1295, + "17": 0.13022, + "18": 0.12847, + "19": 0.12992, + "20": 0.1308, + "21": 0.12844, + "22": 0.13063, + "23": 0.13033, + "24": 0.13003, + "25": 0.12935, + "26": 0.13016, + "27": 0.12989, + "28": 0.12947, + "29": 0.12857, + "30": 0.12949, + "31": 0.12997, + "32": 0.12843, + "33": 0.1291, + "34": 0.12894, + "35": 0.13061, + "36": 0.12974, + "37": 0.12939, + "38": 0.13039, + "39": 0.13034, + "40": 0.13069, + "41": 0.13259, + "42": 0.13109, + "43": 0.13211, + "44": 0.1299, + "45": 0.1295, + "46": 0.13001, + "47": 0.13037, + "48": 0.13043, + "49": 0.13012, + "50": 0.12915, + "51": 0.14665, + "52": 0.12869, + "53": 0.12717, + "54": 0.12709, + "55": 0.12611, + "56": 0.12645, + "57": 0.12711, + "58": 0.12728, + "59": 0.1269, + "60": 0.12701, + "61": 0.1281, + "62": 0.12781, + "63": 0.12842, + "64": 0.12745, + "65": 0.12897, + "66": 0.12786, + "67": 0.12983, + "68": 0.13068, + "69": 0.1284, + "70": 0.12896, + "71": 0.1288, + "72": 0.13026, + "73": 0.13011, + "74": 0.12891, + "75": 0.12798, + "76": 0.12866, + "77": 0.12994, + "78": 0.12957, + "79": 0.12765, + "80": 0.12884, + "81": 0.12898, + "82": 0.12927, + "83": 0.12848, + "84": 0.12845, + "85": 0.12849, + "86": 0.12983, + "87": 0.1303, + "88": 0.12961, + "89": 0.13093, + "90": 0.12951, + "91": 0.12818, + "92": 0.12902, + "93": 0.12967, + "94": 0.13419, + "95": 0.14029, + "96": 0.1429, + "97": 0.14018, + "98": 0.13632, + "99": 0.14128, + "100": 0.14034 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..dd354f801de --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.90259, + "52": 9.79281, + "53": 10.11536, + "54": 9.99216, + "55": 9.91665, + "56": 9.66015, + "57": 9.52038, + "58": 9.87094, + "59": 9.6209, + "60": 9.54952, + "61": 9.70012, + "62": 10.00629, + "63": 9.42168, + "64": 9.79893, + "65": 8.97548, + "66": 9.73165, + "67": 9.38933, + "68": 9.80066, + "69": 9.81152, + "70": 9.76761, + "71": 9.63356, + "72": 9.59892, + "73": 9.51708, + "74": 8.96512, + "75": 9.43589, + "76": 9.11207, + "77": 10.06881, + "78": 9.72515, + "79": 9.39985, + "80": 9.41154, + "81": 9.50094, + "82": 9.69861, + "83": 9.33578, + "84": 9.4341, + "85": 9.63907, + "86": 9.06166, + "87": 9.60563, + "88": 9.77626, + "89": 9.6243, + "90": 9.82766, + "91": 9.35869, + "92": 9.38066, + "93": 9.09681, + "94": 8.83995, + "95": 9.52751, + "96": 9.53562, + "97": 9.32689, + "98": 9.69354, + "99": 8.88933, + "100": 9.42104 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 22791436.0, + "52": 22748292.0, + "53": 22924772.0, + "54": 22840284.0, + "55": 22517880.0, + "56": 22877730.0, + "57": 23113080.0, + "58": 22845568.0, + "59": 22716022.0, + "60": 22743056.0, + "61": 22724434.0, + "62": 22672316.0, + "63": 22846416.0, + "64": 22823178.0, + "65": 23061654.0, + "66": 22729712.0, + "67": 22908434.0, + "68": 22610444.0, + "69": 22584604.0, + "70": 22828526.0, + "71": 22748442.0, + "72": 22655052.0, + "73": 22740588.0, + "74": 23048316.0, + "75": 23054664.0, + "76": 22901072.0, + "77": 22272198.0, + "78": 22789244.0, + "79": 22743700.0, + "80": 22706576.0, + "81": 22890704.0, + "82": 22778282.0, + "83": 22840256.0, + "84": 23010368.0, + "85": 22711796.0, + "86": 23103236.0, + "87": 22735120.0, + "88": 22636998.0, + "89": 22498612.0, + "90": 22972652.0, + "91": 22767776.0, + "92": 22809424.0, + "93": 22658980.0, + "94": 22911920.0, + "95": 23047890.0, + "96": 22828804.0, + "97": 22608196.0, + "98": 22762820.0, + "99": 22906714.0, + "100": 23016048.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 746444288.0, + "52": 746444288.0, + "53": 746444288.0, + "54": 746444288.0, + "55": 746444288.0, + "56": 746444288.0, + "57": 746444288.0, + "58": 746444288.0, + "59": 746444288.0, + "60": 746444288.0, + "61": 746444288.0, + "62": 746444288.0, + "63": 746444288.0, + "64": 746444288.0, + "65": 746444288.0, + "66": 746444288.0, + "67": 746444288.0, + "68": 746444288.0, + "69": 746444288.0, + "70": 746444288.0, + "71": 746444288.0, + "72": 746444288.0, + "73": 746444288.0, + "74": 746444288.0, + "75": 746444288.0, + "76": 746444288.0, + "77": 746444288.0, + "78": 746444288.0, + "79": 746444288.0, + "80": 746444288.0, + "81": 746444288.0, + "82": 746444288.0, + "83": 746444288.0, + "84": 746444288.0, + "85": 746444288.0, + "86": 746444288.0, + "87": 746444288.0, + "88": 746444288.0, + "89": 746444288.0, + "90": 746444288.0, + "91": 746444288.0, + "92": 746444288.0, + "93": 746444288.0, + "94": 746444288.0, + "95": 746444288.0, + "96": 746444288.0, + "97": 746444288.0, + "98": 746444288.0, + "99": 746444288.0, + "100": 746444288.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2223731712.0, + "52": 2223732736.0, + "53": 2223732736.0, + "54": 2223732736.0, + "55": 2223732736.0, + "56": 2223732736.0, + "57": 2223732736.0, + "58": 2223732736.0, + "59": 2223732736.0, + "60": 2223732736.0, + "61": 2223732736.0, + "62": 2223732736.0, + "63": 2223732736.0, + "64": 2223732736.0, + "65": 2223732736.0, + "66": 2223732736.0, + "67": 2223732736.0, + "68": 2223732736.0, + "69": 2223732736.0, + "70": 2223732736.0, + "71": 2223732736.0, + "72": 2223732736.0, + "73": 2223732736.0, + "74": 2223732736.0, + "75": 2223732736.0, + "76": 2223732736.0, + "77": 2223732736.0, + "78": 2223732736.0, + "79": 2223732736.0, + "80": 2223732736.0, + "81": 2223732736.0, + "82": 2223732736.0, + "83": 2223732736.0, + "84": 2223732736.0, + "85": 2223732736.0, + "86": 2223732736.0, + "87": 2223732736.0, + "88": 2223732736.0, + "89": 2223732736.0, + "90": 2223732736.0, + "91": 2223732736.0, + "92": 2223732736.0, + "93": 2223732736.0, + "94": 2223732736.0, + "95": 2223732736.0, + "96": 2223732736.0, + "97": 2223732736.0, + "98": 2223732736.0, + "99": 2223732736.0, + "100": 2223732736.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.28424, + "53": 0.15724, + "54": 0.14436, + "55": 0.14133, + "56": 0.14939, + "57": 0.15152, + "58": 0.16555, + "59": 0.19478, + "60": 0.13288, + "61": 0.13086, + "62": 0.13088, + "63": 0.13074, + "64": 0.1303, + "65": 0.13189, + "66": 0.13138, + "67": 0.12968, + "68": 0.13118, + "69": 0.13064, + "70": 0.12931, + "71": 0.12915, + "72": 0.12915, + "73": 0.13375, + "74": 0.13641, + "75": 0.13586, + "76": 0.13551, + "77": 0.13604, + "78": 0.13931, + "79": 0.13798, + "80": 0.13724, + "81": 0.13702, + "82": 0.13663, + "83": 0.1357, + "84": 0.13618, + "85": 0.13577, + "86": 0.13569, + "87": 0.13635, + "88": 0.13659, + "89": 0.13724, + "90": 0.13599, + "91": 0.13637, + "92": 0.13565, + "93": 0.13693, + "94": 0.13576, + "95": 0.13566, + "96": 0.13579, + "97": 0.13592, + "98": 0.13631, + "99": 0.13476, + "100": 0.13606 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_gb200.json index 06040458828..2610b7fe2f4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 9.05034, - "2": 0.14876, - "3": 0.14285, - "4": 0.13033, - "5": 0.24651, - "6": 0.19893, - "7": 0.15924, - "8": 0.11963, - "9": 0.12767, - "10": 0.24283, - "11": 0.12856, - "12": 0.13101, - "13": 0.5056, - "14": 0.1222, - "15": 0.23869, - "16": 0.13294, - "17": 0.13193, - "18": 0.14163, - "19": 0.13647, - "20": 0.2257, - "21": 0.13437, - "22": 0.24393, - "23": 0.13446, - "24": 0.23274, - "25": 0.14725, - "26": 0.13804, - "27": 0.14255, - "28": 0.14086, - "29": 0.23437, - "30": 0.25225, - "31": 0.13433, - "32": 0.25099, - "33": 0.14422, - "34": 0.20638, - "35": 0.13575, - "36": 0.13592, - "37": 0.14521, - "38": 0.9985, - "39": 0.14828, - "40": 0.13964, - "41": 0.13609, - "42": 0.33948, - "43": 0.13414, - "44": 0.27111, - "45": 0.14576, - "46": 0.13882, - "47": 0.13432, - "48": 0.14571, - "49": 0.14535, - "50": 0.4444 + "1": "nan", + "2": 2.94258, + "3": 0.12978, + "4": 0.11688, + "5": 0.11937, + "6": 0.12093, + "7": 0.12307, + "8": 0.13062, + "9": 0.12926, + "10": 0.1228, + "11": 0.12859, + "12": 0.12404, + "13": 0.12912, + "14": 0.12318, + "15": 0.12609, + "16": 0.13327, + "17": 0.12859, + "18": 0.12957, + "19": 0.12658, + "20": 0.12929, + "21": 0.12937, + "22": 0.1298, + "23": 0.12888, + "24": 0.12917, + "25": 0.1285, + "26": 0.12864, + "27": 0.13061, + "28": 0.1272, + "29": 0.12953, + "30": 0.12693, + "31": 0.13141, + "32": 0.12786, + "33": 0.12815, + "34": 0.12937, + "35": 0.12957, + "36": 0.12737, + "37": 0.1313, + "38": 0.12977, + "39": 0.12805, + "40": 0.1298, + "41": 0.1296, + "42": 0.13074, + "43": 0.12955, + "44": 0.13171, + "45": 0.13055, + "46": 0.13271, + "47": 0.13004, + "48": 0.12873, + "49": 0.13129, + "50": 0.12858 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json index 110646cd819..10988c85257 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 554160640.0, - "2": 555209216.0, - "3": 554160640.0, - "4": 554160640.0, - "5": 554160640.0, - "6": 554160640.0, - "7": 554160640.0, - "8": 554160640.0, - "9": 555209216.0, - "10": 554160640.0, - "11": 554160640.0, - "12": 554160640.0, - "13": 554160640.0, - "14": 554160640.0, - "15": 554160640.0, - "16": 554160640.0, - "17": 554160640.0, - "18": 554160640.0, - "19": 554160640.0, - "20": 554160640.0, - "21": 554160640.0, - "22": 554160640.0, - "23": 554160640.0, - "24": 554160640.0, - "25": 554160640.0, - "26": 554160640.0, - "27": 554160640.0, - "28": 554160640.0, - "29": 554160640.0, - "30": 554160640.0, - "31": 554160640.0, - "32": 554160640.0, - "33": 554160640.0, - "34": 554160640.0, - "35": 554160640.0, - "36": 554160640.0, - "37": 554160640.0, - "38": 554160640.0, - "39": 554160640.0, - "40": 554160640.0, - "41": 554160640.0, - "42": 555209216.0, - "43": 554160640.0, - "44": 554160640.0, - "45": 554160640.0, - "46": 554160640.0, - "47": 554160640.0, - "48": 554160640.0, - "49": 554160640.0, - "50": 554160640.0 + "1": 552193536.0, + "2": 552193536.0, + "3": 553242112.0, + "4": 553242112.0, + "5": 552193536.0, + "6": 553242112.0, + "7": 553242112.0, + "8": 553242112.0, + "9": 553242112.0, + "10": 553242112.0, + "11": 553242112.0, + "12": 552193536.0, + "13": 552193536.0, + "14": 552193536.0, + "15": 552193536.0, + "16": 553242112.0, + "17": 553242112.0, + "18": 552193536.0, + "19": 553242112.0, + "20": 553242112.0, + "21": 553242112.0, + "22": 552193536.0, + "23": 553242112.0, + "24": 553242112.0, + "25": 553242112.0, + "26": 553242112.0, + "27": 553242112.0, + "28": 553242112.0, + "29": 553242112.0, + "30": 553242112.0, + "31": 552193536.0, + "32": 552193536.0, + "33": 553242112.0, + "34": 553242112.0, + "35": 552193536.0, + "36": 553242112.0, + "37": 552193536.0, + "38": 552193536.0, + "39": 552193536.0, + "40": 552193536.0, + "41": 552193536.0, + "42": 552193536.0, + "43": 552193536.0, + "44": 552193536.0, + "45": 552193536.0, + "46": 552193536.0, + "47": 552193536.0, + "48": 552193536.0, + "49": 552193536.0, + "50": 553242112.0 } }, "mem-max-allocated-bytes": { @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 3798208000.0, - "2": 3944053248.0, - "3": 3944053248.0, - "4": 3944053248.0, - "5": 3944053248.0, - "6": 3944053248.0, - "7": 3944053248.0, - "8": 3944053248.0, - "9": 3944053248.0, - "10": 3944053248.0, - "11": 3944053248.0, - "12": 3944053248.0, - "13": 3944053248.0, - "14": 3944053248.0, - "15": 3944053248.0, - "16": 3944053248.0, - "17": 3944053248.0, - "18": 3944053248.0, - "19": 3944053248.0, - "20": 3944053248.0, - "21": 3944053248.0, - "22": 3944053248.0, - "23": 3944053248.0, - "24": 3944053248.0, - "25": 3944053248.0, - "26": 3944053248.0, - "27": 3944053248.0, - "28": 3944053248.0, - "29": 3944053248.0, - "30": 3944053248.0, - "31": 3944053248.0, - "32": 3944053248.0, - "33": 3944053248.0, - "34": 3944053248.0, - "35": 3944053248.0, - "36": 3944053248.0, - "37": 3944053248.0, - "38": 3944053248.0, - "39": 3944053248.0, - "40": 3944053248.0, - "41": 3944053248.0, - "42": 3944053248.0, - "43": 3944053248.0, - "44": 3944053248.0, - "45": 3944053248.0, - "46": 3944053248.0, - "47": 3944053248.0, - "48": 3944053248.0, - "49": 3944053248.0, - "50": 3944053248.0 + "2": 3942086144.0, + "3": 3942086144.0, + "4": 3942086144.0, + "5": 3942086144.0, + "6": 3942086144.0, + "7": 3942086144.0, + "8": 3942086144.0, + "9": 3942086144.0, + "10": 3942086144.0, + "11": 3942086144.0, + "12": 3942086144.0, + "13": 3942086144.0, + "14": 3942086144.0, + "15": 3942086144.0, + "16": 3942086144.0, + "17": 3942086144.0, + "18": 3942086144.0, + "19": 3942086144.0, + "20": 3942086144.0, + "21": 3942086144.0, + "22": 3942086144.0, + "23": 3942086144.0, + "24": 3942086144.0, + "25": 3942086144.0, + "26": 3942086144.0, + "27": 3942086144.0, + "28": 3942086144.0, + "29": 3942086144.0, + "30": 3942086144.0, + "31": 3942086144.0, + "32": 3942086144.0, + "33": 3942086144.0, + "34": 3942086144.0, + "35": 3942086144.0, + "36": 3942086144.0, + "37": 3942086144.0, + "38": 3942086144.0, + "39": 3942086144.0, + "40": 3942086144.0, + "41": 3942086144.0, + "42": 3942086144.0, + "43": 3942086144.0, + "44": 3942086144.0, + "45": 3942086144.0, + "46": 3942086144.0, + "47": 3942086144.0, + "48": 3942086144.0, + "49": 3942086144.0, + "50": 3942086144.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 7.76857, - "2": 0.14325, - "3": 0.13072, - "4": 0.11885, - "5": 0.11896, - "6": 0.1181, - "7": 0.11917, - "8": 0.11807, - "9": 0.11852, - "10": 0.11869, - "11": 0.21274, - "12": 0.11744, - "13": 0.11909, - "14": 0.12072, - "15": 0.11937, - "16": 0.11875, - "17": 0.11813, - "18": 0.117, - "19": 0.11808, - "20": 0.1185, - "21": 0.21315, - "22": 0.11941, - "23": 0.11829, - "24": 0.12018, - "25": 0.11873, - "26": 0.12277, - "27": 0.11624, - "28": 0.11801, - "29": 0.11768, - "30": 0.11811, - "31": 0.21259, - "32": 0.11823, - "33": 0.11857, - "34": 0.11893, - "35": 0.12121, - "36": 0.11984, - "37": 0.12002, - "38": 0.11889, - "39": 0.12151, - "40": 0.11884, - "41": 0.21346, - "42": 0.11706, - "43": 0.12099, - "44": 0.1203, - "45": 0.11997, - "46": 0.12288, - "47": 0.12077, - "48": 0.11925, - "49": 0.11743, - "50": 0.11695 + "1": "nan", + "2": 3.84171, + "3": 0.13294, + "4": 0.11994, + "5": 0.11682, + "6": 0.11799, + "7": 0.12021, + "8": 0.11949, + "9": 0.1195, + "10": 0.12086, + "11": 0.21563, + "12": 0.12013, + "13": 0.1204, + "14": 0.1188, + "15": 0.1192, + "16": 0.11917, + "17": 0.11999, + "18": 0.12006, + "19": 0.11965, + "20": 0.12016, + "21": 0.21525, + "22": 0.11978, + "23": 0.12009, + "24": 0.12004, + "25": 0.12129, + "26": 0.12041, + "27": 0.12075, + "28": 0.12015, + "29": 0.1204, + "30": 0.12048, + "31": 0.21709, + "32": 0.12108, + "33": 0.11972, + "34": 0.12, + "35": 0.11969, + "36": 0.11944, + "37": 0.11946, + "38": 0.12056, + "39": 0.12045, + "40": 0.12052, + "41": 0.21777, + "42": 0.12063, + "43": 0.12165, + "44": 0.1204, + "45": 0.12036, + "46": 0.12154, + "47": 0.12043, + "48": 0.12145, + "49": 0.12079, + "50": 0.12035 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json index 641a00e237a..a34edb3389a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 9.86945, - "2": 0.13101, - "3": 0.70357, - "4": 0.14163, - "5": 0.12855, - "6": 0.38046, - "7": 0.126, - "8": 0.14086, - "9": 0.23777, - "10": 0.1402, - "11": 0.41122, - "12": 0.27395, - "13": 0.10789, - "14": 0.24924, - "15": 0.33411, - "16": 0.24471, - "17": 0.10663, - "18": 0.24551, - "19": 0.10581, - "20": 0.18449, - "21": 0.22744, - "22": 0.10616, - "23": 0.34738, - "24": 0.11037, - "25": 0.11093, - "26": 0.43374, - "27": 0.1067, - "28": 0.10671, - "29": 0.1061, - "30": 0.22031, - "31": 0.11271, - "32": 0.10683, - "33": 0.10556, - "34": 0.25465, - "35": 0.22935, - "36": 0.1072, - "37": 0.10789, - "38": 0.1067, - "39": 0.21523, - "40": 0.1053, - "41": 0.11778, - "42": 0.22642, - "43": 0.10673, - "44": 0.23278, - "45": 0.1046, - "46": 0.22439, - "47": 0.22232, - "48": 0.10912, - "49": 0.10674, - "50": 0.1055, - "51": 0.11049, - "52": 0.1948, - "53": 0.1045, - "54": 0.24019, - "55": 0.10505, - "56": 0.23176, - "57": 0.10745, - "58": 0.10668, - "59": 0.10741, - "60": 0.37464, - "61": 0.10467, - "62": 0.10857, - "63": 0.10767, - "64": 0.10998, - "65": 0.10888, - "66": 0.17063, - "67": 0.36721, - "68": 0.10834, - "69": 0.10693, - "70": 0.24024, - "71": 0.10802, - "72": 0.10696, - "73": 0.10736, - "74": 0.10874, - "75": 0.15339, - "76": 0.18985, - "77": 0.32078, - "78": 0.1062, - "79": 0.29068, - "80": 0.10837, - "81": 0.17251, - "82": 0.10428, - "83": 0.21093, - "84": 0.13349, - "85": 0.23049, - "86": 0.10991, - "87": 0.10573, - "88": 0.10661, - "89": 0.10792, - "90": 0.22654, - "91": 0.31392, - "92": 0.10844, - "93": 0.24022, - "94": 0.111, - "95": 0.10539, - "96": 0.109, - "97": 0.11025, - "98": 0.11065, - "99": 0.44653, - "100": 0.10883 + "1": "nan", + "2": 2.73603, + "3": 0.12344, + "4": 0.10783, + "5": 0.10595, + "6": 0.10649, + "7": 0.10691, + "8": 0.10679, + "9": 0.10607, + "10": 0.10675, + "11": 0.10687, + "12": 0.10636, + "13": 0.10663, + "14": 0.10668, + "15": 0.10696, + "16": 0.10672, + "17": 0.10678, + "18": 0.10603, + "19": 0.10659, + "20": 0.10684, + "21": 0.10766, + "22": 0.10849, + "23": 0.10853, + "24": 0.10805, + "25": 0.10776, + "26": 0.1069, + "27": 0.10818, + "28": 0.10669, + "29": 0.10643, + "30": 0.10634, + "31": 0.10766, + "32": 0.1076, + "33": 0.10583, + "34": 0.10631, + "35": 0.10587, + "36": 0.1054, + "37": 0.10589, + "38": 0.10633, + "39": 0.10593, + "40": 0.10674, + "41": 0.10812, + "42": 0.11127, + "43": 0.11494, + "44": 0.11409, + "45": 0.11538, + "46": 0.11702, + "47": 0.1155, + "48": 0.11481, + "49": 0.11507, + "50": 0.11401, + "51": 0.11655, + "52": 0.11513, + "53": 0.11379, + "54": 0.11378, + "55": 0.11658, + "56": 0.11792, + "57": 0.11792, + "58": 0.11715, + "59": 0.11915, + "60": 0.11642, + "61": 0.11578, + "62": 0.1171, + "63": 0.11758, + "64": 0.11517, + "65": 0.11624, + "66": 0.11434, + "67": 0.11609, + "68": 0.11506, + "69": 0.11568, + "70": 0.11661, + "71": 0.11647, + "72": 0.1166, + "73": 0.11795, + "74": 0.11661, + "75": 0.11785, + "76": 0.11659, + "77": 0.11531, + "78": 0.11705, + "79": 0.11662, + "80": 0.11765, + "81": 0.11829, + "82": 0.11742, + "83": 0.11529, + "84": 0.11678, + "85": 0.11581, + "86": 0.11703, + "87": 0.11699, + "88": 0.11641, + "89": 0.11638, + "90": 0.11586, + "91": 0.11853, + "92": 0.11725, + "93": 0.1178, + "94": 0.11647, + "95": 0.11672, + "96": 0.11702, + "97": 0.11754, + "98": 0.11614, + "99": 0.11757, + "100": 0.11708 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json index 56bb24659d2..0758fd3a8cf 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 9.98291, - "2": 0.12743, - "3": 0.38059, - "4": 0.12841, - "5": 0.14511, - "6": 0.10173, - "7": 0.26971, - "8": 0.10382, - "9": 0.3953, - "10": 0.1058, - "11": 0.10231, - "12": 0.509, - "13": 0.10327, - "14": 0.23202, - "15": 0.12684, - "16": 0.10412, - "17": 0.2441, - "18": 0.10687, - "19": 0.25963, - "20": 0.10498, - "21": 0.25469, - "22": 0.10453, - "23": 0.10222, - "24": 0.25281, - "25": 0.1172, - "26": 0.10348, - "27": 0.29437, - "28": 0.10187, - "29": 0.28228, - "30": 0.1021, - "31": 0.23112, - "32": 0.10338, - "33": 0.24896, - "34": 0.10339, - "35": 0.24587, - "36": 0.22187, - "37": 0.10494, - "38": 0.10356, - "39": 0.10387, - "40": 0.1047, - "41": 0.10726, - "42": 0.10304, - "43": 0.22521, - "44": 0.12908, - "45": 0.21396, - "46": 0.32037, - "47": 0.10321, - "48": 0.10612, - "49": 0.46303, - "50": 0.10477, - "51": 0.11648, - "52": 0.10312, - "53": 0.10274, - "54": 0.10625, - "55": 0.10219, - "56": 0.24603, - "57": 0.10299, - "58": 0.10437, - "59": 0.10386, - "60": 0.10294, - "61": 0.26442, - "62": 0.10245, - "63": 0.17569, - "64": 0.10337, - "65": 0.23811, - "66": 0.10233, - "67": 0.23691, - "68": 0.21983, - "69": 0.19586, - "70": 0.10467, - "71": 0.10454, - "72": 0.1059, - "73": 0.10652, - "74": 0.14966, - "75": 0.10278, - "76": 0.39764, - "77": 0.10176, - "78": 0.23756, - "79": 0.10342, - "80": 0.24469, - "81": 0.10295, - "82": 0.26649, - "83": 0.105, - "84": 0.47883, - "85": 0.10596, - "86": 0.10525, - "87": 0.22714, - "88": 0.10536, - "89": 0.10595, - "90": 0.22588, - "91": 0.10237, - "92": 0.2621, - "93": 0.10543, - "94": 0.21938, - "95": 0.10276, - "96": 0.17373, - "97": 0.10501, - "98": 0.22197, - "99": 0.10635, - "100": 0.1032 + "1": "nan", + "2": 3.0922, + "3": 0.12788, + "4": 0.11451, + "5": 0.11407, + "6": 0.11304, + "7": 0.11565, + "8": 0.11482, + "9": 0.11841, + "10": 0.11916, + "11": 0.11884, + "12": 0.11911, + "13": 0.1155, + "14": 0.12253, + "15": 0.11369, + "16": 0.11887, + "17": 0.11433, + "18": 0.12243, + "19": 0.11544, + "20": 0.11344, + "21": 0.1254, + "22": 0.11712, + "23": 0.12494, + "24": 0.12239, + "25": 0.12344, + "26": 0.11952, + "27": 0.12117, + "28": 0.11916, + "29": 0.11974, + "30": 0.11517, + "31": 0.1219, + "32": 0.12112, + "33": 0.11997, + "34": 0.1133, + "35": 0.12245, + "36": 0.12118, + "37": 0.11239, + "38": 0.12174, + "39": 0.11964, + "40": 0.11993, + "41": 0.12013, + "42": 0.12614, + "43": 0.11697, + "44": 0.11669, + "45": 0.11781, + "46": 0.11776, + "47": 0.11182, + "48": 0.1196, + "49": 0.11814, + "50": 0.11736, + "51": 0.12093, + "52": 0.1107, + "53": 0.11502, + "54": 0.11571, + "55": 0.11493, + "56": 0.11712, + "57": 0.11663, + "58": 0.11203, + "59": 0.11604, + "60": 0.11649, + "61": 0.11616, + "62": 0.11641, + "63": 0.11603, + "64": 0.11613, + "65": 0.11708, + "66": 0.11292, + "67": 0.11356, + "68": 0.11416, + "69": 0.11305, + "70": 0.11582, + "71": 0.11552, + "72": 0.11318, + "73": 0.11798, + "74": 0.11632, + "75": 0.11624, + "76": 0.11602, + "77": 0.11547, + "78": 0.11457, + "79": 0.11402, + "80": 0.11415, + "81": 0.11627, + "82": 0.11295, + "83": 0.11397, + "84": 0.11221, + "85": 0.11326, + "86": 0.11792, + "87": 0.11391, + "88": 0.11365, + "89": 0.11478, + "90": 0.11346, + "91": 0.11213, + "92": 0.11712, + "93": 0.11574, + "94": 0.11724, + "95": 0.11254, + "96": 0.11871, + "97": 0.11957, + "98": 0.11759, + "99": 0.11864, + "100": 0.11833 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..f7efd011023 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.8426, + "52": 9.72578, + "53": 10.05977, + "54": 9.95226, + "55": 9.88321, + "56": 9.61276, + "57": 9.46222, + "58": 9.82313, + "59": 9.57665, + "60": 9.48518, + "61": 9.6788, + "62": 9.97777, + "63": 9.36212, + "64": 9.75714, + "65": 8.93499, + "66": 9.69281, + "67": 9.36709, + "68": 9.78179, + "69": 9.79451, + "70": 9.72295, + "71": 9.62027, + "72": 9.56974, + "73": 9.481, + "74": 8.91241, + "75": 9.40906, + "76": 9.06623, + "77": 10.05808, + "78": 9.72188, + "79": 9.36927, + "80": 9.40027, + "81": 9.47702, + "82": 9.69788, + "83": 9.30742, + "84": 9.41496, + "85": 9.61115, + "86": 9.07104, + "87": 9.59609, + "88": 9.74908, + "89": 9.5961, + "90": 9.82722, + "91": 9.3366, + "92": 9.3558, + "93": 9.08695, + "94": 8.82752, + "95": 9.53066, + "96": 9.52759, + "97": 9.30671, + "98": 9.66909, + "99": 8.89637, + "100": 9.4052 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2587.0, + "52": 2574.0, + "53": 2831.0, + "54": 2602.0, + "55": 2403.0, + "56": 2822.0, + "57": 2223.0, + "58": 2954.0, + "59": 2871.0, + "60": 2518.0, + "61": 2922.0, + "62": 2677.0, + "63": 2533.0, + "64": 3023.0, + "65": 2609.0, + "66": 2960.0, + "67": 2867.0, + "68": 2652.0, + "69": 3053.0, + "70": 3011.0, + "71": 2870.0, + "72": 2460.0, + "73": 3114.0, + "74": 2017.0, + "75": 2527.0, + "76": 2954.0, + "77": 2955.0, + "78": 3055.0, + "79": 3098.0, + "80": 3047.0, + "81": 3362.0, + "82": 3296.0, + "83": 2825.0, + "84": 3113.0, + "85": 3196.0, + "86": 2666.0, + "87": 3583.0, + "88": 2985.0, + "89": 3259.0, + "90": 3220.0, + "91": 2781.0, + "92": 3090.0, + "93": 2686.0, + "94": 3474.0, + "95": 3147.0, + "96": 3418.0, + "97": 3036.0, + "98": 3411.0, + "99": 3152.0, + "100": 3098.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 763221504.0, + "52": 763221504.0, + "53": 763221504.0, + "54": 763221504.0, + "55": 763221504.0, + "56": 763221504.0, + "57": 763221504.0, + "58": 763221504.0, + "59": 763221504.0, + "60": 763221504.0, + "61": 763221504.0, + "62": 763221504.0, + "63": 763221504.0, + "64": 763221504.0, + "65": 763221504.0, + "66": 763221504.0, + "67": 763221504.0, + "68": 763221504.0, + "69": 763221504.0, + "70": 763221504.0, + "71": 763221504.0, + "72": 763221504.0, + "73": 763221504.0, + "74": 763221504.0, + "75": 763221504.0, + "76": 763221504.0, + "77": 763221504.0, + "78": 763221504.0, + "79": 763221504.0, + "80": 763221504.0, + "81": 763221504.0, + "82": 763221504.0, + "83": 763221504.0, + "84": 763221504.0, + "85": 763221504.0, + "86": 763221504.0, + "87": 763221504.0, + "88": 763221504.0, + "89": 763221504.0, + "90": 763221504.0, + "91": 763221504.0, + "92": 763221504.0, + "93": 763221504.0, + "94": 763221504.0, + "95": 763221504.0, + "96": 763221504.0, + "97": 763221504.0, + "98": 763221504.0, + "99": 763221504.0, + "100": 763221504.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2682096640.0, + "52": 2682097664.0, + "53": 2682097664.0, + "54": 2682097664.0, + "55": 2682097664.0, + "56": 2682097664.0, + "57": 2682097664.0, + "58": 2682097664.0, + "59": 2682097664.0, + "60": 2682097664.0, + "61": 2682097664.0, + "62": 2682097664.0, + "63": 2682097664.0, + "64": 2682097664.0, + "65": 2682097664.0, + "66": 2682097664.0, + "67": 2682097664.0, + "68": 2682097664.0, + "69": 2682097664.0, + "70": 2682097664.0, + "71": 2682097664.0, + "72": 2682097664.0, + "73": 2682097664.0, + "74": 2682097664.0, + "75": 2682097664.0, + "76": 2682097664.0, + "77": 2682097664.0, + "78": 2682097664.0, + "79": 2682097664.0, + "80": 2682097664.0, + "81": 2682097664.0, + "82": 2682097664.0, + "83": 2682097664.0, + "84": 2682097664.0, + "85": 2682097664.0, + "86": 2682097664.0, + "87": 2682097664.0, + "88": 2682097664.0, + "89": 2682097664.0, + "90": 2682097664.0, + "91": 2682097664.0, + "92": 2682097664.0, + "93": 2682097664.0, + "94": 2682097664.0, + "95": 2682097664.0, + "96": 2682097664.0, + "97": 2682097664.0, + "98": 2682097664.0, + "99": 2682097664.0, + "100": 2682097664.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 3.06236, + "53": 0.13884, + "54": 0.12077, + "55": 0.12587, + "56": 0.12115, + "57": 0.12166, + "58": 0.12353, + "59": 0.1247, + "60": 0.12221, + "61": 0.12159, + "62": 0.12136, + "63": 0.13043, + "64": 0.12973, + "65": 0.13067, + "66": 0.14918, + "67": 0.11954, + "68": 0.11631, + "69": 0.11511, + "70": 0.11621, + "71": 0.11553, + "72": 0.11537, + "73": 0.11691, + "74": 0.11875, + "75": 0.11769, + "76": 0.11586, + "77": 0.11847, + "78": 0.11896, + "79": 0.11697, + "80": 0.11854, + "81": 0.11758, + "82": 0.11531, + "83": 0.11776, + "84": 0.11613, + "85": 0.11822, + "86": 0.11858, + "87": 0.11763, + "88": 0.11691, + "89": 0.11931, + "90": 0.11678, + "91": 0.11601, + "92": 0.11377, + "93": 0.11692, + "94": 0.11741, + "95": 0.11634, + "96": 0.1145, + "97": 0.12011, + "98": 0.11722, + "99": 0.11609, + "100": 0.11641 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json index 3d38faf23fc..28843c12217 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json @@ -262,10 +262,10 @@ "42": 552325632.0, "43": 552325632.0, "44": 552325632.0, - "45": 552325632.0, + "45": 553374208.0, "46": 552325632.0, "47": 552325632.0, - "48": 552325632.0, + "48": 553374208.0, "49": 552325632.0, "50": 552325632.0, "51": 552325632.0, @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.20216, - "2": 0.13277, - "3": 0.2975, - "4": 0.10754, - "5": 0.10418, - "6": 0.10612, - "7": 0.24272, - "8": 0.11347, - "9": 0.14845, - "10": 0.10733, - "11": 0.10387, - "12": 0.47615, - "13": 0.26992, - "14": 0.10483, - "15": 0.1204, - "16": 0.10696, - "17": 0.10552, - "18": 0.10521, - "19": 0.24005, - "20": 0.17139, - "21": 0.13276, - "22": 0.21348, - "23": 0.10526, - "24": 0.23652, - "25": 0.10695, - "26": 0.105, - "27": 0.1046, - "28": 0.108, - "29": 0.22645, - "30": 0.10764, - "31": 0.37801, - "32": 0.10822, - "33": 0.26043, - "34": 0.10725, - "35": 0.10759, - "36": 0.10627, - "37": 0.10521, - "38": 0.23173, - "39": 0.23132, - "40": 0.10561, - "41": 0.10865, - "42": 0.10488, - "43": 0.10774, - "44": 0.10716, - "45": 0.2275, - "46": 0.10501, - "47": 0.26542, - "48": 0.10561, - "49": 0.10565, - "50": 0.21987, - "51": 0.12154, - "52": 0.10569, - "53": 0.10443, - "54": 0.1047, - "55": 0.10628, - "56": 0.106, - "57": 0.21826, - "58": 0.29942, - "59": 0.10627, - "60": 0.10754, - "61": 0.10422, - "62": 0.10591, - "63": 0.22208, - "64": 0.10704, - "65": 0.10754, - "66": 0.11693, - "67": 0.10619, - "68": 0.10599, - "69": 0.1064, - "70": 0.10712, - "71": 0.20506, - "72": 0.12154, - "73": 0.10701, - "74": 0.10797, - "75": 0.10599, - "76": 0.11118, - "77": 0.22203, - "78": 0.11082, - "79": 0.10971, - "80": 0.10673, - "81": 0.23373, - "82": 0.25241, - "83": 0.10924, - "84": 0.23617, - "85": 0.10907, - "86": 0.10895, - "87": 0.21649, - "88": 0.1977, - "89": 0.1081, - "90": 0.10767, - "91": 0.2306, - "92": 0.1072, - "93": 0.11204, - "94": 0.22079, - "95": 0.10723, - "96": 0.10789, - "97": 0.10605, - "98": 0.10621, - "99": 0.26274, - "100": 0.10674 + "1": "nan", + "2": 3.33855, + "3": 0.12562, + "4": 0.10973, + "5": 0.10864, + "6": 0.10778, + "7": 0.10885, + "8": 0.10884, + "9": 0.10877, + "10": 0.10868, + "11": 0.10997, + "12": 0.10853, + "13": 0.1086, + "14": 0.10927, + "15": 0.10879, + "16": 0.10908, + "17": 0.10873, + "18": 0.10883, + "19": 0.11028, + "20": 0.11031, + "21": 0.11086, + "22": 0.10971, + "23": 0.10987, + "24": 0.1089, + "25": 0.11118, + "26": 0.10952, + "27": 0.1165, + "28": 0.11961, + "29": 0.11977, + "30": 0.11657, + "31": 0.11728, + "32": 0.11689, + "33": 0.11642, + "34": 0.11739, + "35": 0.11665, + "36": 0.11537, + "37": 0.11552, + "38": 0.11544, + "39": 0.11538, + "40": 0.11584, + "41": 0.11597, + "42": 0.11635, + "43": 0.11593, + "44": 0.11678, + "45": 0.11608, + "46": 0.11637, + "47": 0.11572, + "48": 0.11577, + "49": 0.11481, + "50": 0.11561, + "51": 0.1213, + "52": 0.10892, + "53": 0.10742, + "54": 0.10842, + "55": 0.10806, + "56": 0.10869, + "57": 0.11057, + "58": 0.108, + "59": 0.10875, + "60": 0.10969, + "61": 0.1087, + "62": 0.10795, + "63": 0.1094, + "64": 0.10922, + "65": 0.11102, + "66": 0.11016, + "67": 0.10977, + "68": 0.10988, + "69": 0.11029, + "70": 0.11078, + "71": 0.11019, + "72": 0.11727, + "73": 0.11024, + "74": 0.11054, + "75": 0.10949, + "76": 0.11384, + "77": 0.11011, + "78": 0.1101, + "79": 0.10943, + "80": 0.11059, + "81": 0.11173, + "82": 0.10987, + "83": 0.1094, + "84": 0.10956, + "85": 0.11029, + "86": 0.11179, + "87": 0.10953, + "88": 0.11045, + "89": 0.1102, + "90": 0.10897, + "91": 0.11022, + "92": 0.10965, + "93": 0.11042, + "94": 0.11158, + "95": 0.11059, + "96": 0.11046, + "97": 0.11123, + "98": 0.11055, + "99": 0.11178, + "100": 0.11266 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..43ec9ec960f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.84256, + "52": 9.72573, + "53": 10.05974, + "54": 9.95226, + "55": 9.88318, + "56": 9.61275, + "57": 9.46219, + "58": 9.8231, + "59": 9.57666, + "60": 9.48516, + "61": 9.67876, + "62": 9.97782, + "63": 9.36212, + "64": 9.75714, + "65": 8.93494, + "66": 9.69283, + "67": 9.36708, + "68": 9.78178, + "69": 9.79452, + "70": 9.72296, + "71": 9.62031, + "72": 9.56974, + "73": 9.48101, + "74": 8.91241, + "75": 9.40905, + "76": 9.06617, + "77": 10.05809, + "78": 9.72194, + "79": 9.36927, + "80": 9.40029, + "81": 9.47702, + "82": 9.69787, + "83": 9.30742, + "84": 9.41492, + "85": 9.61113, + "86": 9.07103, + "87": 9.5961, + "88": 9.74909, + "89": 9.59604, + "90": 9.82722, + "91": 9.33657, + "92": 9.35582, + "93": 9.08689, + "94": 8.82754, + "95": 9.53065, + "96": 9.5276, + "97": 9.30672, + "98": 9.66905, + "99": 8.89635, + "100": 9.40525 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2552.0, + "52": 2565.0, + "53": 2883.0, + "54": 2710.0, + "55": 2301.0, + "56": 2798.0, + "57": 2334.0, + "58": 2979.0, + "59": 2960.0, + "60": 2451.0, + "61": 2841.0, + "62": 2577.0, + "63": 2516.0, + "64": 2907.0, + "65": 2567.0, + "66": 2862.0, + "67": 2809.0, + "68": 2609.0, + "69": 2965.0, + "70": 2985.0, + "71": 2864.0, + "72": 2613.0, + "73": 3108.0, + "74": 2048.0, + "75": 2563.0, + "76": 3046.0, + "77": 3127.0, + "78": 2959.0, + "79": 3082.0, + "80": 3025.0, + "81": 3400.0, + "82": 3223.0, + "83": 2786.0, + "84": 3180.0, + "85": 3233.0, + "86": 2611.0, + "87": 3542.0, + "88": 3084.0, + "89": 3210.0, + "90": 3271.0, + "91": 2770.0, + "92": 3220.0, + "93": 2662.0, + "94": 3405.0, + "95": 3085.0, + "96": 3336.0, + "97": 3050.0, + "98": 3421.0, + "99": 3271.0, + "100": 3079.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 552064512.0, + "52": 552064512.0, + "53": 552064512.0, + "54": 552064512.0, + "55": 552064512.0, + "56": 552064512.0, + "57": 552064512.0, + "58": 552064512.0, + "59": 552064512.0, + "60": 552064512.0, + "61": 552064512.0, + "62": 552064512.0, + "63": 552064512.0, + "64": 552064512.0, + "65": 552064512.0, + "66": 552064512.0, + "67": 552064512.0, + "68": 552064512.0, + "69": 552064512.0, + "70": 552064512.0, + "71": 552064512.0, + "72": 552064512.0, + "73": 552064512.0, + "74": 552064512.0, + "75": 552064512.0, + "76": 552064512.0, + "77": 552064512.0, + "78": 552064512.0, + "79": 552064512.0, + "80": 552064512.0, + "81": 552064512.0, + "82": 552064512.0, + "83": 552064512.0, + "84": 552064512.0, + "85": 552064512.0, + "86": 552064512.0, + "87": 552064512.0, + "88": 552064512.0, + "89": 552064512.0, + "90": 552064512.0, + "91": 552064512.0, + "92": 552064512.0, + "93": 552064512.0, + "94": 552064512.0, + "95": 552064512.0, + "96": 552064512.0, + "97": 552064512.0, + "98": 552064512.0, + "99": 552064512.0, + "100": 552064512.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2470941696.0, + "52": 2470942208.0, + "53": 2470942208.0, + "54": 2470942208.0, + "55": 2470942208.0, + "56": 2470942208.0, + "57": 2470942208.0, + "58": 2470942208.0, + "59": 2470942208.0, + "60": 2470942208.0, + "61": 2470942720.0, + "62": 2470942720.0, + "63": 2470942720.0, + "64": 2470942720.0, + "65": 2470942720.0, + "66": 2470942720.0, + "67": 2470942720.0, + "68": 2470942720.0, + "69": 2470942720.0, + "70": 2470942720.0, + "71": 2470942720.0, + "72": 2470942720.0, + "73": 2470942720.0, + "74": 2470942720.0, + "75": 2470942720.0, + "76": 2470942720.0, + "77": 2470942720.0, + "78": 2470942720.0, + "79": 2470942720.0, + "80": 2470942720.0, + "81": 2470942720.0, + "82": 2470942720.0, + "83": 2470942720.0, + "84": 2470942720.0, + "85": 2470942720.0, + "86": 2470942720.0, + "87": 2470942720.0, + "88": 2470942720.0, + "89": 2470942720.0, + "90": 2470942720.0, + "91": 2470942720.0, + "92": 2470942720.0, + "93": 2470942720.0, + "94": 2470942720.0, + "95": 2470942720.0, + "96": 2470942720.0, + "97": 2470942720.0, + "98": 2470942720.0, + "99": 2470942720.0, + "100": 2470942720.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 3.27475, + "53": 0.12363, + "54": 0.11177, + "55": 0.1112, + "56": 0.11272, + "57": 0.11284, + "58": 0.11161, + "59": 0.11167, + "60": 0.11262, + "61": 0.11171, + "62": 0.11092, + "63": 0.11143, + "64": 0.11171, + "65": 0.11299, + "66": 0.1124, + "67": 0.1119, + "68": 0.11174, + "69": 0.11252, + "70": 0.11217, + "71": 0.1112, + "72": 0.11653, + "73": 0.11887, + "74": 0.11966, + "75": 0.11921, + "76": 0.12192, + "77": 0.1219, + "78": 0.12342, + "79": 0.12312, + "80": 0.12263, + "81": 0.12762, + "82": 0.1234, + "83": 0.12364, + "84": 0.12458, + "85": 0.12385, + "86": 0.12395, + "87": 0.12307, + "88": 0.12362, + "89": 0.12421, + "90": 0.12452, + "91": 0.12623, + "92": 0.1253, + "93": 0.12482, + "94": 0.12453, + "95": 0.12892, + "96": 0.13902, + "97": 0.12489, + "98": 0.12331, + "99": 0.12522, + "100": 0.12499 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgx_gb200.json index f4999e7c2dd..8a90b6fb7df 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 9.62286, - "2": 0.48262, - "3": 0.20639, - "4": 0.31026, - "5": 0.31827, - "6": 0.29163, - "7": 0.29838, - "8": 0.17787, - "9": 0.27978, - "10": 0.17026, - "11": 0.27026, - "12": 0.1834, - "13": 0.19697, - "14": 0.43123, - "15": 0.18322, - "16": 0.18141, - "17": 0.19707, - "18": 0.4629, - "19": 0.1817, - "20": 0.25096, - "21": 0.18877, - "22": 0.24459, - "23": 0.17984, - "24": 0.20058, - "25": 0.1758, - "26": 0.17872, - "27": 0.17193, - "28": 0.17115, - "29": 0.36031, - "30": 0.2658, - "31": 0.16933, - "32": 0.20868, - "33": 0.17195, - "34": 0.17439, - "35": 0.2501, - "36": 0.17686, - "37": 0.20398, - "38": 0.32448, - "39": 0.1735, - "40": 0.17268, - "41": 0.33455, - "42": 0.23584, - "43": 0.23483, - "44": 0.16767, - "45": 0.17612, - "46": 0.30477, - "47": 0.37075, - "48": 0.18367, - "49": 0.25006, - "50": 0.56439 + "1": "nan", + "2": 3.1265, + "3": 0.15779, + "4": 0.14192, + "5": 0.14446, + "6": 0.14251, + "7": 0.14375, + "8": 0.1446, + "9": 0.14351, + "10": 0.14568, + "11": 0.14477, + "12": 0.14491, + "13": 0.1447, + "14": 0.14656, + "15": 0.14652, + "16": 0.14521, + "17": 0.14638, + "18": 0.14483, + "19": 0.14549, + "20": 0.14457, + "21": 0.14306, + "22": 0.14559, + "23": 0.14596, + "24": 0.14513, + "25": 0.14367, + "26": 0.14368, + "27": 0.14398, + "28": 0.14369, + "29": 0.14435, + "30": 0.14415, + "31": 0.1433, + "32": 0.14342, + "33": 0.1441, + "34": 0.14372, + "35": 0.14431, + "36": 0.1454, + "37": 0.14634, + "38": 0.14514, + "39": 0.14529, + "40": 0.14504, + "41": 0.14496, + "42": 0.14436, + "43": 0.14492, + "44": 0.14452, + "45": 0.14629, + "46": 0.14514, + "47": 0.14578, + "48": 0.1442, + "49": 0.14396, + "50": 0.14376 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json index 605457b437c..da72109d85d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 8.46203, - "2": 0.17159, - "3": 0.30409, - "4": 0.13684, - "5": 0.29184, - "6": 0.13641, - "7": 0.15548, - "8": 0.24827, - "9": 0.13458, - "10": 0.24758, - "11": 0.26919, - "12": 0.15859, - "13": 0.24263, - "14": 0.40638, - "15": 0.14802, - "16": 0.75916, - "17": 0.27027, - "18": 0.41589, - "19": 0.23222, - "20": 0.27356, - "21": 0.38604, - "22": 0.40542, - "23": 0.61332, - "24": 0.36261, - "25": 0.60934, - "26": 0.13901, - "27": 0.23646, - "28": 0.13727, - "29": 0.23988, - "30": 0.13874, - "31": 0.13771, - "32": 0.13771, - "33": 0.13803, - "34": 0.13667, - "35": 0.13906, - "36": 0.13535, - "37": 0.13539, - "38": 0.13547, - "39": 0.13555, - "40": 0.13617, - "41": 0.37768, - "42": 0.1374, - "43": 0.22178, - "44": 0.13712, - "45": 0.13831, - "46": 0.137, - "47": 0.13638, - "48": 0.13731, - "49": 0.21987, - "50": 0.13794 + "1": "nan", + "2": 2.44061, + "3": 0.15375, + "4": 0.14111, + "5": 0.14053, + "6": 0.14023, + "7": 0.14152, + "8": 0.14128, + "9": 0.1417, + "10": 0.14155, + "11": 0.14076, + "12": 0.1405, + "13": 0.14129, + "14": 0.14106, + "15": 0.14101, + "16": 0.14178, + "17": 0.14173, + "18": 0.14103, + "19": 0.14094, + "20": 0.14012, + "21": 0.14153, + "22": 0.14228, + "23": 0.14118, + "24": 0.14079, + "25": 0.14034, + "26": 0.14027, + "27": 0.13947, + "28": 0.13928, + "29": 0.1398, + "30": 0.14085, + "31": 0.14179, + "32": 0.13944, + "33": 0.14174, + "34": 0.1436, + "35": 0.13902, + "36": 0.13933, + "37": 0.13922, + "38": 0.13997, + "39": 0.13881, + "40": 0.13924, + "41": 0.1392, + "42": 0.14092, + "43": 0.14136, + "44": 0.14035, + "45": 0.13841, + "46": 0.1411, + "47": 0.13878, + "48": 0.14005, + "49": 0.13925, + "50": 0.13845 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json index d3f4ebb9b68..28de0d56b1b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 8.84804, - "2": 0.09127, - "3": 0.38568, - "4": 0.10516, - "5": 0.18187, - "6": 0.13288, - "7": 0.17979, - "8": 0.25055, - "9": 0.07376, - "10": 0.06396, - "11": 0.42421, - "12": 0.06524, - "13": 0.06447, - "14": 0.06499, - "15": 0.24593, - "16": 0.06277, - "17": 0.2443, - "18": 0.26141, - "19": 0.06388, + "1": "nan", + "2": 2.91983, + "3": 0.07964, + "4": 0.0755, + "5": 0.06521, + "6": 0.07949, + "7": 0.0691, + "8": 0.06527, + "9": 0.09221, + "10": 0.0948, + "11": 0.07486, + "12": 0.06312, + "13": 0.06422, + "14": 0.0656, + "15": 0.07274, + "16": 0.06384, + "17": 0.06441, + "18": 0.06446, + "19": 0.06349, "20": 0.06319, - "21": 0.44504, - "22": 0.06309, - "23": 0.24094, - "24": 0.06366, - "25": 0.12615, - "26": 0.45347, - "27": 0.06454, - "28": 0.06518, - "29": 0.23896, - "30": 0.06569, - "31": 0.23519, - "32": 0.06271, - "33": 0.06599, - "34": 0.45696, - "35": 0.06614, - "36": 0.24275, - "37": 0.0626, - "38": 0.18028, - "39": 0.07237, - "40": 0.24435, - "41": 0.09656, - "42": 0.258, - "43": 0.09133, - "44": 0.09694, - "45": 0.11452, - "46": 0.08793, - "47": 0.24321, - "48": 0.08548, - "49": 0.0909, - "50": 0.16493 + "21": 0.06302, + "22": 0.06467, + "23": 0.06428, + "24": 0.06395, + "25": 0.06411, + "26": 0.06398, + "27": 0.06336, + "28": 0.06345, + "29": 0.07201, + "30": 0.06458, + "31": 0.06379, + "32": 0.06337, + "33": 0.06262, + "34": 0.06257, + "35": 0.06407, + "36": 0.06308, + "37": 0.06324, + "38": 0.06353, + "39": 0.06346, + "40": 0.06294, + "41": 0.06471, + "42": 0.06426, + "43": 0.06446, + "44": 0.06426, + "45": 0.06337, + "46": 0.06427, + "47": 0.06421, + "48": 0.06315, + "49": 0.0639, + "50": 0.06324 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json index 2de96fdc0a6..a5fecfacf8f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.87037, "2": 10.87119, - "3": 10.84722, - "4": 10.83185, - "5": 10.86876, - "6": 10.88753, - "7": 10.86095, - "8": 10.86864, - "9": 10.85424, - "10": 10.82319, - "11": 10.86739, - "12": 10.8666, - "13": 10.88538, - "14": 10.88994, - "15": 10.81366, - "16": 10.80332, + "3": 10.84723, + "4": 10.83181, + "5": 10.86879, + "6": 10.8876, + "7": 10.86094, + "8": 10.86859, + "9": 10.85425, + "10": 10.82318, + "11": 10.86741, + "12": 10.86662, + "13": 10.88537, + "14": 10.88991, + "15": 10.81369, + "16": 10.80328, "17": 10.77723, - "18": 10.81063, - "19": 10.80524, - "20": 10.70339, - "21": 10.67012, - "22": 10.51209, - "23": 10.69985, + "18": 10.81064, + "19": 10.80526, + "20": 10.7034, + "21": 10.67013, + "22": 10.51206, + "23": 10.69987, "24": 10.56044, - "25": 10.49857, - "26": 10.57872, - "27": 10.56749, - "28": 10.53108, + "25": 10.49854, + "26": 10.57876, + "27": 10.56747, + "28": 10.53107, "29": 10.55838, - "30": 10.32727, - "31": 10.04391, - "32": 10.42571, - "33": 10.4193, - "34": 10.15675, + "30": 10.32726, + "31": 10.04382, + "32": 10.42576, + "33": 10.41931, + "34": 10.15673, "35": 10.21897, "36": 10.16206, - "37": 10.29722, - "38": 10.13231, - "39": 10.35956, - "40": 10.02296, + "37": 10.29717, + "38": 10.1323, + "39": 10.35955, + "40": 10.02292, "41": 10.06592, - "42": 10.15518, - "43": 9.75609, - "44": 9.86983, - "45": 9.75094, - "46": 9.73598, - "47": 10.0747, - "48": 9.77504, - "49": 9.43418, - "50": 9.84339, - "51": 9.78577, - "52": 9.6708, - "53": 10.00723, + "42": 10.15514, + "43": 9.7561, + "44": 9.86986, + "45": 9.75091, + "46": 9.73604, + "47": 10.07473, + "48": 9.77502, + "49": 9.43421, + "50": 9.84343, + "51": 9.78575, + "52": 9.67077, + "53": 10.00722, "54": 9.89701, - "55": 9.82612, + "55": 9.82613, "56": 9.54829, - "57": 9.40077, - "58": 9.77422, + "57": 9.40075, + "58": 9.77419, "59": 9.51686, - "60": 9.42721, - "61": 9.63408, - "62": 9.93879, + "60": 9.42722, + "61": 9.63404, + "62": 9.93883, "63": 9.30503, - "64": 9.71266, - "65": 8.86836, - "66": 9.64474, - "67": 9.31349, - "68": 9.73443, + "64": 9.71265, + "65": 8.86835, + "66": 9.64476, + "67": 9.31344, + "68": 9.73448, "69": 9.755, - "70": 9.68613, - "71": 9.57703, - "72": 9.53066, - "73": 9.43092, - "74": 8.8548, - "75": 9.35819, - "76": 9.01448, - "77": 10.0265, - "78": 9.68108, - "79": 9.33349, - "80": 9.35488, - "81": 9.44135, + "70": 9.68616, + "71": 9.57699, + "72": 9.53063, + "73": 9.43094, + "74": 8.85481, + "75": 9.35821, + "76": 9.01443, + "77": 10.02645, + "78": 9.6811, + "79": 9.33347, + "80": 9.35483, + "81": 9.44132, "82": 9.66188, - "83": 9.26313, - "84": 9.37185, + "83": 9.26309, + "84": 9.37181, "85": 9.57429, - "86": 9.03444, + "86": 9.0344, "87": 9.56188, - "88": 9.71281, - "89": 9.55802, + "88": 9.71279, + "89": 9.55801, "90": 9.79197, - "91": 9.29019, - "92": 9.31615, - "93": 9.04052, - "94": 8.78281, + "91": 9.29017, + "92": 9.31612, + "93": 9.04053, + "94": 8.78283, "95": 9.49395, - "96": 9.48884, - "97": 9.26046, - "98": 9.63128, - "99": 8.85093, - "100": 9.36489 + "96": 9.48877, + "97": 9.26048, + "98": 9.63126, + "99": 8.85095, + "100": 9.36493 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 623.0, - "2": 605.0, - "3": 617.0, - "4": 657.0, - "5": 652.0, - "6": 662.0, - "7": 595.0, - "8": 626.0, - "9": 683.0, - "10": 550.0, - "11": 658.0, - "12": 657.0, - "13": 701.0, - "14": 668.0, - "15": 659.0, - "16": 692.0, - "17": 640.0, - "18": 627.0, - "19": 633.0, - "20": 601.0, - "21": 632.0, - "22": 637.0, - "23": 718.0, - "24": 623.0, - "25": 612.0, - "26": 689.0, - "27": 678.0, - "28": 717.0, - "29": 715.0, - "30": 670.0, - "31": 627.0, - "32": 718.0, - "33": 850.0, - "34": 658.0, - "35": 721.0, - "36": 764.0, - "37": 859.0, - "38": 733.0, - "39": 851.0, - "40": 766.0, - "41": 863.0, - "42": 839.0, - "43": 732.0, - "44": 870.0, - "45": 737.0, - "46": 913.0, - "47": 911.0, - "48": 832.0, - "49": 825.0, - "50": 827.0, - "51": 914.0, - "52": 900.0, - "53": 989.0, - "54": 1021.0, - "55": 874.0, - "56": 985.0, - "57": 841.0, - "58": 938.0, - "59": 1035.0, - "60": 876.0, - "61": 1044.0, - "62": 982.0, - "63": 976.0, - "64": 1071.0, - "65": 1026.0, - "66": 994.0, - "67": 961.0, - "68": 1084.0, - "69": 1108.0, - "70": 1081.0, - "71": 1069.0, - "72": 931.0, - "73": 984.0, - "74": 770.0, - "75": 914.0, - "76": 1050.0, - "77": 1196.0, - "78": 1128.0, - "79": 1048.0, - "80": 1147.0, - "81": 1175.0, - "82": 1112.0, - "83": 988.0, - "84": 1099.0, - "85": 1133.0, - "86": 875.0, - "87": 1189.0, - "88": 1114.0, - "89": 1101.0, - "90": 1124.0, - "91": 1079.0, - "92": 1114.0, - "93": 937.0, - "94": 1106.0, - "95": 1097.0, + "1": 617.0, + "2": 610.0, + "3": 593.0, + "4": 664.0, + "5": 648.0, + "6": 669.0, + "7": 605.0, + "8": 612.0, + "9": 596.0, + "10": 559.0, + "11": 659.0, + "12": 590.0, + "13": 663.0, + "14": 675.0, + "15": 672.0, + "16": 715.0, + "17": 627.0, + "18": 614.0, + "19": 680.0, + "20": 570.0, + "21": 674.0, + "22": 593.0, + "23": 633.0, + "24": 612.0, + "25": 606.0, + "26": 669.0, + "27": 625.0, + "28": 753.0, + "29": 733.0, + "30": 661.0, + "31": 648.0, + "32": 688.0, + "33": 786.0, + "34": 689.0, + "35": 675.0, + "36": 734.0, + "37": 807.0, + "38": 799.0, + "39": 831.0, + "40": 745.0, + "41": 780.0, + "42": 868.0, + "43": 713.0, + "44": 751.0, + "45": 817.0, + "46": 856.0, + "47": 934.0, + "48": 906.0, + "49": 840.0, + "50": 799.0, + "51": 923.0, + "52": 897.0, + "53": 1019.0, + "54": 908.0, + "55": 839.0, + "56": 976.0, + "57": 853.0, + "58": 1024.0, + "59": 988.0, + "60": 870.0, + "61": 1041.0, + "62": 961.0, + "63": 847.0, + "64": 1053.0, + "65": 1004.0, + "66": 1005.0, + "67": 938.0, + "68": 1006.0, + "69": 1110.0, + "70": 985.0, + "71": 1002.0, + "72": 958.0, + "73": 997.0, + "74": 705.0, + "75": 870.0, + "76": 1088.0, + "77": 1153.0, + "78": 1083.0, + "79": 1035.0, + "80": 1122.0, + "81": 1209.0, + "82": 1132.0, + "83": 1024.0, + "84": 1122.0, + "85": 1141.0, + "86": 861.0, + "87": 1190.0, + "88": 1115.0, + "89": 1128.0, + "90": 1107.0, + "91": 1128.0, + "92": 1160.0, + "93": 973.0, + "94": 1117.0, + "95": 1022.0, "96": 1178.0, - "97": 1103.0, - "98": 1260.0, - "99": 1105.0, - "100": 1131.0 + "97": 1068.0, + "98": 1278.0, + "99": 1071.0, + "100": 1175.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 637976064.0, - "2": 637976064.0, - "3": 637976064.0, - "4": 637976064.0, - "5": 637976064.0, - "6": 637976064.0, - "7": 637976064.0, - "8": 637976064.0, - "9": 637976064.0, - "10": 637976064.0, - "11": 637976064.0, - "12": 637976064.0, - "13": 637976064.0, - "14": 637976064.0, - "15": 637976064.0, - "16": 637976064.0, - "17": 637976064.0, - "18": 637976064.0, - "19": 637976064.0, - "20": 637976064.0, - "21": 637976064.0, - "22": 637976064.0, - "23": 637976064.0, - "24": 637976064.0, - "25": 637976064.0, - "26": 637976064.0, - "27": 637976064.0, - "28": 637976064.0, - "29": 637976064.0, - "30": 637976064.0, - "31": 637976064.0, - "32": 637976064.0, - "33": 637976064.0, - "34": 637976064.0, - "35": 637976064.0, - "36": 637976064.0, - "37": 637976064.0, - "38": 637976064.0, - "39": 637976064.0, - "40": 637976064.0, - "41": 637976064.0, - "42": 637976064.0, - "43": 637976064.0, - "44": 637976064.0, - "45": 637976064.0, - "46": 637976064.0, - "47": 637976064.0, - "48": 637976064.0, - "49": 637976064.0, - "50": 637976064.0, - "51": 637976064.0, - "52": 637976064.0, - "53": 637976064.0, - "54": 637976064.0, - "55": 637976064.0, - "56": 637976064.0, - "57": 637976064.0, - "58": 637976064.0, - "59": 637976064.0, - "60": 637976064.0, - "61": 637976064.0, - "62": 637976064.0, - "63": 637976064.0, - "64": 637976064.0, - "65": 637976064.0, - "66": 637976064.0, - "67": 637976064.0, - "68": 637976064.0, - "69": 637976064.0, - "70": 637976064.0, - "71": 637976064.0, - "72": 637976064.0, - "73": 637976064.0, - "74": 637976064.0, - "75": 637976064.0, - "76": 637976064.0, - "77": 637976064.0, - "78": 637976064.0, - "79": 637976064.0, - "80": 637976064.0, - "81": 637976064.0, - "82": 637976064.0, - "83": 637976064.0, - "84": 637976064.0, - "85": 637976064.0, - "86": 637976064.0, - "87": 637976064.0, - "88": 637976064.0, - "89": 637976064.0, - "90": 637976064.0, - "91": 637976064.0, - "92": 637976064.0, - "93": 637976064.0, - "94": 637976064.0, - "95": 637976064.0, - "96": 637976064.0, - "97": 637976064.0, - "98": 637976064.0, - "99": 637976064.0, - "100": 637976064.0 + "1": 638631424.0, + "2": 638631424.0, + "3": 638631424.0, + "4": 638631424.0, + "5": 638631424.0, + "6": 638631424.0, + "7": 638631424.0, + "8": 638631424.0, + "9": 638631424.0, + "10": 638631424.0, + "11": 638631424.0, + "12": 638631424.0, + "13": 638631424.0, + "14": 638631424.0, + "15": 638631424.0, + "16": 638631424.0, + "17": 638631424.0, + "18": 638631424.0, + "19": 638631424.0, + "20": 638631424.0, + "21": 638631424.0, + "22": 638631424.0, + "23": 638631424.0, + "24": 638631424.0, + "25": 638631424.0, + "26": 638631424.0, + "27": 638631424.0, + "28": 638631424.0, + "29": 638631424.0, + "30": 638631424.0, + "31": 638631424.0, + "32": 638631424.0, + "33": 638631424.0, + "34": 638631424.0, + "35": 638631424.0, + "36": 638631424.0, + "37": 638631424.0, + "38": 638631424.0, + "39": 638631424.0, + "40": 638631424.0, + "41": 638631424.0, + "42": 638631424.0, + "43": 638631424.0, + "44": 638631424.0, + "45": 638631424.0, + "46": 638631424.0, + "47": 638631424.0, + "48": 638631424.0, + "49": 638631424.0, + "50": 638631424.0, + "51": 638631424.0, + "52": 638631424.0, + "53": 638631424.0, + "54": 638631424.0, + "55": 638631424.0, + "56": 638631424.0, + "57": 638631424.0, + "58": 638631424.0, + "59": 638631424.0, + "60": 638631424.0, + "61": 638631424.0, + "62": 638631424.0, + "63": 638631424.0, + "64": 638631424.0, + "65": 638631424.0, + "66": 638631424.0, + "67": 638631424.0, + "68": 638631424.0, + "69": 638631424.0, + "70": 638631424.0, + "71": 638631424.0, + "72": 638631424.0, + "73": 638631424.0, + "74": 638631424.0, + "75": 638631424.0, + "76": 638631424.0, + "77": 638631424.0, + "78": 638631424.0, + "79": 638631424.0, + "80": 638631424.0, + "81": 638631424.0, + "82": 638631424.0, + "83": 638631424.0, + "84": 638631424.0, + "85": 638631424.0, + "86": 638631424.0, + "87": 638631424.0, + "88": 638631424.0, + "89": 638631424.0, + "90": 638631424.0, + "91": 638631424.0, + "92": 638631424.0, + "93": 638631424.0, + "94": 638631424.0, + "95": 638631424.0, + "96": 638631424.0, + "97": 638631424.0, + "98": 638631424.0, + "99": 638631424.0, + "100": 638631424.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 908535808.0, - "2": 1167747584.0, - "3": 1168796160.0, - "4": 1168796160.0, - "5": 1179675136.0, - "6": 1179675136.0, - "7": 1179675136.0, - "8": 1179675136.0, - "9": 1179675136.0, - "10": 1179675136.0, - "11": 1179675136.0, - "12": 1179675136.0, - "13": 1179675136.0, - "14": 1179675136.0, - "15": 1179675136.0, - "16": 1179675136.0, - "17": 1180330496.0, - "18": 1180330496.0, - "19": 1180330496.0, - "20": 1180330496.0, - "21": 1180330496.0, - "22": 1180330496.0, - "23": 1180330496.0, - "24": 1180330496.0, - "25": 1180330496.0, - "26": 1180330496.0, - "27": 1180330496.0, - "28": 1180330496.0, - "29": 1180330496.0, - "30": 1180330496.0, - "31": 1180330496.0, - "32": 1180330496.0, - "33": 1180330496.0, - "34": 1180330496.0, - "35": 1180330496.0, - "36": 1180330496.0, - "37": 1180330496.0, - "38": 1180330496.0, - "39": 1180330496.0, - "40": 1180330496.0, - "41": 1180330496.0, - "42": 1180330496.0, - "43": 1180330496.0, - "44": 1180330496.0, - "45": 1180330496.0, - "46": 1180330496.0, - "47": 1180330496.0, - "48": 1180330496.0, - "49": 1180330496.0, - "50": 1180330496.0, - "51": 1180330496.0, - "52": 1180330496.0, - "53": 1180330496.0, - "54": 1180330496.0, - "55": 1180330496.0, - "56": 1180330496.0, - "57": 1180330496.0, - "58": 1180330496.0, - "59": 1180330496.0, - "60": 1180330496.0, - "61": 1180330496.0, - "62": 1180330496.0, - "63": 1180330496.0, - "64": 1180330496.0, - "65": 1180330496.0, - "66": 1180330496.0, - "67": 1180330496.0, - "68": 1180330496.0, - "69": 1180330496.0, - "70": 1180330496.0, - "71": 1180330496.0, - "72": 1180330496.0, - "73": 1180330496.0, - "74": 1180330496.0, - "75": 1180330496.0, - "76": 1180330496.0, - "77": 1180330496.0, - "78": 1180330496.0, - "79": 1180330496.0, - "80": 1180330496.0, - "81": 1180330496.0, - "82": 1180330496.0, - "83": 1180330496.0, - "84": 1180330496.0, - "85": 1180330496.0, - "86": 1180330496.0, - "87": 1180330496.0, - "88": 1180330496.0, - "89": 1180330496.0, - "90": 1180330496.0, - "91": 1180330496.0, - "92": 1180330496.0, - "93": 1180330496.0, - "94": 1180330496.0, - "95": 1180330496.0, - "96": 1180330496.0, - "97": 1180330496.0, - "98": 1180330496.0, - "99": 1180330496.0, - "100": 1180330496.0 + "1": 910633472.0, + "2": 1170499584.0, + "3": 1170499584.0, + "4": 1170499584.0, + "5": 1170499584.0, + "6": 1170499584.0, + "7": 1170500096.0, + "8": 1170500096.0, + "9": 1170500096.0, + "10": 1170500096.0, + "11": 1170761728.0, + "12": 1170761728.0, + "13": 1170761728.0, + "14": 1173644800.0, + "15": 1173644800.0, + "16": 1178888192.0, + "17": 1178888192.0, + "18": 1178888192.0, + "19": 1178888192.0, + "20": 1178888192.0, + "21": 1178888192.0, + "22": 1178888192.0, + "23": 1178888192.0, + "24": 1178888192.0, + "25": 1178888192.0, + "26": 1178888192.0, + "27": 1178888192.0, + "28": 1178888192.0, + "29": 1178888192.0, + "30": 1178888192.0, + "31": 1178888192.0, + "32": 1178888192.0, + "33": 1178888192.0, + "34": 1178888192.0, + "35": 1178888192.0, + "36": 1178888192.0, + "37": 1179936768.0, + "38": 1179936768.0, + "39": 1179936768.0, + "40": 1179936768.0, + "41": 1179936768.0, + "42": 1179936768.0, + "43": 1179936768.0, + "44": 1179936768.0, + "45": 1179936768.0, + "46": 1179936768.0, + "47": 1179936768.0, + "48": 1179936768.0, + "49": 1179936768.0, + "50": 1179936768.0, + "51": 1179936768.0, + "52": 1179936768.0, + "53": 1179936768.0, + "54": 1179936768.0, + "55": 1179936768.0, + "56": 1179936768.0, + "57": 1179936768.0, + "58": 1179936768.0, + "59": 1179936768.0, + "60": 1179936768.0, + "61": 1179936768.0, + "62": 1179936768.0, + "63": 1179936768.0, + "64": 1179936768.0, + "65": 1179936768.0, + "66": 1179936768.0, + "67": 1179936768.0, + "68": 1179936768.0, + "69": 1179936768.0, + "70": 1179936768.0, + "71": 1179936768.0, + "72": 1179936768.0, + "73": 1179936768.0, + "74": 1179936768.0, + "75": 1179936768.0, + "76": 1179936768.0, + "77": 1179936768.0, + "78": 1179936768.0, + "79": 1179936768.0, + "80": 1179936768.0, + "81": 1179936768.0, + "82": 1179936768.0, + "83": 1179936768.0, + "84": 1179936768.0, + "85": 1179936768.0, + "86": 1179936768.0, + "87": 1179936768.0, + "88": 1179936768.0, + "89": 1179936768.0, + "90": 1179936768.0, + "91": 1179936768.0, + "92": 1179936768.0, + "93": 1179936768.0, + "94": 1179936768.0, + "95": 1179936768.0, + "96": 1179936768.0, + "97": 1179936768.0, + "98": 1179936768.0, + "99": 1180984832.0, + "100": 1180984832.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 9.2723, - "2": 0.3877, - "3": 0.37645, - "4": 0.36551, - "5": 0.37045, - "6": 0.36893, - "7": 0.36938, - "8": 0.36753, - "9": 0.36888, - "10": 0.43135, - "11": 0.36252, - "12": 0.37084, - "13": 0.365, - "14": 0.36033, - "15": 0.35887, - "16": 0.36201, - "17": 0.53142, - "18": 0.36699, - "19": 0.36318, - "20": 0.36321, - "21": 0.36209, - "22": 0.72283, - "23": 0.3641, - "24": 0.36359, - "25": 0.36227, - "26": 0.36731, - "27": 0.36879, - "28": 0.36963, - "29": 0.37051, - "30": 0.36794, - "31": 0.37079, - "32": 0.368, - "33": 0.44096, - "34": 0.52072, - "35": 0.48704, - "36": 0.4152, - "37": 0.37792, - "38": 0.37304, - "39": 0.37505, - "40": 0.37438, - "41": 0.3737, - "42": 0.37569, - "43": 0.37181, - "44": 0.37336, - "45": 0.3731, - "46": 0.37229, - "47": 0.37374, - "48": 0.37375, - "49": 0.3719, - "50": 0.37298, - "51": 0.3797, - "52": 0.36304, - "53": 0.36729, - "54": 0.36756, - "55": 0.37134, - "56": 0.37139, - "57": 0.37112, - "58": 0.38383, - "59": 0.3916, - "60": 0.37403, - "61": 0.37341, - "62": 0.37078, - "63": 0.37095, - "64": 0.37149, - "65": 0.37269, - "66": 0.3736, - "67": 0.37255, - "68": 0.36695, - "69": 0.37351, - "70": 0.37443, - "71": 0.3726, - "72": 0.3731, - "73": 0.37353, - "74": 0.3737, - "75": 0.373, - "76": 0.36094, - "77": 0.36374, - "78": 0.36366, - "79": 0.36446, - "80": 0.36414, - "81": 0.36245, - "82": 0.3641, - "83": 0.3627, - "84": 0.36487, - "85": 0.36027, - "86": 0.3602, - "87": 0.3611, - "88": 0.36555, - "89": 0.36571, - "90": 0.36479, - "91": 0.36175, - "92": 0.36215, - "93": 0.36421, - "94": 0.36147, - "95": 0.36348, - "96": 0.36311, - "97": 0.36282, - "98": 0.38328, - "99": 0.40994, - "100": 0.36791 + "1": "nan", + "2": 2.94981, + "3": 0.44844, + "4": 0.45032, + "5": 0.45007, + "6": 0.45413, + "7": 0.45595, + "8": 0.46147, + "9": 0.45703, + "10": 0.45713, + "11": 0.45843, + "12": 0.46481, + "13": 0.46367, + "14": 0.46145, + "15": 0.46144, + "16": 0.45944, + "17": 0.46053, + "18": 0.46234, + "19": 0.46318, + "20": 0.46597, + "21": 0.46872, + "22": 0.46167, + "23": 0.46295, + "24": 0.46293, + "25": 0.4609, + "26": 0.46534, + "27": 0.46202, + "28": 0.46538, + "29": 0.45857, + "30": 0.45499, + "31": 0.45555, + "32": 0.45778, + "33": 0.45371, + "34": 0.4591, + "35": 0.45853, + "36": 0.46033, + "37": 0.46964, + "38": 0.46372, + "39": 0.47429, + "40": 0.4563, + "41": 0.45921, + "42": 0.46933, + "43": 0.45644, + "44": 0.46471, + "45": 0.4574, + "46": 0.46247, + "47": 0.45727, + "48": 0.45962, + "49": 0.45179, + "50": 0.45444, + "51": 0.4599, + "52": 0.44679, + "53": 0.45022, + "54": 0.45041, + "55": 0.45771, + "56": 0.45328, + "57": 0.45098, + "58": 0.44748, + "59": 0.44807, + "60": 0.4538, + "61": 0.45222, + "62": 0.44954, + "63": 0.44907, + "64": 0.45008, + "65": 0.44883, + "66": 0.4485, + "67": 0.44967, + "68": 0.45395, + "69": 0.45369, + "70": 0.45227, + "71": 0.45433, + "72": 0.45362, + "73": 0.45783, + "74": 0.45269, + "75": 0.45513, + "76": 0.45076, + "77": 0.4512, + "78": 0.4499, + "79": 0.45799, + "80": 0.45507, + "81": 0.45882, + "82": 0.46542, + "83": 0.45653, + "84": 0.46726, + "85": 0.47932, + "86": 0.45972, + "87": 0.46195, + "88": 0.46285, + "89": 0.46098, + "90": 0.46499, + "91": 0.46284, + "92": 0.46284, + "93": 0.45889, + "94": 0.45485, + "95": 0.45165, + "96": 0.45389, + "97": 0.45854, + "98": 0.45665, + "99": 0.46287, + "100": 0.47613 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..21e65ea8685 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.78575, + "52": 9.67079, + "53": 10.00718, + "54": 9.897, + "55": 9.82613, + "56": 9.54826, + "57": 9.40078, + "58": 9.77416, + "59": 9.51683, + "60": 9.42721, + "61": 9.63407, + "62": 9.93885, + "63": 9.30502, + "64": 9.71263, + "65": 8.86836, + "66": 9.64475, + "67": 9.31349, + "68": 9.73448, + "69": 9.75501, + "70": 9.68613, + "71": 9.57698, + "72": 9.53067, + "73": 9.43091, + "74": 8.85477, + "75": 9.35819, + "76": 9.01446, + "77": 10.02647, + "78": 9.68112, + "79": 9.33348, + "80": 9.35484, + "81": 9.44135, + "82": 9.66189, + "83": 9.2631, + "84": 9.37182, + "85": 9.57428, + "86": 9.03438, + "87": 9.56188, + "88": 9.7128, + "89": 9.55803, + "90": 9.79197, + "91": 9.2902, + "92": 9.31613, + "93": 9.04053, + "94": 8.78282, + "95": 9.49399, + "96": 9.48876, + "97": 9.2605, + "98": 9.6313, + "99": 8.85096, + "100": 9.36491 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 930.0, + "52": 935.0, + "53": 1000.0, + "54": 919.0, + "55": 868.0, + "56": 1000.0, + "57": 827.0, + "58": 1023.0, + "59": 1019.0, + "60": 876.0, + "61": 1017.0, + "62": 936.0, + "63": 963.0, + "64": 1082.0, + "65": 982.0, + "66": 1037.0, + "67": 986.0, + "68": 1083.0, + "69": 1055.0, + "70": 1040.0, + "71": 999.0, + "72": 883.0, + "73": 1019.0, + "74": 728.0, + "75": 847.0, + "76": 1083.0, + "77": 1150.0, + "78": 1105.0, + "79": 1071.0, + "80": 1139.0, + "81": 1195.0, + "82": 1064.0, + "83": 1012.0, + "84": 1105.0, + "85": 1121.0, + "86": 836.0, + "87": 1193.0, + "88": 1096.0, + "89": 1116.0, + "90": 1162.0, + "91": 1098.0, + "92": 1160.0, + "93": 906.0, + "94": 1177.0, + "95": 1117.0, + "96": 1232.0, + "97": 1115.0, + "98": 1241.0, + "99": 1032.0, + "100": 1132.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 640336384.0, + "52": 640336384.0, + "53": 640336384.0, + "54": 640336384.0, + "55": 640336384.0, + "56": 640336384.0, + "57": 640336384.0, + "58": 640336384.0, + "59": 640336384.0, + "60": 640336384.0, + "61": 640336384.0, + "62": 640336384.0, + "63": 640336384.0, + "64": 640336384.0, + "65": 640336384.0, + "66": 640336384.0, + "67": 640336384.0, + "68": 640336384.0, + "69": 640336384.0, + "70": 640336384.0, + "71": 640336384.0, + "72": 640336384.0, + "73": 640336384.0, + "74": 640336384.0, + "75": 640336384.0, + "76": 640336384.0, + "77": 640336384.0, + "78": 640336384.0, + "79": 640336384.0, + "80": 640336384.0, + "81": 640336384.0, + "82": 640336384.0, + "83": 640336384.0, + "84": 640336384.0, + "85": 640336384.0, + "86": 640336384.0, + "87": 640336384.0, + "88": 640336384.0, + "89": 640336384.0, + "90": 640336384.0, + "91": 640336384.0, + "92": 640336384.0, + "93": 640336384.0, + "94": 640336384.0, + "95": 640336384.0, + "96": 640336384.0, + "97": 640336384.0, + "98": 640336384.0, + "99": 640336384.0, + "100": 640336384.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1171155456.0, + "52": 1174302208.0, + "53": 1176398848.0, + "54": 1176398848.0, + "55": 1176398848.0, + "56": 1176398848.0, + "57": 1176398848.0, + "58": 1176398848.0, + "59": 1178496512.0, + "60": 1178496512.0, + "61": 1178496512.0, + "62": 1178496512.0, + "63": 1178496512.0, + "64": 1178496512.0, + "65": 1178496512.0, + "66": 1178496512.0, + "67": 1178496512.0, + "68": 1178496512.0, + "69": 1178496512.0, + "70": 1178496512.0, + "71": 1178496512.0, + "72": 1178496512.0, + "73": 1178496512.0, + "74": 1178496512.0, + "75": 1178496512.0, + "76": 1178496512.0, + "77": 1178496512.0, + "78": 1178496512.0, + "79": 1178496512.0, + "80": 1178496512.0, + "81": 1178496512.0, + "82": 1178496512.0, + "83": 1178496512.0, + "84": 1178496512.0, + "85": 1178496512.0, + "86": 1178496512.0, + "87": 1178496512.0, + "88": 1178496512.0, + "89": 1178496512.0, + "90": 1178496512.0, + "91": 1178496512.0, + "92": 1178496512.0, + "93": 1178496512.0, + "94": 1178496512.0, + "95": 1178496512.0, + "96": 1178496512.0, + "97": 1178496512.0, + "98": 1178496512.0, + "99": 1178496512.0, + "100": 1178496512.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.54283, + "53": 0.43318, + "54": 0.42496, + "55": 0.42557, + "56": 0.42913, + "57": 0.44479, + "58": 0.45904, + "59": 0.46949, + "60": 0.46125, + "61": 0.46841, + "62": 0.4736, + "63": 0.46017, + "64": 0.45223, + "65": 0.45568, + "66": 0.44984, + "67": 0.44794, + "68": 0.45062, + "69": 0.45415, + "70": 0.46315, + "71": 0.45069, + "72": 0.45122, + "73": 0.45026, + "74": 0.44997, + "75": 0.44929, + "76": 0.45314, + "77": 0.45848, + "78": 0.4566, + "79": 0.45909, + "80": 0.46265, + "81": 0.4592, + "82": 0.47898, + "83": 0.47817, + "84": 0.46757, + "85": 0.46663, + "86": 0.46924, + "87": 0.48331, + "88": 0.46217, + "89": 0.4596, + "90": 0.45471, + "91": 0.45598, + "92": 0.45849, + "93": 0.4626, + "94": 0.46398, + "95": 0.45663, + "96": 0.45814, + "97": 0.45394, + "98": 0.45984, + "99": 0.47284, + "100": 0.46707 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json index 8b51d66847b..52d95069cff 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 6.76973, - "2": 0.17585, - "3": 0.17368, - "4": 0.16152, - "5": 0.19039, - "6": 0.22444, - "7": 0.19405, - "8": 0.19945, - "9": 0.19849, - "10": 0.19715, - "11": 0.26257, - "12": 0.20383, - "13": 0.20656, - "14": 0.16788, - "15": 0.16036, - "16": 0.16063, - "17": 0.28798, - "18": 0.16008, - "19": 0.15785, - "20": 0.15974, - "21": 0.15889, - "22": 0.15943, - "23": 0.15886, - "24": 0.16021, - "25": 0.15915, - "26": 0.16121, - "27": 0.15965, - "28": 0.15981, - "29": 0.16011, - "30": 0.15997, - "31": 0.16048, - "32": 0.15884, - "33": 0.16058, - "34": 0.15945, - "35": 0.15917, - "36": 0.16205, - "37": 0.15947, - "38": 0.16161, - "39": 0.15927, - "40": 0.15876, - "41": 0.159, - "42": 0.47609, - "43": 0.17027, - "44": 0.1644, - "45": 0.16303, - "46": 0.16036, - "47": 0.16029, - "48": 0.16095, - "49": 0.16015, - "50": 0.1603, - "51": 0.21916, - "52": 0.20178, - "53": 0.20344, - "54": 0.22444, - "55": 0.25106, - "56": 0.19763, - "57": 0.21076, - "58": 0.24116, - "59": 0.19345, - "60": 0.1603, - "61": 0.15954, - "62": 0.16062, - "63": 0.20422, - "64": 0.1605, - "65": 0.16211, - "66": 0.16077, - "67": 0.16024, - "68": 0.16099, - "69": 0.16333, - "70": 0.16439, - "71": 0.16108, - "72": 0.16247, - "73": 0.1611, - "74": 0.16235, - "75": 0.16292, - "76": 0.16349, - "77": 0.1636, - "78": 0.16363, - "79": 0.34343, - "80": 0.15998, - "81": 0.15954, - "82": 0.15941, - "83": 0.15965, - "84": 0.16027, - "85": 0.16164, - "86": 0.16113, - "87": 0.16126, - "88": 0.16032, - "89": 0.26526, - "90": 0.15925, - "91": 0.1601, - "92": 0.15972, - "93": 0.15947, - "94": 0.15955, - "95": 0.15981, - "96": 0.15971, - "97": 0.15989, - "98": 0.15959, - "99": 0.15994, - "100": 0.16111 + "1": "nan", + "2": 3.23604, + "3": 0.17137, + "4": 0.15759, + "5": 0.15886, + "6": 0.15986, + "7": 0.16006, + "8": 0.16032, + "9": 0.15956, + "10": 0.15866, + "11": 0.16034, + "12": 0.161, + "13": 0.16092, + "14": 0.16138, + "15": 0.16079, + "16": 0.16106, + "17": 0.16054, + "18": 0.16039, + "19": 0.15987, + "20": 0.1604, + "21": 0.1606, + "22": 0.1605, + "23": 0.16063, + "24": 0.16081, + "25": 0.16081, + "26": 0.16009, + "27": 0.16063, + "28": 0.16056, + "29": 0.16163, + "30": 0.16078, + "31": 0.16052, + "32": 0.16157, + "33": 0.16141, + "34": 0.1609, + "35": 0.16067, + "36": 0.164, + "37": 0.16064, + "38": 0.16086, + "39": 0.16108, + "40": 0.1619, + "41": 0.15987, + "42": 0.16141, + "43": 0.16345, + "44": 0.15987, + "45": 0.16151, + "46": 0.16073, + "47": 0.16034, + "48": 0.15782, + "49": 0.15892, + "50": 0.15976, + "51": 0.17905, + "52": 0.16268, + "53": 0.15809, + "54": 0.15783, + "55": 0.1601, + "56": 0.16197, + "57": 0.16434, + "58": 0.16544, + "59": 0.16658, + "60": 0.16487, + "61": 0.16473, + "62": 0.1655, + "63": 0.16592, + "64": 0.1663, + "65": 0.16721, + "66": 0.16644, + "67": 0.16736, + "68": 0.16682, + "69": 0.16612, + "70": 0.1673, + "71": 0.1652, + "72": 0.16908, + "73": 0.16732, + "74": 0.16542, + "75": 0.16546, + "76": 0.16719, + "77": 0.16547, + "78": 0.16719, + "79": 0.16664, + "80": 0.16167, + "81": 0.163, + "82": 0.16309, + "83": 0.16502, + "84": 0.16364, + "85": 0.1648, + "86": 0.16491, + "87": 0.16514, + "88": 0.16442, + "89": 0.16591, + "90": 0.16301, + "91": 0.16462, + "92": 0.16639, + "93": 0.16358, + "94": 0.16489, + "95": 0.16504, + "96": 0.16457, + "97": 0.163, + "98": 0.16359, + "99": 0.16433, + "100": 0.16527 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..a3bcbb68249 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.85898, + "52": 9.75133, + "53": 10.06617, + "54": 9.95613, + "55": 9.89104, + "56": 9.62508, + "57": 9.47981, + "58": 9.83478, + "59": 9.58498, + "60": 9.49806, + "61": 9.69192, + "62": 9.98825, + "63": 9.37824, + "64": 9.76808, + "65": 8.94514, + "66": 9.70125, + "67": 9.37149, + "68": 9.78313, + "69": 9.79923, + "70": 9.7312, + "71": 9.62753, + "72": 9.58452, + "73": 9.48417, + "74": 8.92523, + "75": 9.4118, + "76": 9.0796, + "77": 10.06083, + "78": 9.7215, + "79": 9.38109, + "80": 9.40161, + "81": 9.48468, + "82": 9.70219, + "83": 9.31549, + "84": 9.41786, + "85": 9.61785, + "86": 9.077, + "87": 9.59967, + "88": 9.75356, + "89": 9.60341, + "90": 9.82789, + "91": 9.33668, + "92": 9.36036, + "93": 9.08765, + "94": 8.83052, + "95": 9.5296, + "96": 9.53024, + "97": 9.30627, + "98": 9.67298, + "99": 8.89917, + "100": 9.40828 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2554.0, + "52": 2466.0, + "53": 2923.0, + "54": 2612.0, + "55": 2351.0, + "56": 2757.0, + "57": 2313.0, + "58": 2798.0, + "59": 2750.0, + "60": 2376.0, + "61": 2848.0, + "62": 2668.0, + "63": 2468.0, + "64": 2818.0, + "65": 2630.0, + "66": 2992.0, + "67": 2802.0, + "68": 2794.0, + "69": 2851.0, + "70": 3059.0, + "71": 2869.0, + "72": 2424.0, + "73": 3035.0, + "74": 2113.0, + "75": 2485.0, + "76": 2782.0, + "77": 3252.0, + "78": 3149.0, + "79": 3192.0, + "80": 3229.0, + "81": 3397.0, + "82": 3297.0, + "83": 2766.0, + "84": 3192.0, + "85": 3206.0, + "86": 2648.0, + "87": 3709.0, + "88": 2962.0, + "89": 3273.0, + "90": 3149.0, + "91": 2825.0, + "92": 3047.0, + "93": 2918.0, + "94": 3432.0, + "95": 3266.0, + "96": 3574.0, + "97": 3190.0, + "98": 3564.0, + "99": 2977.0, + "100": 3249.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 492274176.0, + "52": 492274176.0, + "53": 492274176.0, + "54": 492274176.0, + "55": 492274176.0, + "56": 492274176.0, + "57": 492274176.0, + "58": 492274176.0, + "59": 492274176.0, + "60": 492274176.0, + "61": 492274176.0, + "62": 492274176.0, + "63": 492274176.0, + "64": 492274176.0, + "65": 492274176.0, + "66": 492274176.0, + "67": 492274176.0, + "68": 492274176.0, + "69": 492274176.0, + "70": 492274176.0, + "71": 492274176.0, + "72": 492274176.0, + "73": 492274176.0, + "74": 492274176.0, + "75": 492274176.0, + "76": 492274176.0, + "77": 492274176.0, + "78": 492274176.0, + "79": 492274176.0, + "80": 492274176.0, + "81": 492274176.0, + "82": 492274176.0, + "83": 492274176.0, + "84": 492274176.0, + "85": 492274176.0, + "86": 492274176.0, + "87": 492274176.0, + "88": 492274176.0, + "89": 492274176.0, + "90": 492274176.0, + "91": 492274176.0, + "92": 492274176.0, + "93": 492274176.0, + "94": 492274176.0, + "95": 492274176.0, + "96": 492274176.0, + "97": 492274176.0, + "98": 492274176.0, + "99": 492274176.0, + "100": 492274176.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1709489664.0, + "52": 1709490688.0, + "53": 1709490688.0, + "54": 1709490688.0, + "55": 1709490688.0, + "56": 1709490688.0, + "57": 1709490688.0, + "58": 1709490688.0, + "59": 1709490688.0, + "60": 1709490688.0, + "61": 1709490688.0, + "62": 1709490688.0, + "63": 1709490688.0, + "64": 1709490688.0, + "65": 1709490688.0, + "66": 1709490688.0, + "67": 1709490688.0, + "68": 1709490688.0, + "69": 1709490688.0, + "70": 1709490688.0, + "71": 1709490688.0, + "72": 1709490688.0, + "73": 1709490688.0, + "74": 1709490688.0, + "75": 1709490688.0, + "76": 1709490688.0, + "77": 1709490688.0, + "78": 1709490688.0, + "79": 1709490688.0, + "80": 1709490688.0, + "81": 1709490688.0, + "82": 1709490688.0, + "83": 1709490688.0, + "84": 1709490688.0, + "85": 1709490688.0, + "86": 1709490688.0, + "87": 1709490688.0, + "88": 1709490688.0, + "89": 1709490688.0, + "90": 1709490688.0, + "91": 1709490688.0, + "92": 1709490688.0, + "93": 1709490688.0, + "94": 1709490688.0, + "95": 1709490688.0, + "96": 1709490688.0, + "97": 1709490688.0, + "98": 1709490688.0, + "99": 1709490688.0, + "100": 1709490688.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 3.02213, + "53": 0.17862, + "54": 0.16745, + "55": 0.16648, + "56": 0.16673, + "57": 0.16292, + "58": 0.19638, + "59": 0.16818, + "60": 0.16539, + "61": 0.16364, + "62": 0.16301, + "63": 0.1601, + "64": 0.16122, + "65": 0.16293, + "66": 0.16244, + "67": 0.16253, + "68": 0.16237, + "69": 0.16026, + "70": 0.17045, + "71": 0.15999, + "72": 0.1709, + "73": 0.16315, + "74": 0.1602, + "75": 0.15985, + "76": 0.15963, + "77": 0.15943, + "78": 0.15987, + "79": 0.16, + "80": 0.16033, + "81": 0.16099, + "82": 0.16037, + "83": 0.16139, + "84": 0.16563, + "85": 0.16553, + "86": 0.16519, + "87": 0.16488, + "88": 0.16176, + "89": 0.16185, + "90": 0.16148, + "91": 0.16234, + "92": 0.1601, + "93": 0.16147, + "94": 0.16081, + "95": 0.16284, + "96": 0.16144, + "97": 0.16225, + "98": 0.16162, + "99": 0.16044, + "100": 0.16202 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json index 0c5b41565c8..9066ef0c241 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 463110656.0, + "2": 463110656.0, + "3": 463110656.0, + "4": 463110656.0, + "5": 463110656.0, + "6": 463110656.0, + "7": 463110656.0, + "8": 463110656.0, + "9": 463110656.0, + "10": 463110656.0, + "11": 463110656.0, + "12": 463110656.0, + "13": 463110656.0, + "14": 463110656.0, + "15": 463110656.0, + "16": 463110656.0, + "17": 463110656.0, + "18": 463110656.0, + "19": 463110656.0, + "20": 463110656.0, + "21": 463110656.0, + "22": 463110656.0, + "23": 463110656.0, + "24": 463110656.0, + "25": 463110656.0, + "26": 463110656.0, + "27": 463110656.0, + "28": 463110656.0, + "29": 463110656.0, + "30": 463110656.0, + "31": 463110656.0, + "32": 463110656.0, + "33": 463110656.0, + "34": 463110656.0, + "35": 463110656.0, + "36": 463110656.0, + "37": 463110656.0, + "38": 463110656.0, + "39": 463110656.0, + "40": 463110656.0, + "41": 463110656.0, + "42": 463110656.0, + "43": 463110656.0, + "44": 463110656.0, + "45": 463110656.0, + "46": 463110656.0, + "47": 463110656.0, + "48": 463110656.0, + "49": 463110656.0, + "50": 463110656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 708779008.0, - "2": 882038272.0, - "3": 882562560.0, - "4": 882562560.0, - "5": 882562560.0, - "6": 882562560.0, - "7": 882562560.0, - "8": 882562560.0, - "9": 882562560.0, - "10": 882562560.0, - "11": 882562560.0, - "12": 882562560.0, - "13": 882562560.0, - "14": 882562560.0, - "15": 882562560.0, - "16": 882562560.0, - "17": 882562560.0, - "18": 882562560.0, - "19": 882562560.0, - "20": 882562560.0, - "21": 882562560.0, - "22": 882562560.0, - "23": 882562560.0, - "24": 882562560.0, - "25": 882562560.0, - "26": 882562560.0, - "27": 882562560.0, - "28": 883608576.0, - "29": 883608576.0, - "30": 883608576.0, - "31": 883608576.0, - "32": 883608576.0, - "33": 883608576.0, - "34": 883608576.0, - "35": 883608576.0, - "36": 883608576.0, - "37": 883608576.0, - "38": 883608576.0, - "39": 883608576.0, - "40": 883608576.0, - "41": 883608576.0, - "42": 883608576.0, - "43": 883608576.0, - "44": 883608576.0, - "45": 883608576.0, - "46": 883608576.0, - "47": 883608576.0, - "48": 883608576.0, - "49": 883608576.0, - "50": 883608576.0 + "1": 704587264.0, + "2": 887671296.0, + "3": 887671296.0, + "4": 887671296.0, + "5": 887671296.0, + "6": 887671296.0, + "7": 887671296.0, + "8": 887671296.0, + "9": 887671296.0, + "10": 887671296.0, + "11": 887674368.0, + "12": 887674368.0, + "13": 887674368.0, + "14": 887674368.0, + "15": 887674368.0, + "16": 887674368.0, + "17": 887674368.0, + "18": 887674368.0, + "19": 887674368.0, + "20": 887674368.0, + "21": 887674368.0, + "22": 887674368.0, + "23": 887674368.0, + "24": 887674368.0, + "25": 887674368.0, + "26": 887674368.0, + "27": 887674368.0, + "28": 887674368.0, + "29": 887674368.0, + "30": 887674368.0, + "31": 887674368.0, + "32": 887674368.0, + "33": 887674368.0, + "34": 887674368.0, + "35": 887674368.0, + "36": 887674368.0, + "37": 887674368.0, + "38": 887674368.0, + "39": 887674368.0, + "40": 887674368.0, + "41": 887674368.0, + "42": 887674368.0, + "43": 887674368.0, + "44": 887674368.0, + "45": 887674368.0, + "46": 887674368.0, + "47": 887674368.0, + "48": 887674368.0, + "49": 887674368.0, + "50": 887674368.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.32961, - "2": 0.54797, - "3": 0.51657, - "4": 0.52599, - "5": 0.61023, - "6": 0.69053, - "7": 0.5446, - "8": 0.51966, - "9": 0.52377, - "10": 0.52901, - "11": 0.52742, - "12": 0.53394, - "13": 0.52346, - "14": 0.52257, - "15": 0.51751, - "16": 0.48338, - "17": 0.48757, - "18": 0.52092, - "19": 0.49857, - "20": 0.49815, - "21": 0.49063, - "22": 0.49632, - "23": 0.4849, - "24": 0.49986, - "25": 0.48483, - "26": 0.49826, - "27": 0.48315, - "28": 0.4875, - "29": 0.498, - "30": 0.49611, - "31": 0.4984, - "32": 0.5284, - "33": 0.50276, - "34": 0.49132, - "35": 0.49787, - "36": 0.4947, - "37": 0.48747, - "38": 0.4952, - "39": 0.49214, - "40": 0.49151, - "41": 0.49593, - "42": 0.49285, - "43": 0.49745, - "44": 0.48784, - "45": 0.51195, - "46": 0.53565, - "47": 0.53921, - "48": 0.53697, - "49": 0.5397, - "50": 0.55869 + "1": "nan", + "2": 3.90343, + "3": 0.46196, + "4": 0.44553, + "5": 0.44562, + "6": 0.4419, + "7": 0.44421, + "8": 0.44706, + "9": 0.44217, + "10": 0.44007, + "11": 0.42363, + "12": 0.42376, + "13": 0.41997, + "14": 0.42079, + "15": 0.49345, + "16": 0.51617, + "17": 0.52444, + "18": 0.52822, + "19": 0.53053, + "20": 0.52743, + "21": 0.5289, + "22": 0.52655, + "23": 0.52934, + "24": 0.52619, + "25": 0.52343, + "26": 0.52071, + "27": 0.50241, + "28": 0.48165, + "29": 0.47009, + "30": 0.46549, + "31": 0.46432, + "32": 0.47167, + "33": 0.53326, + "34": 0.49042, + "35": 0.46143, + "36": 0.45859, + "37": 0.45093, + "38": 0.45152, + "39": 0.4443, + "40": 0.44393, + "41": 0.43638, + "42": 0.44204, + "43": 0.43923, + "44": 0.44115, + "45": 0.44017, + "46": 0.47447, + "47": 0.49352, + "48": 0.4943, + "49": 0.49375, + "50": 0.49263 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json index b9bbabe5437..fea64911d52 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 460096000.0, - "2": 460096000.0, - "3": 460096000.0, - "4": 460096000.0, - "5": 460096000.0, - "6": 460096000.0, - "7": 460096000.0, - "8": 460096000.0, - "9": 460096000.0, - "10": 460096000.0, - "11": 460096000.0, - "12": 460096000.0, - "13": 460096000.0, - "14": 460096000.0, - "15": 460096000.0, - "16": 460096000.0, - "17": 460096000.0, - "18": 460096000.0, - "19": 460096000.0, - "20": 460096000.0, - "21": 460096000.0, - "22": 460096000.0, - "23": 460096000.0, - "24": 460096000.0, - "25": 460096000.0, - "26": 460096000.0, - "27": 460096000.0, - "28": 460096000.0, - "29": 460096000.0, - "30": 460096000.0, - "31": 460096000.0, - "32": 460096000.0, - "33": 460096000.0, - "34": 460096000.0, - "35": 460096000.0, - "36": 460096000.0, - "37": 460096000.0, - "38": 460096000.0, - "39": 460096000.0, - "40": 460096000.0, - "41": 460096000.0, - "42": 460096000.0, - "43": 460096000.0, - "44": 460096000.0, - "45": 460096000.0, - "46": 460096000.0, - "47": 460096000.0, - "48": 460096000.0, - "49": 460096000.0, - "50": 460096000.0 + "1": 463110656.0, + "2": 463110656.0, + "3": 463110656.0, + "4": 463110656.0, + "5": 463110656.0, + "6": 463110656.0, + "7": 463110656.0, + "8": 463110656.0, + "9": 463110656.0, + "10": 463110656.0, + "11": 463110656.0, + "12": 463110656.0, + "13": 463110656.0, + "14": 463110656.0, + "15": 463110656.0, + "16": 463110656.0, + "17": 463110656.0, + "18": 463110656.0, + "19": 463110656.0, + "20": 463110656.0, + "21": 463110656.0, + "22": 463110656.0, + "23": 463110656.0, + "24": 463110656.0, + "25": 463110656.0, + "26": 463110656.0, + "27": 463110656.0, + "28": 463110656.0, + "29": 463110656.0, + "30": 463110656.0, + "31": 463110656.0, + "32": 463110656.0, + "33": 463110656.0, + "34": 463110656.0, + "35": 463110656.0, + "36": 463110656.0, + "37": 463110656.0, + "38": 463110656.0, + "39": 463110656.0, + "40": 463110656.0, + "41": 463110656.0, + "42": 463110656.0, + "43": 463110656.0, + "44": 463110656.0, + "45": 463110656.0, + "46": 463110656.0, + "47": 463110656.0, + "48": 463110656.0, + "49": 463110656.0, + "50": 463110656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 704587264.0, - "2": 885184000.0, - "3": 885184000.0, - "4": 885184000.0, - "5": 885184000.0, - "6": 885184000.0, - "7": 886231040.0, - "8": 886231552.0, - "9": 886231552.0, - "10": 886231552.0, - "11": 886231552.0, - "12": 886231552.0, - "13": 886231552.0, - "14": 886231552.0, - "15": 886231552.0, - "16": 886231552.0, - "17": 886231552.0, - "18": 886231552.0, - "19": 886231552.0, - "20": 886231552.0, - "21": 886231552.0, - "22": 886231552.0, - "23": 886231552.0, - "24": 886231552.0, - "25": 886231552.0, - "26": 886231552.0, - "27": 886232064.0, - "28": 886232064.0, - "29": 886232064.0, - "30": 886232064.0, - "31": 886232064.0, - "32": 886232064.0, - "33": 886232064.0, - "34": 886232064.0, - "35": 886232064.0, - "36": 886232064.0, - "37": 886232064.0, - "38": 886232064.0, - "39": 886232064.0, - "40": 886232064.0, - "41": 886232064.0, - "42": 886232064.0, - "43": 886232064.0, - "44": 886232064.0, - "45": 886232064.0, - "46": 886232064.0, - "47": 886232064.0, - "48": 886232064.0, - "49": 886232064.0, - "50": 886232064.0 + "1": 707730944.0, + "2": 887671296.0, + "3": 887671296.0, + "4": 887671296.0, + "5": 887671296.0, + "6": 887671296.0, + "7": 887671296.0, + "8": 887671296.0, + "9": 887671296.0, + "10": 887671296.0, + "11": 887671296.0, + "12": 887671296.0, + "13": 887671296.0, + "14": 887671296.0, + "15": 887671296.0, + "16": 887671296.0, + "17": 887671296.0, + "18": 887671296.0, + "19": 887671296.0, + "20": 887671296.0, + "21": 887671296.0, + "22": 887671296.0, + "23": 887671296.0, + "24": 887671296.0, + "25": 887671296.0, + "26": 887671296.0, + "27": 887671296.0, + "28": 887671296.0, + "29": 887671296.0, + "30": 887671296.0, + "31": 887671296.0, + "32": 887671296.0, + "33": 887671296.0, + "34": 887671296.0, + "35": 887671296.0, + "36": 887671296.0, + "37": 887671296.0, + "38": 887671296.0, + "39": 887671296.0, + "40": 887671296.0, + "41": 887671296.0, + "42": 887671296.0, + "43": 887671296.0, + "44": 887671296.0, + "45": 887671296.0, + "46": 887671296.0, + "47": 887671296.0, + "48": 887671296.0, + "49": 887671296.0, + "50": 887671296.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 13.9895, - "2": 0.51807, - "3": 0.49599, - "4": 0.47064, - "5": 0.48452, - "6": 0.41822, - "7": 0.41485, - "8": 0.4156, - "9": 0.43484, - "10": 0.40847, - "11": 0.5122, - "12": 0.40698, - "13": 0.40749, - "14": 0.49304, - "15": 0.49799, - "16": 0.40895, - "17": 0.41708, - "18": 0.44007, - "19": 0.47716, - "20": 0.47638, - "21": 0.41659, - "22": 0.4125, - "23": 0.41163, - "24": 0.46826, - "25": 0.46402, - "26": 0.42136, - "27": 0.4113, - "28": 0.40612, - "29": 0.61576, - "30": 0.74613, - "31": 0.47263, - "32": 0.48955, - "33": 0.72478, - "34": 0.5927, - "35": 0.6127, - "36": 0.44041, - "37": 0.42799, - "38": 0.46386, - "39": 0.42311, - "40": 0.42142, - "41": 0.42074, - "42": 0.42015, - "43": 0.43664, - "44": 0.41727, - "45": 0.41517, - "46": 0.42041, - "47": 0.58839, - "48": 0.4946, - "49": 0.5046, - "50": 0.50846 + "1": "nan", + "2": 3.59451, + "3": 0.48132, + "4": 0.46344, + "5": 0.46723, + "6": 0.47347, + "7": 0.48756, + "8": 0.49394, + "9": 0.49358, + "10": 0.495, + "11": 0.49567, + "12": 0.49577, + "13": 0.49608, + "14": 0.49247, + "15": 0.49553, + "16": 0.49581, + "17": 0.49335, + "18": 0.5003, + "19": 0.49904, + "20": 0.50095, + "21": 0.49831, + "22": 0.49726, + "23": 0.49738, + "24": 0.50198, + "25": 0.49901, + "26": 0.50161, + "27": 0.50183, + "28": 0.49371, + "29": 0.49579, + "30": 0.49585, + "31": 0.49614, + "32": 0.49424, + "33": 0.49565, + "34": 0.49645, + "35": 0.50022, + "36": 0.50076, + "37": 0.49676, + "38": 0.4972, + "39": 0.49438, + "40": 0.49751, + "41": 0.49485, + "42": 0.49564, + "43": 0.4958, + "44": 0.49763, + "45": 0.49766, + "46": 0.50005, + "47": 0.49885, + "48": 0.50156, + "49": 0.50235, + "50": 0.49766 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic/golden_values_dev_dgx_gb200.json index 8175fe3e6be..bbc822686c6 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.86244, "2": 10.88582, - "3": 10.84736, - "4": 10.85573, - "5": 10.86003, + "3": 10.84735, + "4": 10.85571, + "5": 10.86002, "6": 10.87733, - "7": 10.8656, - "8": 10.84911, + "7": 10.86562, + "8": 10.84914, "9": 10.86609, - "10": 10.82475, - "11": 10.8562, - "12": 10.85373, - "13": 10.86788, - "14": 10.87111, - "15": 10.8223, - "16": 10.79994, - "17": 10.77431, - "18": 10.78343, - "19": 10.79309, - "20": 10.68225, - "21": 10.64708, - "22": 10.50918, - "23": 10.66826, - "24": 10.54194, - "25": 10.49281, - "26": 10.55932, - "27": 10.54239, - "28": 10.51128, + "10": 10.82473, + "11": 10.85616, + "12": 10.85369, + "13": 10.86791, + "14": 10.87114, + "15": 10.82234, + "16": 10.79991, + "17": 10.7743, + "18": 10.78346, + "19": 10.79307, + "20": 10.68222, + "21": 10.64709, + "22": 10.50919, + "23": 10.6683, + "24": 10.54196, + "25": 10.49283, + "26": 10.55931, + "27": 10.5424, + "28": 10.51132, "29": 10.53257, - "30": 10.28989, - "31": 10.02853, - "32": 10.3888, - "33": 10.39592, + "30": 10.28992, + "31": 10.02852, + "32": 10.38881, + "33": 10.39597, "34": 10.13449, - "35": 10.18931, - "36": 10.13352, - "37": 10.27378, - "38": 10.1075, + "35": 10.18929, + "36": 10.13354, + "37": 10.27384, + "38": 10.10753, "39": 10.34011, - "40": 9.98542, - "41": 10.06415, - "42": 10.1375, - "43": 9.73383, - "44": 9.86311, - "45": 9.73726, - "46": 9.71341, - "47": 10.07757, - "48": 9.76762, - "49": 9.4199, - "50": 9.81687 + "40": 9.98544, + "41": 10.06413, + "42": 10.13747, + "43": 9.73382, + "44": 9.86306, + "45": 9.73725, + "46": 9.71343, + "47": 10.07755, + "48": 9.76765, + "49": 9.41988, + "50": 9.81692 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 567.0, - "2": 609.0, - "3": 638.0, - "4": 657.0, - "5": 654.0, - "6": 637.0, - "7": 614.0, - "8": 599.0, - "9": 637.0, - "10": 517.0, - "11": 673.0, - "12": 640.0, - "13": 685.0, - "14": 609.0, - "15": 596.0, - "16": 653.0, - "17": 590.0, - "18": 559.0, - "19": 675.0, - "20": 598.0, - "21": 699.0, - "22": 631.0, - "23": 650.0, - "24": 625.0, - "25": 591.0, - "26": 627.0, - "27": 684.0, - "28": 679.0, - "29": 748.0, - "30": 703.0, - "31": 626.0, - "32": 724.0, - "33": 753.0, - "34": 658.0, - "35": 727.0, - "36": 730.0, - "37": 861.0, - "38": 778.0, - "39": 899.0, - "40": 845.0, - "41": 770.0, - "42": 819.0, - "43": 716.0, - "44": 793.0, - "45": 770.0, - "46": 849.0, - "47": 900.0, - "48": 873.0, - "49": 852.0, - "50": 888.0 + "1": 593.0, + "2": 604.0, + "3": 621.0, + "4": 634.0, + "5": 658.0, + "6": 640.0, + "7": 624.0, + "8": 576.0, + "9": 622.0, + "10": 481.0, + "11": 703.0, + "12": 606.0, + "13": 667.0, + "14": 652.0, + "15": 654.0, + "16": 625.0, + "17": 598.0, + "18": 534.0, + "19": 627.0, + "20": 616.0, + "21": 720.0, + "22": 601.0, + "23": 647.0, + "24": 615.0, + "25": 577.0, + "26": 654.0, + "27": 661.0, + "28": 705.0, + "29": 681.0, + "30": 725.0, + "31": 613.0, + "32": 766.0, + "33": 801.0, + "34": 690.0, + "35": 697.0, + "36": 733.0, + "37": 839.0, + "38": 806.0, + "39": 841.0, + "40": 858.0, + "41": 837.0, + "42": 812.0, + "43": 696.0, + "44": 819.0, + "45": 753.0, + "46": 840.0, + "47": 921.0, + "48": 863.0, + "49": 850.0, + "50": 830.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 461144576.0, - "2": 461144576.0, - "3": 461144576.0, - "4": 461144576.0, - "5": 461144576.0, - "6": 461144576.0, - "7": 461144576.0, - "8": 461144576.0, - "9": 461144576.0, - "10": 461144576.0, - "11": 461144576.0, - "12": 461144576.0, - "13": 461144576.0, - "14": 461144576.0, - "15": 461144576.0, - "16": 461144576.0, - "17": 461144576.0, - "18": 461144576.0, - "19": 461144576.0, - "20": 461144576.0, - "21": 461144576.0, - "22": 461144576.0, - "23": 461144576.0, - "24": 461144576.0, - "25": 461144576.0, - "26": 461144576.0, - "27": 461144576.0, - "28": 461144576.0, - "29": 461144576.0, - "30": 461144576.0, - "31": 461144576.0, - "32": 461144576.0, - "33": 461144576.0, - "34": 461144576.0, - "35": 461144576.0, - "36": 461144576.0, - "37": 461144576.0, - "38": 461144576.0, - "39": 461144576.0, - "40": 461144576.0, - "41": 461144576.0, - "42": 461144576.0, - "43": 461144576.0, - "44": 461144576.0, - "45": 461144576.0, - "46": 461144576.0, - "47": 461144576.0, - "48": 461144576.0, - "49": 461144576.0, - "50": 461144576.0 + "1": 460882432.0, + "2": 460882432.0, + "3": 460882432.0, + "4": 460882432.0, + "5": 460882432.0, + "6": 460882432.0, + "7": 460882432.0, + "8": 460882432.0, + "9": 460882432.0, + "10": 460882432.0, + "11": 460882432.0, + "12": 460882432.0, + "13": 460882432.0, + "14": 460882432.0, + "15": 460882432.0, + "16": 460882432.0, + "17": 460882432.0, + "18": 460882432.0, + "19": 460882432.0, + "20": 460882432.0, + "21": 460882432.0, + "22": 460882432.0, + "23": 460882432.0, + "24": 460882432.0, + "25": 460882432.0, + "26": 460882432.0, + "27": 460882432.0, + "28": 460882432.0, + "29": 460882432.0, + "30": 460882432.0, + "31": 460882432.0, + "32": 460882432.0, + "33": 460882432.0, + "34": 460882432.0, + "35": 460882432.0, + "36": 460882432.0, + "37": 460882432.0, + "38": 460882432.0, + "39": 460882432.0, + "40": 460882432.0, + "41": 460882432.0, + "42": 460882432.0, + "43": 460882432.0, + "44": 460882432.0, + "45": 460882432.0, + "46": 460882432.0, + "47": 460882432.0, + "48": 460882432.0, + "49": 460882432.0, + "50": 460882432.0 } }, "mem-max-allocated-bytes": { @@ -176,55 +176,55 @@ "step_interval": 1, "values": { "1": 705635840.0, - "2": 884659712.0, - "3": 885183488.0, - "4": 885183488.0, - "5": 885707264.0, - "6": 885707264.0, - "7": 885707264.0, - "8": 885707264.0, - "9": 885707264.0, - "10": 885707264.0, - "11": 885707264.0, - "12": 885707264.0, - "13": 885707264.0, - "14": 885707264.0, - "15": 885707264.0, - "16": 885707264.0, - "17": 885707264.0, - "18": 885707264.0, - "19": 885707264.0, - "20": 885707264.0, - "21": 885707264.0, - "22": 885707264.0, - "23": 885707264.0, - "24": 885707264.0, - "25": 885707264.0, - "26": 885707264.0, - "27": 885707264.0, - "28": 885707264.0, - "29": 885707264.0, - "30": 885708288.0, - "31": 885708288.0, - "32": 885708288.0, - "33": 885708288.0, - "34": 885708288.0, - "35": 885708288.0, - "36": 885708288.0, - "37": 885708288.0, - "38": 885708288.0, - "39": 885708288.0, - "40": 885708288.0, - "41": 885708288.0, - "42": 885708288.0, - "43": 885708288.0, - "44": 885708288.0, - "45": 885708288.0, - "46": 885708288.0, - "47": 885708288.0, - "48": 885708288.0, - "49": 885708288.0, - "50": 885708288.0 + "2": 883348992.0, + "3": 883348992.0, + "4": 883348992.0, + "5": 883348992.0, + "6": 883348992.0, + "7": 883348992.0, + "8": 883348992.0, + "9": 883348992.0, + "10": 883348992.0, + "11": 883348992.0, + "12": 883348992.0, + "13": 883348992.0, + "14": 883348992.0, + "15": 883348992.0, + "16": 883348992.0, + "17": 883348992.0, + "18": 883348992.0, + "19": 883348992.0, + "20": 885443584.0, + "21": 885445120.0, + "22": 885445120.0, + "23": 885445120.0, + "24": 885445120.0, + "25": 885445120.0, + "26": 885445120.0, + "27": 885445120.0, + "28": 885445120.0, + "29": 885445120.0, + "30": 885445120.0, + "31": 885446144.0, + "32": 885446144.0, + "33": 885446144.0, + "34": 885446144.0, + "35": 885446144.0, + "36": 885446144.0, + "37": 885446144.0, + "38": 885446144.0, + "39": 885446144.0, + "40": 885446144.0, + "41": 885446144.0, + "42": 885446144.0, + "43": 885446144.0, + "44": 885446144.0, + "45": 885446144.0, + "46": 885446144.0, + "47": 885446144.0, + "48": 885446144.0, + "49": 885446144.0, + "50": 885446144.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.21093, - "2": 0.56501, - "3": 0.71491, - "4": 0.53313, - "5": 0.43082, - "6": 0.4637, - "7": 0.40802, - "8": 0.46193, - "9": 0.40155, - "10": 0.40252, - "11": 0.52711, - "12": 0.4035, - "13": 0.40765, - "14": 0.40187, - "15": 0.40322, - "16": 0.40497, - "17": 0.40698, - "18": 0.40153, - "19": 0.46487, - "20": 0.40131, - "21": 0.4044, - "22": 0.40166, - "23": 0.40177, - "24": 0.40507, - "25": 0.405, - "26": 0.40144, - "27": 0.40453, - "28": 0.40108, - "29": 0.4052, - "30": 0.40603, - "31": 0.40719, - "32": 0.40638, - "33": 0.40514, - "34": 0.44714, - "35": 0.40534, - "36": 0.40221, - "37": 0.3984, - "38": 0.40367, - "39": 0.40221, - "40": 0.43747, - "41": 0.40384, - "42": 0.40404, - "43": 0.40132, - "44": 0.40047, - "45": 0.40017, - "46": 0.40235, - "47": 0.39964, - "48": 0.39919, - "49": 0.40337, - "50": 0.48503 + "1": "nan", + "2": 3.8465, + "3": 0.43585, + "4": 0.41053, + "5": 0.40923, + "6": 0.40815, + "7": 0.41221, + "8": 0.41419, + "9": 0.41058, + "10": 0.40643, + "11": 0.40558, + "12": 0.40602, + "13": 0.4063, + "14": 0.40517, + "15": 0.40811, + "16": 0.40555, + "17": 0.40549, + "18": 0.40655, + "19": 0.40455, + "20": 0.40661, + "21": 0.40594, + "22": 0.406, + "23": 0.40409, + "24": 0.40593, + "25": 0.40476, + "26": 0.40466, + "27": 0.40486, + "28": 0.40491, + "29": 0.41169, + "30": 0.40768, + "31": 0.40772, + "32": 0.40874, + "33": 0.40861, + "34": 0.40706, + "35": 0.40837, + "36": 0.40765, + "37": 0.40963, + "38": 0.40873, + "39": 0.40653, + "40": 0.4068, + "41": 0.40742, + "42": 0.40739, + "43": 0.43116, + "44": 0.40318, + "45": 0.41555, + "46": 0.40362, + "47": 0.40203, + "48": 0.40164, + "49": 0.40396, + "50": 0.40607 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json index aa1e18f88cb..724f3a7c2b8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 460882432.0, + "2": 460882432.0, + "3": 460882432.0, + "4": 460882432.0, + "5": 460882432.0, + "6": 460882432.0, + "7": 460882432.0, + "8": 460882432.0, + "9": 460882432.0, + "10": 460882432.0, + "11": 460882432.0, + "12": 460882432.0, + "13": 460882432.0, + "14": 460882432.0, + "15": 460882432.0, + "16": 460882432.0, + "17": 460882432.0, + "18": 460882432.0, + "19": 460882432.0, + "20": 460882432.0, + "21": 460882432.0, + "22": 460882432.0, + "23": 460882432.0, + "24": 460882432.0, + "25": 460882432.0, + "26": 460882432.0, + "27": 460882432.0, + "28": 460882432.0, + "29": 460882432.0, + "30": 460882432.0, + "31": 460882432.0, + "32": 460882432.0, + "33": 460882432.0, + "34": 460882432.0, + "35": 460882432.0, + "36": 460882432.0, + "37": 460882432.0, + "38": 460882432.0, + "39": 460882432.0, + "40": 460882432.0, + "41": 460882432.0, + "42": 460882432.0, + "43": 460882432.0, + "44": 460882432.0, + "45": 460882432.0, + "46": 460882432.0, + "47": 460882432.0, + "48": 460882432.0, + "49": 460882432.0, + "50": 460882432.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 708781568.0, - "2": 885184000.0, - "3": 885184000.0, - "4": 885184000.0, - "5": 885184000.0, - "6": 885184000.0, - "7": 885184000.0, - "8": 885184000.0, - "9": 885184000.0, - "10": 885184000.0, - "11": 885184000.0, - "12": 885184000.0, - "13": 885184000.0, - "14": 885184000.0, - "15": 885184000.0, - "16": 885184000.0, - "17": 885184000.0, - "18": 885184000.0, - "19": 885184000.0, - "20": 885184000.0, - "21": 886231552.0, - "22": 886231552.0, - "23": 886231552.0, - "24": 886231552.0, - "25": 886231552.0, - "26": 886231552.0, - "27": 886231552.0, - "28": 886231552.0, - "29": 886231552.0, - "30": 886231552.0, - "31": 886231552.0, - "32": 886231552.0, - "33": 886231552.0, - "34": 886231552.0, - "35": 886231552.0, - "36": 886231552.0, - "37": 886231552.0, - "38": 886231552.0, - "39": 886231552.0, - "40": 886231552.0, - "41": 886231552.0, - "42": 886231552.0, - "43": 886231552.0, - "44": 886231552.0, - "45": 886231552.0, - "46": 886231552.0, - "47": 886231552.0, - "48": 886231552.0, - "49": 886231552.0, - "50": 886231552.0 + "1": 706684416.0, + "2": 885445632.0, + "3": 885445632.0, + "4": 885445632.0, + "5": 885445632.0, + "6": 885445632.0, + "7": 885445632.0, + "8": 885445632.0, + "9": 885445632.0, + "10": 885445632.0, + "11": 885445632.0, + "12": 885445632.0, + "13": 885445632.0, + "14": 885445632.0, + "15": 885445632.0, + "16": 885445632.0, + "17": 885445632.0, + "18": 885445632.0, + "19": 885445632.0, + "20": 885445632.0, + "21": 885445632.0, + "22": 885445632.0, + "23": 885445632.0, + "24": 885445632.0, + "25": 885445632.0, + "26": 885445632.0, + "27": 885445632.0, + "28": 885445632.0, + "29": 885445632.0, + "30": 885446144.0, + "31": 885446144.0, + "32": 885446144.0, + "33": 885446144.0, + "34": 885446144.0, + "35": 885446144.0, + "36": 885446144.0, + "37": 885446144.0, + "38": 885446144.0, + "39": 885446144.0, + "40": 885446144.0, + "41": 885446144.0, + "42": 885446144.0, + "43": 885446144.0, + "44": 885446144.0, + "45": 885446144.0, + "46": 885446144.0, + "47": 885446144.0, + "48": 885446144.0, + "49": 885446144.0, + "50": 885446144.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.83536, - "2": 0.50436, - "3": 0.49153, - "4": 0.51839, - "5": 0.41963, - "6": 0.42593, - "7": 0.50539, - "8": 0.43728, - "9": 0.43214, - "10": 0.43276, - "11": 0.43243, - "12": 0.64271, - "13": 0.48613, - "14": 0.47822, - "15": 0.4732, - "16": 0.43011, - "17": 0.49091, - "18": 0.4264, - "19": 0.46987, - "20": 0.48787, - "21": 0.48533, - "22": 0.49433, - "23": 0.42402, - "24": 0.45662, - "25": 0.48851, - "26": 0.55798, - "27": 0.49442, - "28": 0.46841, - "29": 0.45193, - "30": 0.42664, - "31": 0.47172, - "32": 0.42125, - "33": 0.42866, - "34": 0.47761, - "35": 0.42624, - "36": 0.45512, - "37": 0.42405, - "38": 0.45455, - "39": 0.42258, - "40": 0.42354, - "41": 0.42486, - "42": 0.42783, - "43": 0.47508, - "44": 0.42611, - "45": 0.4236, - "46": 0.42862, - "47": 0.42603, - "48": 0.6007, - "49": 0.42833, - "50": 0.42517 + "1": "nan", + "2": 4.33644, + "3": 0.43367, + "4": 0.41796, + "5": 0.42437, + "6": 0.41803, + "7": 0.42777, + "8": 0.49261, + "9": 0.5259, + "10": 0.5165, + "11": 0.50964, + "12": 0.51307, + "13": 0.5209, + "14": 0.52184, + "15": 0.51919, + "16": 0.52047, + "17": 0.52096, + "18": 0.51364, + "19": 0.52433, + "20": 0.5561, + "21": 0.6836, + "22": 0.64032, + "23": 0.47706, + "24": 0.46456, + "25": 0.46618, + "26": 0.46113, + "27": 0.46089, + "28": 0.45645, + "29": 0.44502, + "30": 0.44476, + "31": 0.44646, + "32": 0.44355, + "33": 0.44206, + "34": 0.4428, + "35": 0.43615, + "36": 0.43343, + "37": 0.4428, + "38": 0.43948, + "39": 0.42992, + "40": 0.44781, + "41": 0.44585, + "42": 0.43409, + "43": 0.42263, + "44": 0.41737, + "45": 0.41789, + "46": 0.41449, + "47": 0.41442, + "48": 0.41697, + "49": 0.41486, + "50": 0.41305 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json index 8858c8ab59e..bb38c983224 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.86244, "2": 10.88582, - "3": 10.84735, - "4": 10.85571, - "5": 10.86001, - "6": 10.87728, + "3": 10.84732, + "4": 10.85572, + "5": 10.86003, + "6": 10.87729, "7": 10.86557, "8": 10.84912, - "9": 10.86609, - "10": 10.82474, - "11": 10.8562, - "12": 10.85373, - "13": 10.86791, - "14": 10.87118, - "15": 10.82233, - "16": 10.79992, - "17": 10.77429, - "18": 10.78345, - "19": 10.79312, - "20": 10.68225, - "21": 10.64714, - "22": 10.50918, + "9": 10.86604, + "10": 10.82473, + "11": 10.85617, + "12": 10.85371, + "13": 10.86788, + "14": 10.87113, + "15": 10.82235, + "16": 10.79993, + "17": 10.77433, + "18": 10.78348, + "19": 10.79308, + "20": 10.68227, + "21": 10.6471, + "22": 10.50922, "23": 10.66831, - "24": 10.54193, + "24": 10.54194, "25": 10.49281, - "26": 10.5593, - "27": 10.54238, - "28": 10.51129, - "29": 10.53257, - "30": 10.28987, - "31": 10.02852, - "32": 10.38878, - "33": 10.39598, - "34": 10.13455, - "35": 10.18928, - "36": 10.13354, - "37": 10.2738, - "38": 10.1075, - "39": 10.34012, - "40": 9.9854, - "41": 10.06415, + "26": 10.55932, + "27": 10.54243, + "28": 10.51131, + "29": 10.53254, + "30": 10.28988, + "31": 10.02851, + "32": 10.3888, + "33": 10.39597, + "34": 10.13451, + "35": 10.18926, + "36": 10.13351, + "37": 10.27379, + "38": 10.10746, + "39": 10.34007, + "40": 9.98541, + "41": 10.06416, "42": 10.13748, - "43": 9.73384, - "44": 9.86308, - "45": 9.73722, + "43": 9.73386, + "44": 9.86309, + "45": 9.73718, "46": 9.71345, - "47": 10.07752, - "48": 9.76768, - "49": 9.4199, - "50": 9.81691 + "47": 10.07751, + "48": 9.76767, + "49": 9.41988, + "50": 9.81692 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 575.0, - "2": 587.0, - "3": 615.0, - "4": 627.0, - "5": 639.0, - "6": 629.0, - "7": 625.0, - "8": 589.0, - "9": 645.0, - "10": 515.0, - "11": 616.0, - "12": 569.0, - "13": 701.0, - "14": 633.0, - "15": 589.0, - "16": 615.0, - "17": 612.0, - "18": 575.0, - "19": 549.0, - "20": 615.0, - "21": 693.0, - "22": 611.0, - "23": 737.0, - "24": 689.0, - "25": 579.0, - "26": 557.0, - "27": 692.0, - "28": 719.0, - "29": 762.0, - "30": 730.0, - "31": 579.0, + "1": 568.0, + "2": 566.0, + "3": 625.0, + "4": 604.0, + "5": 668.0, + "6": 650.0, + "7": 602.0, + "8": 647.0, + "9": 632.0, + "10": 527.0, + "11": 641.0, + "12": 661.0, + "13": 666.0, + "14": 656.0, + "15": 642.0, + "16": 603.0, + "17": 653.0, + "18": 570.0, + "19": 674.0, + "20": 571.0, + "21": 709.0, + "22": 663.0, + "23": 704.0, + "24": 628.0, + "25": 568.0, + "26": 632.0, + "27": 668.0, + "28": 736.0, + "29": 760.0, + "30": 687.0, + "31": 589.0, "32": 740.0, - "33": 766.0, - "34": 683.0, - "35": 705.0, - "36": 709.0, - "37": 810.0, - "38": 771.0, - "39": 872.0, - "40": 846.0, - "41": 757.0, - "42": 789.0, - "43": 766.0, - "44": 833.0, - "45": 738.0, - "46": 870.0, - "47": 891.0, - "48": 874.0, - "49": 857.0, - "50": 875.0 + "33": 772.0, + "34": 713.0, + "35": 753.0, + "36": 731.0, + "37": 873.0, + "38": 762.0, + "39": 836.0, + "40": 864.0, + "41": 780.0, + "42": 847.0, + "43": 740.0, + "44": 822.0, + "45": 718.0, + "46": 826.0, + "47": 890.0, + "48": 852.0, + "49": 872.0, + "50": 869.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 460882432.0, + "2": 460882432.0, + "3": 460882432.0, + "4": 460882432.0, + "5": 460882432.0, + "6": 460882432.0, + "7": 460882432.0, + "8": 460882432.0, + "9": 460882432.0, + "10": 460882432.0, + "11": 460882432.0, + "12": 460882432.0, + "13": 460882432.0, + "14": 460882432.0, + "15": 460882432.0, + "16": 460882432.0, + "17": 460882432.0, + "18": 460882432.0, + "19": 460882432.0, + "20": 460882432.0, + "21": 460882432.0, + "22": 460882432.0, + "23": 460882432.0, + "24": 460882432.0, + "25": 460882432.0, + "26": 460882432.0, + "27": 460882432.0, + "28": 460882432.0, + "29": 460882432.0, + "30": 460882432.0, + "31": 460882432.0, + "32": 460882432.0, + "33": 460882432.0, + "34": 460882432.0, + "35": 460882432.0, + "36": 460882432.0, + "37": 460882432.0, + "38": 460882432.0, + "39": 460882432.0, + "40": 460882432.0, + "41": 460882432.0, + "42": 460882432.0, + "43": 460882432.0, + "44": 460882432.0, + "45": 460882432.0, + "46": 460882432.0, + "47": 460882432.0, + "48": 460882432.0, + "49": 460882432.0, + "50": 460882432.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 704587264.0, - "2": 883611136.0, - "3": 884135424.0, - "4": 884658176.0, - "5": 884658176.0, - "6": 884658176.0, - "7": 884658176.0, - "8": 884658176.0, - "9": 884658688.0, - "10": 884659200.0, - "11": 884659200.0, - "12": 884659200.0, - "13": 884659200.0, - "14": 884659200.0, - "15": 884659200.0, - "16": 884659712.0, - "17": 884659712.0, - "18": 884659712.0, - "19": 884659712.0, - "20": 884659712.0, - "21": 884659712.0, - "22": 884659712.0, - "23": 884659712.0, - "24": 884659712.0, - "25": 884659712.0, - "26": 884659712.0, - "27": 884659712.0, - "28": 884659712.0, - "29": 884659712.0, - "30": 884659712.0, - "31": 884659712.0, - "32": 884659712.0, - "33": 884659712.0, - "34": 884659712.0, - "35": 884659712.0, - "36": 884659712.0, - "37": 884659712.0, - "38": 884659712.0, - "39": 884659712.0, - "40": 884659712.0, - "41": 884659712.0, - "42": 884659712.0, - "43": 884659712.0, - "44": 884659712.0, - "45": 884659712.0, - "46": 884659712.0, - "47": 884659712.0, - "48": 884659712.0, - "49": 884659712.0, - "50": 884659712.0 + "1": 705634816.0, + "2": 884397568.0, + "3": 885443584.0, + "4": 885445632.0, + "5": 885445632.0, + "6": 885445632.0, + "7": 885445632.0, + "8": 885445632.0, + "9": 885445632.0, + "10": 885445632.0, + "11": 885445632.0, + "12": 885445632.0, + "13": 885445632.0, + "14": 885445632.0, + "15": 885446144.0, + "16": 885446144.0, + "17": 885446144.0, + "18": 885446144.0, + "19": 885446144.0, + "20": 885446144.0, + "21": 885446144.0, + "22": 885446144.0, + "23": 885446144.0, + "24": 885446144.0, + "25": 885446144.0, + "26": 885446144.0, + "27": 885446144.0, + "28": 885446144.0, + "29": 885446144.0, + "30": 885446144.0, + "31": 885446144.0, + "32": 885446144.0, + "33": 885446144.0, + "34": 885446144.0, + "35": 885446144.0, + "36": 885446144.0, + "37": 885446144.0, + "38": 885446144.0, + "39": 885446144.0, + "40": 885446144.0, + "41": 885446144.0, + "42": 885446144.0, + "43": 885446144.0, + "44": 885446144.0, + "45": 885446144.0, + "46": 885446144.0, + "47": 885446144.0, + "48": 885446144.0, + "49": 885446144.0, + "50": 885446144.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.6955, - "2": 0.4755, - "3": 0.50907, - "4": 0.43067, - "5": 0.73714, - "6": 0.4269, - "7": 0.42684, - "8": 0.64221, - "9": 0.48428, - "10": 0.42395, - "11": 0.42943, - "12": 0.49106, - "13": 0.49108, - "14": 0.67522, - "15": 0.42547, - "16": 0.41999, - "17": 0.46662, - "18": 0.45683, - "19": 0.41987, - "20": 0.41746, - "21": 0.41909, - "22": 0.4703, - "23": 0.42675, - "24": 0.62571, - "25": 0.47889, - "26": 0.53722, - "27": 0.49475, - "28": 0.48715, - "29": 0.59996, - "30": 0.4396, - "31": 0.42052, - "32": 0.4463, - "33": 0.45305, - "34": 0.45764, - "35": 0.42178, - "36": 0.4257, - "37": 0.43568, - "38": 0.42736, - "39": 0.42942, - "40": 0.43094, - "41": 0.42609, - "42": 0.42743, - "43": 0.43464, - "44": 0.43647, - "45": 0.46437, - "46": 0.46709, - "47": 0.64826, - "48": 0.44677, - "49": 0.64353, - "50": 0.4369 + "1": "nan", + "2": 4.20529, + "3": 0.43211, + "4": 0.41974, + "5": 0.41788, + "6": 0.41713, + "7": 0.41839, + "8": 0.41778, + "9": 0.41756, + "10": 0.4154, + "11": 0.41682, + "12": 0.41539, + "13": 0.41618, + "14": 0.41668, + "15": 0.41894, + "16": 0.41891, + "17": 0.46526, + "18": 0.46951, + "19": 0.48697, + "20": 0.51157, + "21": 0.44025, + "22": 0.41388, + "23": 0.41164, + "24": 0.41655, + "25": 0.41424, + "26": 0.41687, + "27": 0.41162, + "28": 0.41035, + "29": 0.41184, + "30": 0.40989, + "31": 0.41153, + "32": 0.41143, + "33": 0.41324, + "34": 0.41271, + "35": 0.41107, + "36": 0.41053, + "37": 0.41372, + "38": 0.4128, + "39": 0.41377, + "40": 0.41093, + "41": 0.41375, + "42": 0.40814, + "43": 0.40704, + "44": 0.40632, + "45": 0.41014, + "46": 0.41007, + "47": 0.41057, + "48": 0.41002, + "49": 0.4095, + "50": 0.41018 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json index 746c6b2ba10..4e8e9932015 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last/golden_values_dev_dgx_gb200.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 460882432.0, + "2": 460882432.0, + "3": 460882432.0, + "4": 460882432.0, + "5": 460882432.0, + "6": 460882432.0, + "7": 460882432.0, + "8": 460882432.0, + "9": 460882432.0, + "10": 460882432.0, + "11": 460882432.0, + "12": 460882432.0, + "13": 460882432.0, + "14": 460882432.0, + "15": 460882432.0, + "16": 460882432.0, + "17": 460882432.0, + "18": 460882432.0, + "19": 460882432.0, + "20": 460882432.0, + "21": 460882432.0, + "22": 460882432.0, + "23": 460882432.0, + "24": 460882432.0, + "25": 460882432.0, + "26": 460882432.0, + "27": 460882432.0, + "28": 460882432.0, + "29": 460882432.0, + "30": 460882432.0, + "31": 460882432.0, + "32": 460882432.0, + "33": 460882432.0, + "34": 460882432.0, + "35": 460882432.0, + "36": 460882432.0, + "37": 460882432.0, + "38": 460882432.0, + "39": 460882432.0, + "40": 460882432.0, + "41": 460882432.0, + "42": 460882432.0, + "43": 460882432.0, + "44": 460882432.0, + "45": 460882432.0, + "46": 460882432.0, + "47": 460882432.0, + "48": 460882432.0, + "49": 460882432.0, + "50": 460882432.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 708781568.0, - "2": 885184000.0, - "3": 885184000.0, - "4": 885184000.0, - "5": 885184000.0, - "6": 885184000.0, - "7": 885184000.0, - "8": 885184000.0, - "9": 885184000.0, - "10": 885184000.0, - "11": 885184000.0, - "12": 885184000.0, - "13": 885184000.0, - "14": 885184000.0, - "15": 885184000.0, - "16": 885184000.0, - "17": 885184000.0, - "18": 885184000.0, - "19": 885184000.0, - "20": 885184000.0, - "21": 885184000.0, - "22": 885184000.0, - "23": 885184000.0, - "24": 885184000.0, - "25": 885184000.0, - "26": 885184000.0, - "27": 885184000.0, - "28": 885184000.0, - "29": 885184000.0, - "30": 885184000.0, - "31": 885184000.0, - "32": 885184000.0, - "33": 885184000.0, - "34": 885184000.0, - "35": 885184000.0, - "36": 885184000.0, - "37": 885184000.0, - "38": 885184000.0, - "39": 885184000.0, - "40": 885184000.0, - "41": 885184000.0, - "42": 885184000.0, - "43": 885184000.0, - "44": 885184000.0, - "45": 885184000.0, - "46": 885184000.0, - "47": 885184000.0, - "48": 885184000.0, - "49": 885184000.0, - "50": 885706752.0 + "1": 705635840.0, + "2": 883348992.0, + "3": 883348992.0, + "4": 883348992.0, + "5": 883348992.0, + "6": 883348992.0, + "7": 883348992.0, + "8": 883348992.0, + "9": 883348992.0, + "10": 883348992.0, + "11": 883348992.0, + "12": 883348992.0, + "13": 883348992.0, + "14": 883348992.0, + "15": 883348992.0, + "16": 883348992.0, + "17": 883348992.0, + "18": 883348992.0, + "19": 883348992.0, + "20": 883348992.0, + "21": 883348992.0, + "22": 883348992.0, + "23": 883348992.0, + "24": 883348992.0, + "25": 883348992.0, + "26": 883348992.0, + "27": 883348992.0, + "28": 883348992.0, + "29": 883348992.0, + "30": 883348992.0, + "31": 883348992.0, + "32": 883348992.0, + "33": 883348992.0, + "34": 883348992.0, + "35": 883348992.0, + "36": 883348992.0, + "37": 883348992.0, + "38": 883348992.0, + "39": 883348992.0, + "40": 883348992.0, + "41": 883348992.0, + "42": 883348992.0, + "43": 883348992.0, + "44": 883348992.0, + "45": 883348992.0, + "46": 883348992.0, + "47": 883348992.0, + "48": 883348992.0, + "49": 883348992.0, + "50": 883348992.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.83222, - "2": 0.46295, - "3": 0.52097, - "4": 0.42074, - "5": 0.72217, - "6": 0.70851, - "7": 0.41812, - "8": 0.41893, - "9": 0.47564, - "10": 0.48012, - "11": 0.41406, - "12": 0.43392, - "13": 0.67246, - "14": 0.41498, - "15": 0.47203, - "16": 0.46, - "17": 0.40996, - "18": 0.4104, - "19": 0.66865, - "20": 0.40782, - "21": 0.40774, - "22": 0.49273, - "23": 0.49254, - "24": 0.47511, - "25": 0.64062, - "26": 0.43231, - "27": 0.50003, - "28": 0.46605, - "29": 0.64224, - "30": 0.42576, - "31": 0.40898, - "32": 0.49354, - "33": 0.47014, - "34": 0.4075, - "35": 0.40863, - "36": 0.40508, - "37": 0.42937, - "38": 0.41009, - "39": 0.4116, - "40": 0.40987, - "41": 0.41014, - "42": 0.45949, - "43": 0.40849, - "44": 0.48462, - "45": 0.4567, - "46": 0.40779, - "47": 0.466, - "48": 0.41678, - "49": 0.40871, - "50": 0.41039 + "1": "nan", + "2": 4.21004, + "3": 0.48981, + "4": 0.47344, + "5": 0.47824, + "6": 0.47946, + "7": 0.48311, + "8": 0.4801, + "9": 0.48448, + "10": 0.48375, + "11": 0.48291, + "12": 0.48722, + "13": 0.48237, + "14": 0.48101, + "15": 0.48357, + "16": 0.48502, + "17": 0.48354, + "18": 0.48307, + "19": 0.48204, + "20": 0.48295, + "21": 0.48064, + "22": 0.48504, + "23": 0.48487, + "24": 0.48367, + "25": 0.48061, + "26": 0.48279, + "27": 0.48417, + "28": 0.48173, + "29": 0.48221, + "30": 0.48351, + "31": 0.48309, + "32": 0.48067, + "33": 0.48269, + "34": 0.48404, + "35": 0.48325, + "36": 0.48418, + "37": 0.48111, + "38": 0.4835, + "39": 0.48361, + "40": 0.48287, + "41": 0.4882, + "42": 0.48161, + "43": 0.48229, + "44": 0.48219, + "45": 0.48623, + "46": 0.48196, + "47": 0.48211, + "48": 0.48322, + "49": 0.4833, + "50": 0.48355 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json index cef90be5674..35348e75b0f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.86244, "2": 10.88582, - "3": 10.84731, - "4": 10.85576, - "5": 10.86004, - "6": 10.87726, - "7": 10.86557, - "8": 10.84915, + "3": 10.84734, + "4": 10.85574, + "5": 10.86001, + "6": 10.8773, + "7": 10.86559, + "8": 10.84909, "9": 10.86608, - "10": 10.82477, - "11": 10.85617, - "12": 10.85377, + "10": 10.82475, + "11": 10.85619, + "12": 10.85375, "13": 10.86788, - "14": 10.87113, - "15": 10.82238, - "16": 10.79992, - "17": 10.77432, - "18": 10.78346, - "19": 10.79308, - "20": 10.68227, - "21": 10.64715, - "22": 10.50914, - "23": 10.66831, - "24": 10.54198, - "25": 10.49277, - "26": 10.55935, - "27": 10.54235, + "14": 10.87116, + "15": 10.82232, + "16": 10.79995, + "17": 10.77433, + "18": 10.78345, + "19": 10.79307, + "20": 10.68225, + "21": 10.64714, + "22": 10.50916, + "23": 10.66829, + "24": 10.54192, + "25": 10.49279, + "26": 10.55934, + "27": 10.54241, "28": 10.51128, - "29": 10.53255, + "29": 10.53257, "30": 10.28988, - "31": 10.02851, - "32": 10.38874, - "33": 10.39594, - "34": 10.13449, - "35": 10.18929, - "36": 10.13352, - "37": 10.2738, + "31": 10.02847, + "32": 10.38882, + "33": 10.39596, + "34": 10.13452, + "35": 10.18931, + "36": 10.13354, + "37": 10.27379, "38": 10.10752, - "39": 10.3401, - "40": 9.98541, - "41": 10.06413, - "42": 10.13748, - "43": 9.73382, + "39": 10.34011, + "40": 9.98539, + "41": 10.06415, + "42": 10.13747, + "43": 9.73381, "44": 9.86306, "45": 9.73727, - "46": 9.7134, - "47": 10.07755, - "48": 9.76767, - "49": 9.4199, - "50": 9.81686 + "46": 9.71341, + "47": 10.07754, + "48": 9.76766, + "49": 9.41987, + "50": 9.81689 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 572.0, - "2": 631.0, - "3": 629.0, - "4": 629.0, - "5": 576.0, - "6": 654.0, - "7": 633.0, - "8": 620.0, - "9": 630.0, - "10": 541.0, - "11": 632.0, - "12": 603.0, - "13": 675.0, - "14": 617.0, - "15": 651.0, - "16": 622.0, - "17": 619.0, - "18": 628.0, - "19": 641.0, - "20": 610.0, - "21": 677.0, - "22": 572.0, - "23": 703.0, - "24": 624.0, - "25": 555.0, - "26": 601.0, - "27": 666.0, - "28": 749.0, - "29": 699.0, - "30": 756.0, - "31": 582.0, - "32": 733.0, + "1": 588.0, + "2": 591.0, + "3": 656.0, + "4": 602.0, + "5": 609.0, + "6": 600.0, + "7": 596.0, + "8": 640.0, + "9": 653.0, + "10": 535.0, + "11": 657.0, + "12": 620.0, + "13": 700.0, + "14": 630.0, + "15": 628.0, + "16": 590.0, + "17": 604.0, + "18": 566.0, + "19": 580.0, + "20": 561.0, + "21": 643.0, + "22": 622.0, + "23": 679.0, + "24": 611.0, + "25": 575.0, + "26": 628.0, + "27": 640.0, + "28": 727.0, + "29": 736.0, + "30": 729.0, + "31": 575.0, + "32": 726.0, "33": 773.0, - "34": 655.0, - "35": 710.0, - "36": 762.0, - "37": 863.0, - "38": 786.0, - "39": 846.0, - "40": 789.0, - "41": 795.0, - "42": 902.0, - "43": 758.0, - "44": 804.0, - "45": 751.0, - "46": 895.0, - "47": 815.0, - "48": 842.0, - "49": 851.0, - "50": 835.0 + "34": 634.0, + "35": 720.0, + "36": 690.0, + "37": 818.0, + "38": 730.0, + "39": 754.0, + "40": 809.0, + "41": 787.0, + "42": 849.0, + "43": 757.0, + "44": 861.0, + "45": 825.0, + "46": 881.0, + "47": 915.0, + "48": 846.0, + "49": 853.0, + "50": 816.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 463110656.0, + "2": 463110656.0, + "3": 463110656.0, + "4": 463110656.0, + "5": 463110656.0, + "6": 463110656.0, + "7": 463110656.0, + "8": 463110656.0, + "9": 463110656.0, + "10": 463110656.0, + "11": 463110656.0, + "12": 463110656.0, + "13": 463110656.0, + "14": 463110656.0, + "15": 463110656.0, + "16": 463110656.0, + "17": 463110656.0, + "18": 463110656.0, + "19": 463110656.0, + "20": 463110656.0, + "21": 463110656.0, + "22": 463110656.0, + "23": 463110656.0, + "24": 463110656.0, + "25": 463110656.0, + "26": 463110656.0, + "27": 463110656.0, + "28": 463110656.0, + "29": 463110656.0, + "30": 463110656.0, + "31": 463110656.0, + "32": 463110656.0, + "33": 463110656.0, + "34": 463110656.0, + "35": 463110656.0, + "36": 463110656.0, + "37": 463110656.0, + "38": 463110656.0, + "39": 463110656.0, + "40": 463110656.0, + "41": 463110656.0, + "42": 463110656.0, + "43": 463110656.0, + "44": 463110656.0, + "45": 463110656.0, + "46": 463110656.0, + "47": 463110656.0, + "48": 463110656.0, + "49": 463110656.0, + "50": 463110656.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 708781568.0, - "2": 885183488.0, - "3": 885184000.0, - "4": 885184000.0, - "5": 885184000.0, - "6": 885184000.0, - "7": 885184000.0, - "8": 885184000.0, - "9": 885184000.0, - "10": 885184000.0, - "11": 885184000.0, - "12": 885184000.0, - "13": 885184000.0, - "14": 885184000.0, - "15": 885184000.0, - "16": 885184000.0, - "17": 885184000.0, - "18": 885184000.0, - "19": 885184000.0, - "20": 885184000.0, - "21": 885184000.0, - "22": 885184000.0, - "23": 885184000.0, - "24": 885184000.0, - "25": 885184000.0, - "26": 886232576.0, - "27": 886232576.0, - "28": 886232576.0, - "29": 886232576.0, - "30": 886232576.0, - "31": 886232576.0, - "32": 886232576.0, - "33": 886232576.0, - "34": 886232576.0, - "35": 886232576.0, - "36": 886232576.0, - "37": 886232576.0, - "38": 886232576.0, - "39": 886232576.0, - "40": 886232576.0, - "41": 886232576.0, - "42": 886232576.0, - "43": 886232576.0, - "44": 886232576.0, - "45": 886232576.0, - "46": 886232576.0, - "47": 886232576.0, - "48": 886232576.0, - "49": 886232576.0, - "50": 886232576.0 + "1": 704587264.0, + "2": 887671808.0, + "3": 887671808.0, + "4": 887671808.0, + "5": 887672320.0, + "6": 887672320.0, + "7": 887672320.0, + "8": 887674368.0, + "9": 887674368.0, + "10": 887674368.0, + "11": 887674368.0, + "12": 887674368.0, + "13": 887674368.0, + "14": 887674368.0, + "15": 887674368.0, + "16": 887674368.0, + "17": 887674368.0, + "18": 887674368.0, + "19": 887674368.0, + "20": 887674368.0, + "21": 887674368.0, + "22": 887674368.0, + "23": 887674368.0, + "24": 887674368.0, + "25": 887674368.0, + "26": 887674368.0, + "27": 887674368.0, + "28": 887674368.0, + "29": 887674368.0, + "30": 887674368.0, + "31": 887674368.0, + "32": 887674368.0, + "33": 887674368.0, + "34": 887674368.0, + "35": 887674368.0, + "36": 887674368.0, + "37": 887674368.0, + "38": 887674368.0, + "39": 887674368.0, + "40": 887674368.0, + "41": 887674368.0, + "42": 887674368.0, + "43": 887674368.0, + "44": 887674368.0, + "45": 887674368.0, + "46": 887674368.0, + "47": 887674368.0, + "48": 887674368.0, + "49": 887674368.0, + "50": 887674368.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 14.90548, - "2": 0.59116, - "3": 0.51351, - "4": 0.5889, - "5": 0.44588, - "6": 0.48318, - "7": 0.40946, - "8": 0.41291, - "9": 0.4711, - "10": 0.46604, - "11": 0.41089, - "12": 0.48863, - "13": 0.50268, - "14": 0.46761, - "15": 0.4075, - "16": 0.43179, - "17": 0.40649, - "18": 0.46497, - "19": 0.40807, - "20": 0.40657, - "21": 0.4151, - "22": 0.47269, - "23": 0.61429, - "24": 0.46129, - "25": 0.40977, - "26": 0.40692, - "27": 0.40603, - "28": 0.77632, - "29": 0.40782, - "30": 0.40901, - "31": 0.40545, - "32": 0.47343, - "33": 0.40648, - "34": 0.40452, - "35": 0.40862, - "36": 0.40878, - "37": 0.40927, - "38": 0.4062, - "39": 0.40929, - "40": 0.40755, - "41": 0.4034, - "42": 0.40739, - "43": 0.5793, - "44": 0.42611, - "45": 0.46136, - "46": 0.40554, - "47": 0.45264, - "48": 0.45209, - "49": 0.40299, - "50": 0.40119 + "1": "nan", + "2": 4.32392, + "3": 0.4432, + "4": 0.41997, + "5": 0.42011, + "6": 0.41602, + "7": 0.41695, + "8": 0.42042, + "9": 0.41532, + "10": 0.41013, + "11": 0.40983, + "12": 0.41104, + "13": 0.41182, + "14": 0.41252, + "15": 0.42002, + "16": 0.47232, + "17": 0.49274, + "18": 0.49507, + "19": 0.49112, + "20": 0.48715, + "21": 0.48361, + "22": 0.48476, + "23": 0.4789, + "24": 0.47778, + "25": 0.4792, + "26": 0.48432, + "27": 0.48617, + "28": 0.48159, + "29": 0.48042, + "30": 0.47772, + "31": 0.47956, + "32": 0.47326, + "33": 0.4727, + "34": 0.47303, + "35": 0.46857, + "36": 0.46923, + "37": 0.46968, + "38": 0.4682, + "39": 0.45815, + "40": 0.45997, + "41": 0.45486, + "42": 0.45349, + "43": 0.44331, + "44": 0.44252, + "45": 0.44141, + "46": 0.44016, + "47": 0.43955, + "48": 0.43852, + "49": 0.43914, + "50": 0.43791 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_nondeterministic/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_nondeterministic/golden_values_dev_dgx_gb200.json index 4b1a17aa98b..08b446921f5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_nondeterministic/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cp2_nondeterministic/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.86244, "2": 10.88582, - "3": 10.84732, + "3": 10.84734, "4": 10.85571, - "5": 10.86004, + "5": 10.86005, "6": 10.87729, - "7": 10.8656, - "8": 10.84913, + "7": 10.86561, + "8": 10.84911, "9": 10.86607, - "10": 10.82475, - "11": 10.85616, - "12": 10.85374, - "13": 10.86787, - "14": 10.87114, - "15": 10.82231, + "10": 10.82477, + "11": 10.85618, + "12": 10.85372, + "13": 10.8679, + "14": 10.87115, + "15": 10.82234, "16": 10.79992, - "17": 10.77434, - "18": 10.7835, - "19": 10.79308, - "20": 10.68228, - "21": 10.64713, - "22": 10.50916, - "23": 10.66826, - "24": 10.54197, - "25": 10.49279, - "26": 10.55934, - "27": 10.54238, - "28": 10.51131, - "29": 10.53257, - "30": 10.28989, - "31": 10.0285, - "32": 10.38879, - "33": 10.39594, - "34": 10.13454, - "35": 10.18927, - "36": 10.13356, - "37": 10.27378, - "38": 10.10748, - "39": 10.34013, - "40": 9.98543, - "41": 10.06417, - "42": 10.1375, + "17": 10.7743, + "18": 10.78345, + "19": 10.79307, + "20": 10.68226, + "21": 10.6471, + "22": 10.50919, + "23": 10.66829, + "24": 10.54194, + "25": 10.49284, + "26": 10.55935, + "27": 10.54236, + "28": 10.51129, + "29": 10.53259, + "30": 10.28991, + "31": 10.02854, + "32": 10.38881, + "33": 10.39595, + "34": 10.13448, + "35": 10.18931, + "36": 10.13349, + "37": 10.2738, + "38": 10.10751, + "39": 10.34007, + "40": 9.98538, + "41": 10.06415, + "42": 10.13746, "43": 9.73384, - "44": 9.86307, - "45": 9.7372, - "46": 9.71343, - "47": 10.07757, - "48": 9.76764, - "49": 9.41992, - "50": 9.81691 + "44": 9.86303, + "45": 9.73723, + "46": 9.71344, + "47": 10.07753, + "48": 9.76766, + "49": 9.4199, + "50": 9.8169 } }, "num-zeros": { @@ -61,55 +61,55 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 550.0, - "2": 584.0, - "3": 581.0, - "4": 611.0, - "5": 630.0, - "6": 629.0, - "7": 619.0, - "8": 582.0, - "9": 634.0, - "10": 525.0, - "11": 701.0, - "12": 622.0, - "13": 670.0, - "14": 615.0, - "15": 638.0, - "16": 596.0, - "17": 645.0, - "18": 555.0, - "19": 607.0, - "20": 560.0, - "21": 667.0, - "22": 599.0, - "23": 676.0, - "24": 660.0, - "25": 619.0, - "26": 595.0, - "27": 638.0, - "28": 707.0, - "29": 680.0, - "30": 693.0, - "31": 607.0, - "32": 698.0, - "33": 774.0, - "34": 696.0, - "35": 699.0, - "36": 674.0, - "37": 897.0, - "38": 818.0, - "39": 882.0, - "40": 873.0, - "41": 746.0, - "42": 836.0, - "43": 808.0, - "44": 829.0, - "45": 757.0, - "46": 877.0, - "47": 932.0, - "48": 892.0, - "49": 861.0, + "1": 561.0, + "2": 574.0, + "3": 615.0, + "4": 612.0, + "5": 664.0, + "6": 648.0, + "7": 593.0, + "8": 587.0, + "9": 622.0, + "10": 528.0, + "11": 652.0, + "12": 592.0, + "13": 640.0, + "14": 634.0, + "15": 646.0, + "16": 666.0, + "17": 596.0, + "18": 617.0, + "19": 635.0, + "20": 582.0, + "21": 698.0, + "22": 608.0, + "23": 643.0, + "24": 645.0, + "25": 584.0, + "26": 619.0, + "27": 669.0, + "28": 702.0, + "29": 714.0, + "30": 683.0, + "31": 604.0, + "32": 722.0, + "33": 758.0, + "34": 674.0, + "35": 705.0, + "36": 782.0, + "37": 828.0, + "38": 796.0, + "39": 884.0, + "40": 832.0, + "41": 821.0, + "42": 813.0, + "43": 749.0, + "44": 856.0, + "45": 792.0, + "46": 774.0, + "47": 914.0, + "48": 832.0, + "49": 821.0, "50": 871.0 } }, @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0 + "1": 460882432.0, + "2": 460882432.0, + "3": 460882432.0, + "4": 460882432.0, + "5": 460882432.0, + "6": 460882432.0, + "7": 460882432.0, + "8": 460882432.0, + "9": 460882432.0, + "10": 460882432.0, + "11": 460882432.0, + "12": 460882432.0, + "13": 460882432.0, + "14": 460882432.0, + "15": 460882432.0, + "16": 460882432.0, + "17": 460882432.0, + "18": 460882432.0, + "19": 460882432.0, + "20": 460882432.0, + "21": 460882432.0, + "22": 460882432.0, + "23": 460882432.0, + "24": 460882432.0, + "25": 460882432.0, + "26": 460882432.0, + "27": 460882432.0, + "28": 460882432.0, + "29": 460882432.0, + "30": 460882432.0, + "31": 460882432.0, + "32": 460882432.0, + "33": 460882432.0, + "34": 460882432.0, + "35": 460882432.0, + "36": 460882432.0, + "37": 460882432.0, + "38": 460882432.0, + "39": 460882432.0, + "40": 460882432.0, + "41": 460882432.0, + "42": 460882432.0, + "43": 460882432.0, + "44": 460882432.0, + "45": 460882432.0, + "46": 460882432.0, + "47": 460882432.0, + "48": 460882432.0, + "49": 460882432.0, + "50": 460882432.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 708781568.0, - "2": 885184000.0, - "3": 885184000.0, - "4": 885184000.0, - "5": 885184000.0, - "6": 885184000.0, - "7": 885184000.0, - "8": 885184000.0, - "9": 885184000.0, - "10": 885184000.0, - "11": 885184000.0, - "12": 885184000.0, - "13": 885184000.0, - "14": 885184000.0, - "15": 885184000.0, - "16": 885184000.0, - "17": 885184000.0, - "18": 885184000.0, - "19": 885184000.0, - "20": 885184000.0, - "21": 885184000.0, - "22": 885184000.0, - "23": 886232064.0, - "24": 886232064.0, - "25": 886232064.0, - "26": 886232064.0, - "27": 886232064.0, - "28": 886232064.0, - "29": 886232064.0, - "30": 886232064.0, - "31": 886232064.0, - "32": 886232064.0, - "33": 886232064.0, - "34": 886232064.0, - "35": 886232064.0, - "36": 886232064.0, - "37": 886232064.0, - "38": 886232064.0, - "39": 886232064.0, - "40": 886232064.0, - "41": 886232064.0, - "42": 886232064.0, - "43": 886232064.0, - "44": 886232064.0, - "45": 886232064.0, - "46": 886232064.0, - "47": 886232064.0, - "48": 886232064.0, - "49": 886232064.0, - "50": 886232064.0 + "1": 705635328.0, + "2": 883348992.0, + "3": 884397568.0, + "4": 885445632.0, + "5": 885445632.0, + "6": 885445632.0, + "7": 885445632.0, + "8": 885445632.0, + "9": 885445632.0, + "10": 885445632.0, + "11": 885445632.0, + "12": 885445632.0, + "13": 885445632.0, + "14": 885445632.0, + "15": 885445632.0, + "16": 885445632.0, + "17": 885445632.0, + "18": 885445632.0, + "19": 885445632.0, + "20": 885445632.0, + "21": 885445632.0, + "22": 885445632.0, + "23": 885446144.0, + "24": 885446144.0, + "25": 885446144.0, + "26": 885446144.0, + "27": 885446144.0, + "28": 885446144.0, + "29": 885446144.0, + "30": 885446144.0, + "31": 885446144.0, + "32": 885446144.0, + "33": 885446144.0, + "34": 885446144.0, + "35": 885446144.0, + "36": 886492672.0, + "37": 886492672.0, + "38": 886492672.0, + "39": 886492672.0, + "40": 886492672.0, + "41": 886492672.0, + "42": 886492672.0, + "43": 886492672.0, + "44": 886492672.0, + "45": 886492672.0, + "46": 886492672.0, + "47": 886492672.0, + "48": 886492672.0, + "49": 886493696.0, + "50": 886493696.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 13.80388, - "2": 0.45981, - "3": 0.47688, - "4": 0.46506, - "5": 0.40776, - "6": 0.40391, - "7": 0.40648, - "8": 0.40522, - "9": 0.54467, - "10": 0.40469, - "11": 0.76012, - "12": 0.40772, - "13": 0.40474, - "14": 0.40399, - "15": 0.40126, - "16": 0.40258, - "17": 0.40163, - "18": 0.40308, - "19": 0.40205, - "20": 0.45775, - "21": 0.45253, - "22": 0.40222, - "23": 0.47993, - "24": 0.74746, - "25": 0.54096, - "26": 0.595, - "27": 0.42244, - "28": 0.45559, - "29": 0.40939, - "30": 0.40941, - "31": 0.40631, - "32": 0.40777, - "33": 0.40662, - "34": 0.45082, - "35": 0.40861, - "36": 0.40683, - "37": 0.40916, - "38": 0.40762, - "39": 0.40423, - "40": 0.41411, - "41": 0.40792, - "42": 0.40703, - "43": 0.40488, - "44": 0.40689, - "45": 0.40884, - "46": 0.40591, - "47": 0.40461, - "48": 0.50976, - "49": 0.4042, - "50": 0.40707 + "1": "nan", + "2": 3.84499, + "3": 0.50052, + "4": 0.45861, + "5": 0.44336, + "6": 0.44062, + "7": 0.43954, + "8": 0.44061, + "9": 0.44129, + "10": 0.44028, + "11": 0.44106, + "12": 0.4893, + "13": 0.49424, + "14": 0.49729, + "15": 0.4969, + "16": 0.49673, + "17": 0.49876, + "18": 0.4992, + "19": 0.49565, + "20": 0.48635, + "21": 0.46659, + "22": 0.45563, + "23": 0.44842, + "24": 0.4425, + "25": 0.44486, + "26": 0.43654, + "27": 0.43626, + "28": 0.43493, + "29": 0.43571, + "30": 0.43296, + "31": 0.4336, + "32": 0.43346, + "33": 0.45798, + "34": 0.47046, + "35": 0.47986, + "36": 0.48443, + "37": 0.48862, + "38": 0.48621, + "39": 0.48674, + "40": 0.48663, + "41": 0.48915, + "42": 0.48901, + "43": 0.4567, + "44": 0.46536, + "45": 0.47673, + "46": 0.48141, + "47": 0.48283, + "48": 0.4896, + "49": 0.48736, + "50": 0.50085 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json index 529bad10ded..7fa492bd7eb 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.86836, "2": 10.88595, - "3": 10.8656, + "3": 10.86559, "4": 10.86891, "5": 10.87418, - "6": 10.89058, - "7": 10.87677, - "8": 10.86475, - "9": 10.88236, - "10": 10.84579, - "11": 10.87162, - "12": 10.87422, - "13": 10.88161, - "14": 10.88886, - "15": 10.83932, + "6": 10.89057, + "7": 10.87676, + "8": 10.86476, + "9": 10.88235, + "10": 10.84582, + "11": 10.87163, + "12": 10.87421, + "13": 10.8816, + "14": 10.88885, + "15": 10.83934, "16": 10.82496, - "17": 10.80144, - "18": 10.81234, - "19": 10.82153, - "20": 10.71934, - "21": 10.69091, - "22": 10.57427, - "23": 10.71091, + "17": 10.80146, + "18": 10.81236, + "19": 10.82152, + "20": 10.71935, + "21": 10.69086, + "22": 10.57422, + "23": 10.71096, "24": 10.59783, - "25": 10.55561, - "26": 10.61523, - "27": 10.60449, - "28": 10.56482, - "29": 10.58475, - "30": 10.3595, - "31": 10.12152, - "32": 10.45239, - "33": 10.45725, - "34": 10.21986, - "35": 10.26447, - "36": 10.21035, + "25": 10.55559, + "26": 10.61516, + "27": 10.60451, + "28": 10.56481, + "29": 10.58476, + "30": 10.35947, + "31": 10.12155, + "32": 10.45234, + "33": 10.45724, + "34": 10.21987, + "35": 10.26441, + "36": 10.21037, "37": 10.33955, - "38": 10.18013, - "39": 10.39593, - "40": 10.06628, - "41": 10.14163, - "42": 10.2085, - "43": 9.83126, - "44": 9.9486, - "45": 9.82846, - "46": 9.80461, - "47": 10.14231, - "48": 9.84461, - "49": 9.52191, - "50": 9.88605 + "38": 10.18012, + "39": 10.39592, + "40": 10.06635, + "41": 10.14165, + "42": 10.20849, + "43": 9.83127, + "44": 9.94857, + "45": 9.82845, + "46": 9.80455, + "47": 10.14227, + "48": 9.84463, + "49": 9.52192, + "50": 9.88604 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1554.0, - "2": 1619.0, - "3": 1663.0, + "1": 1673.0, + "2": 1596.0, + "3": 1676.0, "4": 1672.0, - "5": 1771.0, - "6": 1739.0, - "7": 1866.0, - "8": 1590.0, - "9": 1819.0, - "10": 1394.0, - "11": 1786.0, - "12": 1643.0, - "13": 1829.0, - "14": 1672.0, - "15": 1827.0, - "16": 1771.0, - "17": 1797.0, - "18": 1632.0, - "19": 1667.0, - "20": 1670.0, - "21": 1843.0, - "22": 1620.0, - "23": 1889.0, - "24": 1513.0, - "25": 1473.0, - "26": 1619.0, - "27": 1768.0, - "28": 1976.0, - "29": 1898.0, - "30": 1858.0, - "31": 1565.0, - "32": 1890.0, - "33": 2166.0, - "34": 1679.0, - "35": 1825.0, - "36": 1909.0, - "37": 2341.0, - "38": 2029.0, - "39": 2294.0, - "40": 2015.0, - "41": 2181.0, - "42": 2211.0, - "43": 1907.0, - "44": 2140.0, - "45": 1936.0, - "46": 2341.0, - "47": 2472.0, - "48": 2272.0, - "49": 2234.0, - "50": 2457.0 + "5": 1818.0, + "6": 1740.0, + "7": 1845.0, + "8": 1651.0, + "9": 1820.0, + "10": 1351.0, + "11": 1811.0, + "12": 1655.0, + "13": 1748.0, + "14": 1719.0, + "15": 1801.0, + "16": 1829.0, + "17": 1828.0, + "18": 1545.0, + "19": 1727.0, + "20": 1654.0, + "21": 1874.0, + "22": 1567.0, + "23": 1955.0, + "24": 1609.0, + "25": 1474.0, + "26": 1750.0, + "27": 1682.0, + "28": 1927.0, + "29": 1949.0, + "30": 1837.0, + "31": 1606.0, + "32": 1849.0, + "33": 2085.0, + "34": 1799.0, + "35": 1933.0, + "36": 1928.0, + "37": 2325.0, + "38": 2099.0, + "39": 2424.0, + "40": 2112.0, + "41": 2240.0, + "42": 2181.0, + "43": 1934.0, + "44": 2042.0, + "45": 2041.0, + "46": 2183.0, + "47": 2424.0, + "48": 2250.0, + "49": 2208.0, + "50": 2425.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 465601024.0, - "2": 465601024.0, - "3": 465601024.0, - "4": 465601024.0, - "5": 465601024.0, - "6": 465601024.0, - "7": 465601024.0, - "8": 465601024.0, - "9": 465601024.0, - "10": 465601024.0, - "11": 465601024.0, - "12": 465601024.0, - "13": 465601024.0, - "14": 465601024.0, - "15": 465601024.0, - "16": 465601024.0, - "17": 465601024.0, - "18": 465601024.0, - "19": 465601024.0, - "20": 465601024.0, - "21": 465601024.0, - "22": 465601024.0, - "23": 465601024.0, - "24": 465601024.0, - "25": 465601024.0, - "26": 465601024.0, - "27": 465601024.0, - "28": 465601024.0, - "29": 465601024.0, - "30": 465601024.0, - "31": 465601024.0, - "32": 465601024.0, - "33": 465601024.0, - "34": 465601024.0, - "35": 465601024.0, - "36": 465601024.0, - "37": 465601024.0, - "38": 465601024.0, - "39": 465601024.0, - "40": 465601024.0, - "41": 465601024.0, - "42": 465601024.0, - "43": 465601024.0, - "44": 465601024.0, - "45": 465601024.0, - "46": 465601024.0, - "47": 465601024.0, - "48": 465601024.0, - "49": 465601024.0, - "50": 465601024.0 + "1": 465207808.0, + "2": 466256384.0, + "3": 466256384.0, + "4": 466256384.0, + "5": 466256384.0, + "6": 466256384.0, + "7": 466256384.0, + "8": 466256384.0, + "9": 466256384.0, + "10": 466256384.0, + "11": 466256384.0, + "12": 466256384.0, + "13": 466256384.0, + "14": 466256384.0, + "15": 466256384.0, + "16": 466256384.0, + "17": 466256384.0, + "18": 466256384.0, + "19": 466256384.0, + "20": 466256384.0, + "21": 466256384.0, + "22": 466256384.0, + "23": 466256384.0, + "24": 466256384.0, + "25": 466256384.0, + "26": 466256384.0, + "27": 466256384.0, + "28": 466256384.0, + "29": 466256384.0, + "30": 466256384.0, + "31": 466256384.0, + "32": 466256384.0, + "33": 466256384.0, + "34": 466256384.0, + "35": 466256384.0, + "36": 466256384.0, + "37": 466256384.0, + "38": 466256384.0, + "39": 466256384.0, + "40": 466256384.0, + "41": 466256384.0, + "42": 466256384.0, + "43": 466256384.0, + "44": 466256384.0, + "45": 466256384.0, + "46": 466256384.0, + "47": 466256384.0, + "48": 466256384.0, + "49": 466256384.0, + "50": 466256384.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1728999424.0, - "2": 1789405696.0, - "3": 1789405696.0, - "4": 1789405696.0, - "5": 1789405696.0, - "6": 1789405696.0, - "7": 1789405696.0, - "8": 1789405696.0, - "9": 1789405696.0, - "10": 1789405696.0, - "11": 1789405696.0, - "12": 1789405696.0, - "13": 1789405696.0, - "14": 1789405696.0, - "15": 1789405696.0, - "16": 1789405696.0, - "17": 1789405696.0, - "18": 1789405696.0, - "19": 1789405696.0, - "20": 1789405696.0, - "21": 1789405696.0, - "22": 1789405696.0, - "23": 1789405696.0, - "24": 1789405696.0, - "25": 1789405696.0, - "26": 1789405696.0, - "27": 1789405696.0, - "28": 1789405696.0, - "29": 1789405696.0, - "30": 1789405696.0, - "31": 1789405696.0, - "32": 1789405696.0, - "33": 1789405696.0, - "34": 1789405696.0, - "35": 1789405696.0, - "36": 1789405696.0, - "37": 1789405696.0, - "38": 1789405696.0, - "39": 1789405696.0, - "40": 1789405696.0, - "41": 1789405696.0, - "42": 1789405696.0, - "43": 1789405696.0, - "44": 1789405696.0, - "45": 1789405696.0, - "46": 1789405696.0, - "47": 1789405696.0, - "48": 1789405696.0, - "49": 1789405696.0, - "50": 1789405696.0 + "1": 1608546816.0, + "2": 1789536768.0, + "3": 1789536768.0, + "4": 1789536768.0, + "5": 1789536768.0, + "6": 1789536768.0, + "7": 1789536768.0, + "8": 1789536768.0, + "9": 1789536768.0, + "10": 1789536768.0, + "11": 1789536768.0, + "12": 1789536768.0, + "13": 1789536768.0, + "14": 1789536768.0, + "15": 1789536768.0, + "16": 1789536768.0, + "17": 1789536768.0, + "18": 1789536768.0, + "19": 1789536768.0, + "20": 1789536768.0, + "21": 1789536768.0, + "22": 1789536768.0, + "23": 1789536768.0, + "24": 1789536768.0, + "25": 1789536768.0, + "26": 1789536768.0, + "27": 1789536768.0, + "28": 1789536768.0, + "29": 1789536768.0, + "30": 1789536768.0, + "31": 1789536768.0, + "32": 1789536768.0, + "33": 1789536768.0, + "34": 1789536768.0, + "35": 1789536768.0, + "36": 1789536768.0, + "37": 1789536768.0, + "38": 1789536768.0, + "39": 1789536768.0, + "40": 1789536768.0, + "41": 1789536768.0, + "42": 1789536768.0, + "43": 1789536768.0, + "44": 1789536768.0, + "45": 1789536768.0, + "46": 1789536768.0, + "47": 1789536768.0, + "48": 1789536768.0, + "49": 1789536768.0, + "50": 1789536768.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.87728, - "2": 0.19403, - "3": 0.17442, - "4": 0.16292, - "5": 0.28152, - "6": 0.1602, - "7": 0.20711, - "8": 0.4188, - "9": 0.1573, - "10": 0.25278, - "11": 0.44625, - "12": 0.23028, - "13": 0.16929, - "14": 0.15589, - "15": 0.24336, - "16": 0.19322, - "17": 0.19037, - "18": 0.15335, - "19": 0.25153, - "20": 0.20655, - "21": 0.15398, - "22": 0.15177, - "23": 0.25777, - "24": 0.15477, - "25": 0.15108, - "26": 0.25255, - "27": 0.23256, - "28": 0.16156, - "29": 0.33982, - "30": 0.15402, - "31": 0.15482, - "32": 0.15494, - "33": 0.15494, - "34": 0.15448, - "35": 0.15383, - "36": 0.15383, - "37": 0.15343, - "38": 0.15387, - "39": 0.15805, - "40": 0.15334, - "41": 0.24971, - "42": 0.15713, - "43": 0.22532, - "44": 0.15365, - "45": 0.41087, - "46": 0.15392, - "47": 0.15221, - "48": 0.23644, - "49": 0.1534, - "50": 0.15283 + "1": "nan", + "2": 2.74426, + "3": 0.17387, + "4": 0.16297, + "5": 0.16176, + "6": 0.16395, + "7": 0.16386, + "8": 0.16289, + "9": 0.16327, + "10": 0.16443, + "11": 0.16326, + "12": 0.16343, + "13": 0.16293, + "14": 0.16535, + "15": 0.16415, + "16": 0.16437, + "17": 0.16472, + "18": 0.16441, + "19": 0.1632, + "20": 0.16239, + "21": 0.1634, + "22": 0.16387, + "23": 0.16433, + "24": 0.1624, + "25": 0.16133, + "26": 0.16119, + "27": 0.16136, + "28": 0.16141, + "29": 0.1614, + "30": 0.16073, + "31": 0.16146, + "32": 0.16038, + "33": 0.16019, + "34": 0.16109, + "35": 0.16035, + "36": 0.15933, + "37": 0.15978, + "38": 0.17485, + "39": 0.15932, + "40": 0.15877, + "41": 0.15919, + "42": 0.15903, + "43": 0.1594, + "44": 0.15734, + "45": 0.15857, + "46": 0.15791, + "47": 0.15837, + "48": 0.15781, + "49": 0.15813, + "50": 0.15862 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json index dac3e5ef607..31729dd5fe5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6.33065, - "2": 0.20464, - "3": 0.17836, - "4": 0.16429, - "5": 0.1621, - "6": 0.16051, - "7": 0.15983, - "8": 0.16067, - "9": 0.15721, - "10": 0.16774, - "11": 0.16215, - "12": 0.21737, - "13": 0.16028, - "14": 0.16036, - "15": 0.15885, - "16": 0.22707, - "17": 0.16509, - "18": 0.1691, - "19": 0.16736, - "20": 0.23508, - "21": 0.16682, - "22": 0.16204, - "23": 0.16527, - "24": 0.1694, - "25": 0.16972, - "26": 0.17668, - "27": 0.15612, - "28": 0.22357, - "29": 0.15777, - "30": 0.16518, - "31": 0.17111, - "32": 0.17188, - "33": 0.16413, - "34": 0.16509, - "35": 0.16886, - "36": 0.16871, - "37": 0.17188, - "38": 0.16901, - "39": 0.1672, - "40": 0.22409, - "41": 0.16827, - "42": 0.16744, - "43": 0.1668, - "44": 0.16817, - "45": 0.16681, - "46": 0.17004, - "47": 0.1702, - "48": 0.17085, - "49": 0.17174, - "50": 0.16979 + "1": "nan", + "2": 2.75152, + "3": 0.1678, + "4": 0.1543, + "5": 0.15772, + "6": 0.15798, + "7": 0.15886, + "8": 0.16038, + "9": 0.15983, + "10": 0.16009, + "11": 0.15881, + "12": 0.16004, + "13": 0.15648, + "14": 0.15396, + "15": 0.15394, + "16": 0.1544, + "17": 0.15329, + "18": 0.1539, + "19": 0.15442, + "20": 0.1521, + "21": 0.15368, + "22": 0.15287, + "23": 0.15397, + "24": 0.15553, + "25": 0.15617, + "26": 0.15925, + "27": 0.145, + "28": 0.14456, + "29": 0.14869, + "30": 0.15407, + "31": 0.15556, + "32": 0.15651, + "33": 0.15726, + "34": 0.1574, + "35": 0.15981, + "36": 0.16037, + "37": 0.16044, + "38": 0.15744, + "39": 0.15875, + "40": 0.15964, + "41": 0.15984, + "42": 0.1605, + "43": 0.15901, + "44": 0.16037, + "45": 0.1616, + "46": 0.16046, + "47": 0.16125, + "48": 0.16168, + "49": 0.1611, + "50": 0.15977 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..49a5e4f8a21 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_mla/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": 10.9101, + "27": 10.90559, + "28": 10.87901, + "29": 10.87862, + "30": 10.82431, + "31": 10.7917, + "32": 10.85763, + "33": 10.85278, + "34": 10.80465, + "35": 10.81124, + "36": 10.79299, + "37": 10.82161, + "38": 10.74654, + "39": 10.79066, + "40": 10.67639, + "41": 10.71189, + "42": 10.72663, + "43": 10.58635, + "44": 10.63487, + "45": 10.59555, + "46": 10.58202, + "47": 10.67878, + "48": 10.55683, + "49": 10.43321, + "50": 10.57623 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": 22611998.0, + "27": 22532132.0, + "28": 22516960.0, + "29": 22593572.0, + "30": 22695024.0, + "31": 23019244.0, + "32": 22648204.0, + "33": 22623192.0, + "34": 22899922.0, + "35": 22852560.0, + "36": 22652964.0, + "37": 22559866.0, + "38": 22960222.0, + "39": 22864432.0, + "40": 22721420.0, + "41": 22722086.0, + "42": 22730128.0, + "43": 23040178.0, + "44": 22809816.0, + "45": 22738252.0, + "46": 22947510.0, + "47": 22697018.0, + "48": 22992168.0, + "49": 22790946.0, + "50": 22969044.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": 389056000.0, + "27": 389056000.0, + "28": 389056000.0, + "29": 389056000.0, + "30": 389056000.0, + "31": 389056000.0, + "32": 389056000.0, + "33": 389056000.0, + "34": 389056000.0, + "35": 389056000.0, + "36": 389056000.0, + "37": 389056000.0, + "38": 389056000.0, + "39": 389056000.0, + "40": 389056000.0, + "41": 389056000.0, + "42": 389056000.0, + "43": 389056000.0, + "44": 389056000.0, + "45": 389056000.0, + "46": 389056000.0, + "47": 389056000.0, + "48": 389056000.0, + "49": 389056000.0, + "50": 389056000.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": 1247206912.0, + "27": 1247207936.0, + "28": 1247207936.0, + "29": 1247207936.0, + "30": 1247207936.0, + "31": 1247207936.0, + "32": 1247207936.0, + "33": 1247207936.0, + "34": 1247207936.0, + "35": 1247207936.0, + "36": 1247207936.0, + "37": 1247207936.0, + "38": 1247207936.0, + "39": 1247207936.0, + "40": 1247207936.0, + "41": 1247207936.0, + "42": 1247207936.0, + "43": 1247207936.0, + "44": 1247207936.0, + "45": 1247207936.0, + "46": 1247207936.0, + "47": 1247207936.0, + "48": 1247207936.0, + "49": 1247207936.0, + "50": 1247207936.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": 2.4291, + "28": 0.15494, + "29": 0.14099, + "30": 0.13913, + "31": 0.1391, + "32": 0.13835, + "33": 0.13909, + "34": 0.13882, + "35": 0.13918, + "36": 0.13936, + "37": 0.1396, + "38": 0.14038, + "39": 0.14154, + "40": 0.14205, + "41": 0.14186, + "42": 0.1401, + "43": 0.14017, + "44": 0.14, + "45": 0.13933, + "46": 0.13921, + "47": 0.13941, + "48": 0.13867, + "49": 0.14055, + "50": 0.14041 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json index 941c681adde..caa1e54ee64 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 6.23525, - "2": 0.24353, - "3": 0.25343, - "4": 0.21688, - "5": 0.2509, - "6": 0.23286, - "7": 0.23132, - "8": 0.23275, - "9": 0.23174, - "10": 0.28716, - "11": 0.23191, - "12": 0.23535, - "13": 0.26183, - "14": 0.23439, - "15": 0.26372, - "16": 0.27689, - "17": 0.31573, - "18": 0.29419, - "19": 0.22489, - "20": 0.21688, - "21": 0.21286, - "22": 0.21368, - "23": 0.21212, - "24": 0.21109, - "25": 0.21238, - "26": 0.21136, - "27": 0.24254, - "28": 0.21046, - "29": 0.21055, - "30": 0.37172, - "31": 0.20753, - "32": 0.22054, - "33": 0.20088, - "34": 0.20169, - "35": 0.2243, - "36": 0.20027, - "37": 0.20099, - "38": 0.21205, - "39": 0.20018, - "40": 0.19821, - "41": 0.20033, - "42": 0.20078, - "43": 0.19985, - "44": 0.19983, - "45": 0.19756, - "46": 0.19892, - "47": 0.19813, - "48": 0.19885, - "49": 0.19949, - "50": 0.19861, - "51": 0.20481, - "52": 0.18697, - "53": 0.18628, - "54": 0.18383, - "55": 0.22054, - "56": 0.18628, - "57": 0.1865, - "58": 0.23363, - "59": 0.18779, - "60": 0.18548, - "61": 0.23086, - "62": 0.18486, - "63": 0.18676, - "64": 0.18877, - "65": 0.18818, - "66": 0.18785, - "67": 0.18912, - "68": 0.18762, - "69": 0.18502, - "70": 0.2393, - "71": 0.18534, - "72": 0.1866, - "73": 0.18699, - "74": 0.2218, - "75": 0.18851, - "76": 0.18761, - "77": 0.18836, - "78": 0.22737, - "79": 0.18832, - "80": 0.18852, - "81": 0.2185, - "82": 0.18552, - "83": 0.19385, - "84": 0.18774, - "85": 0.1898, - "86": 0.3457, - "87": 0.4164, - "88": 0.18999, - "89": 0.1872, - "90": 0.18803, - "91": 0.22713, - "92": 0.18693, - "93": 0.18603, - "94": 0.18711, - "95": 0.18552, - "96": 0.22396, - "97": 0.18576, - "98": 0.18988, - "99": 0.21054, - "100": 0.21361 + "1": "nan", + "2": 2.48366, + "3": 0.20961, + "4": 0.19355, + "5": 0.19146, + "6": 0.19108, + "7": 0.19236, + "8": 0.19259, + "9": 0.19267, + "10": 0.19436, + "11": 0.19257, + "12": 0.19432, + "13": 0.19332, + "14": 0.19442, + "15": 0.19393, + "16": 0.19417, + "17": 0.19555, + "18": 0.19451, + "19": 0.19452, + "20": 0.19555, + "21": 0.19375, + "22": 0.19402, + "23": 0.19539, + "24": 0.19475, + "25": 0.19576, + "26": 0.19424, + "27": 0.19514, + "28": 0.19519, + "29": 0.19578, + "30": 0.19503, + "31": 0.19394, + "32": 0.19582, + "33": 0.19444, + "34": 0.19405, + "35": 0.19498, + "36": 0.19463, + "37": 0.19572, + "38": 0.19362, + "39": 0.19492, + "40": 0.19487, + "41": 0.19497, + "42": 0.19617, + "43": 0.19571, + "44": 0.19661, + "45": 0.19634, + "46": 0.19537, + "47": 0.19646, + "48": 0.19658, + "49": 0.19727, + "50": 0.19567, + "51": 0.21203, + "52": 0.19551, + "53": 0.19415, + "54": 0.19434, + "55": 0.19584, + "56": 0.19437, + "57": 0.19536, + "58": 0.20364, + "59": 0.20029, + "60": 0.1929, + "61": 0.19274, + "62": 0.19364, + "63": 0.19667, + "64": 0.19406, + "65": 0.19781, + "66": 0.19435, + "67": 0.19308, + "68": 0.1932, + "69": 0.19478, + "70": 0.19591, + "71": 0.19922, + "72": 0.19646, + "73": 0.19646, + "74": 0.19739, + "75": 0.19817, + "76": 0.20056, + "77": 0.19655, + "78": 0.19459, + "79": 0.19478, + "80": 0.19638, + "81": 0.19329, + "82": 0.19254, + "83": 0.19379, + "84": 0.19435, + "85": 0.19517, + "86": 0.19446, + "87": 0.19464, + "88": 0.19501, + "89": 0.19544, + "90": 0.19268, + "91": 0.19425, + "92": 0.1933, + "93": 0.19366, + "94": 0.19328, + "95": 0.19408, + "96": 0.19474, + "97": 0.19719, + "98": 0.19535, + "99": 0.19604, + "100": 0.19554 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..848d772bc72 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.84743, + "52": 9.74209, + "53": 10.05697, + "54": 9.9505, + "55": 9.88145, + "56": 9.61274, + "57": 9.4687, + "58": 9.82193, + "59": 9.57642, + "60": 9.49762, + "61": 9.69189, + "62": 9.9867, + "63": 9.37512, + "64": 9.76679, + "65": 8.94648, + "66": 9.7023, + "67": 9.36326, + "68": 9.7831, + "69": 9.7986, + "70": 9.7317, + "71": 9.62571, + "72": 9.58488, + "73": 9.48967, + "74": 8.9286, + "75": 9.40862, + "76": 9.07925, + "77": 10.0594, + "78": 9.72288, + "79": 9.37784, + "80": 9.40429, + "81": 9.48309, + "82": 9.7004, + "83": 9.31595, + "84": 9.41838, + "85": 9.61685, + "86": 9.07533, + "87": 9.59616, + "88": 9.75215, + "89": 9.60184, + "90": 9.82281, + "91": 9.34037, + "92": 9.35854, + "93": 9.08805, + "94": 8.83037, + "95": 9.5266, + "96": 9.53049, + "97": 9.30389, + "98": 9.67196, + "99": 8.89637, + "100": 9.40644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2491.0, + "52": 2552.0, + "53": 2980.0, + "54": 2680.0, + "55": 2274.0, + "56": 2734.0, + "57": 2319.0, + "58": 2907.0, + "59": 2886.0, + "60": 2566.0, + "61": 2855.0, + "62": 2704.0, + "63": 2370.0, + "64": 2998.0, + "65": 2563.0, + "66": 2868.0, + "67": 2762.0, + "68": 2739.0, + "69": 2730.0, + "70": 3156.0, + "71": 2803.0, + "72": 2506.0, + "73": 2896.0, + "74": 1937.0, + "75": 2450.0, + "76": 2794.0, + "77": 3047.0, + "78": 3104.0, + "79": 3069.0, + "80": 3286.0, + "81": 3543.0, + "82": 3192.0, + "83": 2614.0, + "84": 3273.0, + "85": 3111.0, + "86": 2680.0, + "87": 3654.0, + "88": 3117.0, + "89": 3351.0, + "90": 3086.0, + "91": 2721.0, + "92": 3045.0, + "93": 2672.0, + "94": 3326.0, + "95": 3125.0, + "96": 3309.0, + "97": 3208.0, + "98": 3572.0, + "99": 2980.0, + "100": 3355.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 515933696.0, + "52": 515933696.0, + "53": 515933696.0, + "54": 515933696.0, + "55": 515933696.0, + "56": 515933696.0, + "57": 515933696.0, + "58": 515933696.0, + "59": 515933696.0, + "60": 515933696.0, + "61": 515933696.0, + "62": 515933696.0, + "63": 515933696.0, + "64": 515933696.0, + "65": 515933696.0, + "66": 515933696.0, + "67": 515933696.0, + "68": 515933696.0, + "69": 515933696.0, + "70": 515933696.0, + "71": 515933696.0, + "72": 515933696.0, + "73": 515933696.0, + "74": 515933696.0, + "75": 515933696.0, + "76": 515933696.0, + "77": 515933696.0, + "78": 515933696.0, + "79": 515933696.0, + "80": 515933696.0, + "81": 515933696.0, + "82": 515933696.0, + "83": 515933696.0, + "84": 515933696.0, + "85": 515933696.0, + "86": 515933696.0, + "87": 515933696.0, + "88": 515933696.0, + "89": 515933696.0, + "90": 515933696.0, + "91": 515933696.0, + "92": 515933696.0, + "93": 515933696.0, + "94": 515933696.0, + "95": 515933696.0, + "96": 515933696.0, + "97": 515933696.0, + "98": 515933696.0, + "99": 515933696.0, + "100": 515933696.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1440754176.0, + "52": 1440755200.0, + "53": 1440755200.0, + "54": 1440755200.0, + "55": 1440755200.0, + "56": 1440755200.0, + "57": 1440755200.0, + "58": 1440755200.0, + "59": 1440755200.0, + "60": 1440755200.0, + "61": 1440755200.0, + "62": 1440755200.0, + "63": 1440755200.0, + "64": 1440755200.0, + "65": 1440755200.0, + "66": 1440755200.0, + "67": 1440755200.0, + "68": 1440755200.0, + "69": 1440755200.0, + "70": 1440755200.0, + "71": 1440755200.0, + "72": 1440755200.0, + "73": 1440755200.0, + "74": 1440755200.0, + "75": 1440755200.0, + "76": 1440755200.0, + "77": 1440755200.0, + "78": 1440755200.0, + "79": 1440755200.0, + "80": 1440755200.0, + "81": 1440755200.0, + "82": 1440755200.0, + "83": 1440755200.0, + "84": 1440755200.0, + "85": 1440755200.0, + "86": 1440755200.0, + "87": 1440755200.0, + "88": 1440755200.0, + "89": 1440755200.0, + "90": 1440755200.0, + "91": 1440755200.0, + "92": 1440755200.0, + "93": 1440755200.0, + "94": 1440755200.0, + "95": 1440755200.0, + "96": 1440755200.0, + "97": 1440755200.0, + "98": 1440755200.0, + "99": 1440755200.0, + "100": 1440755200.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.50794, + "53": 0.19671, + "54": 0.18376, + "55": 0.18594, + "56": 0.18674, + "57": 0.18474, + "58": 0.18412, + "59": 0.18456, + "60": 0.18456, + "61": 0.18623, + "62": 0.18524, + "63": 0.18624, + "64": 0.18621, + "65": 0.18695, + "66": 0.18541, + "67": 0.1857, + "68": 0.18575, + "69": 0.18658, + "70": 0.1875, + "71": 0.18753, + "72": 0.18718, + "73": 0.18797, + "74": 0.18972, + "75": 0.18765, + "76": 0.18764, + "77": 0.18827, + "78": 0.18801, + "79": 0.18785, + "80": 0.18903, + "81": 0.18889, + "82": 0.18772, + "83": 0.18876, + "84": 0.18791, + "85": 0.18973, + "86": 0.18948, + "87": 0.18998, + "88": 0.18905, + "89": 0.1898, + "90": 0.1895, + "91": 0.18953, + "92": 0.18969, + "93": 0.18888, + "94": 0.18888, + "95": 0.18773, + "96": 0.18832, + "97": 0.18919, + "98": 0.189, + "99": 0.1888, + "100": 0.188 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json index 500fc1be7cf..bffbd713b20 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.86244, "2": 10.88582, - "3": 10.84735, - "4": 10.85573, - "5": 10.86001, - "6": 10.87731, - "7": 10.86558, - "8": 10.84914, - "9": 10.86606, - "10": 10.82476, - "11": 10.85615, - "12": 10.85374, - "13": 10.8679, - "14": 10.87118, - "15": 10.82236, - "16": 10.79992, - "17": 10.77431, - "18": 10.78349, - "19": 10.79309, + "3": 10.84733, + "4": 10.85571, + "5": 10.86009, + "6": 10.87728, + "7": 10.86559, + "8": 10.84911, + "9": 10.86605, + "10": 10.82478, + "11": 10.85618, + "12": 10.85375, + "13": 10.86786, + "14": 10.87115, + "15": 10.82231, + "16": 10.79989, + "17": 10.77429, + "18": 10.78346, + "19": 10.79306, "20": 10.68226, - "21": 10.64711, - "22": 10.5092, - "23": 10.66829, - "24": 10.54196, - "25": 10.49278, - "26": 10.55935, - "27": 10.54234, + "21": 10.64716, + "22": 10.50918, + "23": 10.66831, + "24": 10.54197, + "25": 10.49277, + "26": 10.55929, + "27": 10.54236, "28": 10.5113, - "29": 10.53259, - "30": 10.28989, - "31": 10.0285, - "32": 10.38878, - "33": 10.39596, - "34": 10.13451, - "35": 10.18928, - "36": 10.13355, - "37": 10.2738, - "38": 10.10751, - "39": 10.3401, - "40": 9.98543, - "41": 10.06416, - "42": 10.13751, - "43": 9.73383, - "44": 9.86311, - "45": 9.73722, - "46": 9.71346, + "29": 10.53258, + "30": 10.28987, + "31": 10.02854, + "32": 10.38881, + "33": 10.39595, + "34": 10.13452, + "35": 10.1893, + "36": 10.13353, + "37": 10.27381, + "38": 10.10749, + "39": 10.34007, + "40": 9.98535, + "41": 10.06411, + "42": 10.13748, + "43": 9.73379, + "44": 9.86307, + "45": 9.73726, + "46": 9.71341, "47": 10.07754, - "48": 9.76768, - "49": 9.41986, - "50": 9.81686, - "51": 9.77423, - "52": 9.66446, - "53": 10.00148, - "54": 9.89157, - "55": 9.8185, - "56": 9.54335, - "57": 9.39451, - "58": 9.76569, - "59": 9.50934, - "60": 9.42824, + "48": 9.76762, + "49": 9.41989, + "50": 9.81685, + "51": 9.7742, + "52": 9.66444, + "53": 10.00151, + "54": 9.89155, + "55": 9.81852, + "56": 9.54337, + "57": 9.39452, + "58": 9.76573, + "59": 9.50935, + "60": 9.42821, "61": 9.63468, - "62": 9.93888, + "62": 9.93891, "63": 9.30458, "64": 9.70984, - "65": 8.86892, - "66": 9.64956, - "67": 9.30818, - "68": 9.73508, - "69": 9.75593, - "70": 9.68707, - "71": 9.57532, - "72": 9.53074, - "73": 9.43675, - "74": 8.85588, - "75": 9.35531, - "76": 9.01375, - "77": 10.0245, - "78": 9.68203, - "79": 9.33141, - "80": 9.35466, - "81": 9.43622, - "82": 9.65854, - "83": 9.26268, - "84": 9.3692, - "85": 9.57098, - "86": 9.03323, - "87": 9.55969, - "88": 9.71078, - "89": 9.5541, + "65": 8.86888, + "66": 9.64952, + "67": 9.30815, + "68": 9.73505, + "69": 9.75596, + "70": 9.68706, + "71": 9.57535, + "72": 9.53075, + "73": 9.43678, + "74": 8.85586, + "75": 9.35532, + "76": 9.01377, + "77": 10.02449, + "78": 9.68205, + "79": 9.33138, + "80": 9.35467, + "81": 9.43621, + "82": 9.65855, + "83": 9.26269, + "84": 9.36921, + "85": 9.57104, + "86": 9.0332, + "87": 9.55973, + "88": 9.71077, + "89": 9.55411, "90": 9.78662, - "91": 9.2909, + "91": 9.29091, "92": 9.31236, "93": 9.03976, - "94": 8.78109, - "95": 9.49172, - "96": 9.49067, - "97": 9.25826, - "98": 9.62998, - "99": 8.84685, - "100": 9.36201 + "94": 8.78112, + "95": 9.49175, + "96": 9.49071, + "97": 9.25827, + "98": 9.63001, + "99": 8.84688, + "100": 9.36199 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 571.0, - "2": 604.0, - "3": 655.0, - "4": 633.0, - "5": 602.0, - "6": 699.0, - "7": 585.0, - "8": 589.0, - "9": 633.0, - "10": 527.0, - "11": 628.0, - "12": 602.0, - "13": 671.0, - "14": 627.0, - "15": 591.0, - "16": 605.0, - "17": 666.0, - "18": 604.0, - "19": 631.0, - "20": 604.0, - "21": 706.0, - "22": 598.0, - "23": 682.0, - "24": 656.0, - "25": 593.0, - "26": 615.0, - "27": 681.0, - "28": 693.0, - "29": 701.0, - "30": 699.0, - "31": 564.0, - "32": 781.0, - "33": 724.0, - "34": 679.0, - "35": 711.0, + "1": 562.0, + "2": 595.0, + "3": 621.0, + "4": 618.0, + "5": 626.0, + "6": 671.0, + "7": 641.0, + "8": 647.0, + "9": 601.0, + "10": 522.0, + "11": 648.0, + "12": 639.0, + "13": 681.0, + "14": 686.0, + "15": 638.0, + "16": 620.0, + "17": 601.0, + "18": 556.0, + "19": 628.0, + "20": 551.0, + "21": 695.0, + "22": 620.0, + "23": 708.0, + "24": 553.0, + "25": 566.0, + "26": 597.0, + "27": 672.0, + "28": 717.0, + "29": 763.0, + "30": 694.0, + "31": 627.0, + "32": 696.0, + "33": 820.0, + "34": 674.0, + "35": 741.0, "36": 733.0, - "37": 858.0, - "38": 794.0, - "39": 789.0, - "40": 857.0, - "41": 739.0, - "42": 856.0, - "43": 742.0, - "44": 798.0, - "45": 772.0, - "46": 872.0, - "47": 941.0, - "48": 838.0, - "49": 799.0, - "50": 840.0, - "51": 961.0, - "52": 952.0, - "53": 1057.0, - "54": 932.0, - "55": 849.0, - "56": 986.0, - "57": 853.0, - "58": 963.0, - "59": 1059.0, - "60": 895.0, - "61": 999.0, - "62": 967.0, + "37": 848.0, + "38": 788.0, + "39": 863.0, + "40": 812.0, + "41": 813.0, + "42": 812.0, + "43": 706.0, + "44": 810.0, + "45": 732.0, + "46": 863.0, + "47": 914.0, + "48": 886.0, + "49": 786.0, + "50": 872.0, + "51": 952.0, + "52": 963.0, + "53": 1095.0, + "54": 956.0, + "55": 844.0, + "56": 969.0, + "57": 831.0, + "58": 985.0, + "59": 1062.0, + "60": 868.0, + "61": 975.0, + "62": 897.0, "63": 928.0, - "64": 1046.0, - "65": 974.0, - "66": 998.0, - "67": 1078.0, - "68": 987.0, - "69": 976.0, - "70": 1112.0, - "71": 1031.0, - "72": 889.0, - "73": 1009.0, - "74": 778.0, - "75": 839.0, - "76": 1017.0, - "77": 1069.0, - "78": 1111.0, - "79": 1041.0, - "80": 1089.0, - "81": 1169.0, - "82": 1034.0, - "83": 951.0, - "84": 1098.0, - "85": 1124.0, - "86": 816.0, - "87": 1218.0, - "88": 1128.0, - "89": 1147.0, - "90": 1130.0, - "91": 1096.0, - "92": 1132.0, - "93": 900.0, - "94": 1119.0, - "95": 1095.0, - "96": 1160.0, - "97": 1006.0, - "98": 1240.0, - "99": 1141.0, - "100": 1108.0 + "64": 1085.0, + "65": 1058.0, + "66": 1068.0, + "67": 966.0, + "68": 999.0, + "69": 1021.0, + "70": 1103.0, + "71": 1068.0, + "72": 884.0, + "73": 1027.0, + "74": 757.0, + "75": 818.0, + "76": 981.0, + "77": 1091.0, + "78": 1135.0, + "79": 1105.0, + "80": 1126.0, + "81": 1181.0, + "82": 1095.0, + "83": 981.0, + "84": 1154.0, + "85": 1139.0, + "86": 804.0, + "87": 1216.0, + "88": 1139.0, + "89": 1113.0, + "90": 1071.0, + "91": 1180.0, + "92": 1100.0, + "93": 846.0, + "94": 1155.0, + "95": 1071.0, + "96": 1123.0, + "97": 1074.0, + "98": 1188.0, + "99": 1161.0, + "100": 1153.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 459571712.0, - "2": 459571712.0, - "3": 459571712.0, - "4": 459571712.0, - "5": 459571712.0, - "6": 459571712.0, - "7": 459571712.0, - "8": 459571712.0, - "9": 459571712.0, - "10": 459571712.0, - "11": 459571712.0, - "12": 459571712.0, - "13": 459571712.0, - "14": 459571712.0, - "15": 459571712.0, - "16": 459571712.0, - "17": 459571712.0, - "18": 459571712.0, - "19": 459571712.0, - "20": 459571712.0, - "21": 459571712.0, - "22": 459571712.0, - "23": 459571712.0, - "24": 459571712.0, - "25": 459571712.0, - "26": 459571712.0, - "27": 459571712.0, - "28": 459571712.0, - "29": 459571712.0, - "30": 459571712.0, - "31": 459571712.0, - "32": 459571712.0, - "33": 459571712.0, - "34": 459571712.0, - "35": 459571712.0, - "36": 459571712.0, - "37": 459571712.0, - "38": 459571712.0, - "39": 459571712.0, - "40": 459571712.0, - "41": 459571712.0, - "42": 459571712.0, - "43": 459571712.0, - "44": 459571712.0, - "45": 459571712.0, - "46": 459571712.0, - "47": 459571712.0, - "48": 459571712.0, - "49": 459571712.0, - "50": 459571712.0, - "51": 459571712.0, - "52": 459571712.0, - "53": 459571712.0, - "54": 459571712.0, - "55": 459571712.0, - "56": 459571712.0, - "57": 459571712.0, - "58": 459571712.0, - "59": 459571712.0, - "60": 459571712.0, - "61": 459571712.0, - "62": 459571712.0, - "63": 459571712.0, - "64": 459571712.0, - "65": 459571712.0, - "66": 459571712.0, - "67": 459571712.0, - "68": 459571712.0, - "69": 459571712.0, - "70": 459571712.0, - "71": 459571712.0, - "72": 459571712.0, - "73": 459571712.0, - "74": 459571712.0, - "75": 459571712.0, - "76": 459571712.0, - "77": 459571712.0, - "78": 459571712.0, - "79": 459571712.0, - "80": 459571712.0, - "81": 459571712.0, - "82": 459571712.0, - "83": 459571712.0, - "84": 459571712.0, - "85": 459571712.0, - "86": 459571712.0, - "87": 459571712.0, - "88": 459571712.0, - "89": 459571712.0, - "90": 459571712.0, - "91": 459571712.0, - "92": 459571712.0, - "93": 459571712.0, - "94": 459571712.0, - "95": 459571712.0, - "96": 459571712.0, - "97": 459571712.0, - "98": 459571712.0, - "99": 459571712.0, - "100": 459571712.0 + "1": 462062080.0, + "2": 462062080.0, + "3": 462062080.0, + "4": 462062080.0, + "5": 462062080.0, + "6": 462062080.0, + "7": 462062080.0, + "8": 462062080.0, + "9": 462062080.0, + "10": 462062080.0, + "11": 462062080.0, + "12": 462062080.0, + "13": 462062080.0, + "14": 462062080.0, + "15": 462062080.0, + "16": 462062080.0, + "17": 462062080.0, + "18": 462062080.0, + "19": 462062080.0, + "20": 462062080.0, + "21": 462062080.0, + "22": 462062080.0, + "23": 462062080.0, + "24": 462062080.0, + "25": 462062080.0, + "26": 462062080.0, + "27": 462062080.0, + "28": 462062080.0, + "29": 462062080.0, + "30": 462062080.0, + "31": 462062080.0, + "32": 462062080.0, + "33": 462062080.0, + "34": 462062080.0, + "35": 462062080.0, + "36": 462062080.0, + "37": 462062080.0, + "38": 462062080.0, + "39": 462062080.0, + "40": 462062080.0, + "41": 462062080.0, + "42": 462062080.0, + "43": 462062080.0, + "44": 462062080.0, + "45": 462062080.0, + "46": 462062080.0, + "47": 462062080.0, + "48": 462062080.0, + "49": 462062080.0, + "50": 462062080.0, + "51": 462062080.0, + "52": 462062080.0, + "53": 462062080.0, + "54": 462062080.0, + "55": 462062080.0, + "56": 462062080.0, + "57": 462062080.0, + "58": 462062080.0, + "59": 462062080.0, + "60": 462062080.0, + "61": 462062080.0, + "62": 462062080.0, + "63": 462062080.0, + "64": 462062080.0, + "65": 462062080.0, + "66": 462062080.0, + "67": 462062080.0, + "68": 462062080.0, + "69": 462062080.0, + "70": 462062080.0, + "71": 462062080.0, + "72": 462062080.0, + "73": 462062080.0, + "74": 462062080.0, + "75": 462062080.0, + "76": 462062080.0, + "77": 462062080.0, + "78": 462062080.0, + "79": 462062080.0, + "80": 462062080.0, + "81": 462062080.0, + "82": 462062080.0, + "83": 462062080.0, + "84": 462062080.0, + "85": 462062080.0, + "86": 462062080.0, + "87": 462062080.0, + "88": 462062080.0, + "89": 462062080.0, + "90": 462062080.0, + "91": 462062080.0, + "92": 462062080.0, + "93": 462062080.0, + "94": 462062080.0, + "95": 462062080.0, + "96": 462062080.0, + "97": 462062080.0, + "98": 462062080.0, + "99": 462062080.0, + "100": 462062080.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 705635840.0, - "2": 883610112.0, - "3": 883610624.0, - "4": 883610624.0, - "5": 883610624.0, - "6": 884657152.0, - "7": 884657152.0, - "8": 884657152.0, - "9": 884657152.0, - "10": 884657152.0, - "11": 884657152.0, - "12": 884657152.0, - "13": 884657152.0, - "14": 884657152.0, - "15": 884659712.0, - "16": 884659712.0, - "17": 884659712.0, - "18": 884659712.0, - "19": 884659712.0, - "20": 884659712.0, - "21": 884659712.0, - "22": 884659712.0, - "23": 884659712.0, - "24": 884659712.0, - "25": 884659712.0, - "26": 884659712.0, - "27": 884659712.0, - "28": 884659712.0, - "29": 884659712.0, - "30": 884659712.0, - "31": 884659712.0, - "32": 884659712.0, - "33": 884659712.0, - "34": 884659712.0, - "35": 884659712.0, - "36": 884659712.0, - "37": 884659712.0, - "38": 884659712.0, - "39": 884659712.0, - "40": 884659712.0, - "41": 884659712.0, - "42": 884659712.0, - "43": 884659712.0, - "44": 884659712.0, - "45": 884659712.0, - "46": 884659712.0, - "47": 884659712.0, - "48": 884659712.0, - "49": 884659712.0, - "50": 884659712.0, - "51": 884659712.0, - "52": 884659712.0, - "53": 884659712.0, - "54": 884659712.0, - "55": 884659712.0, - "56": 884659712.0, - "57": 884659712.0, - "58": 884659712.0, - "59": 884659712.0, - "60": 884659712.0, - "61": 884659712.0, - "62": 884659712.0, - "63": 884659712.0, - "64": 884659712.0, - "65": 884659712.0, - "66": 884659712.0, - "67": 884659712.0, - "68": 884659712.0, - "69": 884659712.0, - "70": 884659712.0, - "71": 884659712.0, - "72": 884659712.0, - "73": 884659712.0, - "74": 884659712.0, - "75": 884659712.0, - "76": 884659712.0, - "77": 884659712.0, - "78": 884659712.0, - "79": 884659712.0, - "80": 884659712.0, - "81": 884659712.0, - "82": 884659712.0, - "83": 884659712.0, - "84": 884659712.0, - "85": 884659712.0, - "86": 884659712.0, - "87": 884659712.0, - "88": 884659712.0, - "89": 884659712.0, - "90": 884659712.0, - "91": 884659712.0, - "92": 884659712.0, - "93": 884659712.0, - "94": 884659712.0, - "95": 884659712.0, - "96": 884659712.0, - "97": 884659712.0, - "98": 884659712.0, - "99": 884659712.0, - "100": 884659712.0 + "1": 703538688.0, + "2": 884528640.0, + "3": 884528640.0, + "4": 884528640.0, + "5": 884528640.0, + "6": 884528640.0, + "7": 884528640.0, + "8": 884528640.0, + "9": 884528640.0, + "10": 884528640.0, + "11": 884528640.0, + "12": 884528640.0, + "13": 884528640.0, + "14": 884528640.0, + "15": 884528640.0, + "16": 884528640.0, + "17": 884528640.0, + "18": 884528640.0, + "19": 884528640.0, + "20": 884528640.0, + "21": 884528640.0, + "22": 884528640.0, + "23": 884528640.0, + "24": 884528640.0, + "25": 884528640.0, + "26": 884528640.0, + "27": 884528640.0, + "28": 884528640.0, + "29": 884528640.0, + "30": 884528640.0, + "31": 884528640.0, + "32": 884528640.0, + "33": 884528640.0, + "34": 884528640.0, + "35": 884528640.0, + "36": 884528640.0, + "37": 884528640.0, + "38": 884528640.0, + "39": 884528640.0, + "40": 884528640.0, + "41": 884528640.0, + "42": 884528640.0, + "43": 884528640.0, + "44": 884528640.0, + "45": 884528640.0, + "46": 884528640.0, + "47": 884528640.0, + "48": 884528640.0, + "49": 884528640.0, + "50": 884528640.0, + "51": 884528640.0, + "52": 884528640.0, + "53": 884528640.0, + "54": 884528640.0, + "55": 884528640.0, + "56": 884528640.0, + "57": 884528640.0, + "58": 884528640.0, + "59": 884528640.0, + "60": 884528640.0, + "61": 884528640.0, + "62": 884528640.0, + "63": 884528640.0, + "64": 884528640.0, + "65": 884528640.0, + "66": 885575168.0, + "67": 885575168.0, + "68": 885575168.0, + "69": 885575168.0, + "70": 885575168.0, + "71": 885575168.0, + "72": 885575168.0, + "73": 885575168.0, + "74": 885575168.0, + "75": 885575168.0, + "76": 885575168.0, + "77": 885575168.0, + "78": 885575168.0, + "79": 885575168.0, + "80": 885575168.0, + "81": 885575168.0, + "82": 885575168.0, + "83": 885575168.0, + "84": 885575168.0, + "85": 885575168.0, + "86": 885575168.0, + "87": 885575168.0, + "88": 885575168.0, + "89": 885575168.0, + "90": 885575168.0, + "91": 885575168.0, + "92": 885575168.0, + "93": 885575168.0, + "94": 885575168.0, + "95": 885575168.0, + "96": 885575168.0, + "97": 885575168.0, + "98": 885575168.0, + "99": 885575168.0, + "100": 885575168.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 13.71622, - "2": 0.68805, - "3": 0.57225, - "4": 0.54755, - "5": 0.48793, - "6": 0.52239, - "7": 0.49126, - "8": 0.49498, - "9": 0.58476, - "10": 0.4973, - "11": 0.49619, - "12": 0.49824, - "13": 0.49835, - "14": 0.49548, - "15": 0.49404, - "16": 0.50855, - "17": 0.76935, - "18": 0.49519, - "19": 0.49579, - "20": 0.50812, - "21": 0.50221, - "22": 0.49623, - "23": 0.50166, - "24": 0.4965, - "25": 0.49653, - "26": 0.56522, - "27": 0.50204, - "28": 0.4912, - "29": 0.49165, - "30": 0.49253, - "31": 0.48561, - "32": 0.50414, - "33": 0.49461, - "34": 0.48721, - "35": 0.50775, - "36": 0.5025, - "37": 0.49896, - "38": 0.50015, - "39": 0.50322, - "40": 0.51086, - "41": 0.51074, - "42": 0.49461, - "43": 0.5049, - "44": 0.47567, - "45": 0.51176, - "46": 0.51628, - "47": 0.50424, - "48": 0.50299, - "49": 0.50456, - "50": 0.51299, - "51": 0.50546, - "52": 0.48547, - "53": 0.48643, - "54": 0.49187, - "55": 0.50244, - "56": 0.5003, - "57": 0.49723, - "58": 0.5007, - "59": 0.50341, - "60": 0.49703, - "61": 0.49913, - "62": 0.48748, - "63": 0.52659, - "64": 0.49384, - "65": 0.48632, - "66": 0.49435, - "67": 0.49537, - "68": 0.49543, - "69": 0.48543, - "70": 0.49128, - "71": 0.49386, - "72": 0.49681, - "73": 0.49076, - "74": 0.50662, - "75": 0.51506, - "76": 0.51539, - "77": 0.51263, - "78": 0.51094, - "79": 0.50786, - "80": 0.85887, - "81": 0.51151, - "82": 0.50586, - "83": 0.51628, - "84": 0.48942, - "85": 0.50794, - "86": 0.45205, - "87": 0.51667, - "88": 0.52246, - "89": 0.51352, - "90": 0.48616, - "91": 0.51165, - "92": 0.52646, - "93": 0.52475, - "94": 0.50978, - "95": 0.50426, - "96": 0.50587, - "97": 0.52063, - "98": 0.52056, - "99": 0.50217, - "100": 0.50666 + "1": "nan", + "2": 3.73344, + "3": 0.435, + "4": 0.42269, + "5": 0.42143, + "6": 0.42169, + "7": 0.42023, + "8": 0.42024, + "9": 0.42222, + "10": 0.42242, + "11": 0.42053, + "12": 0.42268, + "13": 0.41982, + "14": 0.41832, + "15": 0.41832, + "16": 0.41936, + "17": 0.41957, + "18": 0.41869, + "19": 0.4182, + "20": 0.41746, + "21": 0.41737, + "22": 0.40981, + "23": 0.4096, + "24": 0.40573, + "25": 0.40471, + "26": 0.40427, + "27": 0.40639, + "28": 0.40633, + "29": 0.40533, + "30": 0.40576, + "31": 0.40376, + "32": 0.40338, + "33": 0.40605, + "34": 0.40135, + "35": 0.40398, + "36": 0.40309, + "37": 0.40852, + "38": 0.40572, + "39": 0.40092, + "40": 0.40543, + "41": 0.40495, + "42": 0.40518, + "43": 0.40074, + "44": 0.40306, + "45": 0.40179, + "46": 0.40307, + "47": 0.40246, + "48": 0.4024, + "49": 0.40234, + "50": 0.40238, + "51": 0.41546, + "52": 0.39352, + "53": 0.39475, + "54": 0.39652, + "55": 0.40055, + "56": 0.39993, + "57": 0.40166, + "58": 0.40151, + "59": 0.40191, + "60": 0.40194, + "61": 0.40084, + "62": 0.39989, + "63": 0.40157, + "64": 0.40012, + "65": 0.40034, + "66": 0.40082, + "67": 0.40008, + "68": 0.39842, + "69": 0.39844, + "70": 0.40021, + "71": 0.39935, + "72": 0.40145, + "73": 0.39804, + "74": 0.39495, + "75": 0.39605, + "76": 0.39578, + "77": 0.39653, + "78": 0.39694, + "79": 0.95682, + "80": 0.39818, + "81": 0.39646, + "82": 0.39909, + "83": 0.4044, + "84": 0.39893, + "85": 0.39807, + "86": 0.39917, + "87": 0.39513, + "88": 0.39647, + "89": 0.39761, + "90": 0.39883, + "91": 0.39867, + "92": 0.39686, + "93": 0.39611, + "94": 0.39717, + "95": 0.39645, + "96": 0.39632, + "97": 0.39808, + "98": 0.39732, + "99": 0.39829, + "100": 0.39861 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..1bf0169b170 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.7742, + "52": 9.66445, + "53": 10.00151, + "54": 9.89155, + "55": 9.81849, + "56": 9.54335, + "57": 9.39451, + "58": 9.76573, + "59": 9.5093, + "60": 9.42825, + "61": 9.63467, + "62": 9.93887, + "63": 9.30457, + "64": 9.70983, + "65": 8.86882, + "66": 9.64953, + "67": 9.3082, + "68": 9.73505, + "69": 9.7559, + "70": 9.68706, + "71": 9.57534, + "72": 9.53073, + "73": 9.43677, + "74": 8.85587, + "75": 9.35529, + "76": 9.01373, + "77": 10.02452, + "78": 9.68203, + "79": 9.33141, + "80": 9.35469, + "81": 9.43623, + "82": 9.65853, + "83": 9.26266, + "84": 9.36921, + "85": 9.571, + "86": 9.03325, + "87": 9.55972, + "88": 9.71078, + "89": 9.5541, + "90": 9.78661, + "91": 9.29086, + "92": 9.31236, + "93": 9.03977, + "94": 8.78115, + "95": 9.49176, + "96": 9.4907, + "97": 9.25833, + "98": 9.63003, + "99": 8.84687, + "100": 9.36199 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 920.0, + "52": 908.0, + "53": 1052.0, + "54": 889.0, + "55": 846.0, + "56": 985.0, + "57": 838.0, + "58": 1021.0, + "59": 1045.0, + "60": 896.0, + "61": 987.0, + "62": 926.0, + "63": 912.0, + "64": 1081.0, + "65": 991.0, + "66": 1095.0, + "67": 964.0, + "68": 938.0, + "69": 1005.0, + "70": 1013.0, + "71": 1082.0, + "72": 896.0, + "73": 1035.0, + "74": 687.0, + "75": 920.0, + "76": 1063.0, + "77": 1086.0, + "78": 1136.0, + "79": 1065.0, + "80": 1111.0, + "81": 1229.0, + "82": 1100.0, + "83": 944.0, + "84": 1182.0, + "85": 1100.0, + "86": 790.0, + "87": 1132.0, + "88": 1071.0, + "89": 1148.0, + "90": 1121.0, + "91": 1120.0, + "92": 1115.0, + "93": 944.0, + "94": 1126.0, + "95": 1116.0, + "96": 1115.0, + "97": 995.0, + "98": 1234.0, + "99": 1120.0, + "100": 1148.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 461669888.0, + "52": 461669888.0, + "53": 461669888.0, + "54": 461669888.0, + "55": 461669888.0, + "56": 461669888.0, + "57": 461669888.0, + "58": 461669888.0, + "59": 461669888.0, + "60": 461669888.0, + "61": 461669888.0, + "62": 461669888.0, + "63": 461669888.0, + "64": 461669888.0, + "65": 461669888.0, + "66": 461669888.0, + "67": 461669888.0, + "68": 461669888.0, + "69": 461669888.0, + "70": 461669888.0, + "71": 461669888.0, + "72": 461669888.0, + "73": 461669888.0, + "74": 461669888.0, + "75": 461669888.0, + "76": 461669888.0, + "77": 461669888.0, + "78": 461669888.0, + "79": 461669888.0, + "80": 461669888.0, + "81": 461669888.0, + "82": 461669888.0, + "83": 461669888.0, + "84": 461669888.0, + "85": 461669888.0, + "86": 461669888.0, + "87": 461669888.0, + "88": 461669888.0, + "89": 461669888.0, + "90": 461669888.0, + "91": 461669888.0, + "92": 461669888.0, + "93": 461669888.0, + "94": 461669888.0, + "95": 461669888.0, + "96": 461669888.0, + "97": 461669888.0, + "98": 461669888.0, + "99": 461669888.0, + "100": 461669888.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 889377280.0, + "52": 889378304.0, + "53": 889378304.0, + "54": 889378304.0, + "55": 889378304.0, + "56": 889378304.0, + "57": 889378304.0, + "58": 889378304.0, + "59": 889378304.0, + "60": 889378304.0, + "61": 889378304.0, + "62": 889378304.0, + "63": 889378304.0, + "64": 889378304.0, + "65": 889378304.0, + "66": 889378304.0, + "67": 889378304.0, + "68": 889378304.0, + "69": 889378304.0, + "70": 889378304.0, + "71": 889378304.0, + "72": 889378304.0, + "73": 889378304.0, + "74": 889378304.0, + "75": 889378304.0, + "76": 889378304.0, + "77": 889378304.0, + "78": 889378304.0, + "79": 889378304.0, + "80": 889378304.0, + "81": 889378304.0, + "82": 889378304.0, + "83": 889378304.0, + "84": 889378304.0, + "85": 889378304.0, + "86": 889378304.0, + "87": 889378304.0, + "88": 889378304.0, + "89": 889378304.0, + "90": 889378816.0, + "91": 889378816.0, + "92": 889378816.0, + "93": 889378816.0, + "94": 889378816.0, + "95": 889378816.0, + "96": 889378816.0, + "97": 889378816.0, + "98": 889378816.0, + "99": 889378816.0, + "100": 889378816.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 3.77319, + "53": 0.42363, + "54": 0.41071, + "55": 0.41011, + "56": 0.40905, + "57": 0.40957, + "58": 0.41032, + "59": 0.40997, + "60": 0.4109, + "61": 0.4104, + "62": 0.40989, + "63": 0.40974, + "64": 0.40928, + "65": 0.40668, + "66": 0.4076, + "67": 0.41006, + "68": 0.41114, + "69": 0.40437, + "70": 0.40702, + "71": 0.4095, + "72": 0.41064, + "73": 0.40549, + "74": 0.40683, + "75": 0.4055, + "76": 0.40589, + "77": 0.40198, + "78": 0.40196, + "79": 0.40383, + "80": 0.40596, + "81": 0.40678, + "82": 0.40646, + "83": 0.40861, + "84": 0.40858, + "85": 0.40709, + "86": 0.40475, + "87": 0.41028, + "88": 0.40188, + "89": 0.40272, + "90": 0.4034, + "91": 0.40676, + "92": 0.40732, + "93": 0.40103, + "94": 0.40501, + "95": 0.4043, + "96": 0.40452, + "97": 0.40255, + "98": 0.40532, + "99": 0.40632, + "100": 0.4042 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json index c8639e2d542..04cd8d66a75 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.86836, "2": 10.88595, - "3": 10.86559, - "4": 10.86893, - "5": 10.87417, - "6": 10.89061, - "7": 10.87673, - "8": 10.8647, - "9": 10.88231, - "10": 10.84582, - "11": 10.87165, - "12": 10.87421, - "13": 10.88164, - "14": 10.88885, - "15": 10.83927, - "16": 10.825, + "3": 10.86558, + "4": 10.86895, + "5": 10.87421, + "6": 10.89062, + "7": 10.87675, + "8": 10.86479, + "9": 10.88234, + "10": 10.8458, + "11": 10.87162, + "12": 10.8742, + "13": 10.88161, + "14": 10.88884, + "15": 10.83933, + "16": 10.82498, "17": 10.80147, "18": 10.81236, - "19": 10.82153, - "20": 10.71933, + "19": 10.82152, + "20": 10.71934, "21": 10.6909, - "22": 10.57427, - "23": 10.71093, - "24": 10.59784, - "25": 10.5556, + "22": 10.57424, + "23": 10.71092, + "24": 10.5978, + "25": 10.55562, "26": 10.61523, - "27": 10.60454, - "28": 10.56483, - "29": 10.58475, - "30": 10.35945, - "31": 10.12153, - "32": 10.45236, - "33": 10.45724, - "34": 10.21987, - "35": 10.2644, - "36": 10.21038, - "37": 10.33961, - "38": 10.18012, - "39": 10.39589, + "27": 10.60449, + "28": 10.56485, + "29": 10.58474, + "30": 10.35951, + "31": 10.12154, + "32": 10.45235, + "33": 10.4572, + "34": 10.2199, + "35": 10.26443, + "36": 10.21037, + "37": 10.33956, + "38": 10.18016, + "39": 10.39593, "40": 10.0663, - "41": 10.14169, - "42": 10.2085, - "43": 9.83125, - "44": 9.94861, - "45": 9.82847, - "46": 9.80462, + "41": 10.14164, + "42": 10.20852, + "43": 9.8313, + "44": 9.94856, + "45": 9.82849, + "46": 9.80457, "47": 10.14229, - "48": 9.84463, - "49": 9.52194, - "50": 9.88607, - "51": 9.84982, - "52": 9.74429, + "48": 9.84462, + "49": 9.52191, + "50": 9.88601, + "51": 9.8498, + "52": 9.74427, "53": 10.05843, - "54": 9.95129, + "54": 9.95125, "55": 9.88343, - "56": 9.61329, + "56": 9.61327, "57": 9.46899, - "58": 9.82161, - "59": 9.57702, - "60": 9.49786, - "61": 9.69256, - "62": 9.98595, - "63": 9.37403, - "64": 9.76605, - "65": 8.94649, - "66": 9.70105, - "67": 9.36367, - "68": 9.78237, - "69": 9.79879, - "70": 9.73166, - "71": 9.62508, - "72": 9.58312, - "73": 9.48822, - "74": 8.92611, - "75": 9.40725, + "58": 9.82164, + "59": 9.57703, + "60": 9.49784, + "61": 9.69255, + "62": 9.98596, + "63": 9.37402, + "64": 9.76603, + "65": 8.94654, + "66": 9.70099, + "67": 9.36365, + "68": 9.78238, + "69": 9.7988, + "70": 9.73169, + "71": 9.62505, + "72": 9.58309, + "73": 9.4882, + "74": 8.92607, + "75": 9.40727, "76": 9.07708, - "77": 10.05858, - "78": 9.7221, - "79": 9.37662, - "80": 9.40273, - "81": 9.48209, - "82": 9.6995, - "83": 9.31351, + "77": 10.0586, + "78": 9.72209, + "79": 9.37663, + "80": 9.40272, + "81": 9.48207, + "82": 9.69954, + "83": 9.31354, "84": 9.4173, - "85": 9.61584, - "86": 9.07429, - "87": 9.59551, - "88": 9.75065, + "85": 9.61582, + "86": 9.07431, + "87": 9.59556, + "88": 9.75064, "89": 9.6004, - "90": 9.8221, - "91": 9.33876, - "92": 9.3578, - "93": 9.08672, - "94": 8.82958, + "90": 9.82205, + "91": 9.33874, + "92": 9.35779, + "93": 9.08668, + "94": 8.8296, "95": 9.52596, - "96": 9.52973, + "96": 9.52974, "97": 9.30335, "98": 9.67136, - "99": 8.89537, - "100": 9.40568 + "99": 8.89539, + "100": 9.40567 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1535.0, - "2": 1566.0, - "3": 1736.0, - "4": 1667.0, - "5": 1821.0, - "6": 1743.0, - "7": 1753.0, - "8": 1683.0, - "9": 1801.0, - "10": 1363.0, - "11": 1688.0, - "12": 1722.0, - "13": 1831.0, - "14": 1630.0, - "15": 1842.0, - "16": 1763.0, - "17": 1822.0, - "18": 1543.0, - "19": 1709.0, - "20": 1618.0, - "21": 1878.0, - "22": 1591.0, - "23": 1932.0, - "24": 1597.0, - "25": 1549.0, - "26": 1621.0, - "27": 1732.0, - "28": 1921.0, - "29": 1931.0, - "30": 1880.0, - "31": 1483.0, - "32": 1832.0, - "33": 2077.0, - "34": 1814.0, - "35": 1908.0, - "36": 1856.0, - "37": 2378.0, - "38": 2057.0, - "39": 2342.0, - "40": 2151.0, - "41": 2265.0, - "42": 2146.0, - "43": 1897.0, - "44": 2097.0, - "45": 2059.0, - "46": 2303.0, - "47": 2451.0, - "48": 2255.0, - "49": 2310.0, - "50": 2472.0, - "51": 2560.0, - "52": 2622.0, - "53": 2835.0, - "54": 2696.0, - "55": 2322.0, - "56": 2793.0, - "57": 2247.0, - "58": 2951.0, - "59": 2850.0, - "60": 2515.0, - "61": 2874.0, - "62": 2686.0, - "63": 2448.0, - "64": 2936.0, - "65": 2670.0, - "66": 2814.0, - "67": 2782.0, - "68": 2808.0, - "69": 2901.0, - "70": 3044.0, - "71": 2876.0, - "72": 2508.0, - "73": 2893.0, - "74": 1974.0, - "75": 2488.0, - "76": 2881.0, - "77": 3104.0, - "78": 3241.0, - "79": 3196.0, - "80": 3322.0, - "81": 3594.0, - "82": 3215.0, - "83": 2643.0, - "84": 3180.0, - "85": 3159.0, - "86": 2619.0, - "87": 3774.0, - "88": 3025.0, - "89": 3322.0, - "90": 3043.0, - "91": 2830.0, - "92": 3015.0, - "93": 2758.0, - "94": 3190.0, - "95": 3172.0, - "96": 3453.0, - "97": 3176.0, - "98": 3590.0, - "99": 3059.0, - "100": 3290.0 + "1": 1580.0, + "2": 1610.0, + "3": 1625.0, + "4": 1685.0, + "5": 1825.0, + "6": 1771.0, + "7": 1831.0, + "8": 1645.0, + "9": 1814.0, + "10": 1387.0, + "11": 1742.0, + "12": 1649.0, + "13": 1757.0, + "14": 1705.0, + "15": 1827.0, + "16": 1765.0, + "17": 1835.0, + "18": 1602.0, + "19": 1814.0, + "20": 1735.0, + "21": 1895.0, + "22": 1594.0, + "23": 1902.0, + "24": 1633.0, + "25": 1574.0, + "26": 1681.0, + "27": 1676.0, + "28": 1961.0, + "29": 1851.0, + "30": 1863.0, + "31": 1499.0, + "32": 1896.0, + "33": 2118.0, + "34": 1725.0, + "35": 1879.0, + "36": 1880.0, + "37": 2347.0, + "38": 2044.0, + "39": 2283.0, + "40": 2155.0, + "41": 2224.0, + "42": 2169.0, + "43": 1958.0, + "44": 2050.0, + "45": 2130.0, + "46": 2346.0, + "47": 2418.0, + "48": 2243.0, + "49": 2161.0, + "50": 2479.0, + "51": 2480.0, + "52": 2545.0, + "53": 2875.0, + "54": 2652.0, + "55": 2384.0, + "56": 2742.0, + "57": 2201.0, + "58": 2755.0, + "59": 2954.0, + "60": 2367.0, + "61": 2889.0, + "62": 2721.0, + "63": 2438.0, + "64": 2928.0, + "65": 2567.0, + "66": 2751.0, + "67": 2802.0, + "68": 2714.0, + "69": 2884.0, + "70": 3124.0, + "71": 2813.0, + "72": 2504.0, + "73": 2852.0, + "74": 1975.0, + "75": 2429.0, + "76": 2850.0, + "77": 3008.0, + "78": 3110.0, + "79": 3114.0, + "80": 3284.0, + "81": 3574.0, + "82": 3207.0, + "83": 2530.0, + "84": 3169.0, + "85": 3150.0, + "86": 2588.0, + "87": 3845.0, + "88": 3094.0, + "89": 3389.0, + "90": 3077.0, + "91": 2872.0, + "92": 3012.0, + "93": 2685.0, + "94": 3279.0, + "95": 3231.0, + "96": 3422.0, + "97": 3154.0, + "98": 3498.0, + "99": 3043.0, + "100": 3361.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 465601024.0, - "2": 465601024.0, - "3": 465601024.0, - "4": 465601024.0, - "5": 465601024.0, - "6": 465601024.0, - "7": 465601024.0, - "8": 465601024.0, - "9": 465601024.0, - "10": 465601024.0, - "11": 465601024.0, - "12": 465601024.0, - "13": 465601024.0, - "14": 465601024.0, - "15": 465601024.0, - "16": 465601024.0, - "17": 465601024.0, - "18": 465601024.0, - "19": 465601024.0, - "20": 465601024.0, - "21": 465601024.0, - "22": 465601024.0, - "23": 465601024.0, - "24": 465601024.0, - "25": 465601024.0, - "26": 465601024.0, - "27": 465601024.0, - "28": 465601024.0, - "29": 465601024.0, - "30": 465601024.0, - "31": 465601024.0, - "32": 465601024.0, - "33": 465601024.0, - "34": 465601024.0, - "35": 465601024.0, - "36": 465601024.0, - "37": 465601024.0, - "38": 465601024.0, - "39": 465601024.0, - "40": 465601024.0, - "41": 465601024.0, - "42": 465601024.0, - "43": 465601024.0, - "44": 465601024.0, - "45": 465601024.0, - "46": 465601024.0, - "47": 465601024.0, - "48": 465601024.0, - "49": 465601024.0, - "50": 465601024.0, - "51": 465601024.0, - "52": 465601024.0, - "53": 465601024.0, - "54": 465601024.0, - "55": 465601024.0, - "56": 465601024.0, - "57": 465601024.0, - "58": 465601024.0, - "59": 465601024.0, - "60": 465601024.0, - "61": 465601024.0, - "62": 465601024.0, - "63": 465601024.0, - "64": 465601024.0, - "65": 465601024.0, - "66": 465601024.0, - "67": 465601024.0, - "68": 465601024.0, - "69": 465601024.0, - "70": 465601024.0, - "71": 465601024.0, - "72": 465601024.0, - "73": 465601024.0, - "74": 465601024.0, - "75": 465601024.0, - "76": 465601024.0, - "77": 465601024.0, - "78": 465601024.0, - "79": 465601024.0, - "80": 465601024.0, - "81": 465601024.0, - "82": 465601024.0, - "83": 465601024.0, - "84": 465601024.0, - "85": 465601024.0, - "86": 465601024.0, - "87": 465601024.0, - "88": 465601024.0, - "89": 465601024.0, - "90": 465601024.0, - "91": 465601024.0, - "92": 465601024.0, - "93": 465601024.0, - "94": 465601024.0, - "95": 465601024.0, - "96": 465601024.0, - "97": 465601024.0, - "98": 465601024.0, - "99": 465601024.0, - "100": 465601024.0 + "1": 465207808.0, + "2": 466256384.0, + "3": 466256384.0, + "4": 466256384.0, + "5": 466256384.0, + "6": 466256384.0, + "7": 466256384.0, + "8": 466256384.0, + "9": 466256384.0, + "10": 466256384.0, + "11": 466256384.0, + "12": 466256384.0, + "13": 466256384.0, + "14": 466256384.0, + "15": 466256384.0, + "16": 466256384.0, + "17": 466256384.0, + "18": 466256384.0, + "19": 466256384.0, + "20": 466256384.0, + "21": 466256384.0, + "22": 466256384.0, + "23": 466256384.0, + "24": 466256384.0, + "25": 466256384.0, + "26": 466256384.0, + "27": 466256384.0, + "28": 466256384.0, + "29": 466256384.0, + "30": 466256384.0, + "31": 466256384.0, + "32": 466256384.0, + "33": 466256384.0, + "34": 466256384.0, + "35": 466256384.0, + "36": 466256384.0, + "37": 466256384.0, + "38": 466256384.0, + "39": 466256384.0, + "40": 466256384.0, + "41": 466256384.0, + "42": 466256384.0, + "43": 466256384.0, + "44": 466256384.0, + "45": 466256384.0, + "46": 466256384.0, + "47": 466256384.0, + "48": 466256384.0, + "49": 466256384.0, + "50": 466256384.0, + "51": 466256384.0, + "52": 466256384.0, + "53": 466256384.0, + "54": 466256384.0, + "55": 466256384.0, + "56": 466256384.0, + "57": 466256384.0, + "58": 466256384.0, + "59": 466256384.0, + "60": 466256384.0, + "61": 466256384.0, + "62": 466256384.0, + "63": 466256384.0, + "64": 466256384.0, + "65": 466256384.0, + "66": 466256384.0, + "67": 466256384.0, + "68": 466256384.0, + "69": 466256384.0, + "70": 466256384.0, + "71": 466256384.0, + "72": 466256384.0, + "73": 466256384.0, + "74": 466256384.0, + "75": 466256384.0, + "76": 466256384.0, + "77": 466256384.0, + "78": 466256384.0, + "79": 466256384.0, + "80": 466256384.0, + "81": 466256384.0, + "82": 466256384.0, + "83": 466256384.0, + "84": 466256384.0, + "85": 466256384.0, + "86": 466256384.0, + "87": 466256384.0, + "88": 466256384.0, + "89": 466256384.0, + "90": 466256384.0, + "91": 466256384.0, + "92": 466256384.0, + "93": 466256384.0, + "94": 466256384.0, + "95": 466256384.0, + "96": 466256384.0, + "97": 466256384.0, + "98": 466256384.0, + "99": 466256384.0, + "100": 466256384.0 } }, "mem-max-allocated-bytes": { @@ -326,105 +326,105 @@ "step_interval": 1, "values": { "1": 1728999424.0, - "2": 1789405696.0, - "3": 1789405696.0, - "4": 1789405696.0, - "5": 1789405696.0, - "6": 1789405696.0, - "7": 1789405696.0, - "8": 1789405696.0, - "9": 1789405696.0, - "10": 1789405696.0, - "11": 1789405696.0, - "12": 1789405696.0, - "13": 1789405696.0, - "14": 1789405696.0, - "15": 1789405696.0, - "16": 1789405696.0, - "17": 1789405696.0, - "18": 1789405696.0, - "19": 1789405696.0, - "20": 1789405696.0, - "21": 1789405696.0, - "22": 1789405696.0, - "23": 1789405696.0, - "24": 1789405696.0, - "25": 1789405696.0, - "26": 1789405696.0, - "27": 1789405696.0, - "28": 1789405696.0, - "29": 1789405696.0, - "30": 1789405696.0, - "31": 1789405696.0, - "32": 1789405696.0, - "33": 1789405696.0, - "34": 1789405696.0, - "35": 1789405696.0, - "36": 1789405696.0, - "37": 1789405696.0, - "38": 1789405696.0, - "39": 1789405696.0, - "40": 1789405696.0, - "41": 1789405696.0, - "42": 1789405696.0, - "43": 1789405696.0, - "44": 1789405696.0, - "45": 1789405696.0, - "46": 1789405696.0, - "47": 1789405696.0, - "48": 1789405696.0, - "49": 1789405696.0, - "50": 1789405696.0, - "51": 1789405696.0, - "52": 1789405696.0, - "53": 1789405696.0, - "54": 1789405696.0, - "55": 1789405696.0, - "56": 1789405696.0, - "57": 1789405696.0, - "58": 1789405696.0, - "59": 1789405696.0, - "60": 1789405696.0, - "61": 1789405696.0, - "62": 1789405696.0, - "63": 1789405696.0, - "64": 1789405696.0, - "65": 1789405696.0, - "66": 1789405696.0, - "67": 1789405696.0, - "68": 1789405696.0, - "69": 1789405696.0, - "70": 1789405696.0, - "71": 1789405696.0, - "72": 1789405696.0, - "73": 1789405696.0, - "74": 1789405696.0, - "75": 1789405696.0, - "76": 1789405696.0, - "77": 1789405696.0, - "78": 1789405696.0, - "79": 1789405696.0, - "80": 1789405696.0, - "81": 1789405696.0, - "82": 1789405696.0, - "83": 1789405696.0, - "84": 1789405696.0, - "85": 1789405696.0, - "86": 1789405696.0, - "87": 1789405696.0, - "88": 1789405696.0, - "89": 1789405696.0, - "90": 1789405696.0, - "91": 1789405696.0, - "92": 1789405696.0, - "93": 1789405696.0, - "94": 1789405696.0, - "95": 1789405696.0, - "96": 1789405696.0, - "97": 1789405696.0, - "98": 1789405696.0, - "99": 1789405696.0, - "100": 1789405696.0 + "2": 1789536768.0, + "3": 1789536768.0, + "4": 1789536768.0, + "5": 1789536768.0, + "6": 1789536768.0, + "7": 1789536768.0, + "8": 1789536768.0, + "9": 1789536768.0, + "10": 1789536768.0, + "11": 1789536768.0, + "12": 1789536768.0, + "13": 1789536768.0, + "14": 1789536768.0, + "15": 1789536768.0, + "16": 1789536768.0, + "17": 1789536768.0, + "18": 1789536768.0, + "19": 1789536768.0, + "20": 1789536768.0, + "21": 1789536768.0, + "22": 1789536768.0, + "23": 1789536768.0, + "24": 1789536768.0, + "25": 1789536768.0, + "26": 1789536768.0, + "27": 1789536768.0, + "28": 1789536768.0, + "29": 1789536768.0, + "30": 1789536768.0, + "31": 1789536768.0, + "32": 1789536768.0, + "33": 1789536768.0, + "34": 1789536768.0, + "35": 1789536768.0, + "36": 1789536768.0, + "37": 1789536768.0, + "38": 1789536768.0, + "39": 1789536768.0, + "40": 1789536768.0, + "41": 1789536768.0, + "42": 1789536768.0, + "43": 1789536768.0, + "44": 1789536768.0, + "45": 1789536768.0, + "46": 1789536768.0, + "47": 1789536768.0, + "48": 1789536768.0, + "49": 1789536768.0, + "50": 1789536768.0, + "51": 1789536768.0, + "52": 1789536768.0, + "53": 1789536768.0, + "54": 1789536768.0, + "55": 1789536768.0, + "56": 1789536768.0, + "57": 1789536768.0, + "58": 1789536768.0, + "59": 1789536768.0, + "60": 1789536768.0, + "61": 1789536768.0, + "62": 1789536768.0, + "63": 1789536768.0, + "64": 1789536768.0, + "65": 1789536768.0, + "66": 1789536768.0, + "67": 1789536768.0, + "68": 1789536768.0, + "69": 1789536768.0, + "70": 1789536768.0, + "71": 1789536768.0, + "72": 1789536768.0, + "73": 1789536768.0, + "74": 1789536768.0, + "75": 1789536768.0, + "76": 1789536768.0, + "77": 1789536768.0, + "78": 1789536768.0, + "79": 1789536768.0, + "80": 1789536768.0, + "81": 1789536768.0, + "82": 1789536768.0, + "83": 1789536768.0, + "84": 1789536768.0, + "85": 1789536768.0, + "86": 1789536768.0, + "87": 1789536768.0, + "88": 1789536768.0, + "89": 1789536768.0, + "90": 1789536768.0, + "91": 1789536768.0, + "92": 1789536768.0, + "93": 1789536768.0, + "94": 1789536768.0, + "95": 1789536768.0, + "96": 1789536768.0, + "97": 1789536768.0, + "98": 1789536768.0, + "99": 1789536768.0, + "100": 1789536768.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 10.87419, - "2": 0.53001, - "3": 0.2186, - "4": 0.25482, - "5": 0.20138, - "6": 0.19379, - "7": 0.79335, - "8": 0.34845, - "9": 0.55178, - "10": 0.41213, - "11": 0.70514, - "12": 0.42183, - "13": 0.79058, - "14": 0.25823, - "15": 0.17847, - "16": 0.17856, - "17": 0.22517, - "18": 0.17747, - "19": 0.2016, - "20": 0.17788, - "21": 0.2366, - "22": 0.17719, - "23": 0.17889, - "24": 0.17909, - "25": 0.23071, - "26": 0.18878, - "27": 0.17959, - "28": 0.17796, - "29": 0.19707, - "30": 0.17868, - "31": 0.23748, - "32": 0.17977, - "33": 0.1776, - "34": 0.17788, - "35": 0.17714, - "36": 0.17848, - "37": 0.17912, - "38": 0.17729, - "39": 0.20194, - "40": 0.5561, - "41": 0.18404, - "42": 0.21996, - "43": 0.1805, - "44": 0.22997, - "45": 0.17843, - "46": 0.17815, - "47": 0.17755, - "48": 0.21932, - "49": 0.17935, - "50": 0.21536, - "51": 0.18927, - "52": 0.17358, - "53": 0.17366, - "54": 0.19577, - "55": 0.17508, - "56": 0.20037, - "57": 0.17429, - "58": 0.2159, - "59": 0.17615, - "60": 0.17613, - "61": 0.17677, - "62": 0.17726, - "63": 0.22918, - "64": 0.17848, - "65": 0.17926, - "66": 0.17835, - "67": 0.17818, - "68": 0.17977, - "69": 0.17935, - "70": 0.17953, - "71": 0.17922, - "72": 0.17845, - "73": 0.19928, - "74": 0.17885, - "75": 0.20547, - "76": 0.2325, - "77": 0.18027, - "78": 0.17887, - "79": 0.18129, - "80": 0.18884, - "81": 0.1894, - "82": 0.18987, - "83": 0.19315, - "84": 0.19155, - "85": 0.19434, - "86": 0.19122, - "87": 0.1931, - "88": 0.19294, - "89": 0.2106, - "90": 0.19136, - "91": 0.19388, - "92": 0.21142, - "93": 0.19188, - "94": 0.19177, - "95": 0.19125, - "96": 0.1943, - "97": 0.20398, - "98": 0.19536, - "99": 0.19149, - "100": 0.19184 + "1": "nan", + "2": 2.69056, + "3": 0.17999, + "4": 0.16451, + "5": 0.16508, + "6": 0.17235, + "7": 0.16329, + "8": 0.1626, + "9": 0.16188, + "10": 0.16733, + "11": 0.16471, + "12": 0.16323, + "13": 0.16176, + "14": 0.16306, + "15": 0.16415, + "16": 0.16286, + "17": 0.16013, + "18": 0.16147, + "19": 0.17142, + "20": 0.1614, + "21": 0.16056, + "22": 0.16073, + "23": 0.1704, + "24": 0.16109, + "25": 0.16097, + "26": 0.16623, + "27": 0.15978, + "28": 0.17015, + "29": 0.17103, + "30": 0.18177, + "31": 0.18267, + "32": 0.18537, + "33": 0.18546, + "34": 0.18686, + "35": 0.18715, + "36": 0.18598, + "37": 0.18556, + "38": 0.18847, + "39": 0.187, + "40": 0.18548, + "41": 0.19477, + "42": 0.18691, + "43": 0.18628, + "44": 0.18945, + "45": 0.18687, + "46": 0.18766, + "47": 0.18828, + "48": 0.1885, + "49": 0.18744, + "50": 0.18918, + "51": 0.20273, + "52": 0.182, + "53": 0.18, + "54": 0.17575, + "55": 0.17407, + "56": 0.17222, + "57": 0.16988, + "58": 0.17015, + "59": 0.17038, + "60": 0.16865, + "61": 0.16894, + "62": 0.16852, + "63": 0.16574, + "64": 0.16829, + "65": 0.16644, + "66": 0.16896, + "67": 0.16934, + "68": 0.1675, + "69": 0.16535, + "70": 0.16738, + "71": 0.17159, + "72": 0.18394, + "73": 0.18193, + "74": 0.18302, + "75": 0.1832, + "76": 0.18125, + "77": 0.17794, + "78": 0.17778, + "79": 0.17611, + "80": 0.17384, + "81": 0.17173, + "82": 0.16989, + "83": 0.16782, + "84": 0.16781, + "85": 0.16901, + "86": 0.16737, + "87": 0.16701, + "88": 0.16719, + "89": 0.16644, + "90": 0.16551, + "91": 0.16712, + "92": 0.16502, + "93": 0.16672, + "94": 0.1665, + "95": 0.1653, + "96": 0.16686, + "97": 0.16586, + "98": 0.16635, + "99": 0.1655, + "100": 0.16563 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..558ad752f07 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.8498, + "52": 9.74428, + "53": 10.05842, + "54": 9.95129, + "55": 9.88341, + "56": 9.61325, + "57": 9.46897, + "58": 9.82163, + "59": 9.57702, + "60": 9.49787, + "61": 9.69255, + "62": 9.98598, + "63": 9.37403, + "64": 9.76601, + "65": 8.94652, + "66": 9.70103, + "67": 9.36369, + "68": 9.7824, + "69": 9.79882, + "70": 9.73168, + "71": 9.6251, + "72": 9.58313, + "73": 9.4882, + "74": 8.92611, + "75": 9.40723, + "76": 9.07704, + "77": 10.05859, + "78": 9.72209, + "79": 9.37661, + "80": 9.40273, + "81": 9.48205, + "82": 9.69955, + "83": 9.31352, + "84": 9.41732, + "85": 9.61583, + "86": 9.07429, + "87": 9.59556, + "88": 9.75065, + "89": 9.60041, + "90": 9.82204, + "91": 9.33875, + "92": 9.35776, + "93": 9.08668, + "94": 8.82962, + "95": 9.52594, + "96": 9.52969, + "97": 9.30331, + "98": 9.67138, + "99": 8.89538, + "100": 9.40569 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2514.0, + "52": 2584.0, + "53": 3025.0, + "54": 2653.0, + "55": 2268.0, + "56": 2637.0, + "57": 2225.0, + "58": 2904.0, + "59": 2970.0, + "60": 2399.0, + "61": 2925.0, + "62": 2639.0, + "63": 2383.0, + "64": 2889.0, + "65": 2675.0, + "66": 2992.0, + "67": 2764.0, + "68": 2725.0, + "69": 2865.0, + "70": 3077.0, + "71": 2923.0, + "72": 2414.0, + "73": 2906.0, + "74": 1947.0, + "75": 2449.0, + "76": 2976.0, + "77": 3163.0, + "78": 3186.0, + "79": 3172.0, + "80": 3344.0, + "81": 3625.0, + "82": 3289.0, + "83": 2699.0, + "84": 3102.0, + "85": 3227.0, + "86": 2754.0, + "87": 3714.0, + "88": 3004.0, + "89": 3321.0, + "90": 3134.0, + "91": 2714.0, + "92": 3077.0, + "93": 2631.0, + "94": 3309.0, + "95": 3226.0, + "96": 3473.0, + "97": 3216.0, + "98": 3581.0, + "99": 3061.0, + "100": 3419.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 465864192.0, + "52": 465864192.0, + "53": 465864192.0, + "54": 465864192.0, + "55": 465864192.0, + "56": 465864192.0, + "57": 465864192.0, + "58": 465864192.0, + "59": 465864192.0, + "60": 465864192.0, + "61": 465864192.0, + "62": 465864192.0, + "63": 465864192.0, + "64": 465864192.0, + "65": 465864192.0, + "66": 465864192.0, + "67": 465864192.0, + "68": 465864192.0, + "69": 465864192.0, + "70": 465864192.0, + "71": 465864192.0, + "72": 465864192.0, + "73": 465864192.0, + "74": 465864192.0, + "75": 465864192.0, + "76": 465864192.0, + "77": 465864192.0, + "78": 465864192.0, + "79": 465864192.0, + "80": 465864192.0, + "81": 465864192.0, + "82": 465864192.0, + "83": 465864192.0, + "84": 465864192.0, + "85": 465864192.0, + "86": 465864192.0, + "87": 465864192.0, + "88": 465864192.0, + "89": 465864192.0, + "90": 465864192.0, + "91": 465864192.0, + "92": 465864192.0, + "93": 465864192.0, + "94": 465864192.0, + "95": 465864192.0, + "96": 465864192.0, + "97": 465864192.0, + "98": 465864192.0, + "99": 465864192.0, + "100": 465864192.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1911693312.0, + "52": 1911693312.0, + "53": 1911693312.0, + "54": 1911693312.0, + "55": 1911693312.0, + "56": 1911693312.0, + "57": 1911693312.0, + "58": 1911693312.0, + "59": 1911693312.0, + "60": 1911693312.0, + "61": 1911693312.0, + "62": 1911693312.0, + "63": 1911693312.0, + "64": 1911693312.0, + "65": 1911693312.0, + "66": 1911693312.0, + "67": 1911693312.0, + "68": 1911693312.0, + "69": 1911693312.0, + "70": 1911693312.0, + "71": 1911693312.0, + "72": 1911693312.0, + "73": 1911693312.0, + "74": 1911693312.0, + "75": 1911693312.0, + "76": 1911693312.0, + "77": 1911693312.0, + "78": 1911693312.0, + "79": 1911693312.0, + "80": 1911693312.0, + "81": 1911693312.0, + "82": 1911693312.0, + "83": 1911693312.0, + "84": 1911693312.0, + "85": 1911693312.0, + "86": 1911693312.0, + "87": 1911693312.0, + "88": 1911693312.0, + "89": 1911693312.0, + "90": 1911693312.0, + "91": 1911693312.0, + "92": 1911693312.0, + "93": 1911693312.0, + "94": 1911693312.0, + "95": 1911693312.0, + "96": 1911693312.0, + "97": 1911693312.0, + "98": 1911693312.0, + "99": 1911693312.0, + "100": 1911693312.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.81421, + "53": 0.16853, + "54": 0.15862, + "55": 0.15827, + "56": 0.15923, + "57": 0.15884, + "58": 0.15972, + "59": 0.15955, + "60": 0.16137, + "61": 0.15962, + "62": 0.16098, + "63": 0.15948, + "64": 0.15791, + "65": 0.15969, + "66": 0.15933, + "67": 0.17128, + "68": 0.15958, + "69": 0.16526, + "70": 0.15854, + "71": 0.16076, + "72": 0.15949, + "73": 0.1598, + "74": 0.15944, + "75": 0.15956, + "76": 0.1605, + "77": 0.15954, + "78": 0.15934, + "79": 0.16153, + "80": 0.16883, + "81": 0.16008, + "82": 0.16051, + "83": 0.16043, + "84": 0.16049, + "85": 0.16138, + "86": 0.16025, + "87": 0.16089, + "88": 0.15937, + "89": 0.16098, + "90": 0.16047, + "91": 0.16142, + "92": 0.1613, + "93": 0.16027, + "94": 0.16427, + "95": 0.16157, + "96": 0.16144, + "97": 0.16147, + "98": 0.16068, + "99": 0.16024, + "100": 0.15949 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json index a321d71dac5..27eb21de0f0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 7.46014, - "2": 0.22036, - "3": 0.24678, - "4": 0.1906, - "5": 0.23432, - "6": 0.19337, - "7": 0.20952, - "8": 0.20857, - "9": 0.20711, - "10": 0.21582, - "11": 0.20302, - "12": 0.23361, - "13": 0.21601, - "14": 0.19637, - "15": 0.19187, - "16": 0.19595, - "17": 0.19262, - "18": 0.25658, - "19": 0.19382, - "20": 0.23562, - "21": 0.19141, - "22": 0.19045, - "23": 0.25041, - "24": 0.19507, - "25": 0.19119, - "26": 0.25125, - "27": 0.24158, - "28": 0.19174, - "29": 0.19271, - "30": 0.19107, - "31": 0.20992, - "32": 0.19656, - "33": 0.22065, - "34": 0.24506, - "35": 0.26305, - "36": 0.19488, - "37": 0.21539, - "38": 0.19008, - "39": 0.45338, - "40": 0.19345, - "41": 0.19327, - "42": 0.19025, - "43": 0.2339, - "44": 0.19531, - "45": 0.19303, - "46": 0.22612, - "47": 0.19173, - "48": 0.22577, - "49": 0.19067, - "50": 0.23575, - "51": 0.24917, - "52": 0.22723, - "53": 0.22561, - "54": 0.22604, - "55": 0.22405, - "56": 0.22789, - "57": 0.22456, - "58": 0.23947, - "59": 0.24294, - "60": 0.22777, - "61": 0.22508, - "62": 0.2306, - "63": 0.23205, - "64": 0.23143, - "65": 0.23321, - "66": 0.23216, - "67": 0.23316, - "68": 0.23149, - "69": 0.23283, - "70": 0.22854, - "71": 0.24333, - "72": 0.23197, - "73": 0.22937, - "74": 0.23068, - "75": 0.2279, - "76": 0.22968, - "77": 0.25609, - "78": 0.25409, - "79": 0.25184, - "80": 0.22949, - "81": 0.22763, - "82": 0.22592, - "83": 0.22813, - "84": 0.22963, - "85": 0.23411, - "86": 0.22821, - "87": 0.23117, - "88": 0.23326, - "89": 0.22984, - "90": 0.22828, - "91": 0.23148, - "92": 0.23378, - "93": 0.23729, - "94": 0.23173, - "95": 0.23146, - "96": 0.23193, - "97": 0.23076, - "98": 0.33615, - "99": 0.23042, - "100": 0.25353 + "1": "nan", + "2": 2.6815, + "3": 0.23582, + "4": 0.21969, + "5": 0.22399, + "6": 0.21848, + "7": 0.21944, + "8": 0.21989, + "9": 0.22542, + "10": 0.22685, + "11": 0.22859, + "12": 0.22734, + "13": 0.22735, + "14": 0.22682, + "15": 0.22731, + "16": 0.22724, + "17": 0.22774, + "18": 0.2253, + "19": 0.21338, + "20": 0.21612, + "21": 0.22487, + "22": 0.2609, + "23": 0.34495, + "24": 0.40538, + "25": 0.27265, + "26": 0.22852, + "27": 0.23498, + "28": 0.23458, + "29": 0.2356, + "30": 0.23223, + "31": 0.23427, + "32": 0.23193, + "33": 0.23007, + "34": 0.22762, + "35": 0.22604, + "36": 0.22153, + "37": 0.21923, + "38": 0.21718, + "39": 0.2162, + "40": 0.21653, + "41": 0.21673, + "42": 0.21416, + "43": 0.21439, + "44": 0.2141, + "45": 0.21364, + "46": 0.21263, + "47": 0.2139, + "48": 0.21445, + "49": 0.21424, + "50": 0.21381, + "51": 0.21544, + "52": 0.21075, + "53": 0.21292, + "54": 0.21407, + "55": 0.2167, + "56": 0.21877, + "57": 0.21861, + "58": 0.22087, + "59": 0.21999, + "60": 0.21884, + "61": 0.21841, + "62": 0.21988, + "63": 0.21876, + "64": 0.21811, + "65": 0.21795, + "66": 0.2197, + "67": 0.22005, + "68": 0.21994, + "69": 0.21937, + "70": 0.21964, + "71": 0.22007, + "72": 0.221, + "73": 0.22145, + "74": 0.22069, + "75": 0.22126, + "76": 0.21984, + "77": 0.22096, + "78": 0.2231, + "79": 0.22168, + "80": 0.21932, + "81": 0.21748, + "82": 0.21971, + "83": 0.22113, + "84": 0.22096, + "85": 0.22316, + "86": 0.22043, + "87": 0.22198, + "88": 0.2247, + "89": 0.2219, + "90": 0.22258, + "91": 0.22224, + "92": 0.22132, + "93": 0.22182, + "94": 0.22397, + "95": 0.22547, + "96": 0.22177, + "97": 0.22282, + "98": 0.22255, + "99": 0.22417, + "100": 0.22334 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..0889d8315f2 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.84743, + "52": 9.74209, + "53": 10.05697, + "54": 9.9505, + "55": 9.88145, + "56": 9.61274, + "57": 9.4687, + "58": 9.82193, + "59": 9.57642, + "60": 9.49762, + "61": 9.69189, + "62": 9.9867, + "63": 9.37512, + "64": 9.76679, + "65": 8.94648, + "66": 9.7023, + "67": 9.36326, + "68": 9.7831, + "69": 9.7986, + "70": 9.7317, + "71": 9.62571, + "72": 9.58488, + "73": 9.48967, + "74": 8.9286, + "75": 9.40862, + "76": 9.07925, + "77": 10.0594, + "78": 9.72288, + "79": 9.37784, + "80": 9.40429, + "81": 9.48309, + "82": 9.7004, + "83": 9.31595, + "84": 9.41838, + "85": 9.61685, + "86": 9.07533, + "87": 9.59616, + "88": 9.75215, + "89": 9.60184, + "90": 9.82281, + "91": 9.34037, + "92": 9.35854, + "93": 9.08805, + "94": 8.83037, + "95": 9.5266, + "96": 9.53049, + "97": 9.30389, + "98": 9.67196, + "99": 8.89637, + "100": 9.40644 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2491.0, + "52": 2552.0, + "53": 2980.0, + "54": 2680.0, + "55": 2274.0, + "56": 2734.0, + "57": 2319.0, + "58": 2907.0, + "59": 2886.0, + "60": 2566.0, + "61": 2855.0, + "62": 2704.0, + "63": 2370.0, + "64": 2998.0, + "65": 2563.0, + "66": 2868.0, + "67": 2762.0, + "68": 2739.0, + "69": 2730.0, + "70": 3156.0, + "71": 2803.0, + "72": 2506.0, + "73": 2896.0, + "74": 1937.0, + "75": 2450.0, + "76": 2794.0, + "77": 3047.0, + "78": 3104.0, + "79": 3069.0, + "80": 3286.0, + "81": 3543.0, + "82": 3192.0, + "83": 2614.0, + "84": 3273.0, + "85": 3111.0, + "86": 2680.0, + "87": 3654.0, + "88": 3117.0, + "89": 3351.0, + "90": 3086.0, + "91": 2721.0, + "92": 3045.0, + "93": 2672.0, + "94": 3326.0, + "95": 3125.0, + "96": 3309.0, + "97": 3208.0, + "98": 3572.0, + "99": 2980.0, + "100": 3355.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 515933696.0, + "52": 515933696.0, + "53": 515933696.0, + "54": 515933696.0, + "55": 515933696.0, + "56": 515933696.0, + "57": 515933696.0, + "58": 515933696.0, + "59": 515933696.0, + "60": 515933696.0, + "61": 515933696.0, + "62": 515933696.0, + "63": 515933696.0, + "64": 515933696.0, + "65": 515933696.0, + "66": 515933696.0, + "67": 515933696.0, + "68": 515933696.0, + "69": 515933696.0, + "70": 515933696.0, + "71": 515933696.0, + "72": 515933696.0, + "73": 515933696.0, + "74": 515933696.0, + "75": 515933696.0, + "76": 515933696.0, + "77": 515933696.0, + "78": 515933696.0, + "79": 515933696.0, + "80": 515933696.0, + "81": 515933696.0, + "82": 515933696.0, + "83": 515933696.0, + "84": 515933696.0, + "85": 515933696.0, + "86": 515933696.0, + "87": 515933696.0, + "88": 515933696.0, + "89": 515933696.0, + "90": 515933696.0, + "91": 515933696.0, + "92": 515933696.0, + "93": 515933696.0, + "94": 515933696.0, + "95": 515933696.0, + "96": 515933696.0, + "97": 515933696.0, + "98": 515933696.0, + "99": 515933696.0, + "100": 515933696.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1440754176.0, + "52": 1440755200.0, + "53": 1440755200.0, + "54": 1440755200.0, + "55": 1440755200.0, + "56": 1440755200.0, + "57": 1440755200.0, + "58": 1440755200.0, + "59": 1440755200.0, + "60": 1440755200.0, + "61": 1440755200.0, + "62": 1440755200.0, + "63": 1440755200.0, + "64": 1440755200.0, + "65": 1440755200.0, + "66": 1440755200.0, + "67": 1440755200.0, + "68": 1440755200.0, + "69": 1440755200.0, + "70": 1440755200.0, + "71": 1440755200.0, + "72": 1440755200.0, + "73": 1440755200.0, + "74": 1440755200.0, + "75": 1440755200.0, + "76": 1440755200.0, + "77": 1440755200.0, + "78": 1440755200.0, + "79": 1440755200.0, + "80": 1440755200.0, + "81": 1440755200.0, + "82": 1440755200.0, + "83": 1440755200.0, + "84": 1440755200.0, + "85": 1440755200.0, + "86": 1440755200.0, + "87": 1440755200.0, + "88": 1440755200.0, + "89": 1440755200.0, + "90": 1440755200.0, + "91": 1440755200.0, + "92": 1440755200.0, + "93": 1440755200.0, + "94": 1440755200.0, + "95": 1440755200.0, + "96": 1440755200.0, + "97": 1440755200.0, + "98": 1440755200.0, + "99": 1440755200.0, + "100": 1440755200.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.76016, + "53": 0.23476, + "54": 0.223, + "55": 0.22429, + "56": 0.21901, + "57": 0.22278, + "58": 0.22163, + "59": 0.22214, + "60": 0.22201, + "61": 0.2247, + "62": 0.22616, + "63": 0.22396, + "64": 0.23047, + "65": 0.23737, + "66": 0.24455, + "67": 0.23295, + "68": 0.22857, + "69": 0.22662, + "70": 0.22814, + "71": 0.2322, + "72": 0.233, + "73": 0.22777, + "74": 0.22898, + "75": 0.23307, + "76": 0.23163, + "77": 0.23205, + "78": 0.23196, + "79": 0.2324, + "80": 0.23104, + "81": 0.23192, + "82": 0.23206, + "83": 0.22902, + "84": 0.23961, + "85": 0.24378, + "86": 0.24255, + "87": 0.24283, + "88": 0.24429, + "89": 0.24795, + "90": 0.2492, + "91": 0.2493, + "92": 0.24516, + "93": 0.24543, + "94": 0.23595, + "95": 0.23484, + "96": 0.23416, + "97": 0.24493, + "98": 0.24676, + "99": 0.24195, + "100": 0.2459 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json index 49fb0cee006..ff9932eb6ca 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 6.0946, - "2": 0.23434, - "3": 0.25974, - "4": 0.19572, - "5": 0.19385, - "6": 0.23205, - "7": 0.7704, - "8": 0.19849, - "9": 0.1899, - "10": 0.19145, - "11": 0.22929, - "12": 0.19296, - "13": 0.18887, - "14": 0.18975, - "15": 0.19107, - "16": 0.18736, - "17": 0.18574, - "18": 0.22677, - "19": 0.19016, - "20": 0.20891, - "21": 0.18795, - "22": 0.18702, - "23": 0.18879, - "24": 0.23626, - "25": 0.18708, - "26": 0.21783, - "27": 0.3498, - "28": 0.18687, - "29": 0.20508, - "30": 0.1874, - "31": 0.27079, - "32": 0.19016, - "33": 0.18984, - "34": 0.18963, - "35": 0.25952, - "36": 0.21489, - "37": 0.20358, - "38": 0.20254, - "39": 0.2039, - "40": 0.20108, - "41": 0.18536, - "42": 0.18627, - "43": 0.22134, - "44": 0.19018, - "45": 0.18634, - "46": 0.18446, - "47": 0.19975, - "48": 0.18759, - "49": 0.18704, - "50": 0.18617, - "51": 0.20108, - "52": 0.18371, - "53": 0.18371, - "54": 0.18409, - "55": 0.18492, - "56": 0.18608, - "57": 0.33035, - "58": 0.18444, - "59": 0.18479, - "60": 0.2007, - "61": 0.18737, - "62": 0.54423, - "63": 0.18739, - "64": 0.18756, - "65": 0.22855, - "66": 0.1889, - "67": 0.18728, - "68": 0.18737, - "69": 0.1863, - "70": 0.18731, - "71": 0.22911, - "72": 0.18493, - "73": 0.1846, - "74": 0.1919, - "75": 0.21803, - "76": 0.36578, - "77": 0.22572, - "78": 0.20057, - "79": 0.18852, - "80": 0.53951, - "81": 0.42214, - "82": 0.18567, - "83": 0.18702, - "84": 0.1856, - "85": 0.18727, - "86": 0.18505, - "87": 0.18506, - "88": 0.22119, - "89": 0.22551, - "90": 0.18825, - "91": 0.18812, - "92": 0.18805, - "93": 0.18696, - "94": 0.18716, - "95": 0.18779, - "96": 0.41477, - "97": 0.18674, - "98": 0.20738, - "99": 0.18625, - "100": 0.21802 + "1": "nan", + "2": 2.61029, + "3": 0.19483, + "4": 0.18062, + "5": 0.18259, + "6": 0.18221, + "7": 0.18364, + "8": 0.18219, + "9": 0.18127, + "10": 0.18188, + "11": 0.18181, + "12": 0.1831, + "13": 0.18466, + "14": 0.18253, + "15": 0.18081, + "16": 0.18182, + "17": 0.18252, + "18": 0.18383, + "19": 0.18271, + "20": 0.18292, + "21": 0.18157, + "22": 0.18322, + "23": 0.18379, + "24": 0.18403, + "25": 0.18149, + "26": 0.18219, + "27": 0.18279, + "28": 0.18152, + "29": 0.18195, + "30": 0.18138, + "31": 0.18264, + "32": 0.18261, + "33": 0.18358, + "34": 0.18161, + "35": 0.18269, + "36": 0.18132, + "37": 0.18252, + "38": 0.18362, + "39": 0.18411, + "40": 0.18423, + "41": 0.1835, + "42": 0.18364, + "43": 0.18373, + "44": 0.18306, + "45": 0.18336, + "46": 0.18278, + "47": 0.18327, + "48": 0.18332, + "49": 0.18271, + "50": 0.18318, + "51": 0.19806, + "52": 0.1799, + "53": 0.18156, + "54": 0.1807, + "55": 0.17954, + "56": 0.18001, + "57": 0.18039, + "58": 0.181, + "59": 0.18041, + "60": 0.17989, + "61": 0.18137, + "62": 0.18121, + "63": 0.18088, + "64": 0.1801, + "65": 0.18077, + "66": 0.18006, + "67": 0.18112, + "68": 0.18089, + "69": 0.18124, + "70": 0.17966, + "71": 0.18084, + "72": 0.18137, + "73": 0.18132, + "74": 0.18078, + "75": 0.18129, + "76": 0.18079, + "77": 0.1816, + "78": 0.18161, + "79": 0.18227, + "80": 0.18173, + "81": 0.18145, + "82": 0.18143, + "83": 0.18128, + "84": 0.18207, + "85": 0.18121, + "86": 0.18062, + "87": 0.17981, + "88": 0.18098, + "89": 0.18014, + "90": 0.17967, + "91": 0.18153, + "92": 0.18175, + "93": 0.18107, + "94": 0.17803, + "95": 0.17796, + "96": 0.17757, + "97": 0.17815, + "98": 0.17979, + "99": 0.18056, + "100": 0.18044 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..5a291771fe9 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.8474, + "52": 9.74209, + "53": 10.05695, + "54": 9.95048, + "55": 9.88137, + "56": 9.61274, + "57": 9.46865, + "58": 9.82191, + "59": 9.57642, + "60": 9.49763, + "61": 9.6919, + "62": 9.98672, + "63": 9.37511, + "64": 9.76682, + "65": 8.94645, + "66": 9.70228, + "67": 9.36325, + "68": 9.78311, + "69": 9.79861, + "70": 9.73171, + "71": 9.62575, + "72": 9.58482, + "73": 9.48964, + "74": 8.92857, + "75": 9.40863, + "76": 9.07924, + "77": 10.05936, + "78": 9.72284, + "79": 9.37782, + "80": 9.40428, + "81": 9.48314, + "82": 9.70039, + "83": 9.31593, + "84": 9.41835, + "85": 9.61687, + "86": 9.07538, + "87": 9.59618, + "88": 9.75215, + "89": 9.60188, + "90": 9.82284, + "91": 9.34035, + "92": 9.35853, + "93": 9.08806, + "94": 8.83039, + "95": 9.5266, + "96": 9.53046, + "97": 9.30391, + "98": 9.67197, + "99": 8.89638, + "100": 9.40645 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 71.0, + "52": 53.0, + "53": 77.0, + "54": 92.0, + "55": 67.0, + "56": 92.0, + "57": 86.0, + "58": 79.0, + "59": 74.0, + "60": 70.0, + "61": 98.0, + "62": 71.0, + "63": 64.0, + "64": 83.0, + "65": 89.0, + "66": 86.0, + "67": 62.0, + "68": 67.0, + "69": 57.0, + "70": 90.0, + "71": 66.0, + "72": 61.0, + "73": 76.0, + "74": 52.0, + "75": 63.0, + "76": 78.0, + "77": 78.0, + "78": 87.0, + "79": 83.0, + "80": 77.0, + "81": 102.0, + "82": 74.0, + "83": 67.0, + "84": 68.0, + "85": 96.0, + "86": 89.0, + "87": 92.0, + "88": 81.0, + "89": 47.0, + "90": 76.0, + "91": 70.0, + "92": 82.0, + "93": 58.0, + "94": 76.0, + "95": 71.0, + "96": 92.0, + "97": 67.0, + "98": 88.0, + "99": 66.0, + "100": 69.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 544900608.0, + "52": 544900608.0, + "53": 544900608.0, + "54": 544900608.0, + "55": 544900608.0, + "56": 544900608.0, + "57": 544900608.0, + "58": 544900608.0, + "59": 544900608.0, + "60": 544900608.0, + "61": 544900608.0, + "62": 544900608.0, + "63": 544900608.0, + "64": 544900608.0, + "65": 544900608.0, + "66": 544900608.0, + "67": 544900608.0, + "68": 544900608.0, + "69": 544900608.0, + "70": 544900608.0, + "71": 544900608.0, + "72": 544900608.0, + "73": 544900608.0, + "74": 544900608.0, + "75": 544900608.0, + "76": 544900608.0, + "77": 544900608.0, + "78": 544900608.0, + "79": 544900608.0, + "80": 544900608.0, + "81": 544900608.0, + "82": 544900608.0, + "83": 544900608.0, + "84": 544900608.0, + "85": 544900608.0, + "86": 544900608.0, + "87": 544900608.0, + "88": 544900608.0, + "89": 544900608.0, + "90": 544900608.0, + "91": 544900608.0, + "92": 544900608.0, + "93": 544900608.0, + "94": 544900608.0, + "95": 544900608.0, + "96": 544900608.0, + "97": 544900608.0, + "98": 544900608.0, + "99": 544900608.0, + "100": 544900608.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1909074432.0, + "52": 1909075456.0, + "53": 1909075456.0, + "54": 1909075456.0, + "55": 1909075456.0, + "56": 1909075456.0, + "57": 1909075456.0, + "58": 1909075456.0, + "59": 1909075456.0, + "60": 1909075456.0, + "61": 1909075456.0, + "62": 1909075456.0, + "63": 1909075456.0, + "64": 1909075456.0, + "65": 1909075456.0, + "66": 1909075456.0, + "67": 1909075456.0, + "68": 1909075456.0, + "69": 1909075456.0, + "70": 1909075456.0, + "71": 1909075456.0, + "72": 1909075456.0, + "73": 1909075456.0, + "74": 1909075456.0, + "75": 1909075456.0, + "76": 1909075456.0, + "77": 1909075456.0, + "78": 1909075456.0, + "79": 1909075456.0, + "80": 1909075456.0, + "81": 1909075456.0, + "82": 1909075456.0, + "83": 1909075456.0, + "84": 1909075456.0, + "85": 1909075456.0, + "86": 1909075456.0, + "87": 1909075456.0, + "88": 1909075456.0, + "89": 1909075456.0, + "90": 1909075456.0, + "91": 1909075456.0, + "92": 1909075456.0, + "93": 1909075456.0, + "94": 1909075456.0, + "95": 1909075456.0, + "96": 1909075456.0, + "97": 1909075456.0, + "98": 1909075456.0, + "99": 1909075456.0, + "100": 1909075456.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.55843, + "53": 0.19589, + "54": 0.18471, + "55": 0.18349, + "56": 0.18435, + "57": 0.18418, + "58": 0.18539, + "59": 0.18565, + "60": 0.18678, + "61": 0.18442, + "62": 0.18318, + "63": 0.18314, + "64": 0.18315, + "65": 0.18517, + "66": 0.18507, + "67": 0.18538, + "68": 0.18295, + "69": 0.18459, + "70": 0.18533, + "71": 0.1857, + "72": 0.1861, + "73": 0.18507, + "74": 0.18454, + "75": 0.18322, + "76": 0.18308, + "77": 0.18278, + "78": 0.18413, + "79": 0.18228, + "80": 0.18383, + "81": 0.18491, + "82": 0.18405, + "83": 0.18374, + "84": 0.18428, + "85": 0.18358, + "86": 0.18433, + "87": 0.18542, + "88": 0.18544, + "89": 0.1847, + "90": 0.18536, + "91": 0.18553, + "92": 0.18571, + "93": 0.18611, + "94": 0.18506, + "95": 0.18462, + "96": 0.18458, + "97": 0.18459, + "98": 0.18525, + "99": 0.18232, + "100": 0.18404 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200.json index 8325c3b9e5b..e00e8181bf7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.86836, "2": 10.88595, "3": 10.86559, - "4": 10.86889, - "5": 10.87417, - "6": 10.8906, - "7": 10.87677, - "8": 10.86475, - "9": 10.88232, + "4": 10.8689, + "5": 10.8742, + "6": 10.89059, + "7": 10.87676, + "8": 10.86476, + "9": 10.88236, "10": 10.84582, - "11": 10.87162, + "11": 10.87163, "12": 10.87422, "13": 10.88163, - "14": 10.88889, - "15": 10.83931, - "16": 10.82496, - "17": 10.80147, - "18": 10.81234, - "19": 10.82152, - "20": 10.71933, - "21": 10.69091, + "14": 10.88884, + "15": 10.83936, + "16": 10.82495, + "17": 10.80144, + "18": 10.81238, + "19": 10.82157, + "20": 10.71932, + "21": 10.69086, "22": 10.57426, "23": 10.71097, "24": 10.5978, - "25": 10.5556, - "26": 10.61522, - "27": 10.60451, - "28": 10.56484, - "29": 10.58476, - "30": 10.35944, - "31": 10.12157, + "25": 10.55559, + "26": 10.61521, + "27": 10.6045, + "28": 10.56482, + "29": 10.58474, + "30": 10.35947, + "31": 10.12154, "32": 10.45234, - "33": 10.45725, - "34": 10.21989, + "33": 10.45723, + "34": 10.21986, "35": 10.26445, - "36": 10.21036, - "37": 10.33952, - "38": 10.18015, - "39": 10.39589, - "40": 10.06631, + "36": 10.21034, + "37": 10.33954, + "38": 10.18014, + "39": 10.39592, + "40": 10.06632, "41": 10.14164, - "42": 10.20853, - "43": 9.83127, - "44": 9.94861, - "45": 9.82847, - "46": 9.8046, - "47": 10.14233, - "48": 9.84459, - "49": 9.52195, - "50": 9.88603, - "51": 9.84982, - "52": 9.74428, - "53": 10.05844, - "54": 9.95125, + "42": 10.20846, + "43": 9.83129, + "44": 9.94859, + "45": 9.82846, + "46": 9.80459, + "47": 10.1423, + "48": 9.84463, + "49": 9.52196, + "50": 9.88602, + "51": 9.84981, + "52": 9.74433, + "53": 10.05841, + "54": 9.95128, "55": 9.88345, - "56": 9.61327, - "57": 9.469, - "58": 9.82161, - "59": 9.57703, - "60": 9.49786, + "56": 9.61328, + "57": 9.46898, + "58": 9.82164, + "59": 9.577, + "60": 9.49788, "61": 9.69254, - "62": 9.98597, - "63": 9.37405, - "64": 9.76601, - "65": 8.94654, - "66": 9.70099, + "62": 9.98596, + "63": 9.37406, + "64": 9.76602, + "65": 8.94652, + "66": 9.70103, "67": 9.36368, - "68": 9.7824, - "69": 9.7988, - "70": 9.73166, - "71": 9.62509, - "72": 9.58308, - "73": 9.48821, - "74": 8.92607, - "75": 9.40719, - "76": 9.07708, - "77": 10.05856, - "78": 9.72208, - "79": 9.37661, - "80": 9.40273, + "68": 9.78239, + "69": 9.79883, + "70": 9.73167, + "71": 9.62508, + "72": 9.58312, + "73": 9.4882, + "74": 8.92612, + "75": 9.40726, + "76": 9.07709, + "77": 10.05858, + "78": 9.72206, + "79": 9.37662, + "80": 9.40272, "81": 9.48208, - "82": 9.69949, - "83": 9.31353, + "82": 9.69955, + "83": 9.31357, "84": 9.41731, - "85": 9.61581, - "86": 9.07429, + "85": 9.61585, + "86": 9.0743, "87": 9.59556, "88": 9.75063, - "89": 9.60041, - "90": 9.82207, - "91": 9.33877, - "92": 9.35776, - "93": 9.0867, - "94": 8.8296, - "95": 9.52595, - "96": 9.52972, + "89": 9.60037, + "90": 9.82206, + "91": 9.33875, + "92": 9.3578, + "93": 9.08666, + "94": 8.82958, + "95": 9.52592, + "96": 9.52973, "97": 9.30331, - "98": 9.67136, - "99": 8.89539, - "100": 9.40568 + "98": 9.67138, + "99": 8.89537, + "100": 9.40567 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1595.0, - "2": 1608.0, - "3": 1639.0, - "4": 1667.0, - "5": 1811.0, - "6": 1793.0, - "7": 1871.0, - "8": 1579.0, - "9": 1850.0, - "10": 1352.0, - "11": 1853.0, - "12": 1662.0, - "13": 1705.0, - "14": 1659.0, - "15": 1812.0, - "16": 1720.0, - "17": 1682.0, - "18": 1583.0, - "19": 1728.0, - "20": 1655.0, - "21": 1978.0, - "22": 1627.0, - "23": 1863.0, - "24": 1654.0, - "25": 1514.0, - "26": 1697.0, - "27": 1653.0, - "28": 1949.0, - "29": 1931.0, - "30": 1896.0, - "31": 1522.0, - "32": 1915.0, - "33": 2134.0, - "34": 1700.0, - "35": 1860.0, - "36": 1880.0, - "37": 2310.0, - "38": 2101.0, - "39": 2417.0, - "40": 2076.0, - "41": 2319.0, - "42": 2199.0, - "43": 1874.0, - "44": 2080.0, - "45": 1980.0, - "46": 2302.0, - "47": 2470.0, - "48": 2202.0, - "49": 2280.0, - "50": 2439.0, - "51": 2490.0, - "52": 2545.0, - "53": 2999.0, - "54": 2565.0, - "55": 2285.0, - "56": 2699.0, - "57": 2189.0, - "58": 2878.0, - "59": 2978.0, - "60": 2478.0, - "61": 2815.0, - "62": 2666.0, - "63": 2512.0, - "64": 2966.0, - "65": 2533.0, - "66": 2865.0, - "67": 2741.0, - "68": 2760.0, - "69": 2810.0, - "70": 3115.0, - "71": 2918.0, - "72": 2413.0, - "73": 2837.0, - "74": 1901.0, - "75": 2387.0, - "76": 2899.0, - "77": 3019.0, - "78": 3233.0, - "79": 3193.0, - "80": 3288.0, - "81": 3397.0, - "82": 3181.0, - "83": 2672.0, - "84": 3163.0, - "85": 3128.0, - "86": 2647.0, - "87": 3754.0, - "88": 3098.0, - "89": 3372.0, - "90": 2966.0, - "91": 2776.0, - "92": 2983.0, - "93": 2767.0, - "94": 3263.0, - "95": 3238.0, - "96": 3471.0, - "97": 3231.0, - "98": 3528.0, - "99": 3090.0, - "100": 3319.0 + "1": 1621.0, + "2": 1581.0, + "3": 1660.0, + "4": 1639.0, + "5": 1858.0, + "6": 1746.0, + "7": 1789.0, + "8": 1599.0, + "9": 1866.0, + "10": 1400.0, + "11": 1838.0, + "12": 1702.0, + "13": 1844.0, + "14": 1707.0, + "15": 1824.0, + "16": 1828.0, + "17": 1810.0, + "18": 1568.0, + "19": 1747.0, + "20": 1605.0, + "21": 1936.0, + "22": 1586.0, + "23": 1869.0, + "24": 1508.0, + "25": 1506.0, + "26": 1674.0, + "27": 1742.0, + "28": 1978.0, + "29": 1867.0, + "30": 1888.0, + "31": 1551.0, + "32": 1866.0, + "33": 2085.0, + "34": 1816.0, + "35": 1884.0, + "36": 1866.0, + "37": 2390.0, + "38": 2008.0, + "39": 2403.0, + "40": 2077.0, + "41": 2225.0, + "42": 2252.0, + "43": 1924.0, + "44": 2075.0, + "45": 1956.0, + "46": 2175.0, + "47": 2425.0, + "48": 2225.0, + "49": 2244.0, + "50": 2430.0, + "51": 2464.0, + "52": 2572.0, + "53": 2977.0, + "54": 2621.0, + "55": 2248.0, + "56": 2813.0, + "57": 2293.0, + "58": 2874.0, + "59": 2959.0, + "60": 2499.0, + "61": 2762.0, + "62": 2658.0, + "63": 2472.0, + "64": 2840.0, + "65": 2587.0, + "66": 2880.0, + "67": 2813.0, + "68": 2775.0, + "69": 2821.0, + "70": 3127.0, + "71": 2870.0, + "72": 2609.0, + "73": 2835.0, + "74": 1993.0, + "75": 2474.0, + "76": 2896.0, + "77": 3050.0, + "78": 3120.0, + "79": 3093.0, + "80": 3284.0, + "81": 3502.0, + "82": 3169.0, + "83": 2614.0, + "84": 3087.0, + "85": 3140.0, + "86": 2590.0, + "87": 3631.0, + "88": 3120.0, + "89": 3308.0, + "90": 3137.0, + "91": 2801.0, + "92": 2977.0, + "93": 2727.0, + "94": 3180.0, + "95": 3264.0, + "96": 3436.0, + "97": 3329.0, + "98": 3661.0, + "99": 3152.0, + "100": 3367.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 464552448.0, - "2": 464552448.0, - "3": 464552448.0, - "4": 464552448.0, - "5": 464552448.0, - "6": 464552448.0, - "7": 464552448.0, - "8": 464552448.0, - "9": 464552448.0, - "10": 464552448.0, - "11": 464552448.0, - "12": 464552448.0, - "13": 464552448.0, - "14": 464552448.0, - "15": 464552448.0, - "16": 464552448.0, - "17": 464552448.0, - "18": 464552448.0, - "19": 464552448.0, - "20": 464552448.0, - "21": 464552448.0, - "22": 464552448.0, - "23": 464552448.0, - "24": 464552448.0, - "25": 464552448.0, - "26": 464552448.0, - "27": 464552448.0, - "28": 464552448.0, - "29": 464552448.0, - "30": 464552448.0, - "31": 464552448.0, - "32": 464552448.0, - "33": 464552448.0, - "34": 464552448.0, - "35": 464552448.0, - "36": 464552448.0, - "37": 464552448.0, - "38": 464552448.0, - "39": 464552448.0, - "40": 464552448.0, - "41": 464552448.0, - "42": 464552448.0, - "43": 464552448.0, - "44": 464552448.0, - "45": 464552448.0, - "46": 464552448.0, - "47": 464552448.0, - "48": 464552448.0, - "49": 464552448.0, - "50": 464552448.0, - "51": 464552448.0, - "52": 464552448.0, - "53": 464552448.0, - "54": 464552448.0, - "55": 464552448.0, - "56": 464552448.0, - "57": 464552448.0, - "58": 464552448.0, - "59": 464552448.0, - "60": 464552448.0, - "61": 464552448.0, - "62": 464552448.0, - "63": 464552448.0, - "64": 464552448.0, - "65": 464552448.0, - "66": 464552448.0, - "67": 464552448.0, - "68": 464552448.0, - "69": 464552448.0, - "70": 464552448.0, - "71": 464552448.0, - "72": 464552448.0, - "73": 464552448.0, - "74": 464552448.0, - "75": 464552448.0, - "76": 464552448.0, - "77": 464552448.0, - "78": 464552448.0, - "79": 464552448.0, - "80": 464552448.0, - "81": 464552448.0, - "82": 464552448.0, - "83": 464552448.0, - "84": 464552448.0, - "85": 464552448.0, - "86": 464552448.0, - "87": 464552448.0, - "88": 464552448.0, - "89": 464552448.0, - "90": 464552448.0, - "91": 464552448.0, - "92": 464552448.0, - "93": 464552448.0, - "94": 464552448.0, - "95": 464552448.0, - "96": 464552448.0, - "97": 464552448.0, - "98": 464552448.0, - "99": 464552448.0, - "100": 464552448.0 + "1": 465207808.0, + "2": 466256384.0, + "3": 466256384.0, + "4": 466256384.0, + "5": 466256384.0, + "6": 466256384.0, + "7": 466256384.0, + "8": 466256384.0, + "9": 466256384.0, + "10": 466256384.0, + "11": 466256384.0, + "12": 466256384.0, + "13": 466256384.0, + "14": 466256384.0, + "15": 466256384.0, + "16": 466256384.0, + "17": 466256384.0, + "18": 466256384.0, + "19": 466256384.0, + "20": 466256384.0, + "21": 466256384.0, + "22": 466256384.0, + "23": 466256384.0, + "24": 466256384.0, + "25": 466256384.0, + "26": 466256384.0, + "27": 466256384.0, + "28": 466256384.0, + "29": 466256384.0, + "30": 466256384.0, + "31": 466256384.0, + "32": 466256384.0, + "33": 466256384.0, + "34": 466256384.0, + "35": 466256384.0, + "36": 466256384.0, + "37": 466256384.0, + "38": 466256384.0, + "39": 466256384.0, + "40": 466256384.0, + "41": 466256384.0, + "42": 466256384.0, + "43": 466256384.0, + "44": 466256384.0, + "45": 466256384.0, + "46": 466256384.0, + "47": 466256384.0, + "48": 466256384.0, + "49": 466256384.0, + "50": 466256384.0, + "51": 466256384.0, + "52": 466256384.0, + "53": 466256384.0, + "54": 466256384.0, + "55": 466256384.0, + "56": 466256384.0, + "57": 466256384.0, + "58": 466256384.0, + "59": 466256384.0, + "60": 466256384.0, + "61": 466256384.0, + "62": 466256384.0, + "63": 466256384.0, + "64": 466256384.0, + "65": 466256384.0, + "66": 466256384.0, + "67": 466256384.0, + "68": 466256384.0, + "69": 466256384.0, + "70": 466256384.0, + "71": 466256384.0, + "72": 466256384.0, + "73": 466256384.0, + "74": 466256384.0, + "75": 466256384.0, + "76": 466256384.0, + "77": 466256384.0, + "78": 466256384.0, + "79": 466256384.0, + "80": 466256384.0, + "81": 466256384.0, + "82": 466256384.0, + "83": 466256384.0, + "84": 466256384.0, + "85": 466256384.0, + "86": 466256384.0, + "87": 466256384.0, + "88": 466256384.0, + "89": 466256384.0, + "90": 466256384.0, + "91": 466256384.0, + "92": 466256384.0, + "93": 466256384.0, + "94": 466256384.0, + "95": 466256384.0, + "96": 466256384.0, + "97": 466256384.0, + "98": 466256384.0, + "99": 466256384.0, + "100": 466256384.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1196456448.0, - "2": 1374693888.0, - "3": 1374693888.0, - "4": 1374693888.0, - "5": 1374693888.0, - "6": 1374693888.0, - "7": 1374693888.0, - "8": 1374693888.0, - "9": 1374693888.0, - "10": 1374693888.0, - "11": 1374693888.0, - "12": 1374693888.0, - "13": 1374693888.0, - "14": 1374693888.0, - "15": 1374693888.0, - "16": 1374693888.0, - "17": 1374693888.0, - "18": 1374693888.0, - "19": 1374693888.0, - "20": 1374693888.0, - "21": 1374693888.0, - "22": 1374693888.0, - "23": 1374693888.0, - "24": 1374693888.0, - "25": 1374693888.0, - "26": 1374693888.0, - "27": 1374693888.0, - "28": 1374693888.0, - "29": 1374693888.0, - "30": 1374693888.0, - "31": 1374693888.0, - "32": 1374693888.0, - "33": 1374693888.0, - "34": 1374693888.0, - "35": 1374693888.0, - "36": 1374693888.0, - "37": 1374693888.0, - "38": 1374693888.0, - "39": 1374693888.0, - "40": 1374693888.0, - "41": 1374693888.0, - "42": 1374693888.0, - "43": 1374693888.0, - "44": 1374693888.0, - "45": 1374693888.0, - "46": 1374693888.0, - "47": 1374693888.0, - "48": 1374693888.0, - "49": 1374693888.0, - "50": 1374693888.0, - "51": 1374693888.0, - "52": 1374693888.0, - "53": 1374693888.0, - "54": 1374693888.0, - "55": 1374693888.0, - "56": 1374693888.0, - "57": 1374693888.0, - "58": 1374693888.0, - "59": 1374693888.0, - "60": 1374693888.0, - "61": 1374693888.0, - "62": 1374693888.0, - "63": 1374693888.0, - "64": 1374693888.0, - "65": 1374693888.0, - "66": 1374693888.0, - "67": 1374693888.0, - "68": 1374693888.0, - "69": 1374693888.0, - "70": 1374693888.0, - "71": 1374693888.0, - "72": 1374693888.0, - "73": 1374693888.0, - "74": 1374693888.0, - "75": 1374693888.0, - "76": 1374693888.0, - "77": 1374693888.0, - "78": 1374693888.0, - "79": 1374693888.0, - "80": 1374693888.0, - "81": 1374693888.0, - "82": 1374693888.0, - "83": 1374693888.0, - "84": 1374693888.0, - "85": 1374693888.0, - "86": 1374693888.0, - "87": 1374693888.0, - "88": 1374693888.0, - "89": 1374693888.0, - "90": 1374693888.0, - "91": 1374693888.0, - "92": 1374693888.0, - "93": 1374693888.0, - "94": 1374693888.0, - "95": 1374693888.0, - "96": 1374693888.0, - "97": 1374693888.0, - "98": 1374693888.0, - "99": 1374693888.0, - "100": 1374693888.0 + "1": 1195407872.0, + "2": 1376397824.0, + "3": 1376397824.0, + "4": 1376397824.0, + "5": 1376397824.0, + "6": 1376397824.0, + "7": 1376397824.0, + "8": 1376397824.0, + "9": 1376397824.0, + "10": 1376397824.0, + "11": 1376397824.0, + "12": 1376397824.0, + "13": 1376397824.0, + "14": 1376397824.0, + "15": 1376397824.0, + "16": 1376397824.0, + "17": 1376397824.0, + "18": 1376397824.0, + "19": 1376397824.0, + "20": 1376397824.0, + "21": 1376397824.0, + "22": 1376397824.0, + "23": 1376397824.0, + "24": 1376397824.0, + "25": 1376397824.0, + "26": 1376397824.0, + "27": 1376397824.0, + "28": 1376397824.0, + "29": 1376397824.0, + "30": 1376397824.0, + "31": 1376397824.0, + "32": 1376397824.0, + "33": 1376397824.0, + "34": 1376397824.0, + "35": 1376397824.0, + "36": 1376397824.0, + "37": 1376397824.0, + "38": 1376397824.0, + "39": 1376397824.0, + "40": 1376397824.0, + "41": 1376397824.0, + "42": 1376397824.0, + "43": 1376397824.0, + "44": 1376397824.0, + "45": 1376397824.0, + "46": 1376397824.0, + "47": 1376397824.0, + "48": 1376397824.0, + "49": 1376397824.0, + "50": 1376397824.0, + "51": 1376397824.0, + "52": 1376397824.0, + "53": 1376397824.0, + "54": 1376397824.0, + "55": 1376397824.0, + "56": 1376397824.0, + "57": 1376397824.0, + "58": 1376397824.0, + "59": 1376397824.0, + "60": 1376397824.0, + "61": 1376397824.0, + "62": 1376397824.0, + "63": 1376397824.0, + "64": 1376397824.0, + "65": 1376397824.0, + "66": 1376397824.0, + "67": 1376397824.0, + "68": 1376397824.0, + "69": 1376397824.0, + "70": 1376397824.0, + "71": 1376397824.0, + "72": 1376397824.0, + "73": 1376397824.0, + "74": 1376397824.0, + "75": 1376397824.0, + "76": 1376397824.0, + "77": 1376397824.0, + "78": 1376397824.0, + "79": 1376397824.0, + "80": 1376397824.0, + "81": 1376397824.0, + "82": 1376397824.0, + "83": 1376397824.0, + "84": 1376397824.0, + "85": 1376397824.0, + "86": 1376397824.0, + "87": 1376397824.0, + "88": 1376397824.0, + "89": 1376397824.0, + "90": 1376397824.0, + "91": 1376397824.0, + "92": 1376397824.0, + "93": 1376397824.0, + "94": 1376397824.0, + "95": 1376397824.0, + "96": 1376397824.0, + "97": 1376397824.0, + "98": 1376397824.0, + "99": 1376397824.0, + "100": 1376397824.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 9.03488, - "2": 0.20387, - "3": 0.20622, - "4": 0.19336, - "5": 0.19521, - "6": 0.20191, - "7": 0.19444, - "8": 0.23348, - "9": 0.25611, - "10": 0.24659, - "11": 0.19017, - "12": 0.2556, - "13": 0.18852, - "14": 0.18766, - "15": 0.2289, - "16": 0.18803, - "17": 0.18847, - "18": 0.18567, - "19": 0.18706, - "20": 0.18811, - "21": 0.20215, - "22": 0.39605, - "23": 0.18875, - "24": 0.21086, - "25": 0.18732, - "26": 0.18675, - "27": 0.18833, - "28": 0.23402, - "29": 0.18843, - "30": 0.18769, - "31": 0.21593, - "32": 0.21936, - "33": 0.18843, - "34": 0.21993, - "35": 0.18728, - "36": 0.18741, - "37": 0.18775, - "38": 0.22431, - "39": 0.24159, - "40": 0.25325, - "41": 0.18582, - "42": 0.18658, - "43": 0.24562, - "44": 0.30876, - "45": 0.22398, - "46": 0.18667, - "47": 0.18821, - "48": 0.18742, - "49": 0.20501, - "50": 0.18644, - "51": 0.19893, - "52": 0.18375, - "53": 0.18186, - "54": 0.18268, - "55": 0.18616, - "56": 0.32841, - "57": 0.18567, - "58": 0.41637, - "59": 0.25482, - "60": 0.18467, - "61": 0.21026, - "62": 0.18373, - "63": 0.20727, - "64": 0.44141, - "65": 0.18532, - "66": 0.18662, - "67": 0.18805, - "68": 0.1877, - "69": 0.18579, - "70": 0.18644, - "71": 0.20361, - "72": 0.25218, - "73": 0.18582, - "74": 0.21341, - "75": 0.1876, - "76": 0.18385, - "77": 0.18512, - "78": 0.18447, - "79": 0.18604, - "80": 0.44402, - "81": 0.22886, - "82": 0.18502, - "83": 0.18578, - "84": 0.18519, - "85": 0.18624, - "86": 0.18704, - "87": 0.18561, - "88": 0.1864, - "89": 0.18676, - "90": 0.18596, - "91": 0.18759, - "92": 0.18643, - "93": 0.2303, - "94": 0.18509, - "95": 0.18557, - "96": 0.22378, - "97": 0.18724, - "98": 0.18202, - "99": 0.19781, - "100": 0.22613 + "1": "nan", + "2": 2.70208, + "3": 0.19598, + "4": 0.17697, + "5": 0.17606, + "6": 0.17518, + "7": 0.17383, + "8": 0.17622, + "9": 0.17697, + "10": 0.17845, + "11": 0.17811, + "12": 0.17772, + "13": 0.17922, + "14": 0.17797, + "15": 0.17934, + "16": 0.18103, + "17": 0.18059, + "18": 0.17963, + "19": 0.18123, + "20": 0.18073, + "21": 0.18061, + "22": 0.18072, + "23": 0.17975, + "24": 0.18067, + "25": 0.18039, + "26": 0.17929, + "27": 0.17948, + "28": 0.17917, + "29": 0.17952, + "30": 0.17908, + "31": 0.1795, + "32": 0.17963, + "33": 0.17979, + "34": 0.18047, + "35": 0.18032, + "36": 0.18061, + "37": 0.17948, + "38": 0.17887, + "39": 0.17897, + "40": 0.17957, + "41": 0.17982, + "42": 0.1808, + "43": 0.17965, + "44": 0.1807, + "45": 0.18099, + "46": 0.17964, + "47": 0.17861, + "48": 0.17946, + "49": 0.18292, + "50": 0.18089, + "51": 0.18183, + "52": 0.16643, + "53": 0.16906, + "54": 0.16731, + "55": 0.16773, + "56": 0.16957, + "57": 0.1691, + "58": 0.17123, + "59": 0.17207, + "60": 0.17308, + "61": 0.17219, + "62": 0.17353, + "63": 0.17543, + "64": 0.17335, + "65": 0.17469, + "66": 0.17402, + "67": 0.17585, + "68": 0.17421, + "69": 0.17363, + "70": 0.1748, + "71": 0.17377, + "72": 0.17421, + "73": 0.17466, + "74": 0.17508, + "75": 0.17297, + "76": 0.17297, + "77": 0.17289, + "78": 0.17516, + "79": 0.17501, + "80": 0.17483, + "81": 0.17493, + "82": 0.17481, + "83": 0.17496, + "84": 0.17501, + "85": 0.17642, + "86": 0.17507, + "87": 0.17445, + "88": 0.17535, + "89": 0.17531, + "90": 0.17467, + "91": 0.17485, + "92": 0.17537, + "93": 0.17577, + "94": 0.1757, + "95": 0.1752, + "96": 0.17534, + "97": 0.17544, + "98": 0.17458, + "99": 0.17379, + "100": 0.17525 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..bb062f69f88 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.84981, + "52": 9.74427, + "53": 10.05843, + "54": 9.9513, + "55": 9.88338, + "56": 9.61325, + "57": 9.46895, + "58": 9.82166, + "59": 9.57703, + "60": 9.49787, + "61": 9.69257, + "62": 9.98595, + "63": 9.37399, + "64": 9.76604, + "65": 8.94651, + "66": 9.70103, + "67": 9.36368, + "68": 9.78235, + "69": 9.79883, + "70": 9.73165, + "71": 9.62507, + "72": 9.5831, + "73": 9.48817, + "74": 8.92613, + "75": 9.40726, + "76": 9.07706, + "77": 10.0586, + "78": 9.72205, + "79": 9.37661, + "80": 9.4027, + "81": 9.48209, + "82": 9.69951, + "83": 9.31355, + "84": 9.41731, + "85": 9.61584, + "86": 9.07426, + "87": 9.59553, + "88": 9.75065, + "89": 9.60039, + "90": 9.82207, + "91": 9.33876, + "92": 9.35777, + "93": 9.08671, + "94": 8.82959, + "95": 9.52597, + "96": 9.52973, + "97": 9.30334, + "98": 9.67135, + "99": 8.89539, + "100": 9.40569 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2483.0, + "52": 2539.0, + "53": 2798.0, + "54": 2526.0, + "55": 2228.0, + "56": 2738.0, + "57": 2252.0, + "58": 2797.0, + "59": 2893.0, + "60": 2453.0, + "61": 2929.0, + "62": 2698.0, + "63": 2347.0, + "64": 2902.0, + "65": 2556.0, + "66": 2922.0, + "67": 2829.0, + "68": 2669.0, + "69": 2814.0, + "70": 3041.0, + "71": 2872.0, + "72": 2512.0, + "73": 2971.0, + "74": 1870.0, + "75": 2349.0, + "76": 2844.0, + "77": 3121.0, + "78": 3116.0, + "79": 3196.0, + "80": 3164.0, + "81": 3454.0, + "82": 3176.0, + "83": 2613.0, + "84": 3093.0, + "85": 3128.0, + "86": 2792.0, + "87": 3771.0, + "88": 3108.0, + "89": 3297.0, + "90": 3042.0, + "91": 2850.0, + "92": 2873.0, + "93": 2709.0, + "94": 3294.0, + "95": 3282.0, + "96": 3536.0, + "97": 3150.0, + "98": 3479.0, + "99": 3113.0, + "100": 3370.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 465864192.0, + "52": 465864192.0, + "53": 465864192.0, + "54": 465864192.0, + "55": 465864192.0, + "56": 465864192.0, + "57": 465864192.0, + "58": 465864192.0, + "59": 465864192.0, + "60": 465864192.0, + "61": 465864192.0, + "62": 465864192.0, + "63": 465864192.0, + "64": 465864192.0, + "65": 465864192.0, + "66": 465864192.0, + "67": 465864192.0, + "68": 465864192.0, + "69": 465864192.0, + "70": 465864192.0, + "71": 465864192.0, + "72": 465864192.0, + "73": 465864192.0, + "74": 465864192.0, + "75": 465864192.0, + "76": 465864192.0, + "77": 465864192.0, + "78": 465864192.0, + "79": 465864192.0, + "80": 465864192.0, + "81": 465864192.0, + "82": 465864192.0, + "83": 465864192.0, + "84": 465864192.0, + "85": 465864192.0, + "86": 465864192.0, + "87": 465864192.0, + "88": 465864192.0, + "89": 465864192.0, + "90": 465864192.0, + "91": 465864192.0, + "92": 465864192.0, + "93": 465864192.0, + "94": 465864192.0, + "95": 465864192.0, + "96": 465864192.0, + "97": 465864192.0, + "98": 465864192.0, + "99": 465864192.0, + "100": 465864192.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1378101760.0, + "52": 1378102784.0, + "53": 1378102784.0, + "54": 1378102784.0, + "55": 1378102784.0, + "56": 1378102784.0, + "57": 1378102784.0, + "58": 1378102784.0, + "59": 1378102784.0, + "60": 1378102784.0, + "61": 1378102784.0, + "62": 1378102784.0, + "63": 1378102784.0, + "64": 1378102784.0, + "65": 1378102784.0, + "66": 1378102784.0, + "67": 1378102784.0, + "68": 1378102784.0, + "69": 1378102784.0, + "70": 1378102784.0, + "71": 1378102784.0, + "72": 1378102784.0, + "73": 1378102784.0, + "74": 1378102784.0, + "75": 1378102784.0, + "76": 1378102784.0, + "77": 1378102784.0, + "78": 1378102784.0, + "79": 1378102784.0, + "80": 1378102784.0, + "81": 1378102784.0, + "82": 1378102784.0, + "83": 1378102784.0, + "84": 1378102784.0, + "85": 1378102784.0, + "86": 1378102784.0, + "87": 1378102784.0, + "88": 1378102784.0, + "89": 1378102784.0, + "90": 1378102784.0, + "91": 1378102784.0, + "92": 1378102784.0, + "93": 1378102784.0, + "94": 1378102784.0, + "95": 1378102784.0, + "96": 1378102784.0, + "97": 1378102784.0, + "98": 1378102784.0, + "99": 1378102784.0, + "100": 1378102784.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.91958, + "53": 0.19241, + "54": 0.1787, + "55": 0.17896, + "56": 0.17921, + "57": 0.17976, + "58": 0.17909, + "59": 0.18055, + "60": 0.18105, + "61": 0.18154, + "62": 0.18039, + "63": 0.18167, + "64": 0.1811, + "65": 0.18155, + "66": 0.18214, + "67": 0.18228, + "68": 0.18197, + "69": 0.18293, + "70": 0.1812, + "71": 0.18051, + "72": 0.18186, + "73": 0.18056, + "74": 0.17931, + "75": 0.17657, + "76": 0.17103, + "77": 0.16975, + "78": 0.17336, + "79": 0.17356, + "80": 0.17203, + "81": 0.17343, + "82": 0.17407, + "83": 0.17347, + "84": 0.17434, + "85": 0.17348, + "86": 0.17449, + "87": 0.17439, + "88": 0.17349, + "89": 0.17397, + "90": 0.17349, + "91": 0.17383, + "92": 0.17402, + "93": 0.17583, + "94": 0.17507, + "95": 0.17414, + "96": 0.17276, + "97": 0.17329, + "98": 0.17376, + "99": 0.17325, + "100": 0.17482 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json index b3990651f36..7dd5b31f34f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor/golden_values_dev_dgx_gb200.json @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 700103168.0, - "2": 700103168.0, - "3": 700103168.0, - "4": 700103168.0, - "5": 700103168.0, - "6": 700103168.0, - "7": 700103168.0, - "8": 700103168.0, - "9": 700103168.0, - "10": 700103168.0, - "11": 700103168.0, - "12": 700103168.0, - "13": 700103168.0, - "14": 700103168.0, - "15": 700103168.0, - "16": 700103168.0, - "17": 700103168.0, - "18": 700103168.0, - "19": 700103168.0, - "20": 700103168.0, - "21": 700103168.0, - "22": 700103168.0, - "23": 700103168.0, - "24": 700103168.0, - "25": 700103168.0, - "26": 700103168.0, - "27": 700103168.0, - "28": 700103168.0, - "29": 700103168.0, - "30": 700103168.0, - "31": 700103168.0, - "32": 700103168.0, - "33": 700103168.0, - "34": 700103168.0, - "35": 700103168.0, - "36": 700103168.0, - "37": 700103168.0, - "38": 700103168.0, - "39": 700103168.0, - "40": 700103168.0, - "41": 700103168.0, - "42": 700103168.0, - "43": 700103168.0, - "44": 700103168.0, - "45": 700103168.0, - "46": 700103168.0, - "47": 700103168.0, - "48": 700103168.0, - "49": 700103168.0, - "50": 700103168.0, - "51": 700103168.0, - "52": 700103168.0, - "53": 700103168.0, - "54": 700103168.0, - "55": 700103168.0, - "56": 700103168.0, - "57": 700103168.0, - "58": 700103168.0, - "59": 700103168.0, - "60": 700103168.0, - "61": 700103168.0, - "62": 700103168.0, - "63": 700103168.0, - "64": 700103168.0, - "65": 700103168.0, - "66": 700103168.0, - "67": 700103168.0, - "68": 700103168.0, - "69": 700103168.0, - "70": 700103168.0, - "71": 700103168.0, - "72": 700103168.0, - "73": 700103168.0, - "74": 700103168.0, - "75": 700103168.0, - "76": 700103168.0, - "77": 700103168.0, - "78": 700103168.0, - "79": 700103168.0, - "80": 700103168.0, - "81": 700103168.0, - "82": 700103168.0, - "83": 700103168.0, - "84": 700103168.0, - "85": 700103168.0, - "86": 700103168.0, - "87": 700103168.0, - "88": 700103168.0, - "89": 700103168.0, - "90": 700103168.0, - "91": 700103168.0, - "92": 700103168.0, - "93": 700103168.0, - "94": 700103168.0, - "95": 700103168.0, - "96": 700103168.0, - "97": 700103168.0, - "98": 700103168.0, - "99": 700103168.0, - "100": 700103168.0 + "1": 246999552.0, + "2": 246999552.0, + "3": 246999552.0, + "4": 246999552.0, + "5": 246999552.0, + "6": 246999552.0, + "7": 246999552.0, + "8": 246999552.0, + "9": 246999552.0, + "10": 246999552.0, + "11": 246999552.0, + "12": 246999552.0, + "13": 246999552.0, + "14": 246999552.0, + "15": 246999552.0, + "16": 246999552.0, + "17": 246999552.0, + "18": 246999552.0, + "19": 246999552.0, + "20": 246999552.0, + "21": 246999552.0, + "22": 246999552.0, + "23": 246999552.0, + "24": 246999552.0, + "25": 246999552.0, + "26": 246999552.0, + "27": 246999552.0, + "28": 246999552.0, + "29": 246999552.0, + "30": 246999552.0, + "31": 246999552.0, + "32": 246999552.0, + "33": 246999552.0, + "34": 246999552.0, + "35": 246999552.0, + "36": 246999552.0, + "37": 246999552.0, + "38": 246999552.0, + "39": 246999552.0, + "40": 246999552.0, + "41": 246999552.0, + "42": 246999552.0, + "43": 246999552.0, + "44": 246999552.0, + "45": 246999552.0, + "46": 246999552.0, + "47": 246999552.0, + "48": 246999552.0, + "49": 246999552.0, + "50": 246999552.0, + "51": 246999552.0, + "52": 246999552.0, + "53": 246999552.0, + "54": 246999552.0, + "55": 246999552.0, + "56": 246999552.0, + "57": 246999552.0, + "58": 246999552.0, + "59": 246999552.0, + "60": 246999552.0, + "61": 246999552.0, + "62": 246999552.0, + "63": 246999552.0, + "64": 246999552.0, + "65": 246999552.0, + "66": 246999552.0, + "67": 246999552.0, + "68": 246999552.0, + "69": 246999552.0, + "70": 246999552.0, + "71": 246999552.0, + "72": 246999552.0, + "73": 246999552.0, + "74": 246999552.0, + "75": 246999552.0, + "76": 246999552.0, + "77": 246999552.0, + "78": 246999552.0, + "79": 246999552.0, + "80": 246999552.0, + "81": 246999552.0, + "82": 246999552.0, + "83": 246999552.0, + "84": 246999552.0, + "85": 246999552.0, + "86": 246999552.0, + "87": 246999552.0, + "88": 246999552.0, + "89": 246999552.0, + "90": 246999552.0, + "91": 246999552.0, + "92": 246999552.0, + "93": 246999552.0, + "94": 246999552.0, + "95": 246999552.0, + "96": 246999552.0, + "97": 246999552.0, + "98": 246999552.0, + "99": 246999552.0, + "100": 246999552.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1981478400.0, - "2": 1981479424.0, - "3": 1981479424.0, - "4": 1981479424.0, - "5": 1981479424.0, - "6": 1981479424.0, - "7": 1981479424.0, - "8": 1981479424.0, - "9": 1981479424.0, - "10": 1981479424.0, - "11": 1981479424.0, - "12": 1981479424.0, - "13": 1981479424.0, - "14": 1981479424.0, - "15": 1981479424.0, - "16": 1981479424.0, - "17": 1981479424.0, - "18": 1981479424.0, - "19": 1981479424.0, - "20": 1981479424.0, - "21": 1981479424.0, - "22": 1981479424.0, - "23": 1981479424.0, - "24": 1981479424.0, - "25": 1981479424.0, - "26": 1981479424.0, - "27": 1981479424.0, - "28": 1981479424.0, - "29": 1981479424.0, - "30": 1981479424.0, - "31": 1981479424.0, - "32": 1981479424.0, - "33": 1981479424.0, - "34": 1981479424.0, - "35": 1981479424.0, - "36": 1981479424.0, - "37": 1981479424.0, - "38": 1981479424.0, - "39": 1981479424.0, - "40": 1981479424.0, - "41": 1981479424.0, - "42": 1981479424.0, - "43": 1981479424.0, - "44": 1981479424.0, - "45": 1981479424.0, - "46": 1981479424.0, - "47": 1981479424.0, - "48": 1981479424.0, - "49": 1981479424.0, - "50": 1981479424.0, - "51": 1981479424.0, - "52": 1981479424.0, - "53": 1981479424.0, - "54": 1981479424.0, - "55": 1981479424.0, - "56": 1981479424.0, - "57": 1981479424.0, - "58": 1981479424.0, - "59": 1981479424.0, - "60": 1981479424.0, - "61": 1981479424.0, - "62": 1981479424.0, - "63": 1981479424.0, - "64": 1981479424.0, - "65": 1981479424.0, - "66": 1981479424.0, - "67": 1981479424.0, - "68": 1981479424.0, - "69": 1981479424.0, - "70": 1981479424.0, - "71": 1981479424.0, - "72": 1981479424.0, - "73": 1981479424.0, - "74": 1981479424.0, - "75": 1981479424.0, - "76": 1981479424.0, - "77": 1981479424.0, - "78": 1981479424.0, - "79": 1981479424.0, - "80": 1981479424.0, - "81": 1981479424.0, - "82": 1981479424.0, - "83": 1981479424.0, - "84": 1981479424.0, - "85": 1981479424.0, - "86": 1981479424.0, - "87": 1981479424.0, - "88": 1981479424.0, - "89": 1981479424.0, - "90": 1981479424.0, - "91": 1981479424.0, - "92": 1981479424.0, - "93": 1981479424.0, - "94": 1981479424.0, - "95": 1981479424.0, - "96": 1981479424.0, - "97": 1981479424.0, - "98": 1981479424.0, - "99": 1981479424.0, - "100": 1981479424.0 + "1": 1528374784.0, + "2": 1528375808.0, + "3": 1528375808.0, + "4": 1528375808.0, + "5": 1528375808.0, + "6": 1528375808.0, + "7": 1528375808.0, + "8": 1528375808.0, + "9": 1528375808.0, + "10": 1528375808.0, + "11": 1528375808.0, + "12": 1528375808.0, + "13": 1528375808.0, + "14": 1528375808.0, + "15": 1528375808.0, + "16": 1528375808.0, + "17": 1528375808.0, + "18": 1528375808.0, + "19": 1528375808.0, + "20": 1528375808.0, + "21": 1528375808.0, + "22": 1528375808.0, + "23": 1528375808.0, + "24": 1528375808.0, + "25": 1528375808.0, + "26": 1528375808.0, + "27": 1528375808.0, + "28": 1528375808.0, + "29": 1528375808.0, + "30": 1528375808.0, + "31": 1528375808.0, + "32": 1528375808.0, + "33": 1528375808.0, + "34": 1528375808.0, + "35": 1528375808.0, + "36": 1528375808.0, + "37": 1528375808.0, + "38": 1528375808.0, + "39": 1528375808.0, + "40": 1528375808.0, + "41": 1528375808.0, + "42": 1528375808.0, + "43": 1528375808.0, + "44": 1528375808.0, + "45": 1528375808.0, + "46": 1528375808.0, + "47": 1528375808.0, + "48": 1528375808.0, + "49": 1528375808.0, + "50": 1528375808.0, + "51": 1528375808.0, + "52": 1528375808.0, + "53": 1528375808.0, + "54": 1528375808.0, + "55": 1528375808.0, + "56": 1528375808.0, + "57": 1528375808.0, + "58": 1528375808.0, + "59": 1528375808.0, + "60": 1528375808.0, + "61": 1528375808.0, + "62": 1528375808.0, + "63": 1528375808.0, + "64": 1528375808.0, + "65": 1528375808.0, + "66": 1528375808.0, + "67": 1528375808.0, + "68": 1528375808.0, + "69": 1528375808.0, + "70": 1528375808.0, + "71": 1528375808.0, + "72": 1528375808.0, + "73": 1528375808.0, + "74": 1528375808.0, + "75": 1528375808.0, + "76": 1528375808.0, + "77": 1528375808.0, + "78": 1528375808.0, + "79": 1528375808.0, + "80": 1528375808.0, + "81": 1528375808.0, + "82": 1528375808.0, + "83": 1528375808.0, + "84": 1528375808.0, + "85": 1528375808.0, + "86": 1528375808.0, + "87": 1528375808.0, + "88": 1528375808.0, + "89": 1528375808.0, + "90": 1528375808.0, + "91": 1528375808.0, + "92": 1528375808.0, + "93": 1528375808.0, + "94": 1528375808.0, + "95": 1528375808.0, + "96": 1528375808.0, + "97": 1528375808.0, + "98": 1528375808.0, + "99": 1528375808.0, + "100": 1528375808.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 4.30733, - "2": 0.54883, - "3": 0.40227, - "4": 0.3032, - "5": 0.22011, - "6": 0.21873, - "7": 0.21589, - "8": 0.21756, - "9": 0.2177, - "10": 0.21872, - "11": 0.23383, - "12": 0.21745, - "13": 0.21657, - "14": 0.21656, - "15": 0.21713, - "16": 0.21742, - "17": 0.21697, - "18": 0.21201, - "19": 0.21506, - "20": 0.2157, - "21": 0.21772, - "22": 0.21677, - "23": 0.21503, - "24": 0.21505, - "25": 0.21274, - "26": 0.21593, - "27": 0.21499, - "28": 0.21603, - "29": 0.21474, - "30": 0.21468, - "31": 0.21508, - "32": 0.21333, - "33": 0.21573, - "34": 0.21478, - "35": 0.21464, - "36": 0.21568, - "37": 0.21601, - "38": 0.21414, - "39": 0.21389, - "40": 0.21264, - "41": 0.21397, - "42": 0.21475, - "43": 0.21799, - "44": 0.21345, - "45": 0.21458, - "46": 0.21222, - "47": 0.2147, - "48": 0.21568, - "49": 0.21432, - "50": 0.21429, - "51": 0.30696, - "52": 0.26677, - "53": 0.22953, - "54": 0.24163, - "55": 0.25403, - "56": 0.26249, - "57": 0.21297, - "58": 0.21192, - "59": 0.20898, - "60": 0.21257, - "61": 0.21307, - "62": 0.21067, - "63": 0.21212, - "64": 0.21044, - "65": 0.21146, - "66": 0.21291, - "67": 0.21327, - "68": 0.21434, - "69": 0.21106, - "70": 0.21146, - "71": 0.21366, - "72": 0.21359, - "73": 0.21245, - "74": 0.21111, - "75": 0.21327, - "76": 0.21236, - "77": 0.21209, - "78": 0.21155, - "79": 0.2124, - "80": 0.21314, - "81": 0.21341, - "82": 0.21206, - "83": 0.21321, - "84": 0.21124, - "85": 0.21448, - "86": 0.21358, - "87": 0.21637, - "88": 0.21209, - "89": 0.21325, - "90": 0.2136, - "91": 0.21349, - "92": 0.20976, - "93": 0.21241, - "94": 0.21301, - "95": 0.21086, - "96": 0.21278, - "97": 0.21118, - "98": 0.21308, - "99": 0.21572, - "100": 0.21585 + "1": "nan", + "2": 1.71105, + "3": 0.22879, + "4": 0.22169, + "5": 0.21979, + "6": 0.21933, + "7": 0.21836, + "8": 0.22054, + "9": 0.22096, + "10": 0.22079, + "11": 0.22255, + "12": 0.21905, + "13": 0.22266, + "14": 0.22261, + "15": 0.22192, + "16": 0.21928, + "17": 0.22014, + "18": 0.2213, + "19": 0.22242, + "20": 0.22097, + "21": 0.21936, + "22": 0.22091, + "23": 0.22071, + "24": 0.22306, + "25": 0.22073, + "26": 0.22028, + "27": 0.22198, + "28": 0.22294, + "29": 0.22204, + "30": 0.21896, + "31": 0.22144, + "32": 0.22279, + "33": 0.22428, + "34": 0.22247, + "35": 0.22192, + "36": 0.22242, + "37": 0.22321, + "38": 0.22186, + "39": 0.22242, + "40": 0.22098, + "41": 0.22254, + "42": 0.55234, + "43": 0.22432, + "44": 0.22103, + "45": 0.22202, + "46": 0.2216, + "47": 0.22107, + "48": 0.21878, + "49": 0.22338, + "50": 0.22181, + "51": 0.22588, + "52": 0.22221, + "53": 0.22214, + "54": 0.22059, + "55": 0.22088, + "56": 0.22231, + "57": 0.2231, + "58": 0.22228, + "59": 0.22136, + "60": 0.22087, + "61": 0.22171, + "62": 0.22165, + "63": 0.22149, + "64": 0.22165, + "65": 0.22916, + "66": 0.25667, + "67": 0.22585, + "68": 0.2212, + "69": 0.22322, + "70": 0.22332, + "71": 0.22291, + "72": 0.22074, + "73": 0.2214, + "74": 0.22287, + "75": 0.21929, + "76": 0.22246, + "77": 0.22148, + "78": 0.22442, + "79": 0.22465, + "80": 0.22859, + "81": 0.22464, + "82": 0.22391, + "83": 0.22417, + "84": 0.22202, + "85": 0.22369, + "86": 0.22224, + "87": 0.22245, + "88": 0.22255, + "89": 0.22379, + "90": 0.22356, + "91": 0.22229, + "92": 0.22297, + "93": 0.22525, + "94": 0.21956, + "95": 0.22318, + "96": 0.22361, + "97": 0.22246, + "98": 0.22326, + "99": 0.22121, + "100": 0.22214 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json index 3264336647e..85eca8a168b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5.51145, - "2": 0.34832, - "3": 0.34015, - "4": 0.32824, - "5": 0.32875, - "6": 0.32954, - "7": 0.3278, - "8": 0.32782, - "9": 0.33548, - "10": 0.32705, - "11": 0.3306, - "12": 0.649, - "13": 0.32524, - "14": 0.32234, - "15": 0.32194, - "16": 0.32286, - "17": 0.32381, - "18": 0.32317, - "19": 0.32316, - "20": 0.32225, - "21": 0.32237, - "22": 0.32068, - "23": 0.31836, - "24": 0.32077, - "25": 0.32241, - "26": 0.3196, - "27": 0.32484, - "28": 0.3223, - "29": 0.32268, - "30": 0.31921, - "31": 0.31951, - "32": 0.31901, - "33": 0.31776, - "34": 0.31959, - "35": 0.32009, - "36": 0.32217, - "37": 0.31843, - "38": 0.32842, - "39": 0.31803, - "40": 0.32118, - "41": 0.67436, - "42": 0.32184, - "43": 0.31883, - "44": 0.31976, - "45": 0.64044, - "46": 0.38679, - "47": 0.37664, - "48": 0.3844, - "49": 0.38013, - "50": 0.38188 + "1": "nan", + "2": 2.48029, + "3": 0.33127, + "4": 0.31594, + "5": 0.32143, + "6": 0.31919, + "7": 0.31884, + "8": 0.32129, + "9": 0.31988, + "10": 0.32069, + "11": 0.31907, + "12": 0.31959, + "13": 0.32211, + "14": 0.31964, + "15": 0.31855, + "16": 0.32013, + "17": 0.32004, + "18": 0.31786, + "19": 0.31755, + "20": 0.31944, + "21": 0.31998, + "22": 0.32066, + "23": 0.32079, + "24": 0.31728, + "25": 0.31689, + "26": 0.31831, + "27": 0.31727, + "28": 0.31999, + "29": 0.31997, + "30": 0.31824, + "31": 0.31724, + "32": 0.33433, + "33": 0.43748, + "34": 0.63551, + "35": 0.35878, + "36": 0.31703, + "37": 0.31709, + "38": 0.32151, + "39": 0.31762, + "40": 0.3204, + "41": 0.3741, + "42": 0.37991, + "43": 0.3738, + "44": 0.38277, + "45": 0.38, + "46": 0.37409, + "47": 0.36543, + "48": 0.37113, + "49": 0.36281, + "50": 0.36274 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json index 4302879367b..c4a7b5d2ff0 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.73516, - "2": 0.33146, - "3": 0.29953, - "4": 0.28786, - "5": 0.28898, - "6": 0.28943, - "7": 0.60486, - "8": 0.28771, - "9": 0.28863, - "10": 0.33495, - "11": 0.35979, - "12": 0.31826, - "13": 0.31158, - "14": 0.3553, - "15": 0.40525, - "16": 0.29653, - "17": 0.28954, - "18": 0.28746, - "19": 0.28594, - "20": 0.28918, - "21": 0.28811, - "22": 0.28994, - "23": 0.2878, - "24": 0.28704, - "25": 0.28786, - "26": 0.28829, - "27": 0.28723, - "28": 0.28842, - "29": 0.28755, - "30": 0.28856, - "31": 0.28778, - "32": 0.29729, - "33": 0.28622, - "34": 0.28852, - "35": 0.29006, - "36": 0.29076, - "37": 0.28535, - "38": 0.28783, - "39": 0.28843, - "40": 0.29078, - "41": 0.28844, - "42": 0.28652, - "43": 0.28742, - "44": 0.2859, - "45": 0.2849, - "46": 0.28877, - "47": 0.28739, - "48": 0.28758, - "49": 0.28616, - "50": 0.29116, - "51": 0.90295, - "52": 0.37657, - "53": 0.35642, - "54": 0.35986, - "55": 0.36134, - "56": 0.36573, - "57": 0.36411, - "58": 0.36481, - "59": 0.36464, - "60": 0.36272, - "61": 0.36512, - "62": 0.36724, - "63": 0.36476, - "64": 0.36594, - "65": 0.36724, - "66": 0.64822, - "67": 0.36581, - "68": 0.36271, - "69": 0.366, - "70": 0.36762, - "71": 0.36789, - "72": 0.64766, - "73": 0.36425, - "74": 0.36764, - "75": 0.3661, - "76": 0.36465, - "77": 0.36495, - "78": 0.36147, - "79": 0.36669, - "80": 0.36518, - "81": 0.36345, - "82": 0.36631, - "83": 0.36797, - "84": 0.36517, - "85": 0.36573, - "86": 0.36641, - "87": 0.36619, - "88": 0.3675, - "89": 0.3649, - "90": 0.36424, - "91": 0.36515, - "92": 0.36402, - "93": 0.3686, - "94": 0.36775, - "95": 0.36962, - "96": 0.36798, - "97": 0.36651, - "98": 0.36783, - "99": 0.36877, - "100": 0.36479 + "1": "nan", + "2": 2.39769, + "3": 0.36211, + "4": 0.34515, + "5": 0.34772, + "6": 0.34875, + "7": 0.33703, + "8": 0.33347, + "9": 0.33755, + "10": 0.3382, + "11": 0.34227, + "12": 0.34002, + "13": 0.34177, + "14": 0.34923, + "15": 0.34952, + "16": 0.34726, + "17": 0.34899, + "18": 0.34889, + "19": 0.3485, + "20": 0.34828, + "21": 0.35065, + "22": 0.35089, + "23": 0.34992, + "24": 0.34939, + "25": 0.34831, + "26": 0.35035, + "27": 0.3455, + "28": 0.34034, + "29": 0.352, + "30": 0.35218, + "31": 0.3474, + "32": 0.34883, + "33": 0.35242, + "34": 0.35219, + "35": 0.35394, + "36": 0.35651, + "37": 0.35447, + "38": 0.35431, + "39": 0.35341, + "40": 0.35274, + "41": 0.35275, + "42": 0.35259, + "43": 0.35167, + "44": 0.35288, + "45": 0.35066, + "46": 0.3519, + "47": 0.35146, + "48": 0.34993, + "49": 0.35082, + "50": 0.35229, + "51": 0.6173, + "52": 0.34392, + "53": 0.34427, + "54": 0.3476, + "55": 0.34816, + "56": 0.34515, + "57": 0.34404, + "58": 0.34542, + "59": 0.34546, + "60": 0.34635, + "61": 0.35023, + "62": 0.34884, + "63": 0.3484, + "64": 0.34644, + "65": 0.34943, + "66": 0.34821, + "67": 0.34706, + "68": 0.34645, + "69": 0.34888, + "70": 0.34562, + "71": 0.34952, + "72": 0.34911, + "73": 0.34968, + "74": 0.34895, + "75": 0.34861, + "76": 0.34704, + "77": 0.34924, + "78": 0.35302, + "79": 0.35161, + "80": 0.34618, + "81": 0.35136, + "82": 0.3518, + "83": 0.34829, + "84": 0.34739, + "85": 0.34831, + "86": 0.34725, + "87": 0.34629, + "88": 0.35011, + "89": 0.34978, + "90": 0.34956, + "91": 0.34919, + "92": 0.35021, + "93": 0.34979, + "94": 0.35425, + "95": 0.34862, + "96": 0.34704, + "97": 0.34718, + "98": 0.34842, + "99": 0.35045, + "100": 0.349 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..e8ad4bfea94 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.86104, + "52": 9.75109, + "53": 10.06631, + "54": 9.95634, + "55": 9.89354, + "56": 9.637, + "57": 9.49142, + "58": 9.8341, + "59": 9.5931, + "60": 9.51379, + "61": 9.69183, + "62": 9.99162, + "63": 9.39196, + "64": 9.77455, + "65": 8.96319, + "66": 9.70663, + "67": 9.3789, + "68": 9.78328, + "69": 9.79736, + "70": 9.73753, + "71": 9.62711, + "72": 9.58907, + "73": 9.50446, + "74": 8.94975, + "75": 9.4278, + "76": 9.08764, + "77": 10.06759, + "78": 9.72141, + "79": 9.3861, + "80": 9.40495, + "81": 9.48596, + "82": 9.70195, + "83": 9.31553, + "84": 9.41806, + "85": 9.61378, + "86": 9.08145, + "87": 9.59631, + "88": 9.75008, + "89": 9.60386, + "90": 9.82838, + "91": 9.33622, + "92": 9.35764, + "93": 9.08795, + "94": 8.83437, + "95": 9.53352, + "96": 9.53315, + "97": 9.31129, + "98": 9.67176, + "99": 8.89816, + "100": 9.40969 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2431.0, + "52": 2467.0, + "53": 2794.0, + "54": 2675.0, + "55": 2313.0, + "56": 2597.0, + "57": 2278.0, + "58": 2887.0, + "59": 2701.0, + "60": 2190.0, + "61": 2764.0, + "62": 2576.0, + "63": 2405.0, + "64": 2903.0, + "65": 2516.0, + "66": 2885.0, + "67": 2700.0, + "68": 2682.0, + "69": 2987.0, + "70": 3141.0, + "71": 3055.0, + "72": 2413.0, + "73": 2864.0, + "74": 1870.0, + "75": 2450.0, + "76": 3032.0, + "77": 3230.0, + "78": 3125.0, + "79": 2982.0, + "80": 3203.0, + "81": 3657.0, + "82": 3174.0, + "83": 2818.0, + "84": 3190.0, + "85": 3166.0, + "86": 2793.0, + "87": 3635.0, + "88": 3005.0, + "89": 3373.0, + "90": 3066.0, + "91": 2857.0, + "92": 3080.0, + "93": 2533.0, + "94": 3303.0, + "95": 3270.0, + "96": 3416.0, + "97": 3085.0, + "98": 3437.0, + "99": 3243.0, + "100": 3119.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 365196800.0, + "52": 365196800.0, + "53": 365196800.0, + "54": 365196800.0, + "55": 365196800.0, + "56": 365196800.0, + "57": 365196800.0, + "58": 365196800.0, + "59": 365196800.0, + "60": 365196800.0, + "61": 365196800.0, + "62": 365196800.0, + "63": 365196800.0, + "64": 365196800.0, + "65": 365196800.0, + "66": 365196800.0, + "67": 365196800.0, + "68": 365196800.0, + "69": 365196800.0, + "70": 365196800.0, + "71": 365196800.0, + "72": 365196800.0, + "73": 365196800.0, + "74": 365196800.0, + "75": 365196800.0, + "76": 365196800.0, + "77": 365196800.0, + "78": 365196800.0, + "79": 365196800.0, + "80": 365196800.0, + "81": 365196800.0, + "82": 365196800.0, + "83": 365196800.0, + "84": 365196800.0, + "85": 365196800.0, + "86": 365196800.0, + "87": 365196800.0, + "88": 365196800.0, + "89": 365196800.0, + "90": 365196800.0, + "91": 365196800.0, + "92": 365196800.0, + "93": 365196800.0, + "94": 365196800.0, + "95": 365196800.0, + "96": 365196800.0, + "97": 365196800.0, + "98": 365196800.0, + "99": 365196800.0, + "100": 365196800.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1132180992.0, + "52": 1132182016.0, + "53": 1132182016.0, + "54": 1132182016.0, + "55": 1132182016.0, + "56": 1132182016.0, + "57": 1132182016.0, + "58": 1132182016.0, + "59": 1132182016.0, + "60": 1132182016.0, + "61": 1132182016.0, + "62": 1132182016.0, + "63": 1132182016.0, + "64": 1132182016.0, + "65": 1132182016.0, + "66": 1132182016.0, + "67": 1132182016.0, + "68": 1132182016.0, + "69": 1132182016.0, + "70": 1132182016.0, + "71": 1132182016.0, + "72": 1132182016.0, + "73": 1132182016.0, + "74": 1132182016.0, + "75": 1132182016.0, + "76": 1132182016.0, + "77": 1132182016.0, + "78": 1132182016.0, + "79": 1132182016.0, + "80": 1132182016.0, + "81": 1132182016.0, + "82": 1132182016.0, + "83": 1132182016.0, + "84": 1132182016.0, + "85": 1132182016.0, + "86": 1132182016.0, + "87": 1132182016.0, + "88": 1132182016.0, + "89": 1132182016.0, + "90": 1132182016.0, + "91": 1132182016.0, + "92": 1132182016.0, + "93": 1132182016.0, + "94": 1132182016.0, + "95": 1132182016.0, + "96": 1132182016.0, + "97": 1132182016.0, + "98": 1132182016.0, + "99": 1132182016.0, + "100": 1132182016.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.25993, + "53": 0.33819, + "54": 0.33044, + "55": 0.3308, + "56": 0.32801, + "57": 0.33341, + "58": 0.33637, + "59": 0.33863, + "60": 0.33792, + "61": 0.33956, + "62": 0.34024, + "63": 0.33835, + "64": 0.34288, + "65": 0.33962, + "66": 0.34436, + "67": 0.34564, + "68": 0.34571, + "69": 0.34698, + "70": 0.34462, + "71": 0.34382, + "72": 0.3445, + "73": 0.3446, + "74": 0.34215, + "75": 0.34655, + "76": 0.34397, + "77": 0.34288, + "78": 0.34323, + "79": 0.34095, + "80": 0.34228, + "81": 0.34072, + "82": 0.34333, + "83": 0.34118, + "84": 0.34195, + "85": 0.34307, + "86": 0.34341, + "87": 0.34409, + "88": 0.34372, + "89": 0.34284, + "90": 0.34363, + "91": 0.347, + "92": 0.34448, + "93": 0.3445, + "94": 0.34642, + "95": 0.34511, + "96": 0.34515, + "97": 0.34484, + "98": 0.34543, + "99": 0.34503, + "100": 0.34832 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json index e52665efa28..e73d1df6682 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.708, - "2": 0.35089, - "3": 0.30496, - "4": 0.29651, - "5": 0.29659, - "6": 0.29472, - "7": 0.29503, - "8": 0.29691, - "9": 0.29322, - "10": 0.29593, - "11": 0.29761, - "12": 0.30107, - "13": 0.29648, - "14": 0.29634, - "15": 0.29469, - "16": 0.29524, - "17": 0.29548, - "18": 0.29571, - "19": 0.29611, - "20": 0.29461, - "21": 0.60922, - "22": 0.29063, - "23": 0.29253, - "24": 0.29221, - "25": 0.35076, - "26": 0.35448, - "27": 0.40801, - "28": 0.32376, - "29": 0.37315, - "30": 0.36741, - "31": 0.30484, - "32": 0.31503, - "33": 0.33111, - "34": 0.33501, - "35": 0.34146, - "36": 0.33794, - "37": 0.3366, - "38": 0.34, - "39": 0.38047, - "40": 0.34724, - "41": 0.34541, - "42": 0.34988, - "43": 0.34614, - "44": 0.34763, - "45": 0.34809, - "46": 0.3476, - "47": 0.34789, - "48": 0.34502, - "49": 0.34682, - "50": 0.34684, - "51": 0.32661, - "52": 0.30335, - "53": 0.30141, - "54": 0.30091, - "55": 0.30835, - "56": 0.30212, - "57": 0.29749, - "58": 0.29597, - "59": 0.29872, - "60": 0.29657, - "61": 0.2928, - "62": 0.29426, - "63": 0.29212, - "64": 0.29342, - "65": 0.2952, - "66": 0.30066, - "67": 0.32851, - "68": 0.32899, - "69": 0.30542, - "70": 0.29401, - "71": 0.2933, - "72": 0.2929, - "73": 0.29695, - "74": 0.29676, - "75": 0.2973, - "76": 0.29472, - "77": 0.29643, - "78": 0.29471, - "79": 0.29414, - "80": 0.29496, - "81": 0.2934, - "82": 0.2937, - "83": 0.29466, - "84": 0.29244, - "85": 0.29464, - "86": 0.29497, - "87": 0.29568, - "88": 0.29595, - "89": 0.29485, - "90": 0.29357, - "91": 0.29468, - "92": 0.29513, - "93": 0.29741, - "94": 0.29444, - "95": 0.29584, - "96": 0.29461, - "97": 0.29375, - "98": 0.29414, - "99": 0.29269, - "100": 0.29041 + "1": "nan", + "2": 2.50092, + "3": 0.30393, + "4": 0.296, + "5": 0.29464, + "6": 0.29386, + "7": 0.29621, + "8": 0.2946, + "9": 0.29682, + "10": 0.29745, + "11": 0.3056, + "12": 0.30475, + "13": 0.30581, + "14": 0.3052, + "15": 0.31033, + "16": 0.30534, + "17": 0.30586, + "18": 0.3053, + "19": 0.30668, + "20": 0.3062, + "21": 0.31086, + "22": 0.30673, + "23": 0.30645, + "24": 0.30648, + "25": 0.30922, + "26": 0.30442, + "27": 0.30196, + "28": 0.3042, + "29": 0.30389, + "30": 0.30468, + "31": 0.30661, + "32": 0.30468, + "33": 0.30645, + "34": 0.30588, + "35": 0.3037, + "36": 0.30433, + "37": 0.30504, + "38": 0.30676, + "39": 0.30639, + "40": 0.30854, + "41": 0.31017, + "42": 0.30559, + "43": 0.30359, + "44": 0.30728, + "45": 0.30737, + "46": 0.30728, + "47": 0.30866, + "48": 0.30981, + "49": 0.3097, + "50": 0.30633, + "51": 0.31798, + "52": 0.30466, + "53": 0.30302, + "54": 0.30516, + "55": 0.30263, + "56": 0.30315, + "57": 0.30305, + "58": 0.30451, + "59": 0.30443, + "60": 0.30525, + "61": 0.30503, + "62": 0.3063, + "63": 0.30517, + "64": 0.30552, + "65": 0.30685, + "66": 0.30584, + "67": 0.31593, + "68": 0.34589, + "69": 0.30682, + "70": 0.30582, + "71": 0.30682, + "72": 0.30578, + "73": 0.30496, + "74": 0.30689, + "75": 0.30927, + "76": 0.31024, + "77": 0.3125, + "78": 0.31093, + "79": 0.31106, + "80": 0.30717, + "81": 0.30815, + "82": 0.30914, + "83": 0.30911, + "84": 0.30335, + "85": 0.29792, + "86": 0.2997, + "87": 0.3032, + "88": 0.30139, + "89": 0.30675, + "90": 0.30412, + "91": 0.30454, + "92": 0.30497, + "93": 0.30233, + "94": 0.30714, + "95": 0.30673, + "96": 0.30193, + "97": 0.30472, + "98": 0.3103, + "99": 0.30957, + "100": 0.30828 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..235ce034813 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.86104, + "52": 9.75109, + "53": 10.06631, + "54": 9.95634, + "55": 9.89354, + "56": 9.637, + "57": 9.49142, + "58": 9.8341, + "59": 9.5931, + "60": 9.51379, + "61": 9.69183, + "62": 9.99162, + "63": 9.39196, + "64": 9.77455, + "65": 8.96319, + "66": 9.70663, + "67": 9.3789, + "68": 9.78328, + "69": 9.79736, + "70": 9.73753, + "71": 9.62711, + "72": 9.58907, + "73": 9.50446, + "74": 8.94975, + "75": 9.4278, + "76": 9.08764, + "77": 10.06759, + "78": 9.72141, + "79": 9.3861, + "80": 9.40495, + "81": 9.48596, + "82": 9.70195, + "83": 9.31553, + "84": 9.41806, + "85": 9.61378, + "86": 9.08145, + "87": 9.59631, + "88": 9.75008, + "89": 9.60386, + "90": 9.82838, + "91": 9.33622, + "92": 9.35764, + "93": 9.08795, + "94": 8.83437, + "95": 9.53352, + "96": 9.53315, + "97": 9.31129, + "98": 9.67176, + "99": 8.89816, + "100": 9.40969 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2431.0, + "52": 2467.0, + "53": 2794.0, + "54": 2675.0, + "55": 2313.0, + "56": 2597.0, + "57": 2278.0, + "58": 2887.0, + "59": 2701.0, + "60": 2190.0, + "61": 2764.0, + "62": 2576.0, + "63": 2405.0, + "64": 2903.0, + "65": 2516.0, + "66": 2885.0, + "67": 2700.0, + "68": 2682.0, + "69": 2987.0, + "70": 3141.0, + "71": 3055.0, + "72": 2413.0, + "73": 2864.0, + "74": 1870.0, + "75": 2450.0, + "76": 3032.0, + "77": 3230.0, + "78": 3125.0, + "79": 2982.0, + "80": 3203.0, + "81": 3657.0, + "82": 3174.0, + "83": 2818.0, + "84": 3190.0, + "85": 3166.0, + "86": 2793.0, + "87": 3635.0, + "88": 3005.0, + "89": 3373.0, + "90": 3066.0, + "91": 2857.0, + "92": 3080.0, + "93": 2533.0, + "94": 3303.0, + "95": 3270.0, + "96": 3416.0, + "97": 3085.0, + "98": 3437.0, + "99": 3243.0, + "100": 3119.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 299204096.0, + "52": 299204096.0, + "53": 299204096.0, + "54": 299204096.0, + "55": 299204096.0, + "56": 299204096.0, + "57": 299204096.0, + "58": 299204096.0, + "59": 299204096.0, + "60": 299204096.0, + "61": 299204096.0, + "62": 299204096.0, + "63": 299204096.0, + "64": 299204096.0, + "65": 299204096.0, + "66": 299204096.0, + "67": 299204096.0, + "68": 299204096.0, + "69": 299204096.0, + "70": 299204096.0, + "71": 299204096.0, + "72": 299204096.0, + "73": 299204096.0, + "74": 299204096.0, + "75": 299204096.0, + "76": 299204096.0, + "77": 299204096.0, + "78": 299204096.0, + "79": 299204096.0, + "80": 299204096.0, + "81": 299204096.0, + "82": 299204096.0, + "83": 299204096.0, + "84": 299204096.0, + "85": 299204096.0, + "86": 299204096.0, + "87": 299204096.0, + "88": 299204096.0, + "89": 299204096.0, + "90": 299204096.0, + "91": 299204096.0, + "92": 299204096.0, + "93": 299204096.0, + "94": 299204096.0, + "95": 299204096.0, + "96": 299204096.0, + "97": 299204096.0, + "98": 299204096.0, + "99": 299204096.0, + "100": 299204096.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1064091136.0, + "52": 1064092160.0, + "53": 1064092160.0, + "54": 1064092160.0, + "55": 1064092160.0, + "56": 1064092160.0, + "57": 1064092160.0, + "58": 1064092160.0, + "59": 1064092160.0, + "60": 1064092160.0, + "61": 1064092160.0, + "62": 1064092160.0, + "63": 1064092160.0, + "64": 1064092160.0, + "65": 1064092160.0, + "66": 1064092160.0, + "67": 1064092160.0, + "68": 1064092160.0, + "69": 1064092160.0, + "70": 1064092160.0, + "71": 1064092160.0, + "72": 1064092160.0, + "73": 1064092160.0, + "74": 1064092160.0, + "75": 1064092160.0, + "76": 1064092160.0, + "77": 1064092160.0, + "78": 1064092160.0, + "79": 1064092160.0, + "80": 1064092160.0, + "81": 1064092160.0, + "82": 1064092160.0, + "83": 1064092160.0, + "84": 1064092160.0, + "85": 1064092160.0, + "86": 1064092160.0, + "87": 1064092160.0, + "88": 1064092160.0, + "89": 1064092160.0, + "90": 1064092160.0, + "91": 1064092160.0, + "92": 1064092160.0, + "93": 1064092160.0, + "94": 1064092160.0, + "95": 1064092160.0, + "96": 1064092160.0, + "97": 1064092160.0, + "98": 1064092160.0, + "99": 1064092160.0, + "100": 1064092160.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.14914, + "53": 0.55064, + "54": 0.29762, + "55": 0.30112, + "56": 0.30035, + "57": 0.29809, + "58": 0.30057, + "59": 0.29654, + "60": 0.29579, + "61": 0.2954, + "62": 0.29808, + "63": 0.2968, + "64": 0.29639, + "65": 0.29803, + "66": 0.29816, + "67": 0.2978, + "68": 0.29666, + "69": 0.29937, + "70": 0.29987, + "71": 0.29726, + "72": 0.29888, + "73": 0.29841, + "74": 0.29818, + "75": 0.29888, + "76": 0.30018, + "77": 0.29543, + "78": 0.29515, + "79": 0.29942, + "80": 0.30103, + "81": 0.30071, + "82": 0.30152, + "83": 0.30277, + "84": 0.30368, + "85": 0.30349, + "86": 0.30411, + "87": 0.30141, + "88": 0.30339, + "89": 0.3072, + "90": 0.30468, + "91": 0.30297, + "92": 0.30317, + "93": 0.30255, + "94": 0.29992, + "95": 0.30116, + "96": 0.29306, + "97": 0.29403, + "98": 0.29399, + "99": 0.29473, + "100": 0.2958 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json index 4200e3b38a8..65fc98f8dd4 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200.json @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.65464, - "2": 0.60021, - "3": 0.56211, - "4": 0.81567, - "5": 0.51087, - "6": 0.51362, - "7": 0.50868, - "8": 0.51119, - "9": 0.51537, - "10": 0.51491, - "11": 0.51179, - "12": 0.51216, - "13": 0.51208, - "14": 0.52419, - "15": 0.85827, - "16": 0.51731, - "17": 0.51718, - "18": 0.51546, - "19": 0.51334, - "20": 0.5203, - "21": 0.51793, - "22": 0.52901, - "23": 0.51605, - "24": 0.51462, - "25": 0.51195, - "26": 0.50837, - "27": 0.85741, - "28": 0.5083, - "29": 0.50928, - "30": 0.50919, - "31": 0.51059, - "32": 0.5129, - "33": 0.51253, - "34": 0.51142, - "35": 0.50986, - "36": 0.51279, - "37": 0.50996, - "38": 0.50872, - "39": 0.51314, - "40": 0.53857, - "41": 0.87144, - "42": 0.53733, - "43": 0.82532, - "44": 0.50255, - "45": 0.50942, - "46": 0.73489, - "47": 0.82645, - "48": 0.50964, - "49": 0.5094, - "50": 0.51015, - "51": 0.51394, - "52": 0.50874, - "53": 0.51284, - "54": 0.52083, - "55": 0.50789, - "56": 0.49975, - "57": 0.49792, - "58": 0.51444, - "59": 0.51001, - "60": 0.50768, - "61": 0.51346, - "62": 0.51695, - "63": 0.51586, - "64": 0.51965, - "65": 0.52295, - "66": 0.51606, - "67": 0.50646, - "68": 0.51105, - "69": 0.50496, - "70": 0.50887, - "71": 0.51043, - "72": 0.51293, - "73": 0.52108, - "74": 0.51224, - "75": 0.51005, - "76": 0.51268, - "77": 0.51097, - "78": 0.50687, - "79": 0.50729, - "80": 0.5142, - "81": 0.54269, - "82": 0.5267, - "83": 0.51288, - "84": 0.5147, - "85": 0.52025, - "86": 0.52158, - "87": 0.51316, - "88": 0.5178, - "89": 0.55243, - "90": 0.51232, - "91": 0.51784, - "92": 0.5159, - "93": 0.51384, - "94": 0.51504, - "95": 0.51606, - "96": 0.5173, - "97": 0.51802, - "98": 0.51331, - "99": 0.51466, - "100": 0.51281 + "1": "nan", + "2": 2.73217, + "3": 0.44849, + "4": 0.46632, + "5": 0.47877, + "6": 0.48831, + "7": 0.48769, + "8": 0.50745, + "9": 0.48778, + "10": 0.49192, + "11": 0.49758, + "12": 0.5014, + "13": 0.49698, + "14": 0.49958, + "15": 0.49877, + "16": 0.50112, + "17": 0.49678, + "18": 0.49696, + "19": 0.49583, + "20": 0.49823, + "21": 0.50092, + "22": 0.50313, + "23": 0.50157, + "24": 0.50564, + "25": 0.50173, + "26": 0.50691, + "27": 0.5209, + "28": 0.51519, + "29": 0.50283, + "30": 0.50601, + "31": 0.50139, + "32": 0.507, + "33": 0.50335, + "34": 0.50467, + "35": 0.50168, + "36": 0.49771, + "37": 0.49868, + "38": 0.49794, + "39": 0.49729, + "40": 0.4917, + "41": 0.49294, + "42": 0.48867, + "43": 0.49291, + "44": 0.49762, + "45": 0.49672, + "46": 0.50694, + "47": 0.49816, + "48": 0.4942, + "49": 0.5031, + "50": 0.50121, + "51": 0.48839, + "52": 0.49123, + "53": 0.83615, + "54": 0.49979, + "55": 0.50032, + "56": 0.5025, + "57": 0.50465, + "58": 0.5032, + "59": 0.52509, + "60": 0.51125, + "61": 0.50912, + "62": 0.50722, + "63": 0.51052, + "64": 0.50743, + "65": 0.51588, + "66": 0.51203, + "67": 0.51526, + "68": 0.50806, + "69": 0.51012, + "70": 0.51073, + "71": 0.50805, + "72": 0.51001, + "73": 0.52219, + "74": 0.50785, + "75": 0.50971, + "76": 0.50837, + "77": 0.51328, + "78": 0.51109, + "79": 0.50795, + "80": 0.86855, + "81": 0.51135, + "82": 0.50858, + "83": 0.51273, + "84": 0.50989, + "85": 0.51087, + "86": 0.51808, + "87": 0.5247, + "88": 0.51417, + "89": 0.5201, + "90": 0.90988, + "91": 0.54215, + "92": 0.52369, + "93": 0.51835, + "94": 0.52068, + "95": 0.5186, + "96": 0.52052, + "97": 0.51882, + "98": 0.52061, + "99": 0.51758, + "100": 0.51114 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..074f4cc53b8 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.86714, + "52": 9.75686, + "53": 10.06973, + "54": 9.95909, + "55": 9.89872, + "56": 9.63952, + "57": 9.4936, + "58": 9.83608, + "59": 9.59679, + "60": 9.51626, + "61": 9.69468, + "62": 9.99033, + "63": 9.39041, + "64": 9.77374, + "65": 8.96559, + "66": 9.70319, + "67": 9.38057, + "68": 9.78256, + "69": 9.79804, + "70": 9.73697, + "71": 9.62634, + "72": 9.582, + "73": 9.50018, + "74": 8.93897, + "75": 9.42247, + "76": 9.08151, + "77": 10.06555, + "78": 9.71951, + "79": 9.38365, + "80": 9.4005, + "81": 9.48215, + "82": 9.69917, + "83": 9.30951, + "84": 9.41595, + "85": 9.61112, + "86": 9.07822, + "87": 9.59519, + "88": 9.74646, + "89": 9.60078, + "90": 9.82618, + "91": 9.32913, + "92": 9.35518, + "93": 9.08231, + "94": 8.83, + "95": 9.53112, + "96": 9.52889, + "97": 9.30954, + "98": 9.66956, + "99": 8.89675, + "100": 9.4083 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2492.0, + "52": 2583.0, + "53": 2788.0, + "54": 2678.0, + "55": 2301.0, + "56": 2724.0, + "57": 2272.0, + "58": 2999.0, + "59": 2686.0, + "60": 2330.0, + "61": 2852.0, + "62": 2703.0, + "63": 2277.0, + "64": 2990.0, + "65": 2475.0, + "66": 2892.0, + "67": 2646.0, + "68": 2650.0, + "69": 2845.0, + "70": 3145.0, + "71": 2913.0, + "72": 2573.0, + "73": 2850.0, + "74": 1865.0, + "75": 2466.0, + "76": 3055.0, + "77": 3185.0, + "78": 3106.0, + "79": 3053.0, + "80": 3184.0, + "81": 3447.0, + "82": 3296.0, + "83": 2726.0, + "84": 3276.0, + "85": 3336.0, + "86": 2803.0, + "87": 3643.0, + "88": 3013.0, + "89": 3185.0, + "90": 3126.0, + "91": 3076.0, + "92": 3139.0, + "93": 2665.0, + "94": 3302.0, + "95": 3282.0, + "96": 3404.0, + "97": 3215.0, + "98": 3465.0, + "99": 3128.0, + "100": 3231.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 397748736.0, + "52": 397748736.0, + "53": 397748736.0, + "54": 397748736.0, + "55": 397748736.0, + "56": 397748736.0, + "57": 397748736.0, + "58": 397748736.0, + "59": 397748736.0, + "60": 397748736.0, + "61": 397748736.0, + "62": 397748736.0, + "63": 397748736.0, + "64": 397748736.0, + "65": 397748736.0, + "66": 397748736.0, + "67": 397748736.0, + "68": 397748736.0, + "69": 397748736.0, + "70": 397748736.0, + "71": 397748736.0, + "72": 397748736.0, + "73": 397748736.0, + "74": 397748736.0, + "75": 397748736.0, + "76": 397748736.0, + "77": 397748736.0, + "78": 397748736.0, + "79": 397748736.0, + "80": 397748736.0, + "81": 397748736.0, + "82": 397748736.0, + "83": 397748736.0, + "84": 397748736.0, + "85": 397748736.0, + "86": 397748736.0, + "87": 397748736.0, + "88": 397748736.0, + "89": 397748736.0, + "90": 397748736.0, + "91": 397748736.0, + "92": 397748736.0, + "93": 397748736.0, + "94": 397748736.0, + "95": 397748736.0, + "96": 397748736.0, + "97": 397748736.0, + "98": 397748736.0, + "99": 397748736.0, + "100": 397748736.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1191471616.0, + "52": 1191472640.0, + "53": 1191472640.0, + "54": 1191472640.0, + "55": 1191472640.0, + "56": 1191472640.0, + "57": 1191472640.0, + "58": 1191472640.0, + "59": 1191472640.0, + "60": 1191472640.0, + "61": 1191472640.0, + "62": 1191472640.0, + "63": 1191472640.0, + "64": 1191472640.0, + "65": 1191472640.0, + "66": 1191472640.0, + "67": 1191472640.0, + "68": 1191472640.0, + "69": 1191472640.0, + "70": 1191472640.0, + "71": 1191472640.0, + "72": 1191472640.0, + "73": 1191472640.0, + "74": 1191472640.0, + "75": 1191472640.0, + "76": 1191472640.0, + "77": 1191472640.0, + "78": 1191472640.0, + "79": 1191472640.0, + "80": 1191472640.0, + "81": 1191472640.0, + "82": 1191472640.0, + "83": 1191472640.0, + "84": 1191472640.0, + "85": 1191472640.0, + "86": 1191472640.0, + "87": 1191472640.0, + "88": 1191472640.0, + "89": 1191472640.0, + "90": 1191472640.0, + "91": 1191472640.0, + "92": 1191472640.0, + "93": 1191472640.0, + "94": 1191472640.0, + "95": 1191472640.0, + "96": 1191472640.0, + "97": 1191472640.0, + "98": 1191472640.0, + "99": 1191472640.0, + "100": 1191472640.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.49404, + "53": 0.49738, + "54": 0.47517, + "55": 0.48073, + "56": 0.48124, + "57": 0.48531, + "58": 0.46905, + "59": 0.45198, + "60": 0.44375, + "61": 0.45318, + "62": 0.43328, + "63": 0.44116, + "64": 0.43199, + "65": 0.43219, + "66": 0.43466, + "67": 0.43576, + "68": 0.43222, + "69": 0.42871, + "70": 0.42889, + "71": 0.43506, + "72": 0.43141, + "73": 0.42482, + "74": 0.4278, + "75": 0.42933, + "76": 0.42676, + "77": 0.43206, + "78": 0.43106, + "79": 0.43328, + "80": 0.429, + "81": 0.4294, + "82": 0.43619, + "83": 0.42881, + "84": 0.44023, + "85": 0.43778, + "86": 0.4293, + "87": 0.42266, + "88": 0.43088, + "89": 0.4333, + "90": 0.42756, + "91": 0.42474, + "92": 0.43075, + "93": 0.43032, + "94": 0.42748, + "95": 0.43116, + "96": 0.43174, + "97": 0.42434, + "98": 0.42337, + "99": 0.42353, + "100": 0.4287 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200.json index 5b22c8f244c..771860e086a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.90251, "2": 10.9138, "3": 10.90169, - "4": 10.90724, - "5": 10.9045, - "6": 10.91656, - "7": 10.91268, + "4": 10.90727, + "5": 10.90448, + "6": 10.91653, + "7": 10.9127, "8": 10.89505, - "9": 10.91555, - "10": 10.87277, - "11": 10.90376, - "12": 10.90404, - "13": 10.91831, - "14": 10.90742, - "15": 10.87551, - "16": 10.85477, + "9": 10.91558, + "10": 10.87283, + "11": 10.90373, + "12": 10.904, + "13": 10.91828, + "14": 10.9074, + "15": 10.87552, + "16": 10.85479, "17": 10.83186, - "18": 10.84054, - "19": 10.84221, - "20": 10.75039, - "21": 10.73638, - "22": 10.62979, - "23": 10.74023, - "24": 10.64438, - "25": 10.60242, - "26": 10.64922, - "27": 10.64074, - "28": 10.58757, - "29": 10.59165, - "30": 10.38969, - "31": 10.18185, - "32": 10.49227, - "33": 10.48772, - "34": 10.26316, - "35": 10.2923, - "36": 10.25547, - "37": 10.37371, - "38": 10.2355, - "39": 10.42347, - "40": 10.10947, - "41": 10.17531, - "42": 10.2316, - "43": 9.87326, - "44": 9.9918, - "45": 9.86649, - "46": 9.84547, - "47": 10.17367, - "48": 9.87146, - "49": 9.55757, - "50": 9.92547, + "18": 10.84055, + "19": 10.84215, + "20": 10.75044, + "21": 10.73632, + "22": 10.62985, + "23": 10.74027, + "24": 10.64442, + "25": 10.60239, + "26": 10.64921, + "27": 10.64076, + "28": 10.5875, + "29": 10.59166, + "30": 10.38964, + "31": 10.18179, + "32": 10.49231, + "33": 10.48774, + "34": 10.26318, + "35": 10.29231, + "36": 10.25552, + "37": 10.37375, + "38": 10.23555, + "39": 10.42349, + "40": 10.10951, + "41": 10.1753, + "42": 10.23159, + "43": 9.87321, + "44": 9.99176, + "45": 9.8665, + "46": 9.84548, + "47": 10.17372, + "48": 9.87147, + "49": 9.55758, + "50": 9.92554, "51": 9.87398, "52": 9.76585, - "53": 10.08271, - "54": 9.97273, - "55": 9.90735, - "56": 9.64216, + "53": 10.0827, + "54": 9.97272, + "55": 9.90734, + "56": 9.64218, "57": 9.48857, - "58": 9.84273, + "58": 9.8427, "59": 9.60111, - "60": 9.52016, - "61": 9.70058, - "62": 9.99644, - "63": 9.39064, - "64": 9.77614, - "65": 8.96633, - "66": 9.70947, - "67": 9.3877, - "68": 9.78895, + "60": 9.52012, + "61": 9.70057, + "62": 9.99645, + "63": 9.39065, + "64": 9.77613, + "65": 8.96632, + "66": 9.70945, + "67": 9.38769, + "68": 9.78891, "69": 9.80803, "70": 9.74237, - "71": 9.63382, + "71": 9.63381, "72": 9.59118, - "73": 9.50694, - "74": 8.94248, - "75": 9.42903, - "76": 9.08836, - "77": 10.07155, - "78": 9.72684, - "79": 9.38725, - "80": 9.40572, - "81": 9.48703, + "73": 9.50696, + "74": 8.94245, + "75": 9.42902, + "76": 9.0883, + "77": 10.07151, + "78": 9.72685, + "79": 9.38721, + "80": 9.4057, + "81": 9.48702, "82": 9.70482, - "83": 9.31557, - "84": 9.42113, - "85": 9.61467, + "83": 9.31556, + "84": 9.42107, + "85": 9.61466, "86": 9.08461, - "87": 9.59903, - "88": 9.75369, - "89": 9.60597, - "90": 9.83153, - "91": 9.33877, - "92": 9.36033, - "93": 9.0904, - "94": 8.83712, + "87": 9.59902, + "88": 9.75368, + "89": 9.60598, + "90": 9.83154, + "91": 9.33878, + "92": 9.36034, + "93": 9.09038, + "94": 8.83714, "95": 9.53804, - "96": 9.53391, - "97": 9.31319, + "96": 9.5339, + "97": 9.31316, "98": 9.67422, - "99": 8.90345, - "100": 9.41498 + "99": 8.90347, + "100": 9.41497 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1541.0, - "2": 1645.0, - "3": 1629.0, - "4": 1795.0, - "5": 1841.0, - "6": 1779.0, - "7": 1764.0, - "8": 1563.0, - "9": 1825.0, - "10": 1409.0, - "11": 1956.0, - "12": 1760.0, - "13": 1869.0, - "14": 1833.0, - "15": 1958.0, - "16": 1850.0, - "17": 1795.0, - "18": 1781.0, - "19": 1766.0, - "20": 1632.0, - "21": 1866.0, - "22": 1649.0, - "23": 1996.0, - "24": 1722.0, - "25": 1609.0, - "26": 1678.0, - "27": 1752.0, - "28": 1903.0, - "29": 1870.0, - "30": 1851.0, - "31": 1483.0, - "32": 1836.0, - "33": 2084.0, - "34": 1799.0, - "35": 1867.0, - "36": 1846.0, - "37": 2303.0, - "38": 2171.0, - "39": 2173.0, - "40": 2153.0, - "41": 2230.0, - "42": 2211.0, - "43": 1945.0, - "44": 2003.0, - "45": 2027.0, - "46": 2196.0, - "47": 2540.0, - "48": 2320.0, - "49": 2289.0, - "50": 2249.0, - "51": 2418.0, - "52": 2459.0, - "53": 2723.0, - "54": 2710.0, - "55": 2167.0, - "56": 2529.0, - "57": 2306.0, - "58": 2690.0, - "59": 2678.0, - "60": 2241.0, - "61": 2828.0, - "62": 2453.0, - "63": 2330.0, - "64": 2785.0, - "65": 2596.0, - "66": 2886.0, - "67": 2589.0, - "68": 2725.0, - "69": 2800.0, - "70": 3022.0, - "71": 2941.0, - "72": 2388.0, - "73": 2770.0, - "74": 1833.0, - "75": 2434.0, - "76": 2843.0, - "77": 3177.0, - "78": 3085.0, - "79": 3080.0, - "80": 3263.0, - "81": 3512.0, - "82": 3152.0, - "83": 2845.0, - "84": 3114.0, - "85": 3196.0, - "86": 2728.0, - "87": 3511.0, - "88": 2941.0, - "89": 3343.0, - "90": 3094.0, - "91": 2946.0, - "92": 3094.0, - "93": 2707.0, - "94": 3304.0, - "95": 3279.0, - "96": 3562.0, - "97": 2956.0, - "98": 3547.0, - "99": 3119.0, - "100": 3119.0 + "1": 1558.0, + "2": 1700.0, + "3": 1576.0, + "4": 1748.0, + "5": 1809.0, + "6": 1822.0, + "7": 1801.0, + "8": 1568.0, + "9": 1788.0, + "10": 1395.0, + "11": 1909.0, + "12": 1795.0, + "13": 1860.0, + "14": 1765.0, + "15": 1878.0, + "16": 1938.0, + "17": 1760.0, + "18": 1708.0, + "19": 1744.0, + "20": 1706.0, + "21": 1825.0, + "22": 1656.0, + "23": 1933.0, + "24": 1615.0, + "25": 1607.0, + "26": 1586.0, + "27": 1799.0, + "28": 1802.0, + "29": 1886.0, + "30": 1885.0, + "31": 1529.0, + "32": 1823.0, + "33": 1998.0, + "34": 1760.0, + "35": 1891.0, + "36": 1999.0, + "37": 2340.0, + "38": 2149.0, + "39": 2308.0, + "40": 2231.0, + "41": 2153.0, + "42": 2285.0, + "43": 1916.0, + "44": 2014.0, + "45": 1914.0, + "46": 2192.0, + "47": 2491.0, + "48": 2179.0, + "49": 2231.0, + "50": 2285.0, + "51": 2371.0, + "52": 2512.0, + "53": 2624.0, + "54": 2501.0, + "55": 2218.0, + "56": 2649.0, + "57": 2213.0, + "58": 2763.0, + "59": 2526.0, + "60": 2261.0, + "61": 2835.0, + "62": 2497.0, + "63": 2406.0, + "64": 2736.0, + "65": 2546.0, + "66": 2800.0, + "67": 2572.0, + "68": 2686.0, + "69": 2768.0, + "70": 2992.0, + "71": 2834.0, + "72": 2391.0, + "73": 2816.0, + "74": 1859.0, + "75": 2369.0, + "76": 2849.0, + "77": 3224.0, + "78": 3000.0, + "79": 3139.0, + "80": 3215.0, + "81": 3443.0, + "82": 3149.0, + "83": 2715.0, + "84": 3170.0, + "85": 3313.0, + "86": 2748.0, + "87": 3534.0, + "88": 3004.0, + "89": 3336.0, + "90": 3117.0, + "91": 2912.0, + "92": 3082.0, + "93": 2671.0, + "94": 3380.0, + "95": 3185.0, + "96": 3513.0, + "97": 3137.0, + "98": 3523.0, + "99": 3099.0, + "100": 3148.0 } }, "mem-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 700958208.0, - "2": 790286848.0, - "3": 790286848.0, - "4": 790286848.0, - "5": 790286848.0, - "6": 790286848.0, - "7": 790286848.0, - "8": 790286848.0, - "9": 790286848.0, - "10": 790286848.0, - "11": 790286848.0, - "12": 790286848.0, - "13": 790286848.0, - "14": 790286848.0, - "15": 790286848.0, - "16": 790286848.0, - "17": 790286848.0, - "18": 790286848.0, - "19": 790286848.0, - "20": 790286848.0, - "21": 790286848.0, - "22": 790286848.0, - "23": 790286848.0, - "24": 790286848.0, - "25": 790286848.0, - "26": 790286848.0, - "27": 790286848.0, - "28": 790286848.0, - "29": 790286848.0, - "30": 790286848.0, - "31": 790286848.0, - "32": 790286848.0, - "33": 790286848.0, - "34": 790286848.0, - "35": 790286848.0, - "36": 790286848.0, - "37": 790286848.0, - "38": 790286848.0, - "39": 790286848.0, - "40": 790286848.0, - "41": 790286848.0, - "42": 790286848.0, - "43": 790286848.0, - "44": 790286848.0, - "45": 790286848.0, - "46": 790286848.0, - "47": 790286848.0, - "48": 790286848.0, - "49": 790286848.0, - "50": 790286848.0, - "51": 790286848.0, - "52": 790286848.0, - "53": 790286848.0, - "54": 790286848.0, - "55": 790286848.0, - "56": 790286848.0, - "57": 790286848.0, - "58": 790286848.0, - "59": 790286848.0, - "60": 790286848.0, - "61": 790286848.0, - "62": 790286848.0, - "63": 790286848.0, - "64": 790286848.0, - "65": 790286848.0, - "66": 790286848.0, - "67": 790286848.0, - "68": 790286848.0, - "69": 790286848.0, - "70": 790286848.0, - "71": 790286848.0, - "72": 790286848.0, - "73": 790286848.0, - "74": 790286848.0, - "75": 790286848.0, - "76": 790286848.0, - "77": 790286848.0, - "78": 790286848.0, - "79": 790286848.0, - "80": 790286848.0, - "81": 790286848.0, - "82": 790286848.0, - "83": 790286848.0, - "84": 790286848.0, - "85": 790286848.0, - "86": 790286848.0, - "87": 790286848.0, - "88": 790286848.0, - "89": 790286848.0, - "90": 790286848.0, - "91": 790286848.0, - "92": 790286848.0, - "93": 790286848.0, - "94": 790286848.0, - "95": 790286848.0, - "96": 790286848.0, - "97": 790286848.0, - "98": 790286848.0, - "99": 790286848.0, - "100": 790286848.0 + "1": 704366080.0, + "2": 794219008.0, + "3": 794219008.0, + "4": 794219008.0, + "5": 794219008.0, + "6": 794219008.0, + "7": 794219008.0, + "8": 794219008.0, + "9": 794219008.0, + "10": 794219008.0, + "11": 794219008.0, + "12": 794219008.0, + "13": 794219008.0, + "14": 794219008.0, + "15": 794219008.0, + "16": 794219008.0, + "17": 794219008.0, + "18": 794219008.0, + "19": 794219008.0, + "20": 794219008.0, + "21": 794219008.0, + "22": 794219008.0, + "23": 794219008.0, + "24": 794219008.0, + "25": 794219008.0, + "26": 794219008.0, + "27": 794219008.0, + "28": 794219008.0, + "29": 794219008.0, + "30": 794219008.0, + "31": 794219008.0, + "32": 794219008.0, + "33": 794219008.0, + "34": 794219008.0, + "35": 794219008.0, + "36": 794219008.0, + "37": 794219008.0, + "38": 794219008.0, + "39": 794219008.0, + "40": 794219008.0, + "41": 794219008.0, + "42": 794219008.0, + "43": 794219008.0, + "44": 794219008.0, + "45": 794219008.0, + "46": 794219008.0, + "47": 794219008.0, + "48": 794219008.0, + "49": 794219008.0, + "50": 794219008.0, + "51": 794219008.0, + "52": 794219008.0, + "53": 794219008.0, + "54": 794219008.0, + "55": 794219008.0, + "56": 794219008.0, + "57": 794219008.0, + "58": 794219008.0, + "59": 794219008.0, + "60": 794219008.0, + "61": 794219008.0, + "62": 794219008.0, + "63": 794219008.0, + "64": 794219008.0, + "65": 794219008.0, + "66": 794219008.0, + "67": 794219008.0, + "68": 794219008.0, + "69": 794219008.0, + "70": 794219008.0, + "71": 794219008.0, + "72": 794219008.0, + "73": 794219008.0, + "74": 794219008.0, + "75": 794219008.0, + "76": 794219008.0, + "77": 794219008.0, + "78": 794219008.0, + "79": 794219008.0, + "80": 794219008.0, + "81": 794219008.0, + "82": 794219008.0, + "83": 794219008.0, + "84": 794219008.0, + "85": 794219008.0, + "86": 794219008.0, + "87": 794219008.0, + "88": 794219008.0, + "89": 794219008.0, + "90": 794219008.0, + "91": 794219008.0, + "92": 794219008.0, + "93": 794219008.0, + "94": 794219008.0, + "95": 794219008.0, + "96": 794219008.0, + "97": 794219008.0, + "98": 794219008.0, + "99": 794219008.0, + "100": 794219008.0 } }, "iteration-time": { @@ -432,106 +432,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 11.14668, - "2": 0.32935, - "3": 0.46923, - "4": 0.29529, - "5": 0.40702, - "6": 0.40156, - "7": 0.43578, - "8": 0.31407, - "9": 0.43033, - "10": 0.31604, - "11": 0.31738, - "12": 0.31563, - "13": 0.36997, - "14": 0.4371, - "15": 0.35906, - "16": 0.31709, - "17": 0.39045, - "18": 0.31331, - "19": 0.3763, - "20": 0.33238, - "21": 0.31767, - "22": 0.43702, - "23": 0.39383, - "24": 0.3148, - "25": 0.31554, - "26": 0.3135, - "27": 0.34957, - "28": 0.31621, - "29": 0.31661, - "30": 0.31507, - "31": 0.41199, - "32": 0.40737, - "33": 0.31355, - "34": 0.31358, - "35": 0.31287, - "36": 0.31491, - "37": 0.36356, - "38": 0.37435, - "39": 0.33637, - "40": 0.31406, - "41": 0.31613, - "42": 0.35153, - "43": 0.3142, - "44": 0.31623, - "45": 0.31572, - "46": 0.34532, - "47": 0.35769, - "48": 0.36855, - "49": 0.31459, - "50": 0.3144, - "51": 0.32345, - "52": 0.30594, - "53": 0.3111, - "54": 0.31377, - "55": 0.39254, - "56": 0.40899, - "57": 0.48809, - "58": 0.31709, - "59": 0.31541, - "60": 0.3139, - "61": 0.42195, - "62": 0.31636, - "63": 0.31499, - "64": 0.31608, - "65": 0.31718, - "66": 0.31606, - "67": 0.348, - "68": 0.39663, - "69": 0.31776, - "70": 0.31679, - "71": 0.31563, - "72": 0.3148, - "73": 0.31785, - "74": 0.36067, - "75": 0.31679, - "76": 0.31667, - "77": 0.40594, - "78": 0.31863, - "79": 0.31973, - "80": 0.31848, - "81": 0.31801, - "82": 0.31661, - "83": 0.3166, - "84": 0.49879, - "85": 0.31644, - "86": 0.31582, - "87": 0.31672, - "88": 0.31561, - "89": 0.3413, - "90": 0.3984, - "91": 0.31512, - "92": 0.39228, - "93": 0.31251, - "94": 0.311, - "95": 0.31228, - "96": 0.31391, - "97": 0.31003, - "98": 0.31573, - "99": 0.3154, - "100": 0.40105 + "1": "nan", + "2": 3.24816, + "3": 0.43688, + "4": 0.27843, + "5": 0.27555, + "6": 0.27709, + "7": 0.277, + "8": 0.27722, + "9": 0.27493, + "10": 0.277, + "11": 0.27605, + "12": 0.27617, + "13": 0.27539, + "14": 0.27709, + "15": 0.27461, + "16": 0.27313, + "17": 0.27396, + "18": 0.27419, + "19": 0.2729, + "20": 0.27386, + "21": 0.27451, + "22": 0.27496, + "23": 0.27291, + "24": 0.27491, + "25": 0.2752, + "26": 0.27531, + "27": 0.27661, + "28": 0.27544, + "29": 0.27432, + "30": 0.27338, + "31": 0.2738, + "32": 0.27312, + "33": 0.2732, + "34": 0.27439, + "35": 0.2727, + "36": 0.27455, + "37": 0.27368, + "38": 0.27316, + "39": 0.29032, + "40": 0.27694, + "41": 0.27622, + "42": 0.28477, + "43": 0.27626, + "44": 0.27624, + "45": 0.27486, + "46": 0.27565, + "47": 0.2747, + "48": 0.2742, + "49": 0.2754, + "50": 0.27741, + "51": 0.30004, + "52": 0.27365, + "53": 0.27134, + "54": 0.27213, + "55": 0.27342, + "56": 0.27158, + "57": 0.27123, + "58": 0.27216, + "59": 0.27272, + "60": 0.27106, + "61": 0.2721, + "62": 0.27338, + "63": 0.2716, + "64": 0.27194, + "65": 0.27219, + "66": 0.27183, + "67": 0.2734, + "68": 0.27398, + "69": 0.27633, + "70": 0.27236, + "71": 0.27322, + "72": 0.27105, + "73": 0.27181, + "74": 0.27247, + "75": 0.27172, + "76": 0.27237, + "77": 0.2696, + "78": 0.2681, + "79": 0.26821, + "80": 0.26803, + "81": 0.27079, + "82": 0.27045, + "83": 0.27549, + "84": 0.27227, + "85": 0.27313, + "86": 0.27337, + "87": 0.27499, + "88": 0.2754, + "89": 0.2753, + "90": 0.2744, + "91": 0.27474, + "92": 0.27214, + "93": 0.27687, + "94": 0.27473, + "95": 0.27478, + "96": 0.27394, + "97": 0.27801, + "98": 0.27283, + "99": 0.27237, + "100": 0.27512 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..3c3511a921b --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 9.87398, + "52": 9.76584, + "53": 10.08272, + "54": 9.97273, + "55": 9.90736, + "56": 9.64213, + "57": 9.48856, + "58": 9.84268, + "59": 9.60111, + "60": 9.52013, + "61": 9.70058, + "62": 9.99642, + "63": 9.39067, + "64": 9.77612, + "65": 8.96637, + "66": 9.70949, + "67": 9.38771, + "68": 9.78893, + "69": 9.808, + "70": 9.74238, + "71": 9.63382, + "72": 9.59116, + "73": 9.50694, + "74": 8.94251, + "75": 9.42898, + "76": 9.0883, + "77": 10.07153, + "78": 9.72682, + "79": 9.38722, + "80": 9.40571, + "81": 9.48701, + "82": 9.70484, + "83": 9.31557, + "84": 9.42111, + "85": 9.61463, + "86": 9.08465, + "87": 9.59904, + "88": 9.75367, + "89": 9.606, + "90": 9.83155, + "91": 9.3388, + "92": 9.36037, + "93": 9.09036, + "94": 8.83711, + "95": 9.53804, + "96": 9.53392, + "97": 9.3132, + "98": 9.67422, + "99": 8.90347, + "100": 9.415 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2423.0, + "52": 2500.0, + "53": 2764.0, + "54": 2630.0, + "55": 2192.0, + "56": 2558.0, + "57": 2261.0, + "58": 2856.0, + "59": 2691.0, + "60": 2271.0, + "61": 2730.0, + "62": 2517.0, + "63": 2411.0, + "64": 2842.0, + "65": 2476.0, + "66": 2911.0, + "67": 2596.0, + "68": 2658.0, + "69": 2791.0, + "70": 3035.0, + "71": 2882.0, + "72": 2367.0, + "73": 2798.0, + "74": 1871.0, + "75": 2433.0, + "76": 2936.0, + "77": 3145.0, + "78": 2937.0, + "79": 2949.0, + "80": 3208.0, + "81": 3626.0, + "82": 3215.0, + "83": 2746.0, + "84": 3128.0, + "85": 3291.0, + "86": 2686.0, + "87": 3535.0, + "88": 2983.0, + "89": 3431.0, + "90": 3105.0, + "91": 2840.0, + "92": 3101.0, + "93": 2561.0, + "94": 3334.0, + "95": 3249.0, + "96": 3468.0, + "97": 3077.0, + "98": 3515.0, + "99": 3067.0, + "100": 3131.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 261759488.0, + "52": 261759488.0, + "53": 261759488.0, + "54": 261759488.0, + "55": 261759488.0, + "56": 261759488.0, + "57": 261759488.0, + "58": 261759488.0, + "59": 261759488.0, + "60": 261759488.0, + "61": 261759488.0, + "62": 261759488.0, + "63": 261759488.0, + "64": 261759488.0, + "65": 261759488.0, + "66": 261759488.0, + "67": 261759488.0, + "68": 261759488.0, + "69": 261759488.0, + "70": 261759488.0, + "71": 261759488.0, + "72": 261759488.0, + "73": 261759488.0, + "74": 261759488.0, + "75": 261759488.0, + "76": 261759488.0, + "77": 261759488.0, + "78": 261759488.0, + "79": 261759488.0, + "80": 261759488.0, + "81": 261759488.0, + "82": 261759488.0, + "83": 261759488.0, + "84": 261759488.0, + "85": 261759488.0, + "86": 261759488.0, + "87": 261759488.0, + "88": 261759488.0, + "89": 261759488.0, + "90": 261759488.0, + "91": 261759488.0, + "92": 261759488.0, + "93": 261759488.0, + "94": 261759488.0, + "95": 261759488.0, + "96": 261759488.0, + "97": 261759488.0, + "98": 261759488.0, + "99": 261759488.0, + "100": 261759488.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 795135488.0, + "52": 795136512.0, + "53": 795136512.0, + "54": 795136512.0, + "55": 795136512.0, + "56": 795136512.0, + "57": 795136512.0, + "58": 795136512.0, + "59": 795136512.0, + "60": 795136512.0, + "61": 795136512.0, + "62": 795136512.0, + "63": 795136512.0, + "64": 795136512.0, + "65": 795136512.0, + "66": 795136512.0, + "67": 795136512.0, + "68": 795136512.0, + "69": 795136512.0, + "70": 795136512.0, + "71": 795136512.0, + "72": 795136512.0, + "73": 795136512.0, + "74": 795136512.0, + "75": 795136512.0, + "76": 795136512.0, + "77": 795136512.0, + "78": 795136512.0, + "79": 795136512.0, + "80": 795136512.0, + "81": 795136512.0, + "82": 795136512.0, + "83": 795136512.0, + "84": 795136512.0, + "85": 795136512.0, + "86": 795136512.0, + "87": 795136512.0, + "88": 795136512.0, + "89": 795136512.0, + "90": 795136512.0, + "91": 795136512.0, + "92": 795136512.0, + "93": 795136512.0, + "94": 795136512.0, + "95": 795136512.0, + "96": 795136512.0, + "97": 795136512.0, + "98": 795136512.0, + "99": 795136512.0, + "100": 795136512.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 3.36268, + "53": 0.28919, + "54": 0.2725, + "55": 0.27972, + "56": 0.2728, + "57": 0.27382, + "58": 0.27288, + "59": 0.27294, + "60": 0.27575, + "61": 0.27075, + "62": 0.27057, + "63": 0.27211, + "64": 0.26991, + "65": 0.27298, + "66": 0.27045, + "67": 0.27231, + "68": 0.27315, + "69": 0.26969, + "70": 0.27037, + "71": 0.27028, + "72": 0.27191, + "73": 0.2714, + "74": 0.27082, + "75": 0.2722, + "76": 0.27153, + "77": 0.27331, + "78": 0.27142, + "79": 0.27368, + "80": 0.27144, + "81": 0.26895, + "82": 0.27139, + "83": 0.26946, + "84": 0.27033, + "85": 0.2702, + "86": 0.26955, + "87": 0.2686, + "88": 0.27213, + "89": 0.2709, + "90": 0.27061, + "91": 0.27274, + "92": 0.26989, + "93": 0.27031, + "94": 0.27054, + "95": 0.269, + "96": 0.27187, + "97": 0.26915, + "98": 0.2696, + "99": 0.27075, + "100": 0.26802 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json index 3d9cf662b8f..12b113ac52d 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200.json @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 5.23658, - "2": 0.11342, - "3": 0.10424, - "4": 0.0896, - "5": 0.08891, - "6": 0.08841, - "7": 0.0882, - "8": 0.08856, - "9": 0.08635, - "10": 0.08776, - "11": 0.08701, - "12": 0.08694, - "13": 0.08552, - "14": 0.08635, - "15": 0.08742, - "16": 0.08423, - "17": 0.08309, - "18": 0.11719, - "19": 0.0929, - "20": 0.1101, - "21": 0.08669, - "22": 0.08719, - "23": 0.08582, - "24": 0.08654, - "25": 0.08603, - "26": 0.08535, - "27": 0.08439, - "28": 0.08545, - "29": 0.08496, - "30": 0.08412, - "31": 0.08316, - "32": 0.08329, - "33": 0.08342, - "34": 0.08511, - "35": 0.0834, - "36": 0.08316, - "37": 0.08223, - "38": 0.08202, - "39": 0.08221, - "40": 0.07703, - "41": 0.08264, - "42": 0.08192, - "43": 0.0814, - "44": 0.08107, - "45": 0.08098, - "46": 0.08419, - "47": 0.08114, - "48": 0.22032, - "49": 0.0833, - "50": 0.08014, - "51": 0.10352, - "52": 0.08063, - "53": 0.07904, - "54": 0.08003, - "55": 0.08622, - "56": 0.08065, - "57": 0.08879, - "58": 0.08111, - "59": 0.08093, - "60": 0.08098, - "61": 0.08226, - "62": 0.08281, - "63": 0.08189, - "64": 0.08714, - "65": 0.08455, - "66": 0.0857, - "67": 0.08236, - "68": 0.08336, - "69": 0.08227, - "70": 0.0833, - "71": 0.08157, - "72": 0.08485, - "73": 0.08177, - "74": 0.08349, - "75": 0.0828, - "76": 0.08429, - "77": 0.08256, - "78": 0.08362, - "79": 0.08272, - "80": 0.08394, - "81": 0.08197, - "82": 0.08345, - "83": 0.08164, - "84": 0.08343, - "85": 0.08257, - "86": 0.08443, - "87": 0.08437, - "88": 0.08308, - "89": 0.08326, - "90": 0.08136, - "91": 0.08197, - "92": 0.08322, - "93": 0.08598, - "94": 0.08404, - "95": 0.08296, - "96": 0.08331, - "97": 0.08342, - "98": 0.08389, - "99": 0.0902, - "100": 0.09282 + "1": "nan", + "2": 2.48216, + "3": 0.10038, + "4": 0.08588, + "5": 0.08467, + "6": 0.08488, + "7": 0.08474, + "8": 0.08597, + "9": 0.0851, + "10": 0.08607, + "11": 0.08606, + "12": 0.08597, + "13": 0.08652, + "14": 0.08774, + "15": 0.08628, + "16": 0.0847, + "17": 0.08455, + "18": 0.13013, + "19": 0.09834, + "20": 0.09375, + "21": 0.09358, + "22": 0.09463, + "23": 0.094, + "24": 0.09339, + "25": 0.09356, + "26": 0.09394, + "27": 0.095, + "28": 0.09502, + "29": 0.09472, + "30": 0.0953, + "31": 0.09574, + "32": 0.09524, + "33": 0.09617, + "34": 0.09524, + "35": 0.09477, + "36": 0.09409, + "37": 0.09554, + "38": 0.09481, + "39": 0.09427, + "40": 0.08957, + "41": 0.0952, + "42": 0.09493, + "43": 0.09445, + "44": 0.09424, + "45": 0.09619, + "46": 0.09583, + "47": 0.09462, + "48": 0.09189, + "49": 0.09344, + "50": 0.09111, + "51": 0.09793, + "52": 0.08604, + "53": 0.08487, + "54": 0.08374, + "55": 0.0848, + "56": 0.08313, + "57": 0.08774, + "58": 0.08284, + "59": 0.08383, + "60": 0.08368, + "61": 0.08436, + "62": 0.08368, + "63": 0.08273, + "64": 0.08275, + "65": 0.0846, + "66": 0.08337, + "67": 0.08515, + "68": 0.08341, + "69": 0.08418, + "70": 0.08487, + "71": 0.08388, + "72": 0.08281, + "73": 0.08364, + "74": 0.0827, + "75": 0.08268, + "76": 0.08269, + "77": 0.08411, + "78": 0.09377, + "79": 0.08743, + "80": 0.08891, + "81": 0.08977, + "82": 0.0911, + "83": 0.09108, + "84": 0.09091, + "85": 0.09179, + "86": 0.09209, + "87": 0.09134, + "88": 0.09198, + "89": 0.09153, + "90": 0.09199, + "91": 0.09404, + "92": 0.09022, + "93": 0.09001, + "94": 0.09097, + "95": 0.09188, + "96": 0.09181, + "97": 0.09309, + "98": 0.0924, + "99": 0.09355, + "100": 0.09423 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..f9d44558b50 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 10.20107, + "52": 10.10861, + "53": 10.3403, + "54": 10.23893, + "55": 10.19008, + "56": 9.96159, + "57": 9.82417, + "58": 10.10904, + "59": 9.9041, + "60": 9.82045, + "61": 9.96789, + "62": 10.19934, + "63": 9.66196, + "64": 10.00416, + "65": 9.2675, + "66": 9.92466, + "67": 9.62367, + "68": 9.98499, + "69": 9.98524, + "70": 9.92553, + "71": 9.81785, + "72": 9.77816, + "73": 9.67402, + "74": 9.16615, + "75": 9.59935, + "76": 9.2754, + "77": 10.18639, + "78": 9.86592, + "79": 9.52838, + "80": 9.55132, + "81": 9.63037, + "82": 9.82843, + "83": 9.47009, + "84": 9.5424, + "85": 9.74228, + "86": 9.20711, + "87": 9.70433, + "88": 9.86745, + "89": 9.72062, + "90": 9.9304, + "91": 9.471, + "92": 9.47539, + "93": 9.21193, + "94": 8.94879, + "95": 9.62951, + "96": 9.63936, + "97": 9.40708, + "98": 9.77232, + "99": 9.01139, + "100": 9.51718 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1418.0, + "52": 1355.0, + "53": 1814.0, + "54": 1567.0, + "55": 1492.0, + "56": 1408.0, + "57": 1401.0, + "58": 1724.0, + "59": 1654.0, + "60": 1416.0, + "61": 1780.0, + "62": 1852.0, + "63": 1560.0, + "64": 1837.0, + "65": 1520.0, + "66": 1649.0, + "67": 1660.0, + "68": 1716.0, + "69": 1815.0, + "70": 2017.0, + "71": 2026.0, + "72": 1579.0, + "73": 1962.0, + "74": 1321.0, + "75": 1782.0, + "76": 1942.0, + "77": 2128.0, + "78": 2057.0, + "79": 1905.0, + "80": 2153.0, + "81": 2320.0, + "82": 2468.0, + "83": 1951.0, + "84": 2184.0, + "85": 2301.0, + "86": 1971.0, + "87": 2900.0, + "88": 2175.0, + "89": 2357.0, + "90": 2515.0, + "91": 1929.0, + "92": 2680.0, + "93": 2160.0, + "94": 2213.0, + "95": 2280.0, + "96": 2563.0, + "97": 2522.0, + "98": 2470.0, + "99": 2266.0, + "100": 2099.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 518880768.0, + "52": 518880768.0, + "53": 518880768.0, + "54": 518880768.0, + "55": 518880768.0, + "56": 518880768.0, + "57": 518880768.0, + "58": 518880768.0, + "59": 518880768.0, + "60": 518880768.0, + "61": 518880768.0, + "62": 518880768.0, + "63": 518880768.0, + "64": 518880768.0, + "65": 518880768.0, + "66": 518880768.0, + "67": 518880768.0, + "68": 518880768.0, + "69": 518880768.0, + "70": 518880768.0, + "71": 518880768.0, + "72": 518880768.0, + "73": 518880768.0, + "74": 518880768.0, + "75": 518880768.0, + "76": 518880768.0, + "77": 518880768.0, + "78": 518880768.0, + "79": 518880768.0, + "80": 518880768.0, + "81": 518880768.0, + "82": 518880768.0, + "83": 518880768.0, + "84": 518880768.0, + "85": 518880768.0, + "86": 518880768.0, + "87": 518880768.0, + "88": 518880768.0, + "89": 518880768.0, + "90": 518880768.0, + "91": 518880768.0, + "92": 518880768.0, + "93": 518880768.0, + "94": 518880768.0, + "95": 518880768.0, + "96": 518880768.0, + "97": 518880768.0, + "98": 518880768.0, + "99": 518880768.0, + "100": 518880768.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 4575260160.0, + "52": 4608815616.0, + "53": 4608815616.0, + "54": 4608815616.0, + "55": 4608815616.0, + "56": 4608815616.0, + "57": 4608815616.0, + "58": 4608815616.0, + "59": 4608815616.0, + "60": 4608815616.0, + "61": 4608815616.0, + "62": 4608815616.0, + "63": 4608815616.0, + "64": 4608815616.0, + "65": 4608815616.0, + "66": 4608815616.0, + "67": 4608815616.0, + "68": 4608815616.0, + "69": 4608815616.0, + "70": 4608815616.0, + "71": 4608815616.0, + "72": 4608815616.0, + "73": 4608815616.0, + "74": 4608815616.0, + "75": 4608815616.0, + "76": 4608815616.0, + "77": 4608815616.0, + "78": 4608815616.0, + "79": 4608815616.0, + "80": 4608815616.0, + "81": 4608815616.0, + "82": 4608815616.0, + "83": 4608815616.0, + "84": 4608815616.0, + "85": 4608815616.0, + "86": 4608815616.0, + "87": 4608815616.0, + "88": 4608815616.0, + "89": 4608815616.0, + "90": 4608815616.0, + "91": 4608815616.0, + "92": 4608815616.0, + "93": 4608815616.0, + "94": 4608815616.0, + "95": 4608815616.0, + "96": 4608815616.0, + "97": 4608815616.0, + "98": 4608815616.0, + "99": 4608815616.0, + "100": 4608815616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.23866, + "53": 0.10658, + "54": 0.0966, + "55": 0.09594, + "56": 0.0952, + "57": 0.09585, + "58": 0.09524, + "59": 0.09415, + "60": 0.09333, + "61": 0.09407, + "62": 0.09371, + "63": 0.09249, + "64": 0.09383, + "65": 0.09373, + "66": 0.09367, + "67": 0.09283, + "68": 0.09259, + "69": 0.09221, + "70": 0.09229, + "71": 0.09427, + "72": 0.09373, + "73": 0.09768, + "74": 0.09797, + "75": 0.09776, + "76": 0.09553, + "77": 0.09265, + "78": 0.09359, + "79": 0.09433, + "80": 0.09424, + "81": 0.09558, + "82": 0.09536, + "83": 0.09601, + "84": 0.09284, + "85": 0.09339, + "86": 0.09417, + "87": 0.09462, + "88": 0.09587, + "89": 0.09335, + "90": 0.0933, + "91": 0.09312, + "92": 0.09369, + "93": 0.0928, + "94": 0.09412, + "95": 0.09343, + "96": 0.09295, + "97": 0.09368, + "98": 0.09289, + "99": 0.09643, + "100": 0.09451 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json index 51ebcb618e4..063c93b8168 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 7.39667, - "2": 0.15731, - "3": 0.14531, - "4": 0.13151, - "5": 0.13018, - "6": 0.12885, - "7": 0.13069, - "8": 0.13596, - "9": 0.12969, - "10": 0.12994, - "11": 0.1314, - "12": 0.12886, - "13": 0.13009, - "14": 0.1305, - "15": 0.13493, - "16": 0.13341, - "17": 0.13216, - "18": 0.13045, - "19": 0.1359, - "20": 0.13207, - "21": 0.13248, - "22": 0.12979, - "23": 0.12948, - "24": 0.13047, - "25": 0.12963, - "26": 0.13113, - "27": 0.13172, - "28": 0.14017, - "29": 0.13059, - "30": 0.12871, - "31": 0.12957, - "32": 0.1298, - "33": 0.13011, - "34": 0.12939, - "35": 0.12965, - "36": 0.13039, - "37": 0.13099, - "38": 0.13051, - "39": 0.12932, - "40": 0.13052, - "41": 0.13052, - "42": 0.13104, - "43": 0.12938, - "44": 0.13063, - "45": 0.13204, - "46": 0.13075, - "47": 0.13071, - "48": 0.12984, - "49": 0.12965, - "50": 0.12987 + "1": "nan", + "2": 2.42188, + "3": 0.13001, + "4": 0.11595, + "5": 0.1162, + "6": 0.11616, + "7": 0.11716, + "8": 0.11796, + "9": 0.1186, + "10": 0.11774, + "11": 0.11769, + "12": 0.12298, + "13": 0.11717, + "14": 0.11738, + "15": 0.11771, + "16": 0.11772, + "17": 0.11737, + "18": 0.11751, + "19": 0.11697, + "20": 0.11822, + "21": 0.11647, + "22": 0.11724, + "23": 0.11721, + "24": 0.11677, + "25": 0.1171, + "26": 0.11775, + "27": 0.11748, + "28": 0.11705, + "29": 0.11727, + "30": 0.11693, + "31": 0.11818, + "32": 0.11738, + "33": 0.11726, + "34": 0.11675, + "35": 0.11722, + "36": 0.11753, + "37": 0.11779, + "38": 0.11683, + "39": 0.11725, + "40": 0.11779, + "41": 0.11794, + "42": 0.11724, + "43": 0.11807, + "44": 0.11744, + "45": 0.12537, + "46": 0.11752, + "47": 0.11739, + "48": 0.11765, + "49": 0.11763, + "50": 0.11812 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json index 94a972ee670..aa94d697c53 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 6.71621, - "2": 0.15096, - "3": 0.1401, - "4": 0.12751, - "5": 0.12582, - "6": 0.12762, - "7": 0.29588, - "8": 0.12501, - "9": 0.12257, - "10": 0.1231, - "11": 0.12179, - "12": 0.12146, - "13": 0.1218, - "14": 0.12189, - "15": 0.11937, - "16": 0.11701, - "17": 0.16358, - "18": 0.1329, - "19": 0.12356, - "20": 0.1223, - "21": 0.11887, - "22": 0.10873, - "23": 0.11776, - "24": 0.11791, - "25": 0.11708, - "26": 0.11725, - "27": 0.12727, - "28": 0.2171, - "29": 0.1145, - "30": 0.11344, - "31": 0.11497, - "32": 0.11511, - "33": 0.1157, - "34": 0.11565, - "35": 0.11684, - "36": 0.11679, - "37": 0.11675, - "38": 0.11549, - "39": 0.3291, - "40": 0.4913, - "41": 0.12148, - "42": 0.11374, - "43": 0.11395, - "44": 0.11452, - "45": 0.11465, - "46": 0.11512, - "47": 0.11552, - "48": 0.11487, - "49": 0.11358, - "50": 0.11314, - "51": 0.14003, - "52": 0.11456, - "53": 0.11604, - "54": 0.11224, - "55": 0.12526, - "56": 0.11247, - "57": 0.11315, - "58": 0.11222, - "59": 0.11353, - "60": 0.1122, - "61": 0.11312, - "62": 0.11183, - "63": 0.1147, - "64": 0.11171, - "65": 0.11298, - "66": 0.11177, - "67": 0.11322, - "68": 0.11115, - "69": 0.11243, - "70": 0.11245, - "71": 0.1128, - "72": 0.1133, - "73": 0.11263, - "74": 0.11369, - "75": 0.11191, - "76": 0.11291, - "77": 0.11243, - "78": 0.11353, - "79": 0.1277, - "80": 0.11295, - "81": 0.11234, - "82": 0.1138, - "83": 0.11202, - "84": 0.11873, - "85": 0.11198, - "86": 0.11416, - "87": 0.11434, - "88": 0.11401, - "89": 0.11423, - "90": 0.11109, - "91": 0.11252, - "92": 0.11221, - "93": 0.11285, - "94": 0.11189, - "95": 0.11269, - "96": 0.12639, - "97": 0.12758, - "98": 0.12878, - "99": 0.1295, - "100": 0.15151 + "1": "nan", + "2": 2.16002, + "3": 0.12086, + "4": 0.10679, + "5": 0.10684, + "6": 0.10764, + "7": 0.10756, + "8": 0.10782, + "9": 0.10736, + "10": 0.10699, + "11": 0.10758, + "12": 0.1082, + "13": 0.10728, + "14": 0.10701, + "15": 0.10751, + "16": 0.10779, + "17": 0.1525, + "18": 0.12315, + "19": 0.11877, + "20": 0.11834, + "21": 0.11693, + "22": 0.10869, + "23": 0.11758, + "24": 0.11876, + "25": 0.1157, + "26": 0.11704, + "27": 0.11721, + "28": 0.11734, + "29": 0.11883, + "30": 0.11662, + "31": 0.11713, + "32": 0.11867, + "33": 0.11765, + "34": 0.11637, + "35": 0.1163, + "36": 0.11685, + "37": 0.11693, + "38": 0.11691, + "39": 0.11511, + "40": 0.11608, + "41": 0.11592, + "42": 0.11755, + "43": 0.11699, + "44": 0.1167, + "45": 0.11741, + "46": 0.11687, + "47": 0.11681, + "48": 0.11668, + "49": 0.11692, + "50": 0.11769, + "51": 0.13585, + "52": 0.11734, + "53": 0.11685, + "54": 0.11694, + "55": 0.11628, + "56": 0.11632, + "57": 0.11669, + "58": 0.11528, + "59": 0.11499, + "60": 0.11541, + "61": 0.11621, + "62": 0.11572, + "63": 0.11627, + "64": 0.11666, + "65": 0.11549, + "66": 0.11562, + "67": 0.11651, + "68": 0.11467, + "69": 0.11506, + "70": 0.11625, + "71": 0.11703, + "72": 0.11635, + "73": 0.11771, + "74": 0.1156, + "75": 0.11766, + "76": 0.11632, + "77": 0.11535, + "78": 0.11674, + "79": 0.11793, + "80": 0.1173, + "81": 0.11677, + "82": 0.11736, + "83": 0.11611, + "84": 0.11798, + "85": 0.11839, + "86": 0.11892, + "87": 0.11724, + "88": 0.11663, + "89": 0.11722, + "90": 0.11751, + "91": 0.11711, + "92": 0.11773, + "93": 0.11853, + "94": 0.11655, + "95": 0.11767, + "96": 0.11808, + "97": 0.11824, + "98": 0.11911, + "99": 0.11735, + "100": 0.11751 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..a0ffc9cfd0d --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp2_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 10.13864, + "52": 10.03803, + "53": 10.2953, + "54": 10.19383, + "55": 10.14359, + "56": 9.8908, + "57": 9.73702, + "58": 10.05022, + "59": 9.83828, + "60": 9.74551, + "61": 9.90679, + "62": 10.16216, + "63": 9.59842, + "64": 9.95194, + "65": 9.18904, + "66": 9.87164, + "67": 9.56047, + "68": 9.94233, + "69": 9.94285, + "70": 9.8854, + "71": 9.77852, + "72": 9.73861, + "73": 9.63511, + "74": 9.10351, + "75": 9.55716, + "76": 9.23197, + "77": 10.16792, + "78": 9.83943, + "79": 9.49691, + "80": 9.52327, + "81": 9.60219, + "82": 9.8054, + "83": 9.43936, + "84": 9.51953, + "85": 9.72086, + "86": 9.18604, + "87": 9.68762, + "88": 9.84868, + "89": 9.70441, + "90": 9.91638, + "91": 9.45088, + "92": 9.45495, + "93": 9.1952, + "94": 8.93245, + "95": 9.61119, + "96": 9.62586, + "97": 9.39727, + "98": 9.76341, + "99": 8.99611, + "100": 9.50318 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2321.0, + "52": 2270.0, + "53": 2929.0, + "54": 2493.0, + "55": 2470.0, + "56": 2387.0, + "57": 2321.0, + "58": 2774.0, + "59": 2339.0, + "60": 2654.0, + "61": 2810.0, + "62": 2863.0, + "63": 2582.0, + "64": 2851.0, + "65": 2686.0, + "66": 2969.0, + "67": 2680.0, + "68": 2913.0, + "69": 2669.0, + "70": 2988.0, + "71": 2881.0, + "72": 2465.0, + "73": 3188.0, + "74": 2209.0, + "75": 2665.0, + "76": 3308.0, + "77": 3227.0, + "78": 3393.0, + "79": 3433.0, + "80": 3273.0, + "81": 3620.0, + "82": 3491.0, + "83": 3140.0, + "84": 3225.0, + "85": 3622.0, + "86": 3290.0, + "87": 4023.0, + "88": 3187.0, + "89": 3975.0, + "90": 3576.0, + "91": 2689.0, + "92": 3474.0, + "93": 3202.0, + "94": 3608.0, + "95": 3510.0, + "96": 3634.0, + "97": 3500.0, + "98": 3933.0, + "99": 3502.0, + "100": 3134.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1044076032.0, + "52": 1044076032.0, + "53": 1044076032.0, + "54": 1044076032.0, + "55": 1044076032.0, + "56": 1044076032.0, + "57": 1044076032.0, + "58": 1044076032.0, + "59": 1044076032.0, + "60": 1044076032.0, + "61": 1044076032.0, + "62": 1044076032.0, + "63": 1044076032.0, + "64": 1044076032.0, + "65": 1044076032.0, + "66": 1044076032.0, + "67": 1044076032.0, + "68": 1044076032.0, + "69": 1044076032.0, + "70": 1044076032.0, + "71": 1044076032.0, + "72": 1044076032.0, + "73": 1044076032.0, + "74": 1044076032.0, + "75": 1044076032.0, + "76": 1044076032.0, + "77": 1044076032.0, + "78": 1044076032.0, + "79": 1044076032.0, + "80": 1044076032.0, + "81": 1044076032.0, + "82": 1044076032.0, + "83": 1044076032.0, + "84": 1044076032.0, + "85": 1044076032.0, + "86": 1044076032.0, + "87": 1044076032.0, + "88": 1044076032.0, + "89": 1044076032.0, + "90": 1044076032.0, + "91": 1044076032.0, + "92": 1044076032.0, + "93": 1044076032.0, + "94": 1044076032.0, + "95": 1044076032.0, + "96": 1044076032.0, + "97": 1044076032.0, + "98": 1044076032.0, + "99": 1044076032.0, + "100": 1044076032.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 3546769920.0, + "52": 3546770944.0, + "53": 3546770944.0, + "54": 3546770944.0, + "55": 3546770944.0, + "56": 3546770944.0, + "57": 3546770944.0, + "58": 3546770944.0, + "59": 3546770944.0, + "60": 3546770944.0, + "61": 3546770944.0, + "62": 3546770944.0, + "63": 3546770944.0, + "64": 3546770944.0, + "65": 3546770944.0, + "66": 3546770944.0, + "67": 3546770944.0, + "68": 3546770944.0, + "69": 3546770944.0, + "70": 3546770944.0, + "71": 3546770944.0, + "72": 3546770944.0, + "73": 3546770944.0, + "74": 3546770944.0, + "75": 3546770944.0, + "76": 3546770944.0, + "77": 3546770944.0, + "78": 3546770944.0, + "79": 3546770944.0, + "80": 3546770944.0, + "81": 3546770944.0, + "82": 3546770944.0, + "83": 3546770944.0, + "84": 3546770944.0, + "85": 3546770944.0, + "86": 3546770944.0, + "87": 3546770944.0, + "88": 3546770944.0, + "89": 3546770944.0, + "90": 3546770944.0, + "91": 3546770944.0, + "92": 3546770944.0, + "93": 3546770944.0, + "94": 3546770944.0, + "95": 3546770944.0, + "96": 3546770944.0, + "97": 3546770944.0, + "98": 3546770944.0, + "99": 3546770944.0, + "100": 3546770944.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.47868, + "53": 0.1279, + "54": 0.11872, + "55": 0.11801, + "56": 0.11659, + "57": 0.11822, + "58": 0.11942, + "59": 0.11781, + "60": 0.11931, + "61": 0.11745, + "62": 0.11823, + "63": 0.11864, + "64": 0.11936, + "65": 0.11981, + "66": 0.11828, + "67": 0.11814, + "68": 0.11767, + "69": 0.11723, + "70": 0.11851, + "71": 0.11778, + "72": 0.1171, + "73": 0.11843, + "74": 0.11879, + "75": 0.11904, + "76": 0.11937, + "77": 0.11851, + "78": 0.11863, + "79": 0.11746, + "80": 0.11801, + "81": 0.11841, + "82": 0.1189, + "83": 0.11865, + "84": 0.11762, + "85": 0.11705, + "86": 0.11788, + "87": 0.11804, + "88": 0.11814, + "89": 0.11967, + "90": 0.11938, + "91": 0.11768, + "92": 0.1176, + "93": 0.1189, + "94": 0.1188, + "95": 0.11753, + "96": 0.1179, + "97": 0.12698, + "98": 0.13715, + "99": 0.1402, + "100": 0.13853 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json index ba4bf2c3eaf..78a164b057a 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4/golden_values_dev_dgx_gb200.json @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 7.95704, - "2": 0.23282, - "3": 0.22573, - "4": 0.14406, - "5": 0.14176, - "6": 0.14066, - "7": 0.14191, - "8": 0.13977, - "9": 0.23575, - "10": 0.14253, - "11": 0.14269, - "12": 0.24047, - "13": 0.18824, - "14": 0.18624, - "15": 0.30512, - "16": 0.14193, - "17": 0.2268, - "18": 0.14073, - "19": 0.23385, - "20": 0.20206, - "21": 0.1413, - "22": 0.13909, - "23": 0.35016, - "24": 0.14315, - "25": 0.22043, - "26": 0.14108, - "27": 0.14032, - "28": 0.14199, - "29": 0.38987, - "30": 0.14061, - "31": 0.14114, - "32": 0.14198, - "33": 0.21726, - "34": 0.14506, - "35": 0.14599, - "36": 0.14386, - "37": 0.14357, - "38": 0.22005, - "39": 0.14191, - "40": 0.14088, - "41": 0.23965, - "42": 0.14104, - "43": 0.21167, - "44": 0.13993, - "45": 0.2299, - "46": 0.24126, - "47": 0.14128, - "48": 0.14024, - "49": 0.22136, - "50": 0.14147 + "1": "nan", + "2": 2.22981, + "3": 0.15128, + "4": 0.13923, + "5": 0.13787, + "6": 0.13801, + "7": 0.13851, + "8": 0.13805, + "9": 0.13877, + "10": 0.14054, + "11": 0.14025, + "12": 0.13996, + "13": 0.13989, + "14": 0.13978, + "15": 0.14117, + "16": 0.14293, + "17": 0.14179, + "18": 0.14229, + "19": 0.14245, + "20": 0.14412, + "21": 0.14397, + "22": 0.1442, + "23": 0.14329, + "24": 0.14358, + "25": 0.14351, + "26": 0.14424, + "27": 0.14406, + "28": 0.1431, + "29": 0.14373, + "30": 0.14433, + "31": 0.14377, + "32": 0.14346, + "33": 0.14433, + "34": 0.14352, + "35": 0.1446, + "36": 0.14442, + "37": 0.14373, + "38": 0.14265, + "39": 0.14371, + "40": 0.14411, + "41": 0.14415, + "42": 0.14342, + "43": 0.14536, + "44": 0.14415, + "45": 0.14252, + "46": 0.14463, + "47": 0.1438, + "48": 0.14396, + "49": 0.14369, + "50": 0.14335 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json index 51e39254e9a..1a14c45ef7f 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 7.92082, - "2": 0.15881, - "3": 0.1483, - "4": 0.13026, - "5": 0.247, - "6": 0.1291, - "7": 0.24882, - "8": 0.12936, - "9": 0.24806, - "10": 0.21162, - "11": 0.12953, - "12": 0.22212, - "13": 0.12944, - "14": 0.12909, - "15": 0.23074, - "16": 0.1288, - "17": 0.28155, - "18": 0.14734, - "19": 0.13796, - "20": 0.13594, - "21": 0.23092, - "22": 0.21716, - "23": 0.13639, - "24": 0.1359, - "25": 0.2221, - "26": 0.16419, - "27": 0.14414, - "28": 0.21146, - "29": 0.13469, - "30": 0.22441, - "31": 0.13661, - "32": 0.13647, - "33": 0.13579, - "34": 0.13549, - "35": 0.13504, - "36": 0.13513, - "37": 0.13527, - "38": 0.19634, - "39": 0.3711, - "40": 0.1353, - "41": 0.13666, - "42": 0.21568, - "43": 0.13653, - "44": 0.13523, - "45": 0.13504, - "46": 0.13584, - "47": 0.13676, - "48": 0.13449, - "49": 0.22259, - "50": 0.14061, - "51": 0.34203, - "52": 0.44673, - "53": 0.30462, - "54": 0.34485, - "55": 0.36971, - "56": 0.37478, - "57": 0.3581, - "58": 0.46665, - "59": 0.47512, - "60": 0.38197, - "61": 0.40684, - "62": 0.48548, - "63": 0.32955, - "64": 0.28002, - "65": 0.1858, - "66": 0.1488, - "67": 0.21555, - "68": 0.17819, - "69": 0.24009, - "70": 0.18827, - "71": 0.17896, - "72": 0.18197, - "73": 0.13026, - "74": 0.21407, - "75": 0.13008, - "76": 0.12912, - "77": 0.12908, - "78": 0.13051, - "79": 0.12938, - "80": 0.13039, - "81": 0.1314, - "82": 0.40745, - "83": 0.12931, - "84": 0.13085, - "85": 0.13025, - "86": 0.13101, - "87": 0.12901, - "88": 0.12981, - "89": 0.12874, - "90": 0.12891, - "91": 0.13086, - "92": 0.19117, - "93": 0.1298, - "94": 0.13035, - "95": 0.12884, - "96": 0.12875, - "97": 0.13072, - "98": 0.14893, - "99": 0.13089, - "100": 0.13044 + "1": "nan", + "2": 2.57843, + "3": 0.14384, + "4": 0.12903, + "5": 0.12836, + "6": 0.12816, + "7": 0.12916, + "8": 0.12856, + "9": 0.12834, + "10": 0.12806, + "11": 0.12799, + "12": 0.1275, + "13": 0.12711, + "14": 0.12751, + "15": 0.129, + "16": 0.12848, + "17": 0.16638, + "18": 0.14237, + "19": 0.1354, + "20": 0.13451, + "21": 0.13484, + "22": 0.1296, + "23": 0.134, + "24": 0.13542, + "25": 0.13555, + "26": 0.13391, + "27": 0.13338, + "28": 0.13321, + "29": 0.13407, + "30": 0.13362, + "31": 0.13566, + "32": 0.13345, + "33": 0.13445, + "34": 0.13432, + "35": 0.13463, + "36": 0.1333, + "37": 0.13493, + "38": 0.1341, + "39": 0.13366, + "40": 0.14828, + "41": 0.15021, + "42": 0.14974, + "43": 0.15118, + "44": 0.15264, + "45": 0.15167, + "46": 0.15228, + "47": 0.15164, + "48": 0.15268, + "49": 0.15149, + "50": 0.15349, + "51": 0.18359, + "52": 0.15225, + "53": 0.14909, + "54": 0.1498, + "55": 0.14962, + "56": 0.14941, + "57": 0.14896, + "58": 0.14931, + "59": 0.149, + "60": 0.14965, + "61": 0.15012, + "62": 0.15003, + "63": 0.14869, + "64": 0.14833, + "65": 0.14957, + "66": 0.14978, + "67": 0.14987, + "68": 0.15037, + "69": 0.15122, + "70": 0.15108, + "71": 0.14994, + "72": 0.1507, + "73": 0.15127, + "74": 0.15073, + "75": 0.14986, + "76": 0.15012, + "77": 0.15071, + "78": 0.15091, + "79": 0.15129, + "80": 0.15073, + "81": 0.15107, + "82": 0.15091, + "83": 0.15083, + "84": 0.15069, + "85": 0.15272, + "86": 0.1517, + "87": 0.15055, + "88": 0.15217, + "89": 0.15281, + "90": 0.14893, + "91": 0.16006, + "92": 0.15632, + "93": 0.15975, + "94": 0.1591, + "95": 0.15873, + "96": 0.15918, + "97": 0.15958, + "98": 0.15854, + "99": 0.15737, + "100": 0.15785 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..8d115483589 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp1_pp4_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 10.18063, + "52": 10.07636, + "53": 10.32773, + "54": 10.23662, + "55": 10.17779, + "56": 9.93459, + "57": 9.79047, + "58": 10.09308, + "59": 9.88561, + "60": 9.79776, + "61": 9.94517, + "62": 10.19094, + "63": 9.64683, + "64": 9.98455, + "65": 9.23395, + "66": 9.90453, + "67": 9.59582, + "68": 9.97649, + "69": 9.97495, + "70": 9.91345, + "71": 9.81704, + "72": 9.7724, + "73": 9.6613, + "74": 9.13276, + "75": 9.5758, + "76": 9.25498, + "77": 10.18582, + "78": 9.86011, + "79": 9.51637, + "80": 9.54101, + "81": 9.61959, + "82": 9.8199, + "83": 9.45715, + "84": 9.53646, + "85": 9.73396, + "86": 9.19313, + "87": 9.70118, + "88": 9.85742, + "89": 9.71286, + "90": 9.92642, + "91": 9.46223, + "92": 9.46428, + "93": 9.20456, + "94": 8.93882, + "95": 9.61804, + "96": 9.62982, + "97": 9.40186, + "98": 9.76277, + "99": 9.00132, + "100": 9.50913 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2281.0, + "52": 2534.0, + "53": 3604.0, + "54": 2989.0, + "55": 2624.0, + "56": 2547.0, + "57": 2287.0, + "58": 3322.0, + "59": 2730.0, + "60": 2919.0, + "61": 3007.0, + "62": 3131.0, + "63": 3226.0, + "64": 3219.0, + "65": 2422.0, + "66": 3741.0, + "67": 2805.0, + "68": 3215.0, + "69": 2871.0, + "70": 3597.0, + "71": 3045.0, + "72": 2952.0, + "73": 3559.0, + "74": 2232.0, + "75": 2889.0, + "76": 3802.0, + "77": 3635.0, + "78": 3762.0, + "79": 4000.0, + "80": 3383.0, + "81": 4629.0, + "82": 3435.0, + "83": 3254.0, + "84": 3786.0, + "85": 3895.0, + "86": 3338.0, + "87": 4169.0, + "88": 3498.0, + "89": 4065.0, + "90": 3825.0, + "91": 3040.0, + "92": 4399.0, + "93": 3899.0, + "94": 4449.0, + "95": 4017.0, + "96": 3820.0, + "97": 4268.0, + "98": 5094.0, + "99": 3940.0, + "100": 3369.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 853400064.0, + "52": 853400064.0, + "53": 853400064.0, + "54": 853400064.0, + "55": 853400064.0, + "56": 853400064.0, + "57": 853400064.0, + "58": 853400064.0, + "59": 853400064.0, + "60": 853400064.0, + "61": 853400064.0, + "62": 853400064.0, + "63": 853400064.0, + "64": 853400064.0, + "65": 853400064.0, + "66": 853400064.0, + "67": 853400064.0, + "68": 853400064.0, + "69": 853400064.0, + "70": 853400064.0, + "71": 853400064.0, + "72": 853400064.0, + "73": 853400064.0, + "74": 853400064.0, + "75": 853400064.0, + "76": 853400064.0, + "77": 853400064.0, + "78": 853400064.0, + "79": 853400064.0, + "80": 853400064.0, + "81": 853400064.0, + "82": 853400064.0, + "83": 853400064.0, + "84": 853400064.0, + "85": 853400064.0, + "86": 853400064.0, + "87": 853400064.0, + "88": 853400064.0, + "89": 853400064.0, + "90": 853400064.0, + "91": 853400064.0, + "92": 853400064.0, + "93": 853400064.0, + "94": 853400064.0, + "95": 853400064.0, + "96": 853400064.0, + "97": 853400064.0, + "98": 853400064.0, + "99": 853400064.0, + "100": 853400064.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2677025280.0, + "52": 2677026304.0, + "53": 2677026304.0, + "54": 2677026304.0, + "55": 2677026304.0, + "56": 2677026304.0, + "57": 2677026304.0, + "58": 2677026304.0, + "59": 2677026304.0, + "60": 2677026304.0, + "61": 2677026304.0, + "62": 2677026304.0, + "63": 2677026304.0, + "64": 2677026304.0, + "65": 2677026304.0, + "66": 2677026304.0, + "67": 2677026304.0, + "68": 2677026304.0, + "69": 2677026304.0, + "70": 2677026304.0, + "71": 2677026304.0, + "72": 2677026304.0, + "73": 2677026304.0, + "74": 2677026304.0, + "75": 2677026304.0, + "76": 2677026304.0, + "77": 2677026304.0, + "78": 2677026304.0, + "79": 2677026304.0, + "80": 2677026304.0, + "81": 2677026304.0, + "82": 2677026304.0, + "83": 2677026304.0, + "84": 2677026304.0, + "85": 2677026304.0, + "86": 2677026304.0, + "87": 2677026304.0, + "88": 2677026304.0, + "89": 2677026304.0, + "90": 2677026304.0, + "91": 2677026304.0, + "92": 2677026304.0, + "93": 2677026304.0, + "94": 2677026304.0, + "95": 2677026304.0, + "96": 2677026304.0, + "97": 2677026304.0, + "98": 2677026304.0, + "99": 2677026304.0, + "100": 2677026304.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 2.52584, + "53": 0.20392, + "54": 0.16414, + "55": 0.13847, + "56": 0.13338, + "57": 0.13357, + "58": 0.13375, + "59": 0.13394, + "60": 0.1334, + "61": 0.1341, + "62": 0.13391, + "63": 0.13459, + "64": 0.13308, + "65": 0.13494, + "66": 0.13405, + "67": 0.13432, + "68": 0.13481, + "69": 0.13446, + "70": 0.13476, + "71": 0.13398, + "72": 0.13448, + "73": 0.13508, + "74": 0.13535, + "75": 0.13502, + "76": 0.13588, + "77": 0.13483, + "78": 0.13626, + "79": 0.13542, + "80": 0.13571, + "81": 0.13587, + "82": 0.13461, + "83": 0.13533, + "84": 0.13399, + "85": 0.13532, + "86": 0.13468, + "87": 0.13492, + "88": 0.13529, + "89": 0.13664, + "90": 0.13526, + "91": 0.13377, + "92": 0.13367, + "93": 0.13265, + "94": 0.13355, + "95": 0.13376, + "96": 0.13303, + "97": 0.13448, + "98": 0.13371, + "99": 0.13395, + "100": 0.1334 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json index 49586883019..0322a9120bd 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200.json @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 4.10681, - "2": 0.30693, - "3": 0.30146, - "4": 0.29106, - "5": 0.29089, - "6": 0.29112, - "7": 0.29159, - "8": 0.29115, - "9": 0.29086, - "10": 0.63125, - "11": 0.2989, - "12": 0.29641, - "13": 0.29201, - "14": 0.29541, - "15": 0.29003, - "16": 0.36384, - "17": 0.29423, - "18": 0.30498, - "19": 0.30687, - "20": 0.30695, - "21": 0.30562, - "22": 0.29047, - "23": 0.30755, - "24": 0.30627, - "25": 0.65941, - "26": 0.30667, - "27": 0.31536, - "28": 0.30722, - "29": 0.30542, - "30": 0.30564, - "31": 0.3045, - "32": 0.30472, - "33": 0.30551, - "34": 0.30423, - "35": 0.3045, - "36": 0.30479, - "37": 0.30596, - "38": 0.30404, - "39": 0.30411, - "40": 0.30491, - "41": 0.3071, - "42": 0.30318, - "43": 0.30217, - "44": 0.30293, - "45": 0.3041, - "46": 0.30338, - "47": 0.3038, - "48": 0.30224, - "49": 0.30264, - "50": 0.3024, - "51": 0.36516, - "52": 0.42479, - "53": 0.43225, - "54": 0.37389, - "55": 0.34351, - "56": 0.66697, - "57": 0.30412, - "58": 0.30714, - "59": 0.31209, - "60": 0.33472, - "61": 0.36046, - "62": 0.39323, - "63": 0.4363, - "64": 0.46158, - "65": 0.43859, - "66": 0.3596, - "67": 0.34843, - "68": 0.69171, - "69": 0.35185, - "70": 0.34317, - "71": 0.34189, - "72": 0.3408, - "73": 0.34132, - "74": 0.33999, - "75": 0.33341, - "76": 0.339, - "77": 0.34005, - "78": 0.33524, - "79": 0.65413, - "80": 0.3407, - "81": 0.33061, - "82": 0.33345, - "83": 0.3333, - "84": 0.33362, - "85": 0.33251, - "86": 0.3337, - "87": 0.33386, - "88": 0.6509, - "89": 0.33263, - "90": 0.32972, - "91": 0.32543, - "92": 0.32519, - "93": 0.32484, - "94": 0.32156, - "95": 0.32526, - "96": 0.32111, - "97": 0.32404, - "98": 0.31936, - "99": 0.31881, - "100": 0.31797 + "1": "nan", + "2": 2.03297, + "3": 0.32699, + "4": 0.31597, + "5": 0.31552, + "6": 0.3147, + "7": 0.31402, + "8": 0.31397, + "9": 0.31445, + "10": 0.31986, + "11": 0.31186, + "12": 0.30888, + "13": 0.3063, + "14": 0.30631, + "15": 0.30635, + "16": 0.38476, + "17": 0.31287, + "18": 0.32251, + "19": 0.32317, + "20": 0.31607, + "21": 0.31688, + "22": 0.30059, + "23": 0.31719, + "24": 0.3226, + "25": 0.31854, + "26": 0.31658, + "27": 0.31847, + "28": 0.3176, + "29": 0.31877, + "30": 0.31787, + "31": 0.31758, + "32": 0.31563, + "33": 0.31399, + "34": 0.31522, + "35": 0.31891, + "36": 0.3187, + "37": 0.32382, + "38": 0.32042, + "39": 0.32237, + "40": 0.32145, + "41": 0.31906, + "42": 0.31768, + "43": 0.32041, + "44": 0.31937, + "45": 0.32089, + "46": 0.31845, + "47": 0.31938, + "48": 0.31552, + "49": 0.31733, + "50": 0.3196, + "51": 0.31383, + "52": 0.31412, + "53": 0.31249, + "54": 0.3127, + "55": 0.31762, + "56": 0.31919, + "57": 0.31929, + "58": 0.32119, + "59": 0.31667, + "60": 0.32308, + "61": 0.31738, + "62": 0.32278, + "63": 0.31714, + "64": 0.31073, + "65": 0.30929, + "66": 0.30856, + "67": 0.31027, + "68": 0.3103, + "69": 0.31124, + "70": 0.30886, + "71": 0.30892, + "72": 0.31237, + "73": 0.31647, + "74": 0.31733, + "75": 0.31764, + "76": 0.3165, + "77": 0.31656, + "78": 0.3176, + "79": 0.31747, + "80": 0.3171, + "81": 0.31656, + "82": 0.3168, + "83": 0.31697, + "84": 0.3181, + "85": 0.31755, + "86": 0.31749, + "87": 0.31765, + "88": 0.31775, + "89": 0.31806, + "90": 0.31417, + "91": 0.64575, + "92": 0.3228, + "93": 0.3237, + "94": 0.32187, + "95": 0.32154, + "96": 0.32116, + "97": 0.33046, + "98": 0.35266, + "99": 0.32136, + "100": 0.32174 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..121b3fe11b7 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 10.23426, + "52": 10.13488, + "53": 10.35279, + "54": 10.26189, + "55": 10.20983, + "56": 9.99599, + "57": 9.87962, + "58": 10.13391, + "59": 9.92304, + "60": 9.85379, + "61": 9.97314, + "62": 10.211, + "63": 9.70514, + "64": 10.01457, + "65": 9.30759, + "66": 9.9366, + "67": 9.63221, + "68": 9.98219, + "69": 9.98048, + "70": 9.92986, + "71": 9.81575, + "72": 9.79602, + "73": 9.69104, + "74": 9.20049, + "75": 9.61228, + "76": 9.28906, + "77": 10.19068, + "78": 9.86601, + "79": 9.53855, + "80": 9.5578, + "81": 9.63332, + "82": 9.82853, + "83": 9.47188, + "84": 9.54101, + "85": 9.74266, + "86": 9.2142, + "87": 9.7016, + "88": 9.86604, + "89": 9.72339, + "90": 9.92767, + "91": 9.47045, + "92": 9.46809, + "93": 9.21217, + "94": 8.94887, + "95": 9.62787, + "96": 9.6406, + "97": 9.40839, + "98": 9.77147, + "99": 9.00853, + "100": 9.51225 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2080.0, + "52": 2200.0, + "53": 3412.0, + "54": 2641.0, + "55": 2229.0, + "56": 2244.0, + "57": 2057.0, + "58": 3223.0, + "59": 2431.0, + "60": 2650.0, + "61": 2712.0, + "62": 2995.0, + "63": 2816.0, + "64": 2860.0, + "65": 2015.0, + "66": 3176.0, + "67": 2529.0, + "68": 3108.0, + "69": 2873.0, + "70": 3540.0, + "71": 2904.0, + "72": 2693.0, + "73": 3253.0, + "74": 1981.0, + "75": 2780.0, + "76": 3465.0, + "77": 3649.0, + "78": 3593.0, + "79": 3981.0, + "80": 3458.0, + "81": 5181.0, + "82": 3334.0, + "83": 2956.0, + "84": 3527.0, + "85": 3711.0, + "86": 3209.0, + "87": 4133.0, + "88": 3443.0, + "89": 4295.0, + "90": 3801.0, + "91": 2958.0, + "92": 4311.0, + "93": 3544.0, + "94": 4264.0, + "95": 4042.0, + "96": 3849.0, + "97": 3974.0, + "98": 4971.0, + "99": 4071.0, + "100": 3363.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 414416384.0, + "52": 414416384.0, + "53": 414416384.0, + "54": 414416384.0, + "55": 414416384.0, + "56": 414416384.0, + "57": 414416384.0, + "58": 414416384.0, + "59": 414416384.0, + "60": 414416384.0, + "61": 414416384.0, + "62": 414416384.0, + "63": 414416384.0, + "64": 414416384.0, + "65": 414416384.0, + "66": 414416384.0, + "67": 414416384.0, + "68": 414416384.0, + "69": 414416384.0, + "70": 414416384.0, + "71": 414416384.0, + "72": 414416384.0, + "73": 414416384.0, + "74": 414416384.0, + "75": 414416384.0, + "76": 414416384.0, + "77": 414416384.0, + "78": 414416384.0, + "79": 414416384.0, + "80": 414416384.0, + "81": 414416384.0, + "82": 414416384.0, + "83": 414416384.0, + "84": 414416384.0, + "85": 414416384.0, + "86": 414416384.0, + "87": 414416384.0, + "88": 414416384.0, + "89": 414416384.0, + "90": 414416384.0, + "91": 414416384.0, + "92": 414416384.0, + "93": 414416384.0, + "94": 414416384.0, + "95": 414416384.0, + "96": 414416384.0, + "97": 414416384.0, + "98": 414416384.0, + "99": 414416384.0, + "100": 414416384.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1595256320.0, + "52": 1595257344.0, + "53": 1595257344.0, + "54": 1595257344.0, + "55": 1595257344.0, + "56": 1595257344.0, + "57": 1595257344.0, + "58": 1595257344.0, + "59": 1595257344.0, + "60": 1595257344.0, + "61": 1595257344.0, + "62": 1595257344.0, + "63": 1595257344.0, + "64": 1595257344.0, + "65": 1595257344.0, + "66": 1595257344.0, + "67": 1595257344.0, + "68": 1595257344.0, + "69": 1595257344.0, + "70": 1595257344.0, + "71": 1595257344.0, + "72": 1595257344.0, + "73": 1595257344.0, + "74": 1595257344.0, + "75": 1595257344.0, + "76": 1595257344.0, + "77": 1595257344.0, + "78": 1595257344.0, + "79": 1595257344.0, + "80": 1595257344.0, + "81": 1595257344.0, + "82": 1595257344.0, + "83": 1595257344.0, + "84": 1595257344.0, + "85": 1595257344.0, + "86": 1595257344.0, + "87": 1595257344.0, + "88": 1595257344.0, + "89": 1595257344.0, + "90": 1595257344.0, + "91": 1595257344.0, + "92": 1595257344.0, + "93": 1595257344.0, + "94": 1595257344.0, + "95": 1595257344.0, + "96": 1595257344.0, + "97": 1595257344.0, + "98": 1595257344.0, + "99": 1595257344.0, + "100": 1595257344.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 1.90434, + "53": 0.34839, + "54": 0.46608, + "55": 0.31197, + "56": 0.30786, + "57": 0.30592, + "58": 0.30764, + "59": 0.30985, + "60": 0.30933, + "61": 0.30847, + "62": 0.30963, + "63": 0.31127, + "64": 0.30722, + "65": 0.31035, + "66": 0.31029, + "67": 0.30982, + "68": 0.30966, + "69": 0.30823, + "70": 0.30879, + "71": 0.3091, + "72": 0.30555, + "73": 0.30938, + "74": 0.3098, + "75": 0.31215, + "76": 0.30896, + "77": 0.31066, + "78": 0.31001, + "79": 0.30673, + "80": 0.30699, + "81": 0.30771, + "82": 0.3072, + "83": 0.30839, + "84": 0.30892, + "85": 0.30911, + "86": 0.30528, + "87": 0.30757, + "88": 0.30812, + "89": 0.3083, + "90": 0.30825, + "91": 0.30638, + "92": 0.30467, + "93": 0.30582, + "94": 0.30847, + "95": 0.30633, + "96": 0.30614, + "97": 0.30644, + "98": 0.30185, + "99": 0.30221, + "100": 0.30191 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json index 3def3c8618f..b1ec74d48d3 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200.json @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 4.43718, - "2": 0.32141, - "3": 0.34228, - "4": 0.3338, - "5": 0.33423, - "6": 0.33597, - "7": 0.33749, - "8": 0.33831, - "9": 0.34047, - "10": 0.33938, - "11": 0.3381, - "12": 0.34241, - "13": 0.35311, - "14": 0.35495, - "15": 0.33902, - "16": 0.42658, - "17": 0.3452, - "18": 0.35813, - "19": 0.35538, - "20": 0.36232, - "21": 0.36626, - "22": 0.3555, - "23": 0.36916, - "24": 0.3744, - "25": 0.37348, - "26": 0.36915, - "27": 0.37147, - "28": 0.36445, - "29": 0.36069, - "30": 0.35961, - "31": 0.35274, - "32": 0.35514, - "33": 0.35563, - "34": 0.35744, - "35": 0.35843, - "36": 0.35512, - "37": 0.35839, - "38": 0.35761, - "39": 0.35765, - "40": 0.62747, - "41": 0.35467, - "42": 0.35928, - "43": 0.35301, - "44": 0.35215, - "45": 0.35947, - "46": 0.35676, - "47": 0.65816, - "48": 0.35624, - "49": 0.35833, - "50": 0.35593, - "51": 0.38053, - "52": 0.74045, - "53": 0.36063, - "54": 0.36054, - "55": 0.363, - "56": 0.36264, - "57": 0.36262, - "58": 0.36213, - "59": 0.36223, - "60": 0.35979, - "61": 0.36002, - "62": 0.36456, - "63": 0.36092, - "64": 0.36222, - "65": 0.36214, - "66": 0.36393, - "67": 0.36348, - "68": 0.36404, - "69": 0.36256, - "70": 0.36106, - "71": 0.36265, - "72": 0.36127, - "73": 0.37126, - "74": 0.3637, - "75": 0.36407, - "76": 0.36415, - "77": 0.36331, - "78": 0.3641, - "79": 0.36546, - "80": 0.36427, - "81": 0.35664, - "82": 0.36196, - "83": 0.36259, - "84": 0.36282, - "85": 0.36131, - "86": 0.35889, - "87": 0.36236, - "88": 0.35979, - "89": 0.36186, - "90": 0.36471, - "91": 0.36565, - "92": 0.36403, - "93": 0.365, - "94": 0.36272, - "95": 0.36119, - "96": 0.36129, - "97": 0.36262, - "98": 0.36263, - "99": 0.36514, - "100": 0.36392 + "1": "nan", + "2": 2.06149, + "3": 0.29865, + "4": 0.28631, + "5": 0.28736, + "6": 0.28526, + "7": 0.29059, + "8": 0.28917, + "9": 0.28615, + "10": 0.29015, + "11": 0.28709, + "12": 0.28726, + "13": 0.28539, + "14": 0.28815, + "15": 0.28809, + "16": 0.36186, + "17": 0.29264, + "18": 0.30232, + "19": 0.30149, + "20": 0.3025, + "21": 0.30312, + "22": 0.28815, + "23": 0.30003, + "24": 0.30279, + "25": 0.30076, + "26": 0.31198, + "27": 0.30137, + "28": 0.30174, + "29": 0.30053, + "30": 0.2998, + "31": 0.30129, + "32": 0.30266, + "33": 0.30179, + "34": 0.30183, + "35": 0.30504, + "36": 0.30132, + "37": 0.30289, + "38": 0.30052, + "39": 0.29983, + "40": 0.29924, + "41": 0.30028, + "42": 0.29816, + "43": 0.30081, + "44": 0.3016, + "45": 0.30595, + "46": 0.30403, + "47": 0.30454, + "48": 0.30318, + "49": 0.30105, + "50": 0.30174, + "51": 0.3175, + "52": 0.29652, + "53": 0.29581, + "54": 0.29555, + "55": 0.29696, + "56": 0.29449, + "57": 0.64732, + "58": 0.30369, + "59": 0.30167, + "60": 0.30075, + "61": 0.29981, + "62": 0.30078, + "63": 0.31092, + "64": 0.30144, + "65": 0.29891, + "66": 0.2987, + "67": 0.29963, + "68": 0.30439, + "69": 0.29787, + "70": 0.3036, + "71": 0.30595, + "72": 0.29733, + "73": 0.29745, + "74": 0.30071, + "75": 0.29706, + "76": 0.2969, + "77": 0.29903, + "78": 0.29958, + "79": 0.29754, + "80": 0.30059, + "81": 0.29879, + "82": 0.30486, + "83": 0.29801, + "84": 0.29892, + "85": 0.2996, + "86": 0.29869, + "87": 0.30043, + "88": 0.29951, + "89": 0.29614, + "90": 0.29973, + "91": 0.30029, + "92": 0.29926, + "93": 0.29973, + "94": 0.29969, + "95": 0.30108, + "96": 0.29798, + "97": 0.29923, + "98": 0.29982, + "99": 0.29854, + "100": 0.3007 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json new file mode 100644 index 00000000000..ae8f01b3327 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_tp4_pp1_resume_torch_dist/golden_values_dev_dgx_gb200_2nd.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 10.23426, + "52": 10.13488, + "53": 10.35279, + "54": 10.26189, + "55": 10.20983, + "56": 9.99599, + "57": 9.87962, + "58": 10.13391, + "59": 9.92304, + "60": 9.85379, + "61": 9.97314, + "62": 10.211, + "63": 9.70514, + "64": 10.01457, + "65": 9.30759, + "66": 9.9366, + "67": 9.63221, + "68": 9.98219, + "69": 9.98048, + "70": 9.92986, + "71": 9.81575, + "72": 9.79602, + "73": 9.69104, + "74": 9.20049, + "75": 9.61228, + "76": 9.28906, + "77": 10.19068, + "78": 9.86601, + "79": 9.53855, + "80": 9.5578, + "81": 9.63332, + "82": 9.82853, + "83": 9.47188, + "84": 9.54101, + "85": 9.74266, + "86": 9.2142, + "87": 9.7016, + "88": 9.86604, + "89": 9.72339, + "90": 9.92767, + "91": 9.47045, + "92": 9.46809, + "93": 9.21217, + "94": 8.94887, + "95": 9.62787, + "96": 9.6406, + "97": 9.40839, + "98": 9.77147, + "99": 9.00853, + "100": 9.51225 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 2080.0, + "52": 2200.0, + "53": 3412.0, + "54": 2641.0, + "55": 2229.0, + "56": 2244.0, + "57": 2057.0, + "58": 3223.0, + "59": 2431.0, + "60": 2650.0, + "61": 2712.0, + "62": 2995.0, + "63": 2816.0, + "64": 2860.0, + "65": 2015.0, + "66": 3176.0, + "67": 2529.0, + "68": 3108.0, + "69": 2873.0, + "70": 3540.0, + "71": 2904.0, + "72": 2693.0, + "73": 3253.0, + "74": 1981.0, + "75": 2780.0, + "76": 3465.0, + "77": 3649.0, + "78": 3593.0, + "79": 3981.0, + "80": 3458.0, + "81": 5181.0, + "82": 3334.0, + "83": 2956.0, + "84": 3527.0, + "85": 3711.0, + "86": 3209.0, + "87": 4133.0, + "88": 3443.0, + "89": 4295.0, + "90": 3801.0, + "91": 2958.0, + "92": 4311.0, + "93": 3544.0, + "94": 4264.0, + "95": 4042.0, + "96": 3849.0, + "97": 3974.0, + "98": 4971.0, + "99": 4071.0, + "100": 3363.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 415464960.0, + "52": 415464960.0, + "53": 415464960.0, + "54": 415464960.0, + "55": 415464960.0, + "56": 415464960.0, + "57": 415464960.0, + "58": 415464960.0, + "59": 415464960.0, + "60": 415464960.0, + "61": 415464960.0, + "62": 415464960.0, + "63": 415464960.0, + "64": 415464960.0, + "65": 415464960.0, + "66": 415464960.0, + "67": 415464960.0, + "68": 415464960.0, + "69": 415464960.0, + "70": 415464960.0, + "71": 415464960.0, + "72": 415464960.0, + "73": 415464960.0, + "74": 415464960.0, + "75": 415464960.0, + "76": 415464960.0, + "77": 415464960.0, + "78": 415464960.0, + "79": 415464960.0, + "80": 415464960.0, + "81": 415464960.0, + "82": 415464960.0, + "83": 415464960.0, + "84": 415464960.0, + "85": 415464960.0, + "86": 415464960.0, + "87": 415464960.0, + "88": 415464960.0, + "89": 415464960.0, + "90": 415464960.0, + "91": 415464960.0, + "92": 415464960.0, + "93": 415464960.0, + "94": 415464960.0, + "95": 415464960.0, + "96": 415464960.0, + "97": 415464960.0, + "98": 415464960.0, + "99": 415464960.0, + "100": 415464960.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": 1596304896.0, + "52": 1596305920.0, + "53": 1596305920.0, + "54": 1596305920.0, + "55": 1596305920.0, + "56": 1596305920.0, + "57": 1596305920.0, + "58": 1596305920.0, + "59": 1596305920.0, + "60": 1596305920.0, + "61": 1596305920.0, + "62": 1596305920.0, + "63": 1596305920.0, + "64": 1596305920.0, + "65": 1596305920.0, + "66": 1596305920.0, + "67": 1596305920.0, + "68": 1596305920.0, + "69": 1596305920.0, + "70": 1596305920.0, + "71": 1596305920.0, + "72": 1596305920.0, + "73": 1596305920.0, + "74": 1596305920.0, + "75": 1596305920.0, + "76": 1596305920.0, + "77": 1596305920.0, + "78": 1596305920.0, + "79": 1596305920.0, + "80": 1596305920.0, + "81": 1596305920.0, + "82": 1596305920.0, + "83": 1596305920.0, + "84": 1596305920.0, + "85": 1596305920.0, + "86": 1596305920.0, + "87": 1596305920.0, + "88": 1596305920.0, + "89": 1596305920.0, + "90": 1596305920.0, + "91": 1596305920.0, + "92": 1596305920.0, + "93": 1596305920.0, + "94": 1596305920.0, + "95": 1596305920.0, + "96": 1596305920.0, + "97": 1596305920.0, + "98": 1596305920.0, + "99": 1596305920.0, + "100": 1596305920.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": "nan", + "3": "nan", + "4": "nan", + "5": "nan", + "6": "nan", + "7": "nan", + "8": "nan", + "9": "nan", + "10": "nan", + "11": "nan", + "12": "nan", + "13": "nan", + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan", + "51": "nan", + "52": 1.8615, + "53": 0.31355, + "54": 0.3044, + "55": 0.30701, + "56": 0.30736, + "57": 0.31773, + "58": 0.30739, + "59": 0.30021, + "60": 0.30271, + "61": 0.31062, + "62": 0.30743, + "63": 0.30357, + "64": 0.30729, + "65": 0.3028, + "66": 0.30068, + "67": 0.30021, + "68": 0.30051, + "69": 0.30257, + "70": 0.30291, + "71": 0.30183, + "72": 0.30259, + "73": 0.30032, + "74": 0.3007, + "75": 0.30192, + "76": 0.30188, + "77": 0.30125, + "78": 0.30028, + "79": 0.3024, + "80": 0.3115, + "81": 0.3014, + "82": 0.3023, + "83": 0.30861, + "84": 0.30129, + "85": 0.30185, + "86": 0.29936, + "87": 0.30094, + "88": 0.3001, + "89": 0.2993, + "90": 0.2987, + "91": 0.30006, + "92": 0.30091, + "93": 0.30097, + "94": 0.29909, + "95": 0.30113, + "96": 0.29925, + "97": 0.29979, + "98": 0.30241, + "99": 0.30073, + "100": 0.30251 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..5c353a70683 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81386, + "2": 10.8312, + "3": 10.80976, + "4": 10.82429, + "5": 10.84553, + "6": 10.85179, + "7": 10.83654, + "8": 10.83644, + "9": 10.84704, + "10": 10.78251, + "11": 10.85041, + "12": 10.84824, + "13": 10.86042, + "14": 10.86568, + "15": 10.81163, + "16": 10.79013, + "17": 10.76156, + "18": 10.78107, + "19": 10.78131, + "20": 10.70284, + "21": 10.67616, + "22": 10.51867, + "23": 10.70469, + "24": 10.57188, + "25": 10.51628, + "26": 10.58103, + "27": 10.59482, + "28": 10.56661, + "29": 10.58588, + "30": 10.33634, + "31": 10.08426, + "32": 10.4506, + "33": 10.4457, + "34": 10.19772, + "35": 10.25794, + "36": 10.21991, + "37": 10.34564, + "38": 10.18704, + "39": 10.39388, + "40": 10.08233, + "41": 10.13235, + "42": 10.21151, + "43": 9.83045, + "44": 9.94704, + "45": 9.84037, + "46": 9.81454, + "47": 10.12979, + "48": 9.85142, + "49": 9.52861, + "50": 9.91131 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4720.0, + "2": 4914.0, + "3": 4776.0, + "4": 4671.0, + "5": 5518.0, + "6": 5824.0, + "7": 4980.0, + "8": 4796.0, + "9": 5167.0, + "10": 4355.0, + "11": 5673.0, + "12": 5426.0, + "13": 5583.0, + "14": 5744.0, + "15": 5108.0, + "16": 5421.0, + "17": 5053.0, + "18": 5299.0, + "19": 5132.0, + "20": 4876.0, + "21": 5310.0, + "22": 4667.0, + "23": 5540.0, + "24": 5085.0, + "25": 4723.0, + "26": 5278.0, + "27": 5336.0, + "28": 5707.0, + "29": 6154.0, + "30": 5376.0, + "31": 4689.0, + "32": 5934.0, + "33": 6223.0, + "34": 5379.0, + "35": 5828.0, + "36": 5708.0, + "37": 6494.0, + "38": 6186.0, + "39": 6680.0, + "40": 6110.0, + "41": 6110.0, + "42": 6339.0, + "43": 5869.0, + "44": 5905.0, + "45": 6036.0, + "46": 5862.0, + "47": 6757.0, + "48": 6445.0, + "49": 6445.0, + "50": 6776.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1144114688.0, + "2": 1144112128.0, + "3": 1144114688.0, + "4": 1144114176.0, + "5": 1144115200.0, + "6": 1144113664.0, + "7": 1144114688.0, + "8": 1144112128.0, + "9": 1144112640.0, + "10": 1144114688.0, + "11": 1144114176.0, + "12": 1144115200.0, + "13": 1144113664.0, + "14": 1144116224.0, + "15": 1144114688.0, + "16": 1144113664.0, + "17": 1144111616.0, + "18": 1144113664.0, + "19": 1144114176.0, + "20": 1144114688.0, + "21": 1144112128.0, + "22": 1144114176.0, + "23": 1144112640.0, + "24": 1144111616.0, + "25": 1144115712.0, + "26": 1144116736.0, + "27": 1144114688.0, + "28": 1144112128.0, + "29": 1144112640.0, + "30": 1144114176.0, + "31": 1144110080.0, + "32": 1144113152.0, + "33": 1144113664.0, + "34": 1144113664.0, + "35": 1144111104.0, + "36": 1144113664.0, + "37": 1144115200.0, + "38": 1144114176.0, + "39": 1144112128.0, + "40": 1144112128.0, + "41": 1144110080.0, + "42": 1144111616.0, + "43": 1144108544.0, + "44": 1144111616.0, + "45": 1144114688.0, + "46": 1144113664.0, + "47": 1144112128.0, + "48": 1144111616.0, + "49": 1144113664.0, + "50": 1144111616.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1592113152.0, + "2": 2049260032.0, + "3": 2049305088.0, + "4": 2049981952.0, + "5": 2049981952.0, + "6": 2049981952.0, + "7": 2049981952.0, + "8": 2049981952.0, + "9": 2049981952.0, + "10": 2050618880.0, + "11": 2050618880.0, + "12": 2050618880.0, + "13": 2050618880.0, + "14": 2050618880.0, + "15": 2050618880.0, + "16": 2050618880.0, + "17": 2050618880.0, + "18": 2050618880.0, + "19": 2050618880.0, + "20": 2050618880.0, + "21": 2050618880.0, + "22": 2050618880.0, + "23": 2050618880.0, + "24": 2050618880.0, + "25": 2050618880.0, + "26": 2050618880.0, + "27": 2050618880.0, + "28": 2050618880.0, + "29": 2050618880.0, + "30": 2050618880.0, + "31": 2050618880.0, + "32": 2050618880.0, + "33": 2050618880.0, + "34": 2050618880.0, + "35": 2050618880.0, + "36": 2050618880.0, + "37": 2050618880.0, + "38": 2050618880.0, + "39": 2050618880.0, + "40": 2050618880.0, + "41": 2050618880.0, + "42": 2050618880.0, + "43": 2050618880.0, + "44": 2050618880.0, + "45": 2050618880.0, + "46": 2050618880.0, + "47": 2050618880.0, + "48": 2050618880.0, + "49": 2050618880.0, + "50": 2050618880.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 7.71822, + "3": 0.41543, + "4": 0.48385, + "5": 0.43238, + "6": 0.4005, + "7": 0.37702, + "8": 0.34877, + "9": 0.34747, + "10": 0.33405, + "11": 0.35433, + "12": 0.4949, + "13": 0.78498, + "14": 0.34468, + "15": 0.34552, + "16": 0.34055, + "17": 0.34051, + "18": 0.33811, + "19": 0.34319, + "20": 0.33811, + "21": 0.34085, + "22": 0.35398, + "23": 0.33545, + "24": 0.3393, + "25": 0.34161, + "26": 0.33744, + "27": 0.33573, + "28": 0.33954, + "29": 0.33344, + "30": 0.33741, + "31": 0.34691, + "32": 0.33581, + "33": 0.3395, + "34": 0.34333, + "35": 0.3424, + "36": 0.34673, + "37": 0.33697, + "38": 0.33705, + "39": 0.33394, + "40": 0.33964, + "41": 0.34276, + "42": 0.3401, + "43": 0.34688, + "44": 0.3413, + "45": 0.33867, + "46": 0.34719, + "47": 0.34606, + "48": 0.35149, + "49": 0.34219, + "50": 0.33349 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..51442760296 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.81386, + "2": 10.8312, + "3": 10.80993, + "4": 10.82435, + "5": 10.84542, + "6": 10.85174, + "7": 10.8364, + "8": 10.83623, + "9": 10.84765, + "10": 10.7827, + "11": 10.85034, + "12": 10.84795, + "13": 10.86022, + "14": 10.86553, + "15": 10.81126, + "16": 10.78981, + "17": 10.76199, + "18": 10.78188, + "19": 10.78067, + "20": 10.7031, + "21": 10.67657, + "22": 10.51865, + "23": 10.70519, + "24": 10.57167, + "25": 10.51611, + "26": 10.58127, + "27": 10.59422, + "28": 10.56658, + "29": 10.58518, + "30": 10.33581, + "31": 10.08412, + "32": 10.45077, + "33": 10.4461, + "34": 10.19766, + "35": 10.2585, + "36": 10.21965, + "37": 10.34543, + "38": 10.18739, + "39": 10.39385, + "40": 10.0823, + "41": 10.13221, + "42": 10.21174, + "43": 9.83034, + "44": 9.9469, + "45": 9.84028, + "46": 9.81421, + "47": 10.12976, + "48": 9.85137, + "49": 9.52825, + "50": 9.91126 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4660.0, + "2": 4808.0, + "3": 4883.0, + "4": 4655.0, + "5": 5526.0, + "6": 5649.0, + "7": 4873.0, + "8": 4733.0, + "9": 5165.0, + "10": 4367.0, + "11": 5583.0, + "12": 5440.0, + "13": 5659.0, + "14": 5572.0, + "15": 5101.0, + "16": 5484.0, + "17": 5115.0, + "18": 5200.0, + "19": 5406.0, + "20": 4960.0, + "21": 5420.0, + "22": 4791.0, + "23": 5566.0, + "24": 5019.0, + "25": 4679.0, + "26": 5246.0, + "27": 5433.0, + "28": 5907.0, + "29": 6065.0, + "30": 5409.0, + "31": 4827.0, + "32": 5809.0, + "33": 6243.0, + "34": 5520.0, + "35": 5592.0, + "36": 5754.0, + "37": 6732.0, + "38": 6330.0, + "39": 6779.0, + "40": 6198.0, + "41": 6001.0, + "42": 6274.0, + "43": 5876.0, + "44": 6046.0, + "45": 6084.0, + "46": 5925.0, + "47": 6772.0, + "48": 6415.0, + "49": 6494.0, + "50": 6648.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1144114688.0, + "2": 1144112128.0, + "3": 1144114688.0, + "4": 1144115200.0, + "5": 1144115200.0, + "6": 1144113664.0, + "7": 1144114688.0, + "8": 1144113152.0, + "9": 1144112640.0, + "10": 1144114688.0, + "11": 1144114176.0, + "12": 1144115200.0, + "13": 1144114176.0, + "14": 1144115712.0, + "15": 1144115712.0, + "16": 1144113664.0, + "17": 1144111616.0, + "18": 1144113664.0, + "19": 1144114688.0, + "20": 1144113664.0, + "21": 1144113152.0, + "22": 1144114176.0, + "23": 1144113664.0, + "24": 1144111616.0, + "25": 1144115712.0, + "26": 1144116224.0, + "27": 1144114688.0, + "28": 1144112128.0, + "29": 1144113152.0, + "30": 1144114176.0, + "31": 1144109568.0, + "32": 1144113152.0, + "33": 1144114176.0, + "34": 1144113664.0, + "35": 1144111104.0, + "36": 1144113664.0, + "37": 1144115200.0, + "38": 1144114688.0, + "39": 1144112128.0, + "40": 1144111616.0, + "41": 1144110080.0, + "42": 1144113152.0, + "43": 1144109568.0, + "44": 1144111616.0, + "45": 1144115200.0, + "46": 1144112640.0, + "47": 1144112128.0, + "48": 1144111616.0, + "49": 1144113664.0, + "50": 1144111616.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1592113152.0, + "2": 2049344000.0, + "3": 2049344000.0, + "4": 2049852928.0, + "5": 2050200576.0, + "6": 2050200576.0, + "7": 2050200576.0, + "8": 2050200576.0, + "9": 2050200576.0, + "10": 2050613760.0, + "11": 2050613760.0, + "12": 2050613760.0, + "13": 2050613760.0, + "14": 2050613760.0, + "15": 2050613760.0, + "16": 2050613760.0, + "17": 2050613760.0, + "18": 2050613760.0, + "19": 2050613760.0, + "20": 2050613760.0, + "21": 2050613760.0, + "22": 2050613760.0, + "23": 2050613760.0, + "24": 2050613760.0, + "25": 2050613760.0, + "26": 2050613760.0, + "27": 2050613760.0, + "28": 2050613760.0, + "29": 2050613760.0, + "30": 2050613760.0, + "31": 2050613760.0, + "32": 2050613760.0, + "33": 2050613760.0, + "34": 2050613760.0, + "35": 2050613760.0, + "36": 2050613760.0, + "37": 2050613760.0, + "38": 2050613760.0, + "39": 2050613760.0, + "40": 2050613760.0, + "41": 2050613760.0, + "42": 2050613760.0, + "43": 2050613760.0, + "44": 2050613760.0, + "45": 2050613760.0, + "46": 2050613760.0, + "47": 2050613760.0, + "48": 2050613760.0, + "49": 2050613760.0, + "50": 2050613760.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 6.272, + "3": 0.37646, + "4": 0.38139, + "5": 0.36006, + "6": 0.35438, + "7": 0.37532, + "8": 0.34896, + "9": 0.34666, + "10": 0.34575, + "11": 0.35016, + "12": 0.34334, + "13": 0.34313, + "14": 0.34121, + "15": 0.34333, + "16": 0.33917, + "17": 0.34414, + "18": 0.34158, + "19": 0.33904, + "20": 0.34192, + "21": 0.34305, + "22": 0.35491, + "23": 0.34584, + "24": 0.34162, + "25": 0.34733, + "26": 0.34153, + "27": 0.34246, + "28": 0.34, + "29": 0.33893, + "30": 0.34315, + "31": 0.3468, + "32": 0.34193, + "33": 0.33765, + "34": 0.34671, + "35": 0.33955, + "36": 0.34134, + "37": 0.33879, + "38": 0.34103, + "39": 0.33784, + "40": 0.33992, + "41": 0.3506, + "42": 0.33836, + "43": 0.34282, + "44": 0.33978, + "45": 0.339, + "46": 0.34898, + "47": 0.34512, + "48": 0.35552, + "49": 0.34616, + "50": 0.33258 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..b804ba57a90 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81772, + "2": 10.82538, + "3": 10.82315, + "4": 10.7893, + "5": 10.84414, + "6": 10.85629, + "7": 10.82859, + "8": 10.8301, + "9": 10.84207, + "10": 10.78601, + "11": 10.85733, + "12": 10.84663, + "13": 10.86562, + "14": 10.86914, + "15": 10.81232, + "16": 10.80865, + "17": 10.77965, + "18": 10.80508, + "19": 10.79288, + "20": 10.74264, + "21": 10.72495, + "22": 10.58933, + "23": 10.73854, + "24": 10.63021, + "25": 10.58736, + "26": 10.63591, + "27": 10.66781, + "28": 10.64617, + "29": 10.65842, + "30": 10.44627, + "31": 10.21299, + "32": 10.53989, + "33": 10.52788, + "34": 10.30656, + "35": 10.35429, + "36": 10.31411, + "37": 10.43313, + "38": 10.29142, + "39": 10.47282, + "40": 10.18336, + "41": 10.24081, + "42": 10.30294, + "43": 9.95174, + "44": 10.05781, + "45": 9.9572, + "46": 9.93655, + "47": 10.22836, + "48": 9.95329, + "49": 9.6607, + "50": 9.99855, + "51": 9.94973, + "52": 9.84349, + "53": 10.14413, + "54": 10.04737, + "55": 9.98385, + "56": 9.71898, + "57": 9.5883, + "58": 9.92285, + "59": 9.67628, + "60": 9.60379, + "61": 9.78734, + "62": 10.06656, + "63": 9.47521, + "64": 9.85036, + "65": 9.03212, + "66": 9.78289, + "67": 9.44253, + "68": 9.85795, + "69": 9.85298, + "70": 9.7992, + "71": 9.6974, + "72": 9.66103, + "73": 9.56335, + "74": 9.05976, + "75": 9.50058, + "76": 9.18716, + "77": 10.12117, + "78": 9.78252, + "79": 9.44971, + "80": 9.47021, + "81": 9.54374, + "82": 9.75396, + "83": 9.39966, + "84": 9.46977, + "85": 9.67727, + "86": 9.13918, + "87": 9.64053, + "88": 9.81152, + "89": 9.6769, + "90": 9.8722, + "91": 9.41711, + "92": 9.42414, + "93": 9.1643, + "94": 8.903, + "95": 9.57911, + "96": 9.5909, + "97": 9.35398, + "98": 9.73253, + "99": 8.96675, + "100": 9.46267 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 31327.0, + "2": 32688.0, + "3": 32823.0, + "4": 31506.0, + "5": 36384.0, + "6": 37608.0, + "7": 35129.0, + "8": 31069.0, + "9": 34564.0, + "10": 29747.0, + "11": 38781.0, + "12": 35732.0, + "13": 37194.0, + "14": 38338.0, + "15": 35011.0, + "16": 36251.0, + "17": 34823.0, + "18": 35457.0, + "19": 35578.0, + "20": 32707.0, + "21": 33645.0, + "22": 30136.0, + "23": 38476.0, + "24": 32292.0, + "25": 30916.0, + "26": 34382.0, + "27": 36106.0, + "28": 37666.0, + "29": 38421.0, + "30": 33015.0, + "31": 30489.0, + "32": 36688.0, + "33": 38306.0, + "34": 33425.0, + "35": 34129.0, + "36": 35506.0, + "37": 38441.0, + "38": 35394.0, + "39": 38939.0, + "40": 36115.0, + "41": 36452.0, + "42": 37245.0, + "43": 34000.0, + "44": 33879.0, + "45": 36293.0, + "46": 37265.0, + "47": 40947.0, + "48": 36423.0, + "49": 35090.0, + "50": 40022.0, + "51": 37599.0, + "52": 36874.0, + "53": 42329.0, + "54": 40750.0, + "55": 37208.0, + "56": 39947.0, + "57": 36219.0, + "58": 42369.0, + "59": 39714.0, + "60": 39697.0, + "61": 40288.0, + "62": 44682.0, + "63": 37743.0, + "64": 43466.0, + "65": 40862.0, + "66": 45025.0, + "67": 40213.0, + "68": 40169.0, + "69": 40885.0, + "70": 45480.0, + "71": 41411.0, + "72": 40544.0, + "73": 45712.0, + "74": 34875.0, + "75": 39109.0, + "76": 45477.0, + "77": 45742.0, + "78": 47634.0, + "79": 48400.0, + "80": 46578.0, + "81": 50032.0, + "82": 49469.0, + "83": 45158.0, + "84": 45794.0, + "85": 49099.0, + "86": 45075.0, + "87": 49153.0, + "88": 47648.0, + "89": 49368.0, + "90": 49965.0, + "91": 44550.0, + "92": 46072.0, + "93": 46606.0, + "94": 47182.0, + "95": 47865.0, + "96": 50348.0, + "97": 46303.0, + "98": 49697.0, + "99": 48948.0, + "100": 44134.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 964398080.0, + "2": 964398592.0, + "3": 964397056.0, + "4": 964396032.0, + "5": 964397056.0, + "6": 964397568.0, + "7": 964397056.0, + "8": 964396544.0, + "9": 964396544.0, + "10": 964396032.0, + "11": 964397056.0, + "12": 964393984.0, + "13": 964396544.0, + "14": 964399104.0, + "15": 964396544.0, + "16": 964397568.0, + "17": 964399104.0, + "18": 964397568.0, + "19": 964397056.0, + "20": 964398080.0, + "21": 964397056.0, + "22": 964394496.0, + "23": 964396032.0, + "24": 964395520.0, + "25": 964395008.0, + "26": 964396032.0, + "27": 964397568.0, + "28": 964396032.0, + "29": 964398080.0, + "30": 964397056.0, + "31": 964390912.0, + "32": 965370880.0, + "33": 964398080.0, + "34": 964396544.0, + "35": 964395008.0, + "36": 964395008.0, + "37": 964396032.0, + "38": 964397056.0, + "39": 964397056.0, + "40": 964398080.0, + "41": 964390400.0, + "42": 964396032.0, + "43": 964393472.0, + "44": 964394496.0, + "45": 964396032.0, + "46": 964390912.0, + "47": 964396032.0, + "48": 964389888.0, + "49": 964392960.0, + "50": 964396032.0, + "51": 964395008.0, + "52": 964391936.0, + "53": 964392960.0, + "54": 964390912.0, + "55": 964390400.0, + "56": 964393984.0, + "57": 964384768.0, + "58": 964389888.0, + "59": 964388352.0, + "60": 964390912.0, + "61": 964396032.0, + "62": 964393472.0, + "63": 964391424.0, + "64": 964388864.0, + "65": 964380672.0, + "66": 964391936.0, + "67": 964391936.0, + "68": 964396032.0, + "69": 964390400.0, + "70": 964392448.0, + "71": 964392448.0, + "72": 964388352.0, + "73": 964390912.0, + "74": 964385792.0, + "75": 964396032.0, + "76": 964396544.0, + "77": 964395008.0, + "78": 964386816.0, + "79": 964391936.0, + "80": 964388864.0, + "81": 964390400.0, + "82": 964391936.0, + "83": 964390912.0, + "84": 964388352.0, + "85": 964391424.0, + "86": 964390912.0, + "87": 964393984.0, + "88": 964390400.0, + "89": 964391424.0, + "90": 964391936.0, + "91": 964391424.0, + "92": 964391424.0, + "93": 964391936.0, + "94": 964392448.0, + "95": 964391936.0, + "96": 964388352.0, + "97": 964390400.0, + "98": 964392448.0, + "99": 964390912.0, + "100": 964389888.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2512260096.0, + "2": 2777459200.0, + "3": 2777459200.0, + "4": 2777459200.0, + "5": 2777459200.0, + "6": 2778216960.0, + "7": 2778216960.0, + "8": 2778216960.0, + "9": 2778216960.0, + "10": 2778216960.0, + "11": 2778216960.0, + "12": 2778216960.0, + "13": 2778216960.0, + "14": 2778216960.0, + "15": 2778216960.0, + "16": 2778216960.0, + "17": 2778216960.0, + "18": 2778216960.0, + "19": 2778216960.0, + "20": 2778216960.0, + "21": 2778216960.0, + "22": 2778216960.0, + "23": 2778216960.0, + "24": 2778216960.0, + "25": 2778216960.0, + "26": 2778216960.0, + "27": 2778216960.0, + "28": 2778216960.0, + "29": 2778216960.0, + "30": 2778216960.0, + "31": 2778216960.0, + "32": 2778216960.0, + "33": 2778216960.0, + "34": 2778216960.0, + "35": 2778216960.0, + "36": 2778216960.0, + "37": 2778216960.0, + "38": 2778216960.0, + "39": 2778216960.0, + "40": 2778216960.0, + "41": 2778216960.0, + "42": 2778216960.0, + "43": 2778216960.0, + "44": 2778216960.0, + "45": 2778216960.0, + "46": 2778216960.0, + "47": 2778216960.0, + "48": 2778216960.0, + "49": 2778216960.0, + "50": 2778216960.0, + "51": 2778216960.0, + "52": 2778216960.0, + "53": 2778216960.0, + "54": 2778216960.0, + "55": 2778216960.0, + "56": 2778216960.0, + "57": 2778216960.0, + "58": 2778216960.0, + "59": 2778216960.0, + "60": 2778216960.0, + "61": 2778216960.0, + "62": 2778216960.0, + "63": 2778216960.0, + "64": 2778216960.0, + "65": 2778216960.0, + "66": 2778216960.0, + "67": 2778216960.0, + "68": 2778216960.0, + "69": 2778216960.0, + "70": 2778216960.0, + "71": 2778216960.0, + "72": 2778216960.0, + "73": 2778216960.0, + "74": 2778216960.0, + "75": 2778216960.0, + "76": 2778216960.0, + "77": 2778216960.0, + "78": 2778216960.0, + "79": 2778216960.0, + "80": 2778216960.0, + "81": 2778216960.0, + "82": 2778216960.0, + "83": 2778216960.0, + "84": 2778216960.0, + "85": 2778216960.0, + "86": 2778216960.0, + "87": 2778216960.0, + "88": 2778216960.0, + "89": 2778216960.0, + "90": 2778216960.0, + "91": 2778216960.0, + "92": 2778216960.0, + "93": 2778216960.0, + "94": 2778216960.0, + "95": 2778216960.0, + "96": 2778216960.0, + "97": 2778216960.0, + "98": 2778216960.0, + "99": 2778216960.0, + "100": 2778216960.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 6.33917, + "3": 0.2744, + "4": 0.25641, + "5": 0.24878, + "6": 0.24537, + "7": 0.2438, + "8": 0.23878, + "9": 0.24702, + "10": 0.23746, + "11": 0.23512, + "12": 0.22956, + "13": 0.2347, + "14": 0.23605, + "15": 0.24017, + "16": 0.23204, + "17": 0.2327, + "18": 0.23371, + "19": 0.23278, + "20": 0.23324, + "21": 0.2375, + "22": 0.2357, + "23": 0.23341, + "24": 0.23508, + "25": 0.23292, + "26": 0.23763, + "27": 0.23487, + "28": 0.23071, + "29": 0.23154, + "30": 0.23464, + "31": 0.23829, + "32": 0.22989, + "33": 0.23328, + "34": 0.23409, + "35": 0.23024, + "36": 0.23774, + "37": 0.23416, + "38": 0.23657, + "39": 0.23087, + "40": 0.23163, + "41": 0.23724, + "42": 0.23245, + "43": 0.23545, + "44": 0.23041, + "45": 0.23512, + "46": 0.23935, + "47": 0.23571, + "48": 0.2329, + "49": 0.25544, + "50": 0.22697, + "51": 0.27515, + "52": 0.69001, + "53": 0.24129, + "54": 0.23155, + "55": 0.24045, + "56": 0.24512, + "57": 0.24802, + "58": 0.23433, + "59": 0.3274, + "60": 0.23221, + "61": 0.23713, + "62": 0.24042, + "63": 0.25806, + "64": 0.2355, + "65": 0.27386, + "66": 0.68273, + "67": 0.30343, + "68": 0.26428, + "69": 0.25274, + "70": 0.24031, + "71": 0.25644, + "72": 0.24947, + "73": 0.2737, + "74": 0.26515, + "75": 0.25101, + "76": 0.27258, + "77": 0.65643, + "78": 0.25055, + "79": 0.26819, + "80": 0.24291, + "81": 0.24807, + "82": 0.24385, + "83": 0.24932, + "84": 0.24366, + "85": 0.25449, + "86": 0.28807, + "87": 0.25052, + "88": 0.25388, + "89": 0.24876, + "90": 0.24712, + "91": 0.27209, + "92": 0.25942, + "93": 0.26516, + "94": 0.27795, + "95": 0.25093, + "96": 0.58451, + "97": 0.26354, + "98": 0.24591, + "99": 0.2477, + "100": 0.24515 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..3dd007cc9ec --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.7915, + "2": 10.8072, + "3": 10.79214, + "4": 10.77571, + "5": 10.82495, + "6": 10.83191, + "7": 10.82037, + "8": 10.81565, + "9": 10.81976, + "10": 10.7695, + "11": 10.8454, + "12": 10.83063, + "13": 10.83651, + "14": 10.85696, + "15": 10.80631, + "16": 10.78763, + "17": 10.75856, + "18": 10.79234, + "19": 10.78331, + "20": 10.73181, + "21": 10.71017, + "22": 10.57574, + "23": 10.71599, + "24": 10.62049, + "25": 10.58266, + "26": 10.61764, + "27": 10.65105, + "28": 10.63303, + "29": 10.63022, + "30": 10.44254, + "31": 10.20049, + "32": 10.52014, + "33": 10.50814, + "34": 10.29535, + "35": 10.33643, + "36": 10.30247, + "37": 10.41766, + "38": 10.28067, + "39": 10.46149, + "40": 10.18213, + "41": 10.21349, + "42": 10.28426, + "43": 9.9557, + "44": 10.05793, + "45": 9.9574, + "46": 9.93571, + "47": 10.22719, + "48": 9.96561, + "49": 9.66581, + "50": 10.00922, + "51": 9.94826, + "52": 9.84653, + "53": 10.14876, + "54": 10.03737, + "55": 9.97454, + "56": 9.71384, + "57": 9.5955, + "58": 9.92044, + "59": 9.67604, + "60": 9.61264, + "61": 9.79194, + "62": 10.05699, + "63": 9.47838, + "64": 9.84479, + "65": 9.03861, + "66": 9.78386, + "67": 9.43595, + "68": 9.85188, + "69": 9.84445, + "70": 9.79288, + "71": 9.69163, + "72": 9.64893, + "73": 9.55502, + "74": 9.04736, + "75": 9.49186, + "76": 9.17766, + "77": 10.11289, + "78": 9.7687, + "79": 9.43966, + "80": 9.45416, + "81": 9.53142, + "82": 9.7541, + "83": 9.38201, + "84": 9.46121, + "85": 9.66928, + "86": 9.13531, + "87": 9.63413, + "88": 9.8011, + "89": 9.66658, + "90": 9.86173, + "91": 9.39963, + "92": 9.41066, + "93": 9.14665, + "94": 8.8869, + "95": 9.56959, + "96": 9.57609, + "97": 9.34309, + "98": 9.72749, + "99": 8.96222, + "100": 9.44903 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 31217.0, + "2": 33106.0, + "3": 33596.0, + "4": 31946.0, + "5": 36783.0, + "6": 37252.0, + "7": 35314.0, + "8": 31970.0, + "9": 34937.0, + "10": 29900.0, + "11": 38039.0, + "12": 34886.0, + "13": 37108.0, + "14": 37755.0, + "15": 35069.0, + "16": 36687.0, + "17": 34887.0, + "18": 35219.0, + "19": 35710.0, + "20": 32682.0, + "21": 33456.0, + "22": 30216.0, + "23": 37780.0, + "24": 32298.0, + "25": 30789.0, + "26": 34549.0, + "27": 35611.0, + "28": 36806.0, + "29": 37955.0, + "30": 32950.0, + "31": 30468.0, + "32": 36291.0, + "33": 37916.0, + "34": 32820.0, + "35": 34371.0, + "36": 34957.0, + "37": 38282.0, + "38": 35878.0, + "39": 38974.0, + "40": 36048.0, + "41": 35988.0, + "42": 37320.0, + "43": 33909.0, + "44": 33889.0, + "45": 35577.0, + "46": 37076.0, + "47": 40966.0, + "48": 35327.0, + "49": 34682.0, + "50": 39871.0, + "51": 36802.0, + "52": 36445.0, + "53": 41968.0, + "54": 40797.0, + "55": 36920.0, + "56": 40345.0, + "57": 36961.0, + "58": 41622.0, + "59": 37988.0, + "60": 40534.0, + "61": 40456.0, + "62": 43543.0, + "63": 37438.0, + "64": 42659.0, + "65": 39924.0, + "66": 44122.0, + "67": 40136.0, + "68": 40005.0, + "69": 41675.0, + "70": 45011.0, + "71": 40746.0, + "72": 41647.0, + "73": 44080.0, + "74": 35412.0, + "75": 39478.0, + "76": 46254.0, + "77": 44764.0, + "78": 47985.0, + "79": 48646.0, + "80": 46686.0, + "81": 50102.0, + "82": 50188.0, + "83": 44717.0, + "84": 46114.0, + "85": 49347.0, + "86": 45770.0, + "87": 49671.0, + "88": 46449.0, + "89": 49666.0, + "90": 51087.0, + "91": 45827.0, + "92": 48163.0, + "93": 46547.0, + "94": 47562.0, + "95": 48540.0, + "96": 50182.0, + "97": 46055.0, + "98": 50271.0, + "99": 48494.0, + "100": 45373.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 892874752.0, + "2": 892866560.0, + "3": 892869120.0, + "4": 892876800.0, + "5": 892869120.0, + "6": 892870656.0, + "7": 892874240.0, + "8": 892868608.0, + "9": 892869632.0, + "10": 892868608.0, + "11": 892869632.0, + "12": 892867072.0, + "13": 892872192.0, + "14": 892873216.0, + "15": 892870656.0, + "16": 892868608.0, + "17": 892879360.0, + "18": 892867072.0, + "19": 892870656.0, + "20": 892867072.0, + "21": 892871168.0, + "22": 892874752.0, + "23": 892877824.0, + "24": 892869120.0, + "25": 892877312.0, + "26": 892873216.0, + "27": 892865024.0, + "28": 892870144.0, + "29": 892869632.0, + "30": 892871680.0, + "31": 892881920.0, + "32": 892874752.0, + "33": 892870144.0, + "34": 892872192.0, + "35": 892874240.0, + "36": 892869632.0, + "37": 892868096.0, + "38": 892867072.0, + "39": 892871168.0, + "40": 892869120.0, + "41": 892873728.0, + "42": 892868608.0, + "43": 892871168.0, + "44": 892871680.0, + "45": 892869632.0, + "46": 892876800.0, + "47": 892869632.0, + "48": 892875264.0, + "49": 892872704.0, + "50": 892869120.0, + "51": 892872192.0, + "52": 892875776.0, + "53": 892868096.0, + "54": 892872192.0, + "55": 892867072.0, + "56": 892865024.0, + "57": 892876288.0, + "58": 892869120.0, + "59": 892871680.0, + "60": 892869120.0, + "61": 892869120.0, + "62": 892869632.0, + "63": 892870656.0, + "64": 892865536.0, + "65": 892872192.0, + "66": 892864512.0, + "67": 892862464.0, + "68": 892867584.0, + "69": 892861952.0, + "70": 892867072.0, + "71": 892870656.0, + "72": 892862464.0, + "73": 892861440.0, + "74": 892849664.0, + "75": 892868096.0, + "76": 892869632.0, + "77": 892868096.0, + "78": 892859392.0, + "79": 892865024.0, + "80": 892855296.0, + "81": 892856320.0, + "82": 892860416.0, + "83": 892869632.0, + "84": 892852736.0, + "85": 892871680.0, + "86": 892861952.0, + "87": 892869120.0, + "88": 892869632.0, + "89": 892859392.0, + "90": 892867072.0, + "91": 892865536.0, + "92": 892865536.0, + "93": 892861440.0, + "94": 892860928.0, + "95": 892869120.0, + "96": 892866560.0, + "97": 892856320.0, + "98": 892869120.0, + "99": 892864512.0, + "100": 892864000.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1948027904.0, + "2": 2183897088.0, + "3": 2184431104.0, + "4": 2190589952.0, + "5": 2190589952.0, + "6": 2190589952.0, + "7": 2190589952.0, + "8": 2190589952.0, + "9": 2190589952.0, + "10": 2190589952.0, + "11": 2190589952.0, + "12": 2190589952.0, + "13": 2190589952.0, + "14": 2190589952.0, + "15": 2190589952.0, + "16": 2190589952.0, + "17": 2194543104.0, + "18": 2194543104.0, + "19": 2194543104.0, + "20": 2194543104.0, + "21": 2194543104.0, + "22": 2194543104.0, + "23": 2194543104.0, + "24": 2194543104.0, + "25": 2194543104.0, + "26": 2194543104.0, + "27": 2194543104.0, + "28": 2194543104.0, + "29": 2194543104.0, + "30": 2194543104.0, + "31": 2195852288.0, + "32": 2195852288.0, + "33": 2195852288.0, + "34": 2195852288.0, + "35": 2195852288.0, + "36": 2195852288.0, + "37": 2195852288.0, + "38": 2195852288.0, + "39": 2195852288.0, + "40": 2195852288.0, + "41": 2195852288.0, + "42": 2195852288.0, + "43": 2195852288.0, + "44": 2195852288.0, + "45": 2195852288.0, + "46": 2195852288.0, + "47": 2195852288.0, + "48": 2195852288.0, + "49": 2195852288.0, + "50": 2195852288.0, + "51": 2195852288.0, + "52": 2195852288.0, + "53": 2195852288.0, + "54": 2195852288.0, + "55": 2195852288.0, + "56": 2195852288.0, + "57": 2195852288.0, + "58": 2195852288.0, + "59": 2195852288.0, + "60": 2195852288.0, + "61": 2195852288.0, + "62": 2195852288.0, + "63": 2195852288.0, + "64": 2195852288.0, + "65": 2195852288.0, + "66": 2195852288.0, + "67": 2195852288.0, + "68": 2195852288.0, + "69": 2195852288.0, + "70": 2195852288.0, + "71": 2195852288.0, + "72": 2195852288.0, + "73": 2195852288.0, + "74": 2195852288.0, + "75": 2195852288.0, + "76": 2195852288.0, + "77": 2195852288.0, + "78": 2195852288.0, + "79": 2195852288.0, + "80": 2195852288.0, + "81": 2195852288.0, + "82": 2195852288.0, + "83": 2195852288.0, + "84": 2195852288.0, + "85": 2195852288.0, + "86": 2195852288.0, + "87": 2195852288.0, + "88": 2195852288.0, + "89": 2195852288.0, + "90": 2195852288.0, + "91": 2195852288.0, + "92": 2195852288.0, + "93": 2195852288.0, + "94": 2195852288.0, + "95": 2195852288.0, + "96": 2195852288.0, + "97": 2195852288.0, + "98": 2195852288.0, + "99": 2195852288.0, + "100": 2195852288.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 3.86481, + "3": 0.3588, + "4": 0.35276, + "5": 0.33575, + "6": 0.3344, + "7": 0.3406, + "8": 0.33551, + "9": 0.33157, + "10": 0.32814, + "11": 0.32882, + "12": 0.3298, + "13": 0.32887, + "14": 0.32898, + "15": 0.33409, + "16": 0.32679, + "17": 0.34317, + "18": 0.33153, + "19": 0.32828, + "20": 0.33077, + "21": 0.32713, + "22": 0.32603, + "23": 0.32819, + "24": 0.33158, + "25": 0.32832, + "26": 0.32593, + "27": 0.33086, + "28": 0.32481, + "29": 0.32607, + "30": 0.33032, + "31": 0.33561, + "32": 0.33149, + "33": 0.32643, + "34": 0.34262, + "35": 0.32889, + "36": 0.32749, + "37": 0.32097, + "38": 0.33036, + "39": 0.69454, + "40": 0.33723, + "41": 0.3284, + "42": 0.32735, + "43": 0.33334, + "44": 0.3333, + "45": 0.33315, + "46": 0.33505, + "47": 0.32976, + "48": 0.32918, + "49": 0.34661, + "50": 0.32681, + "51": 0.3427, + "52": 0.3299, + "53": 0.32454, + "54": 0.3251, + "55": 0.32968, + "56": 0.34696, + "57": 0.33819, + "58": 0.32649, + "59": 0.3341, + "60": 0.33324, + "61": 0.33925, + "62": 0.33532, + "63": 0.34334, + "64": 0.34963, + "65": 0.38392, + "66": 0.33805, + "67": 0.3728, + "68": 0.33745, + "69": 0.33504, + "70": 0.33581, + "71": 0.35385, + "72": 0.34934, + "73": 0.34952, + "74": 0.35756, + "75": 0.35105, + "76": 0.34933, + "77": 0.33518, + "78": 0.34556, + "79": 0.34603, + "80": 0.36355, + "81": 0.34186, + "82": 0.34271, + "83": 0.39765, + "84": 0.36927, + "85": 0.33938, + "86": 0.35142, + "87": 0.34329, + "88": 0.33135, + "89": 0.34535, + "90": 0.33856, + "91": 0.3522, + "92": 0.33934, + "93": 0.38169, + "94": 0.36358, + "95": 0.33846, + "96": 0.33554, + "97": 0.34438, + "98": 0.32586, + "99": 0.43185, + "100": 0.33974 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..31167be6de5 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.7915, + "2": 10.8072, + "3": 10.79214, + "4": 10.77571, + "5": 10.82495, + "6": 10.83193, + "7": 10.82077, + "8": 10.81496, + "9": 10.81973, + "10": 10.7692, + "11": 10.84519, + "12": 10.83101, + "13": 10.83652, + "14": 10.85771, + "15": 10.80581, + "16": 10.78733, + "17": 10.75844, + "18": 10.79297, + "19": 10.78295, + "20": 10.73199, + "21": 10.70953, + "22": 10.57675, + "23": 10.71651, + "24": 10.61983, + "25": 10.58207, + "26": 10.61694, + "27": 10.6509, + "28": 10.63261, + "29": 10.63024, + "30": 10.4432, + "31": 10.19983, + "32": 10.52048, + "33": 10.5079, + "34": 10.29565, + "35": 10.33536, + "36": 10.30278, + "37": 10.41788, + "38": 10.28121, + "39": 10.46185, + "40": 10.18169, + "41": 10.21391, + "42": 10.28457, + "43": 9.95538, + "44": 10.05751, + "45": 9.95713, + "46": 9.93528, + "47": 10.22675, + "48": 9.96521, + "49": 9.66603, + "50": 10.009, + "51": 9.94789, + "52": 9.84665, + "53": 10.14887, + "54": 10.03772, + "55": 9.97445, + "56": 9.71378, + "57": 9.59509, + "58": 9.92081, + "59": 9.67609, + "60": 9.61253, + "61": 9.79221, + "62": 10.05653, + "63": 9.47849, + "64": 9.84455, + "65": 9.03889, + "66": 9.78399, + "67": 9.43609, + "68": 9.85203, + "69": 9.84438, + "70": 9.7933, + "71": 9.69163, + "72": 9.64909, + "73": 9.55528, + "74": 9.04743, + "75": 9.49185, + "76": 9.178, + "77": 10.11275, + "78": 9.76838, + "79": 9.4398, + "80": 9.45421, + "81": 9.53191, + "82": 9.75402, + "83": 9.38186, + "84": 9.46162, + "85": 9.66959, + "86": 9.1349, + "87": 9.6343, + "88": 9.80083, + "89": 9.66682, + "90": 9.86175, + "91": 9.39987, + "92": 9.41063, + "93": 9.14654, + "94": 8.88648, + "95": 9.56986, + "96": 9.57642, + "97": 9.34305, + "98": 9.72786, + "99": 8.96203, + "100": 9.44942 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 31217.0, + "2": 33106.0, + "3": 33596.0, + "4": 31946.0, + "5": 36783.0, + "6": 37377.0, + "7": 35362.0, + "8": 31711.0, + "9": 34749.0, + "10": 29758.0, + "11": 38348.0, + "12": 35446.0, + "13": 37087.0, + "14": 37869.0, + "15": 35242.0, + "16": 36520.0, + "17": 35190.0, + "18": 35191.0, + "19": 35614.0, + "20": 32571.0, + "21": 33220.0, + "22": 30518.0, + "23": 37619.0, + "24": 32547.0, + "25": 30591.0, + "26": 34546.0, + "27": 35275.0, + "28": 36936.0, + "29": 37531.0, + "30": 33354.0, + "31": 30754.0, + "32": 36331.0, + "33": 38273.0, + "34": 32645.0, + "35": 34237.0, + "36": 35092.0, + "37": 37931.0, + "38": 35480.0, + "39": 39175.0, + "40": 36296.0, + "41": 35902.0, + "42": 37609.0, + "43": 33748.0, + "44": 34027.0, + "45": 35215.0, + "46": 37108.0, + "47": 41056.0, + "48": 35765.0, + "49": 35087.0, + "50": 39734.0, + "51": 36712.0, + "52": 36176.0, + "53": 41774.0, + "54": 40447.0, + "55": 37071.0, + "56": 39975.0, + "57": 36828.0, + "58": 41815.0, + "59": 37962.0, + "60": 40415.0, + "61": 39921.0, + "62": 43840.0, + "63": 37890.0, + "64": 42699.0, + "65": 40347.0, + "66": 44159.0, + "67": 40057.0, + "68": 39563.0, + "69": 42246.0, + "70": 44867.0, + "71": 40910.0, + "72": 40982.0, + "73": 44363.0, + "74": 35672.0, + "75": 39602.0, + "76": 46157.0, + "77": 44919.0, + "78": 48134.0, + "79": 48666.0, + "80": 46770.0, + "81": 50144.0, + "82": 49680.0, + "83": 44991.0, + "84": 45912.0, + "85": 49371.0, + "86": 45600.0, + "87": 49292.0, + "88": 46411.0, + "89": 49710.0, + "90": 51008.0, + "91": 45796.0, + "92": 47991.0, + "93": 46847.0, + "94": 47360.0, + "95": 48680.0, + "96": 50369.0, + "97": 46162.0, + "98": 49921.0, + "99": 48235.0, + "100": 45390.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1254511616.0, + "2": 1254503424.0, + "3": 1254505984.0, + "4": 1254513664.0, + "5": 1254505984.0, + "6": 1254507520.0, + "7": 1254511104.0, + "8": 1254505472.0, + "9": 1254505984.0, + "10": 1254506496.0, + "11": 1254507520.0, + "12": 1254503936.0, + "13": 1254509568.0, + "14": 1254510080.0, + "15": 1254506496.0, + "16": 1254505984.0, + "17": 1254516224.0, + "18": 1254503424.0, + "19": 1254506496.0, + "20": 1254504960.0, + "21": 1254508032.0, + "22": 1254510592.0, + "23": 1254512640.0, + "24": 1254505472.0, + "25": 1254513664.0, + "26": 1254512128.0, + "27": 1254501888.0, + "28": 1254509056.0, + "29": 1254508032.0, + "30": 1254509056.0, + "31": 1254519296.0, + "32": 1254512128.0, + "33": 1254507008.0, + "34": 1254509056.0, + "35": 1254510080.0, + "36": 1254507008.0, + "37": 1254504448.0, + "38": 1254505472.0, + "39": 1254508032.0, + "40": 1254505984.0, + "41": 1254512128.0, + "42": 1254504960.0, + "43": 1254507008.0, + "44": 1254508032.0, + "45": 1254506496.0, + "46": 1254513664.0, + "47": 1254507008.0, + "48": 1254511616.0, + "49": 1254508032.0, + "50": 1254506496.0, + "51": 1254508032.0, + "52": 1254513152.0, + "53": 1254505984.0, + "54": 1254508544.0, + "55": 1254503936.0, + "56": 1254502912.0, + "57": 1254515200.0, + "58": 1254503936.0, + "59": 1254508544.0, + "60": 1254503936.0, + "61": 1254507008.0, + "62": 1254508032.0, + "63": 1254507520.0, + "64": 1254502400.0, + "65": 1254509568.0, + "66": 1254501376.0, + "67": 1254499328.0, + "68": 1254503936.0, + "69": 1254499328.0, + "70": 1254502912.0, + "71": 1254507520.0, + "72": 1254499328.0, + "73": 1254497280.0, + "74": 1254486016.0, + "75": 1254504960.0, + "76": 1254507008.0, + "77": 1254504448.0, + "78": 1254496256.0, + "79": 1254500864.0, + "80": 1254491648.0, + "81": 1254493696.0, + "82": 1254497280.0, + "83": 1254505984.0, + "84": 1254489600.0, + "85": 1254505984.0, + "86": 1254500352.0, + "87": 1254505472.0, + "88": 1254506496.0, + "89": 1254498304.0, + "90": 1254504448.0, + "91": 1254501888.0, + "92": 1254501888.0, + "93": 1254499328.0, + "94": 1254494720.0, + "95": 1254504960.0, + "96": 1254503424.0, + "97": 1254492672.0, + "98": 1254505984.0, + "99": 1254499328.0, + "100": 1254501888.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2066381824.0, + "2": 2543623168.0, + "3": 2544637440.0, + "4": 2550310912.0, + "5": 2550310912.0, + "6": 2550310912.0, + "7": 2550310912.0, + "8": 2550310912.0, + "9": 2550310912.0, + "10": 2550310912.0, + "11": 2550310912.0, + "12": 2550310912.0, + "13": 2550310912.0, + "14": 2550310912.0, + "15": 2550310912.0, + "16": 2550310912.0, + "17": 2554848768.0, + "18": 2554848768.0, + "19": 2554848768.0, + "20": 2554848768.0, + "21": 2554848768.0, + "22": 2554848768.0, + "23": 2554848768.0, + "24": 2554848768.0, + "25": 2554848768.0, + "26": 2554848768.0, + "27": 2554848768.0, + "28": 2554848768.0, + "29": 2554848768.0, + "30": 2554848768.0, + "31": 2556047872.0, + "32": 2556047872.0, + "33": 2556047872.0, + "34": 2556047872.0, + "35": 2556047872.0, + "36": 2556047872.0, + "37": 2556047872.0, + "38": 2556047872.0, + "39": 2556047872.0, + "40": 2556047872.0, + "41": 2556047872.0, + "42": 2556047872.0, + "43": 2556047872.0, + "44": 2556047872.0, + "45": 2556047872.0, + "46": 2556047872.0, + "47": 2556047872.0, + "48": 2556047872.0, + "49": 2556047872.0, + "50": 2556047872.0, + "51": 2556047872.0, + "52": 2556047872.0, + "53": 2556047872.0, + "54": 2556047872.0, + "55": 2556047872.0, + "56": 2556047872.0, + "57": 2556047872.0, + "58": 2556047872.0, + "59": 2556047872.0, + "60": 2556047872.0, + "61": 2556047872.0, + "62": 2556047872.0, + "63": 2556047872.0, + "64": 2556047872.0, + "65": 2556047872.0, + "66": 2556047872.0, + "67": 2556047872.0, + "68": 2556047872.0, + "69": 2556047872.0, + "70": 2556047872.0, + "71": 2556047872.0, + "72": 2556047872.0, + "73": 2556047872.0, + "74": 2556047872.0, + "75": 2556047872.0, + "76": 2556047872.0, + "77": 2556047872.0, + "78": 2556047872.0, + "79": 2556047872.0, + "80": 2556047872.0, + "81": 2556047872.0, + "82": 2556047872.0, + "83": 2556047872.0, + "84": 2556047872.0, + "85": 2556047872.0, + "86": 2556047872.0, + "87": 2556047872.0, + "88": 2556047872.0, + "89": 2556047872.0, + "90": 2556047872.0, + "91": 2556047872.0, + "92": 2556047872.0, + "93": 2556047872.0, + "94": 2556047872.0, + "95": 2556047872.0, + "96": 2556047872.0, + "97": 2556047872.0, + "98": 2556047872.0, + "99": 2556047872.0, + "100": 2556047872.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 4.80614, + "3": 0.71249, + "4": 0.40839, + "5": 0.39358, + "6": 0.3911, + "7": 0.39032, + "8": 0.38318, + "9": 0.70382, + "10": 0.34707, + "11": 0.34403, + "12": 0.34043, + "13": 0.33959, + "14": 0.33461, + "15": 0.34767, + "16": 0.33495, + "17": 0.34839, + "18": 0.33673, + "19": 0.33335, + "20": 0.33161, + "21": 0.32643, + "22": 0.33565, + "23": 0.33625, + "24": 0.33009, + "25": 0.33065, + "26": 0.33344, + "27": 0.33552, + "28": 0.33047, + "29": 0.33011, + "30": 0.33358, + "31": 0.34631, + "32": 0.33536, + "33": 0.33271, + "34": 0.33949, + "35": 0.33073, + "36": 0.32877, + "37": 0.32806, + "38": 0.33111, + "39": 0.33408, + "40": 0.33428, + "41": 0.34927, + "42": 1.47745, + "43": 0.48012, + "44": 0.33077, + "45": 0.33262, + "46": 0.34066, + "47": 0.33152, + "48": 0.33512, + "49": 0.34429, + "50": 0.33697, + "51": 0.34656, + "52": 0.337, + "53": 0.33133, + "54": 0.33172, + "55": 0.33188, + "56": 0.35163, + "57": 0.34162, + "58": 0.33258, + "59": 0.7122, + "60": 0.33979, + "61": 0.33569, + "62": 0.33523, + "63": 0.33864, + "64": 0.34776, + "65": 0.37658, + "66": 0.3377, + "67": 0.36916, + "68": 0.3452, + "69": 0.33854, + "70": 0.34023, + "71": 0.3544, + "72": 0.34395, + "73": 0.3567, + "74": 0.35025, + "75": 0.35164, + "76": 0.35012, + "77": 0.3364, + "78": 0.34491, + "79": 0.34789, + "80": 0.35388, + "81": 0.34075, + "82": 0.34743, + "83": 0.34211, + "84": 0.34722, + "85": 0.33956, + "86": 0.35402, + "87": 0.34301, + "88": 0.34056, + "89": 0.35764, + "90": 0.33476, + "91": 0.3539, + "92": 0.34448, + "93": 0.34895, + "94": 0.3624, + "95": 0.34001, + "96": 0.3382, + "97": 0.35217, + "98": 0.33252, + "99": 0.34909, + "100": 0.34966 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..ce3d79128b1 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80509, + "2": 10.82386, + "3": 10.80196, + "4": 10.79424, + "5": 10.8277, + "6": 10.84005, + "7": 10.8365, + "8": 10.82828, + "9": 10.83477, + "10": 10.77496, + "11": 10.85204, + "12": 10.83903, + "13": 10.85207, + "14": 10.85914, + "15": 10.81681, + "16": 10.79456, + "17": 10.77491, + "18": 10.80399, + "19": 10.79956, + "20": 10.73801, + "21": 10.72487, + "22": 10.59177, + "23": 10.73098, + "24": 10.6406, + "25": 10.59018, + "26": 10.63555, + "27": 10.66245, + "28": 10.6472, + "29": 10.64163, + "30": 10.4518, + "31": 10.22249, + "32": 10.52995, + "33": 10.51998, + "34": 10.31247, + "35": 10.34796, + "36": 10.31677, + "37": 10.42804, + "38": 10.29194, + "39": 10.46881, + "40": 10.19257, + "41": 10.23159, + "42": 10.29766, + "43": 9.97363, + "44": 10.07169, + "45": 9.97015, + "46": 9.94713, + "47": 10.23179, + "48": 9.97593, + "49": 9.67748, + "50": 10.0144 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31182.0, + "2": 33013.0, + "3": 33646.0, + "4": 32202.0, + "5": 36913.0, + "6": 37554.0, + "7": 35184.0, + "8": 32207.0, + "9": 34523.0, + "10": 29945.0, + "11": 38237.0, + "12": 35346.0, + "13": 37426.0, + "14": 38358.0, + "15": 35140.0, + "16": 36293.0, + "17": 35645.0, + "18": 35117.0, + "19": 35648.0, + "20": 32896.0, + "21": 33511.0, + "22": 30704.0, + "23": 38149.0, + "24": 32677.0, + "25": 31055.0, + "26": 34700.0, + "27": 35410.0, + "28": 37268.0, + "29": 37953.0, + "30": 33210.0, + "31": 30482.0, + "32": 36908.0, + "33": 38308.0, + "34": 33125.0, + "35": 34341.0, + "36": 34925.0, + "37": 38767.0, + "38": 35780.0, + "39": 38955.0, + "40": 36485.0, + "41": 36015.0, + "42": 37638.0, + "43": 33689.0, + "44": 33688.0, + "45": 35448.0, + "46": 36810.0, + "47": 40858.0, + "48": 35696.0, + "49": 34729.0, + "50": 39077.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1027095040.0, + "2": 1027093504.0, + "3": 1027094528.0, + "4": 1027095040.0, + "5": 1027091968.0, + "6": 1027091968.0, + "7": 1027098112.0, + "8": 1027097600.0, + "9": 1027094528.0, + "10": 1027094016.0, + "11": 1027098624.0, + "12": 1027094528.0, + "13": 1027092480.0, + "14": 1027095040.0, + "15": 1027095040.0, + "16": 1027091456.0, + "17": 1027101184.0, + "18": 1027096064.0, + "19": 1027093504.0, + "20": 1027093504.0, + "21": 1027097088.0, + "22": 1027100160.0, + "23": 1027100160.0, + "24": 1027095552.0, + "25": 1027097088.0, + "26": 1027098112.0, + "27": 1027091456.0, + "28": 1027090944.0, + "29": 1027091968.0, + "30": 1027099648.0, + "31": 1027109888.0, + "32": 1027095552.0, + "33": 1027090944.0, + "34": 1027098112.0, + "35": 1027103744.0, + "36": 1027098112.0, + "37": 1027092480.0, + "38": 1027091456.0, + "39": 1027095040.0, + "40": 1027095040.0, + "41": 1027100160.0, + "42": 1027091968.0, + "43": 1027098624.0, + "44": 1027098624.0, + "45": 1027096064.0, + "46": 1027104256.0, + "47": 1027093504.0, + "48": 1027101184.0, + "49": 1027096064.0, + "50": 1027095552.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3059586560.0, + "2": 3299159040.0, + "3": 3299482112.0, + "4": 3302137344.0, + "5": 3302137344.0, + "6": 3302137344.0, + "7": 3303535104.0, + "8": 3303535104.0, + "9": 3303535104.0, + "10": 3303535104.0, + "11": 3303535104.0, + "12": 3303535104.0, + "13": 3303535104.0, + "14": 3303535104.0, + "15": 3303535104.0, + "16": 3303535104.0, + "17": 3306910208.0, + "18": 3306910208.0, + "19": 3306910208.0, + "20": 3306910208.0, + "21": 3306910208.0, + "22": 3306910208.0, + "23": 3306910208.0, + "24": 3306910208.0, + "25": 3306910208.0, + "26": 3306910208.0, + "27": 3306910208.0, + "28": 3306910208.0, + "29": 3306910208.0, + "30": 3306910208.0, + "31": 3312495616.0, + "32": 3312495616.0, + "33": 3312495616.0, + "34": 3312495616.0, + "35": 3312495616.0, + "36": 3312495616.0, + "37": 3312495616.0, + "38": 3312495616.0, + "39": 3312495616.0, + "40": 3312495616.0, + "41": 3312495616.0, + "42": 3312495616.0, + "43": 3312495616.0, + "44": 3312495616.0, + "45": 3312495616.0, + "46": 3312495616.0, + "47": 3312495616.0, + "48": 3312495616.0, + "49": 3312495616.0, + "50": 3312495616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 3.82645, + "3": 0.34371, + "4": 0.32704, + "5": 0.31536, + "6": 0.32001, + "7": 0.31919, + "8": 0.31719, + "9": 0.31876, + "10": 0.31015, + "11": 0.31546, + "12": 0.31198, + "13": 0.31518, + "14": 0.40567, + "15": 0.31856, + "16": 0.30868, + "17": 0.31352, + "18": 0.31536, + "19": 0.31164, + "20": 0.31286, + "21": 0.35519, + "22": 0.30985, + "23": 0.31256, + "24": 0.31727, + "25": 0.36651, + "26": 0.47287, + "27": 0.57438, + "28": 0.3575, + "29": 0.71431, + "30": 0.31163, + "31": 0.31877, + "32": 0.34436, + "33": 0.51773, + "34": 0.32292, + "35": 0.31651, + "36": 0.34162, + "37": 0.31339, + "38": 0.30524, + "39": 0.63856, + "40": 0.31883, + "41": 0.31475, + "42": 0.67365, + "43": 0.33393, + "44": 0.31389, + "45": 0.65089, + "46": 0.6524, + "47": 0.3061, + "48": 0.30487, + "49": 0.3295, + "50": 0.30784 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..f62929eef31 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80509, + "2": 10.82386, + "3": 10.80196, + "4": 10.79424, + "5": 10.8277, + "6": 10.84005, + "7": 10.8365, + "8": 10.82828, + "9": 10.83477, + "10": 10.77496, + "11": 10.85204, + "12": 10.83903, + "13": 10.85207, + "14": 10.85914, + "15": 10.81681, + "16": 10.79456, + "17": 10.77491, + "18": 10.80399, + "19": 10.79956, + "20": 10.73801, + "21": 10.72487, + "22": 10.59177, + "23": 10.73098, + "24": 10.6406, + "25": 10.59018, + "26": 10.63555, + "27": 10.66245, + "28": 10.6472, + "29": 10.64163, + "30": 10.4518, + "31": 10.22249, + "32": 10.52995, + "33": 10.51998, + "34": 10.31247, + "35": 10.34796, + "36": 10.31677, + "37": 10.42804, + "38": 10.29194, + "39": 10.46881, + "40": 10.19257, + "41": 10.23159, + "42": 10.29766, + "43": 9.97363, + "44": 10.07169, + "45": 9.97015, + "46": 9.94713, + "47": 10.23179, + "48": 9.97593, + "49": 9.67748, + "50": 10.0144 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 31182.0, + "2": 33013.0, + "3": 33646.0, + "4": 32202.0, + "5": 36913.0, + "6": 37554.0, + "7": 35184.0, + "8": 32207.0, + "9": 34523.0, + "10": 29945.0, + "11": 38237.0, + "12": 35346.0, + "13": 37426.0, + "14": 38358.0, + "15": 35140.0, + "16": 36293.0, + "17": 35645.0, + "18": 35117.0, + "19": 35648.0, + "20": 32896.0, + "21": 33511.0, + "22": 30704.0, + "23": 38149.0, + "24": 32677.0, + "25": 31055.0, + "26": 34700.0, + "27": 35410.0, + "28": 37268.0, + "29": 37953.0, + "30": 33210.0, + "31": 30482.0, + "32": 36908.0, + "33": 38308.0, + "34": 33125.0, + "35": 34341.0, + "36": 34925.0, + "37": 38767.0, + "38": 35780.0, + "39": 38955.0, + "40": 36485.0, + "41": 36015.0, + "42": 37638.0, + "43": 33689.0, + "44": 33688.0, + "45": 35448.0, + "46": 36810.0, + "47": 40858.0, + "48": 35696.0, + "49": 34729.0, + "50": 39077.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1027095040.0, + "2": 1027093504.0, + "3": 1027094528.0, + "4": 1027095040.0, + "5": 1027091968.0, + "6": 1027091968.0, + "7": 1027098112.0, + "8": 1027097600.0, + "9": 1027094528.0, + "10": 1027094016.0, + "11": 1027098624.0, + "12": 1027094528.0, + "13": 1027092480.0, + "14": 1027095040.0, + "15": 1027095040.0, + "16": 1027091456.0, + "17": 1027101184.0, + "18": 1027096064.0, + "19": 1027093504.0, + "20": 1027093504.0, + "21": 1027097088.0, + "22": 1027100160.0, + "23": 1027100160.0, + "24": 1027095552.0, + "25": 1027097088.0, + "26": 1027098112.0, + "27": 1027091456.0, + "28": 1027090944.0, + "29": 1027091968.0, + "30": 1027099648.0, + "31": 1027109888.0, + "32": 1027095552.0, + "33": 1027090944.0, + "34": 1027098112.0, + "35": 1027103744.0, + "36": 1027098112.0, + "37": 1027092480.0, + "38": 1027091456.0, + "39": 1027095040.0, + "40": 1027095040.0, + "41": 1027100160.0, + "42": 1027091968.0, + "43": 1027098624.0, + "44": 1027098624.0, + "45": 1027096064.0, + "46": 1027104256.0, + "47": 1027093504.0, + "48": 1027101184.0, + "49": 1027096064.0, + "50": 1027095552.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3059586560.0, + "2": 3299159040.0, + "3": 3299482112.0, + "4": 3302137344.0, + "5": 3302137344.0, + "6": 3302137344.0, + "7": 3303535104.0, + "8": 3303535104.0, + "9": 3303535104.0, + "10": 3303535104.0, + "11": 3303535104.0, + "12": 3303535104.0, + "13": 3303535104.0, + "14": 3303535104.0, + "15": 3303535104.0, + "16": 3303535104.0, + "17": 3306910208.0, + "18": 3306910208.0, + "19": 3306910208.0, + "20": 3306910208.0, + "21": 3306910208.0, + "22": 3306910208.0, + "23": 3306910208.0, + "24": 3306910208.0, + "25": 3306910208.0, + "26": 3306910208.0, + "27": 3306910208.0, + "28": 3306910208.0, + "29": 3306910208.0, + "30": 3306910208.0, + "31": 3312495616.0, + "32": 3312495616.0, + "33": 3312495616.0, + "34": 3312495616.0, + "35": 3312495616.0, + "36": 3312495616.0, + "37": 3312495616.0, + "38": 3312495616.0, + "39": 3312495616.0, + "40": 3312495616.0, + "41": 3312495616.0, + "42": 3312495616.0, + "43": 3312495616.0, + "44": 3312495616.0, + "45": 3312495616.0, + "46": 3312495616.0, + "47": 3312495616.0, + "48": 3312495616.0, + "49": 3312495616.0, + "50": 3312495616.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 3.6307, + "3": 0.3854, + "4": 0.38116, + "5": 0.36866, + "6": 0.36756, + "7": 0.37196, + "8": 0.37096, + "9": 0.36719, + "10": 0.36516, + "11": 0.36882, + "12": 0.37126, + "13": 0.36294, + "14": 0.36799, + "15": 0.3669, + "16": 0.36835, + "17": 0.37548, + "18": 0.37236, + "19": 0.36274, + "20": 0.36388, + "21": 0.36581, + "22": 0.3703, + "23": 0.36921, + "24": 0.35712, + "25": 0.36049, + "26": 0.36512, + "27": 0.36657, + "28": 0.36074, + "29": 0.41887, + "30": 0.45698, + "31": 0.54747, + "32": 0.4695, + "33": 0.67157, + "34": 0.4186, + "35": 0.39703, + "36": 0.40139, + "37": 0.39345, + "38": 0.38789, + "39": 1.0807, + "40": 0.42023, + "41": 0.3945, + "42": 0.39312, + "43": 0.41319, + "44": 0.40657, + "45": 0.4003, + "46": 0.3986, + "47": 0.38501, + "48": 0.38618, + "49": 0.38586, + "50": 0.38297 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..682fa44a64d --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.82137, + "2": 10.8271, + "3": 10.81279, + "4": 10.80424, + "5": 10.84481, + "6": 10.85159, + "7": 10.82705, + "8": 10.83127, + "9": 10.8396, + "10": 10.79638, + "11": 10.85834, + "12": 10.8443, + "13": 10.8625, + "14": 10.86559, + "15": 10.8001, + "16": 10.78718, + "17": 10.7639, + "18": 10.78578, + "19": 10.78836, + "20": 10.71249, + "21": 10.68241, + "22": 10.54353, + "23": 10.69825, + "24": 10.58633, + "25": 10.52721, + "26": 10.58871, + "27": 10.60408, + "28": 10.57696, + "29": 10.57897, + "30": 10.36401, + "31": 10.10796, + "32": 10.44854, + "33": 10.4401, + "34": 10.20252, + "35": 10.25069, + "36": 10.21055, + "37": 10.32849, + "38": 10.17511, + "39": 10.38336, + "40": 10.05674, + "41": 10.10841, + "42": 10.18865, + "43": 9.80582, + "44": 9.91887, + "45": 9.79924, + "46": 9.78948, + "47": 10.11342, + "48": 9.82499, + "49": 9.49844, + "50": 9.87311 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 27059.0, + "2": 29311.0, + "3": 28886.0, + "4": 27768.0, + "5": 32694.0, + "6": 33260.0, + "7": 31409.0, + "8": 27342.0, + "9": 30401.0, + "10": 25524.0, + "11": 33805.0, + "12": 31146.0, + "13": 33161.0, + "14": 33991.0, + "15": 31160.0, + "16": 32445.0, + "17": 30974.0, + "18": 31151.0, + "19": 31742.0, + "20": 28624.0, + "21": 29115.0, + "22": 26827.0, + "23": 34472.0, + "24": 29096.0, + "25": 27239.0, + "26": 30910.0, + "27": 31915.0, + "28": 33968.0, + "29": 36017.0, + "30": 30702.0, + "31": 27384.0, + "32": 33681.0, + "33": 35476.0, + "34": 30160.0, + "35": 31419.0, + "36": 32568.0, + "37": 36189.0, + "38": 33607.0, + "39": 37731.0, + "40": 34463.0, + "41": 33229.0, + "42": 35616.0, + "43": 32361.0, + "44": 31908.0, + "45": 33571.0, + "46": 33618.0, + "47": 38873.0, + "48": 35034.0, + "49": 34407.0, + "50": 37669.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1559412224.0, + "2": 1558616064.0, + "3": 1558495744.0, + "4": 1559267328.0, + "5": 1558842880.0, + "6": 1559098368.0, + "7": 1558495744.0, + "8": 1558546432.0, + "9": 1558495744.0, + "10": 1558546432.0, + "11": 1558597120.0, + "12": 1558546432.0, + "13": 1558597120.0, + "14": 1558546432.0, + "15": 1558904320.0, + "16": 1558647808.0, + "17": 1558597120.0, + "18": 1558889472.0, + "19": 1558597120.0, + "20": 1559229440.0, + "21": 1558597120.0, + "22": 1558758400.0, + "23": 1559698944.0, + "24": 1559078912.0, + "25": 1559052800.0, + "26": 1558647808.0, + "27": 1559382528.0, + "28": 1558749184.0, + "29": 1558830592.0, + "30": 1558749184.0, + "31": 1558915584.0, + "32": 1559541760.0, + "33": 1558698496.0, + "34": 1558749184.0, + "35": 1559422464.0, + "36": 1558863872.0, + "37": 1558799872.0, + "38": 1558749184.0, + "39": 1559397888.0, + "40": 1559002112.0, + "41": 1558799872.0, + "42": 1558850560.0, + "43": 1559724544.0, + "44": 1558850560.0, + "45": 1558901248.0, + "46": 1559175168.0, + "47": 1558901248.0, + "48": 1558850560.0, + "49": 1558901248.0, + "50": 1559632896.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 3495116800.0, + "2": 4054579712.0, + "3": 4062724096.0, + "4": 4062724096.0, + "5": 4062724096.0, + "6": 4062724096.0, + "7": 4070930432.0, + "8": 4070930432.0, + "9": 4073446400.0, + "10": 4073446400.0, + "11": 4073446400.0, + "12": 4073446400.0, + "13": 4073446400.0, + "14": 4075493888.0, + "15": 4075493888.0, + "16": 4075493888.0, + "17": 4075493888.0, + "18": 4075493888.0, + "19": 4075493888.0, + "20": 4075493888.0, + "21": 4075493888.0, + "22": 4079303168.0, + "23": 4096666624.0, + "24": 4096666624.0, + "25": 4096666624.0, + "26": 4096666624.0, + "27": 4096666624.0, + "28": 4096666624.0, + "29": 4096666624.0, + "30": 4096666624.0, + "31": 4105302016.0, + "32": 4105302016.0, + "33": 4105302016.0, + "34": 4105302016.0, + "35": 4105302016.0, + "36": 4105302016.0, + "37": 4105302016.0, + "38": 4105302016.0, + "39": 4105302016.0, + "40": 4105302016.0, + "41": 4105302016.0, + "42": 4105302016.0, + "43": 4105302016.0, + "44": 4105302016.0, + "45": 4105302016.0, + "46": 4105302016.0, + "47": 4105302016.0, + "48": 4105302016.0, + "49": 4105302016.0, + "50": 4105302016.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 6.04776, + "3": 0.43191, + "4": 0.39355, + "5": 0.39556, + "6": 0.39818, + "7": 0.39915, + "8": 0.39139, + "9": 0.41074, + "10": 0.45245, + "11": 0.45849, + "12": 0.46806, + "13": 0.46943, + "14": 0.47411, + "15": 0.48525, + "16": 0.47939, + "17": 0.47872, + "18": 0.4715, + "19": 0.4792, + "20": 0.46531, + "21": 0.46809, + "22": 0.46348, + "23": 0.47875, + "24": 0.83175, + "25": 0.50009, + "26": 0.4884, + "27": 0.82926, + "28": 0.50184, + "29": 0.50509, + "30": 0.49725, + "31": 0.50602, + "32": 0.84607, + "33": 0.50581, + "34": 0.49849, + "35": 0.50057, + "36": 0.5007, + "37": 0.50598, + "38": 0.50147, + "39": 0.51593, + "40": 0.51491, + "41": 0.50337, + "42": 0.48945, + "43": 0.49729, + "44": 0.49341, + "45": 0.4898, + "46": 0.49624, + "47": 0.51146, + "48": 0.49582, + "49": 0.49624, + "50": 0.49469 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..113a491b0ba --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81199, + "2": 10.82649, + "3": 10.81384, + "4": 10.79509, + "5": 10.83534, + "6": 10.84275, + "7": 10.83571, + "8": 10.83439, + "9": 10.83696, + "10": 10.78957, + "11": 10.85974, + "12": 10.84264, + "13": 10.84986, + "14": 10.86378, + "15": 10.80482, + "16": 10.79204, + "17": 10.7636, + "18": 10.78823, + "19": 10.78841, + "20": 10.70796, + "21": 10.68628, + "22": 10.53299, + "23": 10.691, + "24": 10.58061, + "25": 10.5289, + "26": 10.57723, + "27": 10.58971, + "28": 10.5643, + "29": 10.56693, + "30": 10.35124, + "31": 10.09414, + "32": 10.43287, + "33": 10.43231, + "34": 10.19673, + "35": 10.23457, + "36": 10.19059, + "37": 10.31658, + "38": 10.16469, + "39": 10.37482, + "40": 10.05031, + "41": 10.10005, + "42": 10.1774, + "43": 9.79407, + "44": 9.91934, + "45": 9.7932, + "46": 9.78104, + "47": 10.10607, + "48": 9.8118, + "49": 9.48096, + "50": 9.86752, + "51": 9.8069, + "52": 9.70296, + "53": 10.03508, + "54": 9.92052, + "55": 9.84588, + "56": 9.58072, + "57": 9.43445, + "58": 9.79856, + "59": 9.54419, + "60": 9.45288, + "61": 9.65801, + "62": 9.95366, + "63": 9.34015, + "64": 9.73433, + "65": 8.90213, + "66": 9.6667, + "67": 9.33687, + "68": 9.7563, + "69": 9.77598, + "70": 9.70281, + "71": 9.60206, + "72": 9.543, + "73": 9.4557, + "74": 8.87804, + "75": 9.37677, + "76": 9.03816, + "77": 10.03912, + "78": 9.69714, + "79": 9.35195, + "80": 9.37278, + "81": 9.45649, + "82": 9.6802, + "83": 9.27723, + "84": 9.39341, + "85": 9.58928, + "86": 9.05151, + "87": 9.57623, + "88": 9.72869, + "89": 9.57637, + "90": 9.80884, + "91": 9.30719, + "92": 9.33823, + "93": 9.05712, + "94": 8.80375, + "95": 9.5091, + "96": 9.50777, + "97": 9.27751, + "98": 9.65271, + "99": 8.87009, + "100": 9.38142 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 26865.0, + "2": 29306.0, + "3": 29361.0, + "4": 28339.0, + "5": 32501.0, + "6": 33051.0, + "7": 31429.0, + "8": 27274.0, + "9": 30849.0, + "10": 25253.0, + "11": 34123.0, + "12": 30710.0, + "13": 33513.0, + "14": 33611.0, + "15": 31132.0, + "16": 32283.0, + "17": 31523.0, + "18": 30937.0, + "19": 31324.0, + "20": 28686.0, + "21": 29644.0, + "22": 27366.0, + "23": 34392.0, + "24": 29052.0, + "25": 27947.0, + "26": 31335.0, + "27": 31669.0, + "28": 33909.0, + "29": 35204.0, + "30": 30468.0, + "31": 27904.0, + "32": 33358.0, + "33": 35896.0, + "34": 30365.0, + "35": 31692.0, + "36": 32966.0, + "37": 35992.0, + "38": 33308.0, + "39": 38061.0, + "40": 34579.0, + "41": 33534.0, + "42": 36447.0, + "43": 32600.0, + "44": 32178.0, + "45": 34034.0, + "46": 34910.0, + "47": 39009.0, + "48": 34943.0, + "49": 34977.0, + "50": 38519.0, + "51": 36877.0, + "52": 36443.0, + "53": 43145.0, + "54": 41676.0, + "55": 38684.0, + "56": 41454.0, + "57": 35771.0, + "58": 41538.0, + "59": 39697.0, + "60": 56137.0, + "61": 59394.0, + "62": 2137056.0, + "63": 36401.0, + "64": 50930.0, + "65": 43788.0, + "66": 2139459.0, + "67": 2137025.0, + "68": 2137005.0, + "69": 2139555.0, + "70": 2140268.0, + "71": 2138613.0, + "72": 2139093.0, + "73": 2141321.0, + "74": 2137048.0, + "75": 2136852.0, + "76": 2140757.0, + "77": 2140654.0, + "78": 2141929.0, + "79": 2142543.0, + "80": 2142157.0, + "81": 2145547.0, + "82": 2144670.0, + "83": 2140858.0, + "84": 2140984.0, + "85": 2145921.0, + "86": 149825.0, + "87": 2144700.0, + "88": 2142479.0, + "89": 2140988.0, + "90": 2144684.0, + "91": 2143848.0, + "92": 2142027.0, + "93": 2139531.0, + "94": 2145775.0, + "95": 2143141.0, + "96": 2146259.0, + "97": 2140268.0, + "98": 2143316.0, + "99": 2144369.0, + "100": 2143057.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 787591680.0, + "2": 787578880.0, + "3": 787593728.0, + "4": 787568128.0, + "5": 787563008.0, + "6": 787585536.0, + "7": 787578368.0, + "8": 787582976.0, + "9": 787581952.0, + "10": 787592192.0, + "11": 787569152.0, + "12": 787570688.0, + "13": 787579392.0, + "14": 787582976.0, + "15": 787565568.0, + "16": 787572224.0, + "17": 787566592.0, + "18": 787547648.0, + "19": 787566592.0, + "20": 787537408.0, + "21": 787540992.0, + "22": 787540480.0, + "23": 787548672.0, + "24": 787542016.0, + "25": 787534336.0, + "26": 787548672.0, + "27": 787509760.0, + "28": 787504640.0, + "29": 787499520.0, + "30": 787494912.0, + "31": 787510784.0, + "32": 787501056.0, + "33": 787482624.0, + "34": 787486208.0, + "35": 787483136.0, + "36": 787482624.0, + "37": 787460608.0, + "38": 787457536.0, + "39": 787461632.0, + "40": 787457536.0, + "41": 787466752.0, + "42": 787432448.0, + "43": 787450368.0, + "44": 787436032.0, + "45": 787411456.0, + "46": 787460608.0, + "47": 787412992.0, + "48": 787440128.0, + "49": 787409920.0, + "50": 787396096.0, + "51": 787388416.0, + "52": 787415040.0, + "53": 787377664.0, + "54": 787403264.0, + "55": 787375104.0, + "56": 787362304.0, + "57": 787405824.0, + "58": 787356160.0, + "59": 787378688.0, + "60": 787380224.0, + "61": 787337216.0, + "62": 787331584.0, + "63": 787368960.0, + "64": 787339264.0, + "65": 787403776.0, + "66": 787330048.0, + "67": 787337728.0, + "68": 787324416.0, + "69": 787335680.0, + "70": 787328512.0, + "71": 787331584.0, + "72": 787341312.0, + "73": 787353088.0, + "74": 787366400.0, + "75": 787342848.0, + "76": 787344384.0, + "77": 787345920.0, + "78": 787371520.0, + "79": 787366400.0, + "80": 787390464.0, + "81": 787385344.0, + "82": 787395584.0, + "83": 787403776.0, + "84": 787397632.0, + "85": 787398144.0, + "86": 787411968.0, + "87": 787389952.0, + "88": 787387904.0, + "89": 787400704.0, + "90": 787379712.0, + "91": 787401216.0, + "92": 787399168.0, + "93": 787391488.0, + "94": 787392000.0, + "95": 787398656.0, + "96": 787395584.0, + "97": 787403776.0, + "98": 787396608.0, + "99": 787406848.0, + "100": 787410432.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2662647296.0, + "2": 2662647296.0, + "3": 2665052672.0, + "4": 2665052672.0, + "5": 2665052672.0, + "6": 2665052672.0, + "7": 2665052672.0, + "8": 2665052672.0, + "9": 2665052672.0, + "10": 2665052672.0, + "11": 2665052672.0, + "12": 2665052672.0, + "13": 2665052672.0, + "14": 2665052672.0, + "15": 2665052672.0, + "16": 2665052672.0, + "17": 2665052672.0, + "18": 2665052672.0, + "19": 2665052672.0, + "20": 2665052672.0, + "21": 2665052672.0, + "22": 2665052672.0, + "23": 2665052672.0, + "24": 2665052672.0, + "25": 2665052672.0, + "26": 2665052672.0, + "27": 2665052672.0, + "28": 2665052672.0, + "29": 2665052672.0, + "30": 2665052672.0, + "31": 2665052672.0, + "32": 2665052672.0, + "33": 2665052672.0, + "34": 2665052672.0, + "35": 2665052672.0, + "36": 2665052672.0, + "37": 2665052672.0, + "38": 2665052672.0, + "39": 2665052672.0, + "40": 2665052672.0, + "41": 2665052672.0, + "42": 2665052672.0, + "43": 2665052672.0, + "44": 2665052672.0, + "45": 2665052672.0, + "46": 2665052672.0, + "47": 2665052672.0, + "48": 2665052672.0, + "49": 2665052672.0, + "50": 2665052672.0, + "51": 2665052672.0, + "52": 2665052672.0, + "53": 2665052672.0, + "54": 2665052672.0, + "55": 2665052672.0, + "56": 2665052672.0, + "57": 2665052672.0, + "58": 2665052672.0, + "59": 2665052672.0, + "60": 2665052672.0, + "61": 2665052672.0, + "62": 2665052672.0, + "63": 2665052672.0, + "64": 2665052672.0, + "65": 2665052672.0, + "66": 2665052672.0, + "67": 2665052672.0, + "68": 2665052672.0, + "69": 2665052672.0, + "70": 2665052672.0, + "71": 2665052672.0, + "72": 2665052672.0, + "73": 2665052672.0, + "74": 2665052672.0, + "75": 2665052672.0, + "76": 2665052672.0, + "77": 2665052672.0, + "78": 2665052672.0, + "79": 2665052672.0, + "80": 2665052672.0, + "81": 2665052672.0, + "82": 2665052672.0, + "83": 2665052672.0, + "84": 2665052672.0, + "85": 2665052672.0, + "86": 2665052672.0, + "87": 2665052672.0, + "88": 2665052672.0, + "89": 2665052672.0, + "90": 2665052672.0, + "91": 2665052672.0, + "92": 2665052672.0, + "93": 2665052672.0, + "94": 2665052672.0, + "95": 2665052672.0, + "96": 2665052672.0, + "97": 2665052672.0, + "98": 2665052672.0, + "99": 2665052672.0, + "100": 2665052672.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 4.78579, + "3": 0.53829, + "4": 0.5501, + "5": 0.52877, + "6": 0.53341, + "7": 0.53101, + "8": 0.52594, + "9": 0.52656, + "10": 0.52721, + "11": 0.51907, + "12": 0.52113, + "13": 0.52417, + "14": 0.52392, + "15": 0.53475, + "16": 0.52116, + "17": 0.52656, + "18": 0.52034, + "19": 0.52016, + "20": 0.52199, + "21": 0.53183, + "22": 0.53661, + "23": 0.54084, + "24": 0.52495, + "25": 0.53128, + "26": 0.52735, + "27": 0.54335, + "28": 0.52654, + "29": 0.53834, + "30": 0.53606, + "31": 0.53938, + "32": 0.53598, + "33": 0.53326, + "34": 0.54444, + "35": 0.53164, + "36": 0.5404, + "37": 0.54568, + "38": 0.54552, + "39": 0.5366, + "40": 0.54027, + "41": 0.53525, + "42": 0.55075, + "43": 0.53886, + "44": 0.53665, + "45": 0.55089, + "46": 0.5331, + "47": 0.54482, + "48": 0.53151, + "49": 0.53493, + "50": 0.53302, + "51": 0.52424, + "52": 0.52434, + "53": 0.51687, + "54": 0.52816, + "55": 0.53022, + "56": 0.53577, + "57": 0.53245, + "58": 0.53568, + "59": 0.54753, + "60": 0.53813, + "61": 0.53815, + "62": 0.5366, + "63": 0.54423, + "64": 0.5344, + "65": 0.53864, + "66": 0.54089, + "67": 0.53579, + "68": 0.54777, + "69": 0.54032, + "70": 0.54348, + "71": 0.5411, + "72": 0.54019, + "73": 0.53851, + "74": 0.54021, + "75": 0.53784, + "76": 0.53954, + "77": 0.54237, + "78": 0.53049, + "79": 0.57915, + "80": 0.57307, + "81": 0.56876, + "82": 0.56781, + "83": 0.56481, + "84": 0.55385, + "85": 0.56577, + "86": 0.569, + "87": 0.5621, + "88": 0.56698, + "89": 0.55835, + "90": 0.85395, + "91": 0.56888, + "92": 0.55621, + "93": 0.57143, + "94": 0.5584, + "95": 0.56204, + "96": 0.5656, + "97": 0.5491, + "98": 0.56348, + "99": 0.5607, + "100": 0.56258 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..bfea64b8438 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79193, + "2": 10.81245, + "3": 10.79181, + "4": 10.78209, + "5": 10.82295, + "6": 10.83309, + "7": 10.81351, + "8": 10.81215, + "9": 10.81457, + "10": 10.76068, + "11": 10.84185, + "12": 10.82404, + "13": 10.83895, + "14": 10.84433, + "15": 10.79974, + "16": 10.78654, + "17": 10.76789, + "18": 10.77495, + "19": 10.77669, + "20": 10.71893, + "21": 10.69691, + "22": 10.5691, + "23": 10.7131, + "24": 10.59975, + "25": 10.56123, + "26": 10.60735, + "27": 10.63093, + "28": 10.6064, + "29": 10.61213, + "30": 10.39823, + "31": 10.16422, + "32": 10.49019, + "33": 10.48385, + "34": 10.26645, + "35": 10.31743, + "36": 10.28264, + "37": 10.39002, + "38": 10.25116, + "39": 10.43811, + "40": 10.1403, + "41": 10.19191, + "42": 10.25886, + "43": 9.91588, + "44": 10.02837, + "45": 9.91815, + "46": 9.89353, + "47": 10.20144, + "48": 9.92509, + "49": 9.62973, + "50": 9.97857 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5192.0, + "2": 5510.0, + "3": 5508.0, + "4": 5240.0, + "5": 6136.0, + "6": 6180.0, + "7": 5549.0, + "8": 5242.0, + "9": 5717.0, + "10": 4818.0, + "11": 6299.0, + "12": 5746.0, + "13": 6110.0, + "14": 6165.0, + "15": 5683.0, + "16": 5805.0, + "17": 5758.0, + "18": 5546.0, + "19": 5787.0, + "20": 5231.0, + "21": 5741.0, + "22": 5126.0, + "23": 6019.0, + "24": 5410.0, + "25": 5100.0, + "26": 5630.0, + "27": 5627.0, + "28": 6146.0, + "29": 6174.0, + "30": 5570.0, + "31": 4768.0, + "32": 5926.0, + "33": 6348.0, + "34": 5389.0, + "35": 5856.0, + "36": 5741.0, + "37": 6611.0, + "38": 6262.0, + "39": 6971.0, + "40": 6094.0, + "41": 6227.0, + "42": 6622.0, + "43": 5761.0, + "44": 5929.0, + "45": 5769.0, + "46": 6141.0, + "47": 6909.0, + "48": 6650.0, + "49": 6100.0, + "50": 6753.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 627716608.0, + "2": 627719168.0, + "3": 627717632.0, + "4": 627719680.0, + "5": 627717120.0, + "6": 627717120.0, + "7": 627719680.0, + "8": 627716608.0, + "9": 627718144.0, + "10": 627718144.0, + "11": 627717632.0, + "12": 627718144.0, + "13": 627719168.0, + "14": 627718144.0, + "15": 627722240.0, + "16": 627718144.0, + "17": 627720704.0, + "18": 627719680.0, + "19": 627719168.0, + "20": 627718144.0, + "21": 627718656.0, + "22": 627723264.0, + "23": 627720192.0, + "24": 627719680.0, + "25": 627718144.0, + "26": 627719168.0, + "27": 627719168.0, + "28": 627718144.0, + "29": 627718144.0, + "30": 627719168.0, + "31": 627719168.0, + "32": 627719168.0, + "33": 627717632.0, + "34": 627719680.0, + "35": 627721216.0, + "36": 627717120.0, + "37": 627719168.0, + "38": 627721216.0, + "39": 627719168.0, + "40": 627718656.0, + "41": 627718144.0, + "42": 627717632.0, + "43": 627717120.0, + "44": 627718656.0, + "45": 627717632.0, + "46": 627717120.0, + "47": 627719168.0, + "48": 627718144.0, + "49": 627716608.0, + "50": 627716096.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 879803392.0, + "2": 1114769920.0, + "3": 1114769920.0, + "4": 1116293632.0, + "5": 1116293632.0, + "6": 1116293632.0, + "7": 1116293632.0, + "8": 1116293632.0, + "9": 1116293632.0, + "10": 1116293632.0, + "11": 1116293632.0, + "12": 1116293632.0, + "13": 1116293632.0, + "14": 1116293632.0, + "15": 1116293632.0, + "16": 1116293632.0, + "17": 1116293632.0, + "18": 1116293632.0, + "19": 1116293632.0, + "20": 1116293632.0, + "21": 1116293632.0, + "22": 1116293632.0, + "23": 1116293632.0, + "24": 1116293632.0, + "25": 1116293632.0, + "26": 1116293632.0, + "27": 1116293632.0, + "28": 1116293632.0, + "29": 1116293632.0, + "30": 1116293632.0, + "31": 1116293632.0, + "32": 1116293632.0, + "33": 1116293632.0, + "34": 1116293632.0, + "35": 1116293632.0, + "36": 1116293632.0, + "37": 1116293632.0, + "38": 1116293632.0, + "39": 1116293632.0, + "40": 1116293632.0, + "41": 1116293632.0, + "42": 1116293632.0, + "43": 1116293632.0, + "44": 1116293632.0, + "45": 1116293632.0, + "46": 1116293632.0, + "47": 1116293632.0, + "48": 1116293632.0, + "49": 1116293632.0, + "50": 1116293632.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 7.71846, + "3": 0.76188, + "4": 0.74577, + "5": 0.73403, + "6": 0.73193, + "7": 0.73107, + "8": 0.72199, + "9": 0.726, + "10": 0.71891, + "11": 0.72723, + "12": 0.71504, + "13": 0.71448, + "14": 0.71551, + "15": 0.71936, + "16": 0.71512, + "17": 0.73948, + "18": 0.83787, + "19": 0.94178, + "20": 0.98096, + "21": 0.71399, + "22": 0.87302, + "23": 0.71359, + "24": 0.7104, + "25": 0.70807, + "26": 0.71636, + "27": 0.70864, + "28": 0.72237, + "29": 0.7163, + "30": 0.7153, + "31": 0.71793, + "32": 0.70846, + "33": 0.7079, + "34": 0.71058, + "35": 0.71492, + "36": 0.72031, + "37": 0.71537, + "38": 0.70333, + "39": 0.70449, + "40": 0.71725, + "41": 0.72322, + "42": 0.7105, + "43": 0.70421, + "44": 0.70441, + "45": 0.70449, + "46": 0.7091, + "47": 0.70989, + "48": 0.70781, + "49": 0.71985, + "50": 0.70534 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..2bcdb30bc50 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79193, + "2": 10.81245, + "3": 10.79181, + "4": 10.78209, + "5": 10.82295, + "6": 10.83309, + "7": 10.81351, + "8": 10.81215, + "9": 10.81457, + "10": 10.76068, + "11": 10.84185, + "12": 10.82404, + "13": 10.83895, + "14": 10.84433, + "15": 10.79974, + "16": 10.78654, + "17": 10.76789, + "18": 10.77495, + "19": 10.77669, + "20": 10.71893, + "21": 10.69691, + "22": 10.5691, + "23": 10.7131, + "24": 10.59975, + "25": 10.56123, + "26": 10.60735, + "27": 10.63093, + "28": 10.6064, + "29": 10.61213, + "30": 10.39823, + "31": 10.16422, + "32": 10.49019, + "33": 10.48385, + "34": 10.26645, + "35": 10.31743, + "36": 10.28264, + "37": 10.39002, + "38": 10.25116, + "39": 10.43811, + "40": 10.1403, + "41": 10.19191, + "42": 10.25886, + "43": 9.91588, + "44": 10.02837, + "45": 9.91815, + "46": 9.89353, + "47": 10.20144, + "48": 9.92509, + "49": 9.62973, + "50": 9.97857 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5192.0, + "2": 5510.0, + "3": 5508.0, + "4": 5240.0, + "5": 6136.0, + "6": 6180.0, + "7": 5549.0, + "8": 5242.0, + "9": 5717.0, + "10": 4818.0, + "11": 6299.0, + "12": 5746.0, + "13": 6110.0, + "14": 6165.0, + "15": 5683.0, + "16": 5805.0, + "17": 5758.0, + "18": 5546.0, + "19": 5787.0, + "20": 5231.0, + "21": 5741.0, + "22": 5126.0, + "23": 6019.0, + "24": 5410.0, + "25": 5100.0, + "26": 5630.0, + "27": 5627.0, + "28": 6146.0, + "29": 6174.0, + "30": 5570.0, + "31": 4768.0, + "32": 5926.0, + "33": 6348.0, + "34": 5389.0, + "35": 5856.0, + "36": 5741.0, + "37": 6611.0, + "38": 6262.0, + "39": 6971.0, + "40": 6094.0, + "41": 6227.0, + "42": 6622.0, + "43": 5761.0, + "44": 5929.0, + "45": 5769.0, + "46": 6141.0, + "47": 6909.0, + "48": 6650.0, + "49": 6100.0, + "50": 6753.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 627716608.0, + "2": 627719168.0, + "3": 627717632.0, + "4": 627719680.0, + "5": 627717120.0, + "6": 627717120.0, + "7": 627719680.0, + "8": 627716608.0, + "9": 627718144.0, + "10": 627718144.0, + "11": 627717632.0, + "12": 627718144.0, + "13": 627719168.0, + "14": 627718144.0, + "15": 627722240.0, + "16": 627718144.0, + "17": 627720704.0, + "18": 627719680.0, + "19": 627719168.0, + "20": 627718144.0, + "21": 627718656.0, + "22": 627723264.0, + "23": 627720192.0, + "24": 627719680.0, + "25": 627718144.0, + "26": 627719168.0, + "27": 627719168.0, + "28": 627718144.0, + "29": 627718144.0, + "30": 627719168.0, + "31": 627719168.0, + "32": 627719168.0, + "33": 627717632.0, + "34": 627719680.0, + "35": 627721216.0, + "36": 627717120.0, + "37": 627719168.0, + "38": 627721216.0, + "39": 627719168.0, + "40": 627718656.0, + "41": 627718144.0, + "42": 627717632.0, + "43": 627717120.0, + "44": 627718656.0, + "45": 627717632.0, + "46": 627717120.0, + "47": 627719168.0, + "48": 627718144.0, + "49": 627716608.0, + "50": 627716096.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 879803392.0, + "2": 1114769920.0, + "3": 1114769920.0, + "4": 1116293632.0, + "5": 1116293632.0, + "6": 1116293632.0, + "7": 1116293632.0, + "8": 1116293632.0, + "9": 1116293632.0, + "10": 1116293632.0, + "11": 1116293632.0, + "12": 1116293632.0, + "13": 1116293632.0, + "14": 1116293632.0, + "15": 1116293632.0, + "16": 1116293632.0, + "17": 1116293632.0, + "18": 1116293632.0, + "19": 1116293632.0, + "20": 1116293632.0, + "21": 1116293632.0, + "22": 1116293632.0, + "23": 1116293632.0, + "24": 1116293632.0, + "25": 1116293632.0, + "26": 1116293632.0, + "27": 1116293632.0, + "28": 1116293632.0, + "29": 1116293632.0, + "30": 1116293632.0, + "31": 1116293632.0, + "32": 1116293632.0, + "33": 1116293632.0, + "34": 1116293632.0, + "35": 1116293632.0, + "36": 1116293632.0, + "37": 1116293632.0, + "38": 1116293632.0, + "39": 1116293632.0, + "40": 1116293632.0, + "41": 1116293632.0, + "42": 1116293632.0, + "43": 1116293632.0, + "44": 1116293632.0, + "45": 1116293632.0, + "46": 1116293632.0, + "47": 1116293632.0, + "48": 1116293632.0, + "49": 1116293632.0, + "50": 1116293632.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 7.52257, + "3": 0.74502, + "4": 0.74089, + "5": 0.73009, + "6": 0.73041, + "7": 0.73704, + "8": 0.71933, + "9": 0.72466, + "10": 1.0546, + "11": 0.71525, + "12": 0.71298, + "13": 0.71412, + "14": 0.71521, + "15": 0.71883, + "16": 0.71464, + "17": 0.72192, + "18": 1.32991, + "19": 0.92083, + "20": 0.72233, + "21": 0.71533, + "22": 0.7144, + "23": 0.71011, + "24": 0.71396, + "25": 0.70984, + "26": 0.7111, + "27": 0.71496, + "28": 0.71187, + "29": 0.71729, + "30": 0.72095, + "31": 0.71436, + "32": 0.70963, + "33": 0.71384, + "34": 0.71534, + "35": 0.7148, + "36": 0.71389, + "37": 0.71097, + "38": 0.71244, + "39": 0.7048, + "40": 0.715, + "41": 1.08196, + "42": 0.71129, + "43": 0.73716, + "44": 0.72639, + "45": 0.71182, + "46": 0.71576, + "47": 0.72917, + "48": 0.72017, + "49": 0.72166, + "50": 0.70656 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..9a3140994d3 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80801, + "2": 10.80912, + "3": 10.81804, + "4": 10.77924, + "5": 10.83867, + "6": 10.8473, + "7": 10.80366, + "8": 10.81171, + "9": 10.80948, + "10": 10.77389, + "11": 10.85853, + "12": 10.83206, + "13": 10.84778, + "14": 10.86351, + "15": 10.77822, + "16": 10.78665, + "17": 10.74969, + "18": 10.78174, + "19": 10.77893, + "20": 10.71133, + "21": 10.68188, + "22": 10.53221, + "23": 10.70751, + "24": 10.58301, + "25": 10.53686, + "26": 10.59662, + "27": 10.62332, + "28": 10.58807, + "29": 10.61089, + "30": 10.39372, + "31": 10.1118, + "32": 10.4835, + "33": 10.48693, + "34": 10.23859, + "35": 10.29466, + "36": 10.25749, + "37": 10.38723, + "38": 10.24326, + "39": 10.43603, + "40": 10.12881, + "41": 10.18559, + "42": 10.25677, + "43": 9.8808, + "44": 10.00863, + "45": 9.89409, + "46": 9.85423, + "47": 10.1998, + "48": 9.90437, + "49": 9.58703, + "50": 9.96891 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5550.0, + "2": 5910.0, + "3": 5966.0, + "4": 5860.0, + "5": 6749.0, + "6": 6869.0, + "7": 6129.0, + "8": 5905.0, + "9": 6154.0, + "10": 5266.0, + "11": 6792.0, + "12": 6324.0, + "13": 6845.0, + "14": 6862.0, + "15": 6306.0, + "16": 6524.0, + "17": 6567.0, + "18": 6194.0, + "19": 6515.0, + "20": 5979.0, + "21": 6327.0, + "22": 5748.0, + "23": 6749.0, + "24": 5978.0, + "25": 5661.0, + "26": 6206.0, + "27": 6307.0, + "28": 7003.0, + "29": 7124.0, + "30": 6390.0, + "31": 5578.0, + "32": 6783.0, + "33": 7031.0, + "34": 6306.0, + "35": 6516.0, + "36": 6614.0, + "37": 7690.0, + "38": 7193.0, + "39": 7850.0, + "40": 7170.0, + "41": 6880.0, + "42": 7329.0, + "43": 6669.0, + "44": 6616.0, + "45": 6700.0, + "46": 7080.0, + "47": 7661.0, + "48": 7259.0, + "49": 7083.0, + "50": 7418.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 458211840.0, + "2": 458214400.0, + "3": 458210304.0, + "4": 458211840.0, + "5": 458212864.0, + "6": 458211840.0, + "7": 458213376.0, + "8": 458212352.0, + "9": 458213888.0, + "10": 458214400.0, + "11": 458212864.0, + "12": 458211840.0, + "13": 458214400.0, + "14": 458213888.0, + "15": 458214912.0, + "16": 458210816.0, + "17": 458213888.0, + "18": 458212864.0, + "19": 458214400.0, + "20": 458212352.0, + "21": 458214912.0, + "22": 458217472.0, + "23": 458213888.0, + "24": 458215424.0, + "25": 458212864.0, + "26": 458211328.0, + "27": 458213888.0, + "28": 458212864.0, + "29": 458213376.0, + "30": 458211840.0, + "31": 458214400.0, + "32": 458213888.0, + "33": 458213376.0, + "34": 458214400.0, + "35": 458213888.0, + "36": 458213888.0, + "37": 458212352.0, + "38": 458211328.0, + "39": 458212352.0, + "40": 458214912.0, + "41": 458212864.0, + "42": 458214912.0, + "43": 458215936.0, + "44": 458213376.0, + "45": 458212352.0, + "46": 458214400.0, + "47": 458214400.0, + "48": 458214400.0, + "49": 458212864.0, + "50": 458212352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1027753472.0, + "2": 1191298048.0, + "3": 1191298048.0, + "4": 1192127488.0, + "5": 1192127488.0, + "6": 1192127488.0, + "7": 1192127488.0, + "8": 1192127488.0, + "9": 1192127488.0, + "10": 1192127488.0, + "11": 1192546816.0, + "12": 1192546816.0, + "13": 1193283584.0, + "14": 1193283584.0, + "15": 1193283584.0, + "16": 1193283584.0, + "17": 1193283584.0, + "18": 1193283584.0, + "19": 1193283584.0, + "20": 1193283584.0, + "21": 1193283584.0, + "22": 1193556992.0, + "23": 1193556992.0, + "24": 1193556992.0, + "25": 1193556992.0, + "26": 1193556992.0, + "27": 1193556992.0, + "28": 1193556992.0, + "29": 1193556992.0, + "30": 1193556992.0, + "31": 1193556992.0, + "32": 1193556992.0, + "33": 1193556992.0, + "34": 1193556992.0, + "35": 1193556992.0, + "36": 1193556992.0, + "37": 1193556992.0, + "38": 1193556992.0, + "39": 1193556992.0, + "40": 1193556992.0, + "41": 1193556992.0, + "42": 1193556992.0, + "43": 1193556992.0, + "44": 1193556992.0, + "45": 1193556992.0, + "46": 1193556992.0, + "47": 1193556992.0, + "48": 1193556992.0, + "49": 1193556992.0, + "50": 1193556992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 5.55037, + "3": 0.71803, + "4": 1.02899, + "5": 0.69274, + "6": 0.69497, + "7": 0.70712, + "8": 0.70476, + "9": 0.71137, + "10": 0.69978, + "11": 0.69201, + "12": 0.68876, + "13": 0.68954, + "14": 0.69496, + "15": 0.70166, + "16": 0.6946, + "17": 0.69269, + "18": 0.69041, + "19": 0.69219, + "20": 0.69191, + "21": 0.68931, + "22": 0.69642, + "23": 0.7087, + "24": 0.71205, + "25": 0.75075, + "26": 0.71466, + "27": 0.79945, + "28": 0.68459, + "29": 0.69018, + "30": 0.68977, + "31": 0.69421, + "32": 0.68991, + "33": 0.70331, + "34": 0.70581, + "35": 0.69718, + "36": 0.69748, + "37": 0.69248, + "38": 0.69828, + "39": 0.68816, + "40": 0.69315, + "41": 0.69476, + "42": 0.69711, + "43": 0.70588, + "44": 0.69538, + "45": 0.69598, + "46": 0.70429, + "47": 0.69137, + "48": 0.69183, + "49": 0.70009, + "50": 0.69388 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..883b2c99518 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.80801, + "2": 10.80912, + "3": 10.81804, + "4": 10.77924, + "5": 10.83867, + "6": 10.8473, + "7": 10.80366, + "8": 10.81171, + "9": 10.80948, + "10": 10.77389, + "11": 10.85853, + "12": 10.83206, + "13": 10.84778, + "14": 10.86351, + "15": 10.77822, + "16": 10.78665, + "17": 10.74969, + "18": 10.78174, + "19": 10.77893, + "20": 10.71133, + "21": 10.68188, + "22": 10.53221, + "23": 10.70751, + "24": 10.58301, + "25": 10.53686, + "26": 10.59662, + "27": 10.62332, + "28": 10.58807, + "29": 10.61089, + "30": 10.39372, + "31": 10.1118, + "32": 10.4835, + "33": 10.48693, + "34": 10.23859, + "35": 10.29466, + "36": 10.25749, + "37": 10.38723, + "38": 10.24326, + "39": 10.43603, + "40": 10.12881, + "41": 10.18559, + "42": 10.25677, + "43": 9.8808, + "44": 10.00863, + "45": 9.89409, + "46": 9.85423, + "47": 10.1998, + "48": 9.90437, + "49": 9.58703, + "50": 9.96891 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5550.0, + "2": 5910.0, + "3": 5966.0, + "4": 5860.0, + "5": 6749.0, + "6": 6869.0, + "7": 6129.0, + "8": 5905.0, + "9": 6154.0, + "10": 5266.0, + "11": 6792.0, + "12": 6324.0, + "13": 6845.0, + "14": 6862.0, + "15": 6306.0, + "16": 6524.0, + "17": 6567.0, + "18": 6194.0, + "19": 6515.0, + "20": 5979.0, + "21": 6327.0, + "22": 5748.0, + "23": 6749.0, + "24": 5978.0, + "25": 5661.0, + "26": 6206.0, + "27": 6307.0, + "28": 7003.0, + "29": 7124.0, + "30": 6390.0, + "31": 5578.0, + "32": 6783.0, + "33": 7031.0, + "34": 6306.0, + "35": 6516.0, + "36": 6614.0, + "37": 7690.0, + "38": 7193.0, + "39": 7850.0, + "40": 7170.0, + "41": 6880.0, + "42": 7329.0, + "43": 6669.0, + "44": 6616.0, + "45": 6700.0, + "46": 7080.0, + "47": 7661.0, + "48": 7259.0, + "49": 7083.0, + "50": 7418.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 458211840.0, + "2": 458214400.0, + "3": 458210304.0, + "4": 458211840.0, + "5": 458212864.0, + "6": 458211840.0, + "7": 458213376.0, + "8": 458212352.0, + "9": 458213888.0, + "10": 458214400.0, + "11": 458212864.0, + "12": 458211840.0, + "13": 458214400.0, + "14": 458213888.0, + "15": 458214912.0, + "16": 458210816.0, + "17": 458213888.0, + "18": 458212864.0, + "19": 458214400.0, + "20": 458212352.0, + "21": 458214912.0, + "22": 458217472.0, + "23": 458213888.0, + "24": 458215424.0, + "25": 458212864.0, + "26": 458211328.0, + "27": 458213888.0, + "28": 458212864.0, + "29": 458213376.0, + "30": 458211840.0, + "31": 458214400.0, + "32": 458213888.0, + "33": 458213376.0, + "34": 458214400.0, + "35": 458213888.0, + "36": 458213888.0, + "37": 458212352.0, + "38": 458211328.0, + "39": 458212352.0, + "40": 458214912.0, + "41": 458212864.0, + "42": 458214912.0, + "43": 458215936.0, + "44": 458213376.0, + "45": 458212352.0, + "46": 458214400.0, + "47": 458214400.0, + "48": 458214400.0, + "49": 458212864.0, + "50": 458212352.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1027753472.0, + "2": 1191298048.0, + "3": 1191298048.0, + "4": 1192127488.0, + "5": 1192127488.0, + "6": 1192127488.0, + "7": 1192127488.0, + "8": 1192127488.0, + "9": 1192127488.0, + "10": 1192127488.0, + "11": 1192546816.0, + "12": 1192546816.0, + "13": 1193283584.0, + "14": 1193283584.0, + "15": 1193283584.0, + "16": 1193283584.0, + "17": 1193283584.0, + "18": 1193283584.0, + "19": 1193283584.0, + "20": 1193283584.0, + "21": 1193283584.0, + "22": 1193556992.0, + "23": 1193556992.0, + "24": 1193556992.0, + "25": 1193556992.0, + "26": 1193556992.0, + "27": 1193556992.0, + "28": 1193556992.0, + "29": 1193556992.0, + "30": 1193556992.0, + "31": 1193556992.0, + "32": 1193556992.0, + "33": 1193556992.0, + "34": 1193556992.0, + "35": 1193556992.0, + "36": 1193556992.0, + "37": 1193556992.0, + "38": 1193556992.0, + "39": 1193556992.0, + "40": 1193556992.0, + "41": 1193556992.0, + "42": 1193556992.0, + "43": 1193556992.0, + "44": 1193556992.0, + "45": 1193556992.0, + "46": 1193556992.0, + "47": 1193556992.0, + "48": 1193556992.0, + "49": 1193556992.0, + "50": 1193556992.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 5.57426, + "3": 0.69219, + "4": 0.73624, + "5": 0.67766, + "6": 0.68248, + "7": 0.69364, + "8": 0.67336, + "9": 0.67768, + "10": 0.64366, + "11": 0.62986, + "12": 0.62576, + "13": 0.618, + "14": 0.61177, + "15": 0.61656, + "16": 0.61633, + "17": 0.61648, + "18": 0.62197, + "19": 0.61422, + "20": 0.61923, + "21": 0.61598, + "22": 0.62583, + "23": 0.62054, + "24": 0.61791, + "25": 0.62065, + "26": 0.61387, + "27": 0.61437, + "28": 0.61372, + "29": 0.65198, + "30": 0.66353, + "31": 0.68179, + "32": 0.67222, + "33": 0.67462, + "34": 0.68277, + "35": 0.66769, + "36": 0.66387, + "37": 0.66002, + "38": 0.63341, + "39": 0.62396, + "40": 0.62802, + "41": 0.62419, + "42": 0.61655, + "43": 0.62362, + "44": 0.61679, + "45": 0.61772, + "46": 0.62253, + "47": 0.61779, + "48": 0.61712, + "49": 0.63575, + "50": 0.67932 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..3e910ef7869 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 10.79574, + "2": 10.81485, + "3": 10.78713, + "4": 10.78269, + "5": 10.82015, + "6": 10.83331, + "7": 10.81116, + "8": 10.81446, + "9": 10.81645, + "10": 10.75997, + "11": 10.8388, + "12": 10.81544, + "13": 10.84141, + "14": 10.8476, + "15": 10.79857, + "16": 10.78544, + "17": 10.77004, + "18": 10.77906, + "19": 10.7689, + "20": 10.71392, + "21": 10.69182, + "22": 10.56438, + "23": 10.70939, + "24": 10.60304, + "25": 10.55748, + "26": 10.60238, + "27": 10.62835, + "28": 10.59772, + "29": 10.61013, + "30": 10.40394, + "31": 10.17092, + "32": 10.49069, + "33": 10.48436, + "34": 10.26719, + "35": 10.31532, + "36": 10.27654, + "37": 10.39353, + "38": 10.24536, + "39": 10.43863, + "40": 10.13998, + "41": 10.19151, + "42": 10.25868, + "43": 9.9191, + "44": 10.03026, + "45": 9.92187, + "46": 9.89763, + "47": 10.1946, + "48": 9.93001, + "49": 9.62787, + "50": 9.97966 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 13025.0, + "2": 14911.0, + "3": 14651.0, + "4": 13760.0, + "5": 16297.0, + "6": 16032.0, + "7": 15521.0, + "8": 13170.0, + "9": 15403.0, + "10": 12605.0, + "11": 16803.0, + "12": 15289.0, + "13": 16415.0, + "14": 16182.0, + "15": 15127.0, + "16": 16135.0, + "17": 15282.0, + "18": 15280.0, + "19": 15379.0, + "20": 13642.0, + "21": 14281.0, + "22": 13476.0, + "23": 16892.0, + "24": 13920.0, + "25": 13236.0, + "26": 15256.0, + "27": 15454.0, + "28": 15973.0, + "29": 16892.0, + "30": 14103.0, + "31": 13113.0, + "32": 16067.0, + "33": 16788.0, + "34": 14559.0, + "35": 14974.0, + "36": 15798.0, + "37": 17569.0, + "38": 16172.0, + "39": 17774.0, + "40": 16088.0, + "41": 16616.0, + "42": 17149.0, + "43": 15487.0, + "44": 15110.0, + "45": 16499.0, + "46": 17407.0, + "47": 19502.0, + "48": 16568.0, + "49": 16613.0, + "50": 18892.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 625796096.0, + "2": 625850368.0, + "3": 625987072.0, + "4": 625831424.0, + "5": 625794048.0, + "6": 625789952.0, + "7": 625830912.0, + "8": 625794048.0, + "9": 625861120.0, + "10": 625806848.0, + "11": 625795584.0, + "12": 626022912.0, + "13": 625802240.0, + "14": 625853952.0, + "15": 625796608.0, + "16": 625793024.0, + "17": 625798144.0, + "18": 625802240.0, + "19": 625792000.0, + "20": 625793536.0, + "21": 626690048.0, + "22": 626176000.0, + "23": 626092032.0, + "24": 625794560.0, + "25": 626540544.0, + "26": 625934848.0, + "27": 625799168.0, + "28": 625801728.0, + "29": 625793536.0, + "30": 626191360.0, + "31": 626149376.0, + "32": 626774016.0, + "33": 625792512.0, + "34": 625793024.0, + "35": 625851904.0, + "36": 625809408.0, + "37": 625794048.0, + "38": 625827328.0, + "39": 625865216.0, + "40": 625831936.0, + "41": 626081280.0, + "42": 626046464.0, + "43": 625792000.0, + "44": 625792000.0, + "45": 626266112.0, + "46": 626042880.0, + "47": 625789440.0, + "48": 625905152.0, + "49": 625883648.0, + "50": 626099712.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 1844879360.0, + "2": 2076399104.0, + "3": 2076399104.0, + "4": 2079827456.0, + "5": 2079827456.0, + "6": 2079827456.0, + "7": 2079827456.0, + "8": 2079827456.0, + "9": 2079827456.0, + "10": 2079827456.0, + "11": 2079827456.0, + "12": 2079827456.0, + "13": 2079827456.0, + "14": 2079827456.0, + "15": 2079827456.0, + "16": 2079827456.0, + "17": 2079827456.0, + "18": 2079827456.0, + "19": 2079827456.0, + "20": 2079827456.0, + "21": 2079827456.0, + "22": 2079827456.0, + "23": 2079827456.0, + "24": 2079827456.0, + "25": 2079827456.0, + "26": 2079827456.0, + "27": 2079827456.0, + "28": 2079827456.0, + "29": 2079827456.0, + "30": 2079827456.0, + "31": 2079827456.0, + "32": 2079827456.0, + "33": 2079827456.0, + "34": 2079827456.0, + "35": 2079827456.0, + "36": 2079827456.0, + "37": 2079827456.0, + "38": 2079827456.0, + "39": 2079827456.0, + "40": 2079827456.0, + "41": 2079827456.0, + "42": 2079827456.0, + "43": 2079827456.0, + "44": 2079827456.0, + "45": 2079827456.0, + "46": 2079827456.0, + "47": 2079827456.0, + "48": 2079827456.0, + "49": 2079827456.0, + "50": 2079827456.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 5.62487, + "3": 0.39428, + "4": 0.3711, + "5": 0.36777, + "6": 0.36423, + "7": 0.35408, + "8": 0.35462, + "9": 0.35588, + "10": 0.35204, + "11": 0.35155, + "12": 0.35049, + "13": 0.35818, + "14": 0.35461, + "15": 0.36874, + "16": 0.367, + "17": 0.37423, + "18": 0.36926, + "19": 0.37139, + "20": 0.37109, + "21": 0.37066, + "22": 0.37237, + "23": 0.37636, + "24": 0.37618, + "25": 0.37461, + "26": 0.37622, + "27": 0.37576, + "28": 0.37551, + "29": 0.3765, + "30": 0.3787, + "31": 0.38695, + "32": 0.37235, + "33": 0.37931, + "34": 0.37817, + "35": 0.3749, + "36": 0.37829, + "37": 0.37774, + "38": 0.3755, + "39": 0.37889, + "40": 0.37688, + "41": 0.38007, + "42": 0.37324, + "43": 0.36948, + "44": 0.37523, + "45": 0.37464, + "46": 0.38496, + "47": 0.3737, + "48": 0.37892, + "49": 0.39066, + "50": 0.37612 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..d9441fb83aa --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80397, + "2": 10.81064, + "3": 10.798, + "4": 10.78655, + "5": 10.8299, + "6": 10.85307, + "7": 10.80967, + "8": 10.8015, + "9": 10.82484, + "10": 10.78237, + "11": 10.83805, + "12": 10.84658, + "13": 10.86137, + "14": 10.86451, + "15": 10.83791, + "16": 10.83385, + "17": 10.81249, + "18": 10.84432, + "19": 10.83623, + "20": 10.8168, + "21": 10.83551, + "22": 10.76274, + "23": 10.85573, + "24": 10.8062, + "25": 10.80042, + "26": 10.8143, + "27": 10.82932, + "28": 10.8546, + "29": 10.86594, + "30": 10.79379, + "31": 10.74776, + "32": 10.84932, + "33": 10.83727, + "34": 10.80597, + "35": 10.80259, + "36": 10.79662, + "37": 10.82559, + "38": 10.79231, + "39": 10.84778, + "40": 10.77804, + "41": 10.79895, + "42": 10.81493, + "43": 10.74316, + "44": 10.76656, + "45": 10.76408, + "46": 10.7768, + "47": 10.79908, + "48": 10.77572, + "49": 10.72207, + "50": 10.78609, + "51": 10.78712, + "52": 10.7653, + "53": 10.81235, + "54": 10.79776, + "55": 10.8072, + "56": 10.7562, + "57": 10.71334, + "58": 10.78166, + "59": 10.75039, + "60": 10.72977, + "61": 10.76435, + "62": 10.81299, + "63": 10.69266, + "64": 10.76646, + "65": 10.62484, + "66": 10.75371, + "67": 10.69118, + "68": 10.77122, + "69": 10.76048, + "70": 10.76506, + "71": 10.73497, + "72": 10.72999, + "73": 10.71715, + "74": 10.57819, + "75": 10.68208, + "76": 10.6133, + "77": 10.80786, + "78": 10.73142, + "79": 10.66063, + "80": 10.68014, + "81": 10.69828, + "82": 10.72277, + "83": 10.64104, + "84": 10.66223, + "85": 10.70251, + "86": 10.57982, + "87": 10.69083, + "88": 10.73435, + "89": 10.67796, + "90": 10.74299, + "91": 10.62241, + "92": 10.64011, + "93": 10.56628, + "94": 10.49922, + "95": 10.65675, + "96": 10.65892, + "97": 10.57941, + "98": 10.67242, + "99": 10.47965, + "100": 10.59346 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1158.0, + "2": 1181.0, + "3": 1227.0, + "4": 1263.0, + "5": 1308.0, + "6": 1594.0, + "7": 1345.0, + "8": 1253.0, + "9": 1239.0, + "10": 1232.0, + "11": 1255.0, + "12": 1250.0, + "13": 1516.0, + "14": 1270.0, + "15": 1335.0, + "16": 1260.0, + "17": 1249.0, + "18": 1261.0, + "19": 1099.0, + "20": 1387.0, + "21": 1298.0, + "22": 1286.0, + "23": 1315.0, + "24": 1131.0, + "25": 1157.0, + "26": 1166.0, + "27": 1133.0, + "28": 1243.0, + "29": 1348.0, + "30": 1235.0, + "31": 1090.0, + "32": 1272.0, + "33": 1355.0, + "34": 1161.0, + "35": 1159.0, + "36": 1146.0, + "37": 1222.0, + "38": 1418.0, + "39": 1273.0, + "40": 1198.0, + "41": 1160.0, + "42": 1285.0, + "43": 1094.0, + "44": 1127.0, + "45": 1130.0, + "46": 1183.0, + "47": 1312.0, + "48": 1238.0, + "49": 1068.0, + "50": 1163.0, + "51": 1234.0, + "52": 1284.0, + "53": 1380.0, + "54": 1191.0, + "55": 1099.0, + "56": 1298.0, + "57": 1241.0, + "58": 1203.0, + "59": 1324.0, + "60": 1254.0, + "61": 1120.0, + "62": 1362.0, + "63": 1213.0, + "64": 1330.0, + "65": 1057.0, + "66": 1171.0, + "67": 1208.0, + "68": 1320.0, + "69": 1304.0, + "70": 1122.0, + "71": 1259.0, + "72": 1254.0, + "73": 1203.0, + "74": 1125.0, + "75": 1413.0, + "76": 1217.0, + "77": 1412.0, + "78": 1291.0, + "79": 1020.0, + "80": 1143.0, + "81": 1243.0, + "82": 1154.0, + "83": 1052.0, + "84": 1219.0, + "85": 1360.0, + "86": 1072.0, + "87": 1319.0, + "88": 1347.0, + "89": 1127.0, + "90": 1474.0, + "91": 1140.0, + "92": 1110.0, + "93": 924.0, + "94": 1062.0, + "95": 1147.0, + "96": 1128.0, + "97": 1099.0, + "98": 1191.0, + "99": 1071.0, + "100": 1214.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 993802240.0, + "2": 993814528.0, + "3": 993790464.0, + "4": 993820160.0, + "5": 993728512.0, + "6": 993758208.0, + "7": 993780224.0, + "8": 993795584.0, + "9": 993800704.0, + "10": 993771520.0, + "11": 993752064.0, + "12": 993737216.0, + "13": 993804800.0, + "14": 993734144.0, + "15": 993773056.0, + "16": 993763840.0, + "17": 993744384.0, + "18": 993743872.0, + "19": 993764864.0, + "20": 993719296.0, + "21": 993772544.0, + "22": 993796096.0, + "23": 993748992.0, + "24": 993819136.0, + "25": 993792512.0, + "26": 993776640.0, + "27": 993804800.0, + "28": 993701888.0, + "29": 993717248.0, + "30": 993766912.0, + "31": 993802240.0, + "32": 993705984.0, + "33": 993759744.0, + "34": 993780224.0, + "35": 993740288.0, + "36": 993748480.0, + "37": 993697280.0, + "38": 993763840.0, + "39": 993747456.0, + "40": 993781248.0, + "41": 993752064.0, + "42": 993757696.0, + "43": 993793024.0, + "44": 993833984.0, + "45": 993780736.0, + "46": 993798144.0, + "47": 993789440.0, + "48": 993793024.0, + "49": 993743360.0, + "50": 993754624.0, + "51": 993786368.0, + "52": 993749504.0, + "53": 993812992.0, + "54": 993750528.0, + "55": 993732608.0, + "56": 993777664.0, + "57": 993859584.0, + "58": 993849856.0, + "59": 993761792.0, + "60": 993774592.0, + "61": 993771520.0, + "62": 993786880.0, + "63": 993787904.0, + "64": 993761280.0, + "65": 993792000.0, + "66": 993746432.0, + "67": 993782784.0, + "68": 993783808.0, + "69": 993741824.0, + "70": 993747968.0, + "71": 993736192.0, + "72": 993762304.0, + "73": 993784832.0, + "74": 993809920.0, + "75": 993753088.0, + "76": 993797632.0, + "77": 993720832.0, + "78": 993729536.0, + "79": 993730560.0, + "80": 993763840.0, + "81": 993728000.0, + "82": 993711616.0, + "83": 993772544.0, + "84": 993782784.0, + "85": 993787392.0, + "86": 993804288.0, + "87": 993737728.0, + "88": 993731584.0, + "89": 993755136.0, + "90": 993742848.0, + "91": 993763840.0, + "92": 993774080.0, + "93": 993792000.0, + "94": 993779712.0, + "95": 993723904.0, + "96": 993714688.0, + "97": 993752064.0, + "98": 993708544.0, + "99": 993792000.0, + "100": 993812992.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3074847232.0, + "2": 3373968896.0, + "3": 3373968896.0, + "4": 3378071552.0, + "5": 3378071552.0, + "6": 3378071552.0, + "7": 3378071552.0, + "8": 3378071552.0, + "9": 3378071552.0, + "10": 3378071552.0, + "11": 3378071552.0, + "12": 3378071552.0, + "13": 3378071552.0, + "14": 3378071552.0, + "15": 3378071552.0, + "16": 3378071552.0, + "17": 3378071552.0, + "18": 3378071552.0, + "19": 3378071552.0, + "20": 3378071552.0, + "21": 3378071552.0, + "22": 3378071552.0, + "23": 3378071552.0, + "24": 3378894848.0, + "25": 3378894848.0, + "26": 3378894848.0, + "27": 3378894848.0, + "28": 3378894848.0, + "29": 3378894848.0, + "30": 3378894848.0, + "31": 3378894848.0, + "32": 3378894848.0, + "33": 3378894848.0, + "34": 3378894848.0, + "35": 3378894848.0, + "36": 3378894848.0, + "37": 3378894848.0, + "38": 3378894848.0, + "39": 3378894848.0, + "40": 3378894848.0, + "41": 3378894848.0, + "42": 3378894848.0, + "43": 3378894848.0, + "44": 3392753152.0, + "45": 3392753152.0, + "46": 3392753152.0, + "47": 3392753152.0, + "48": 3392753152.0, + "49": 3392753152.0, + "50": 3392753152.0, + "51": 3392753152.0, + "52": 3392753152.0, + "53": 3392753152.0, + "54": 3392753152.0, + "55": 3392753152.0, + "56": 3392753152.0, + "57": 3407671808.0, + "58": 3407671808.0, + "59": 3407671808.0, + "60": 3407671808.0, + "61": 3407671808.0, + "62": 3407671808.0, + "63": 3407671808.0, + "64": 3407671808.0, + "65": 3407671808.0, + "66": 3407671808.0, + "67": 3407671808.0, + "68": 3407671808.0, + "69": 3407671808.0, + "70": 3407671808.0, + "71": 3407671808.0, + "72": 3407671808.0, + "73": 3407671808.0, + "74": 3407671808.0, + "75": 3407671808.0, + "76": 3407671808.0, + "77": 3407671808.0, + "78": 3407671808.0, + "79": 3407671808.0, + "80": 3407671808.0, + "81": 3407671808.0, + "82": 3407671808.0, + "83": 3407671808.0, + "84": 3407671808.0, + "85": 3407671808.0, + "86": 3407671808.0, + "87": 3407671808.0, + "88": 3407671808.0, + "89": 3407671808.0, + "90": 3407671808.0, + "91": 3407671808.0, + "92": 3407671808.0, + "93": 3407671808.0, + "94": 3407671808.0, + "95": 3407671808.0, + "96": 3407671808.0, + "97": 3407671808.0, + "98": 3407671808.0, + "99": 3407671808.0, + "100": 3407671808.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 3.79579, + "3": 0.25267, + "4": 0.22623, + "5": 0.22831, + "6": 0.57471, + "7": 0.22109, + "8": 0.21732, + "9": 0.2127, + "10": 0.20981, + "11": 0.21771, + "12": 0.23313, + "13": 0.20775, + "14": 0.19946, + "15": 0.21125, + "16": 0.2099, + "17": 0.20543, + "18": 0.19972, + "19": 0.20265, + "20": 0.20005, + "21": 0.20188, + "22": 0.19675, + "23": 0.19822, + "24": 0.19828, + "25": 0.19827, + "26": 0.19789, + "27": 0.20238, + "28": 0.19366, + "29": 0.19297, + "30": 0.19521, + "31": 0.19886, + "32": 0.19176, + "33": 0.19628, + "34": 0.19156, + "35": 0.19683, + "36": 0.19061, + "37": 0.19031, + "38": 0.19383, + "39": 0.1966, + "40": 0.19152, + "41": 0.18691, + "42": 0.1917, + "43": 0.20258, + "44": 0.19552, + "45": 0.20203, + "46": 0.18769, + "47": 0.18872, + "48": 0.18493, + "49": 0.18884, + "50": 0.18824, + "51": 0.20579, + "52": 0.18818, + "53": 0.18827, + "54": 0.19298, + "55": 0.57299, + "56": 0.18813, + "57": 0.18557, + "58": 0.18597, + "59": 0.18577, + "60": 0.18756, + "61": 0.18972, + "62": 0.18872, + "63": 0.18937, + "64": 0.1888, + "65": 0.19262, + "66": 0.1879, + "67": 0.18498, + "68": 0.18535, + "69": 0.19492, + "70": 0.1923, + "71": 0.18822, + "72": 0.19191, + "73": 0.19457, + "74": 0.19765, + "75": 0.19091, + "76": 0.73064, + "77": 0.19543, + "78": 0.19034, + "79": 0.18715, + "80": 0.19339, + "81": 0.19135, + "82": 0.18703, + "83": 0.19082, + "84": 0.18783, + "85": 0.1926, + "86": 0.19556, + "87": 0.19127, + "88": 0.19028, + "89": 0.56083, + "90": 0.19223, + "91": 0.18622, + "92": 0.18536, + "93": 0.19063, + "94": 0.18804, + "95": 0.18711, + "96": 0.1883, + "97": 0.19006, + "98": 0.18897, + "99": 0.60361, + "100": 0.19278 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..35841a4eaa1 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80397, + "2": 10.81064, + "3": 10.79794, + "4": 10.78736, + "5": 10.82943, + "6": 10.85276, + "7": 10.80881, + "8": 10.79936, + "9": 10.82143, + "10": 10.77719, + "11": 10.8324, + "12": 10.8353, + "13": 10.84919, + "14": 10.85282, + "15": 10.79747, + "16": 10.78882, + "17": 10.75342, + "18": 10.78784, + "19": 10.77563, + "20": 10.70852, + "21": 10.6913, + "22": 10.53886, + "23": 10.69977, + "24": 10.58969, + "25": 10.54168, + "26": 10.60115, + "27": 10.61421, + "28": 10.59031, + "29": 10.60749, + "30": 10.38665, + "31": 10.13235, + "32": 10.49407, + "33": 10.47454, + "34": 10.23691, + "35": 10.28682, + "36": 10.26871, + "37": 10.38314, + "38": 10.23703, + "39": 10.43401, + "40": 10.1257, + "41": 10.17238, + "42": 10.24995, + "43": 9.85773, + "44": 9.98944, + "45": 9.87376, + "46": 9.84256, + "47": 10.1623, + "48": 9.89144, + "49": 9.57738, + "50": 9.96171, + "51": 9.88785, + "52": 9.76989, + "53": 10.10483, + "54": 9.99665, + "55": 9.92216, + "56": 9.67616, + "57": 9.51879, + "58": 9.89053, + "59": 9.63068, + "60": 9.55149, + "61": 9.72264, + "62": 10.0414, + "63": 9.43971, + "64": 9.8184, + "65": 8.98176, + "66": 9.75925, + "67": 9.39746, + "68": 9.83254, + "69": 9.81649, + "70": 9.75965, + "71": 9.66402, + "72": 9.63516, + "73": 9.54388, + "74": 9.00071, + "75": 9.465, + "76": 9.13889, + "77": 10.09535, + "78": 9.75814, + "79": 9.41614, + "80": 9.44749, + "81": 9.5168, + "82": 9.73156, + "83": 9.36737, + "84": 9.45017, + "85": 9.65534, + "86": 9.10891, + "87": 9.62042, + "88": 9.79408, + "89": 9.64391, + "90": 9.85314, + "91": 9.39297, + "92": 9.39817, + "93": 9.13664, + "94": 8.86865, + "95": 9.55719, + "96": 9.56146, + "97": 9.33062, + "98": 9.69677, + "99": 8.93672, + "100": 9.43355 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1158.0, + "2": 1181.0, + "3": 1239.0, + "4": 1193.0, + "5": 1380.0, + "6": 1501.0, + "7": 1391.0, + "8": 1239.0, + "9": 1272.0, + "10": 1225.0, + "11": 1337.0, + "12": 1167.0, + "13": 1452.0, + "14": 1254.0, + "15": 1269.0, + "16": 1347.0, + "17": 1234.0, + "18": 1202.0, + "19": 1183.0, + "20": 1141.0, + "21": 1236.0, + "22": 982.0, + "23": 1234.0, + "24": 1135.0, + "25": 1073.0, + "26": 1087.0, + "27": 1008.0, + "28": 1166.0, + "29": 1127.0, + "30": 1094.0, + "31": 929.0, + "32": 1176.0, + "33": 1123.0, + "34": 1084.0, + "35": 1110.0, + "36": 1053.0, + "37": 1208.0, + "38": 1233.0, + "39": 1210.0, + "40": 1229.0, + "41": 1217.0, + "42": 1317.0, + "43": 1212.0, + "44": 1144.0, + "45": 1374.0, + "46": 1249.0, + "47": 1163.0, + "48": 1176.0, + "49": 1335.0, + "50": 1172.0, + "51": 1345.0, + "52": 1334.0, + "53": 1388.0, + "54": 1316.0, + "55": 1263.0, + "56": 1320.0, + "57": 1143.0, + "58": 1050337.0, + "59": 1629.0, + "60": 58535.0, + "61": 1389.0, + "62": 1050420.0, + "63": 1302.0, + "64": 77146.0, + "65": 1058670.0, + "66": 1404.0, + "67": 59730.0, + "68": 50392.0, + "69": 2394.0, + "70": 108916.0, + "71": 46411.0, + "72": 2099307.0, + "73": 98574.0, + "74": 1106325.0, + "75": 4199.0, + "76": 1157870.0, + "77": 2145771.0, + "78": 1106899.0, + "79": 57707.0, + "80": 1105606.0, + "81": 1059706.0, + "82": 96668.0, + "83": 2099291.0, + "84": 1106935.0, + "85": 2099569.0, + "86": 1097473.0, + "87": 2099736.0, + "88": 2154950.0, + "89": 1154961.0, + "90": 3148620.0, + "91": 1157803.0, + "92": 3148576.0, + "93": 3148494.0, + "94": 1160976.0, + "95": 3148697.0, + "96": 3148663.0, + "97": 2147830.0, + "98": 3148776.0, + "99": 2099539.0, + "100": 3148590.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 788258816.0, + "2": 788271104.0, + "3": 788248064.0, + "4": 788275200.0, + "5": 788183552.0, + "6": 788212736.0, + "7": 788235264.0, + "8": 788251648.0, + "9": 788255232.0, + "10": 788226560.0, + "11": 788207104.0, + "12": 788192256.0, + "13": 788256256.0, + "14": 788187136.0, + "15": 788221952.0, + "16": 788214272.0, + "17": 788195328.0, + "18": 788191744.0, + "19": 788210176.0, + "20": 788163072.0, + "21": 788207616.0, + "22": 788235264.0, + "23": 788186112.0, + "24": 788249088.0, + "25": 788213248.0, + "26": 788204032.0, + "27": 788216832.0, + "28": 788117504.0, + "29": 788125696.0, + "30": 788169216.0, + "31": 788190720.0, + "32": 788118016.0, + "33": 788142592.0, + "34": 788156928.0, + "35": 788133376.0, + "36": 788130816.0, + "37": 788082688.0, + "38": 788156416.0, + "39": 788091904.0, + "40": 788120576.0, + "41": 788122112.0, + "42": 788104192.0, + "43": 788170752.0, + "44": 788189696.0, + "45": 788140032.0, + "46": 788148224.0, + "47": 788086272.0, + "48": 788128768.0, + "49": 788078080.0, + "50": 788078592.0, + "51": 788088832.0, + "52": 788065280.0, + "53": 788091392.0, + "54": 788059648.0, + "55": 788009472.0, + "56": 788060672.0, + "57": 788129792.0, + "58": 788124672.0, + "59": 788038656.0, + "60": 788026368.0, + "61": 788004352.0, + "62": 788007936.0, + "63": 788059136.0, + "64": 787972096.0, + "65": 788070400.0, + "66": 788031488.0, + "67": 788046336.0, + "68": 788037632.0, + "69": 787970560.0, + "70": 787992064.0, + "71": 788023808.0, + "72": 788022272.0, + "73": 788058624.0, + "74": 788075520.0, + "75": 788037632.0, + "76": 788094976.0, + "77": 787966464.0, + "78": 787980288.0, + "79": 788018176.0, + "80": 788026880.0, + "81": 787994624.0, + "82": 787986944.0, + "83": 788061696.0, + "84": 787999744.0, + "85": 787995648.0, + "86": 788012544.0, + "87": 787939328.0, + "88": 787957760.0, + "89": 787977728.0, + "90": 787927552.0, + "91": 787998720.0, + "92": 788026368.0, + "93": 788039680.0, + "94": 788032512.0, + "95": 788007424.0, + "96": 787978240.0, + "97": 788036608.0, + "98": 787984384.0, + "99": 788088320.0, + "100": 788081664.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2992141824.0, + "2": 3170010624.0, + "3": 3170010624.0, + "4": 3172966400.0, + "5": 3172966400.0, + "6": 3172966400.0, + "7": 3172966400.0, + "8": 3172966400.0, + "9": 3172966400.0, + "10": 3172966400.0, + "11": 3172966400.0, + "12": 3172966400.0, + "13": 3172966400.0, + "14": 3172966400.0, + "15": 3172966400.0, + "16": 3172966400.0, + "17": 3172966400.0, + "18": 3172966400.0, + "19": 3172966400.0, + "20": 3172966400.0, + "21": 3172966400.0, + "22": 3172966400.0, + "23": 3172966400.0, + "24": 3172966400.0, + "25": 3172966400.0, + "26": 3172966400.0, + "27": 3172966400.0, + "28": 3172966400.0, + "29": 3172966400.0, + "30": 3172966400.0, + "31": 3172966400.0, + "32": 3172966400.0, + "33": 3172966400.0, + "34": 3172966400.0, + "35": 3172966400.0, + "36": 3172966400.0, + "37": 3172966400.0, + "38": 3172966400.0, + "39": 3172966400.0, + "40": 3172966400.0, + "41": 3172966400.0, + "42": 3172966400.0, + "43": 3172966400.0, + "44": 3172966400.0, + "45": 3172966400.0, + "46": 3172966400.0, + "47": 3172966400.0, + "48": 3172966400.0, + "49": 3172966400.0, + "50": 3172966400.0, + "51": 3172966400.0, + "52": 3172966400.0, + "53": 3172966400.0, + "54": 3172966400.0, + "55": 3172966400.0, + "56": 3172966400.0, + "57": 3172966400.0, + "58": 3172966400.0, + "59": 3172966400.0, + "60": 3172966400.0, + "61": 3172966400.0, + "62": 3172966400.0, + "63": 3172966400.0, + "64": 3172966400.0, + "65": 3172966400.0, + "66": 3172966400.0, + "67": 3172966400.0, + "68": 3172966400.0, + "69": 3172966400.0, + "70": 3172966400.0, + "71": 3172966400.0, + "72": 3172966400.0, + "73": 3172966400.0, + "74": 3172966400.0, + "75": 3172966400.0, + "76": 3172966400.0, + "77": 3172966400.0, + "78": 3172966400.0, + "79": 3172966400.0, + "80": 3172966400.0, + "81": 3172966400.0, + "82": 3172966400.0, + "83": 3172966400.0, + "84": 3172966400.0, + "85": 3172966400.0, + "86": 3172966400.0, + "87": 3172966400.0, + "88": 3172966400.0, + "89": 3172966400.0, + "90": 3172966400.0, + "91": 3172966400.0, + "92": 3172966400.0, + "93": 3172966400.0, + "94": 3172966400.0, + "95": 3172966400.0, + "96": 3172966400.0, + "97": 3172966400.0, + "98": 3172966400.0, + "99": 3172966400.0, + "100": 3172966400.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 4.24581, + "3": 0.22984, + "4": 0.19901, + "5": 0.20013, + "6": 0.19557, + "7": 0.19148, + "8": 0.18346, + "9": 0.18886, + "10": 0.18315, + "11": 0.18065, + "12": 0.18158, + "13": 0.17863, + "14": 0.17669, + "15": 0.16949, + "16": 0.1813, + "17": 0.17239, + "18": 0.16839, + "19": 0.17197, + "20": 0.18078, + "21": 0.17159, + "22": 0.17182, + "23": 0.17032, + "24": 0.16416, + "25": 0.16192, + "26": 0.17125, + "27": 0.15863, + "28": 0.16459, + "29": 0.16007, + "30": 0.16611, + "31": 0.16404, + "32": 0.15794, + "33": 0.16011, + "34": 0.15482, + "35": 0.15384, + "36": 0.16644, + "37": 0.15786, + "38": 0.15755, + "39": 0.15631, + "40": 0.15691, + "41": 0.15405, + "42": 0.14989, + "43": 0.1609, + "44": 0.15219, + "45": 0.15611, + "46": 0.1513, + "47": 0.15678, + "48": 0.14912, + "49": 0.14848, + "50": 0.15182, + "51": 0.16313, + "52": 0.14839, + "53": 0.14122, + "54": 0.14422, + "55": 0.14712, + "56": 0.14693, + "57": 0.14795, + "58": 0.14977, + "59": 0.15359, + "60": 0.14668, + "61": 0.15225, + "62": 0.14521, + "63": 0.14464, + "64": 0.14763, + "65": 0.14539, + "66": 0.14694, + "67": 0.1474, + "68": 0.14336, + "69": 0.14734, + "70": 0.14392, + "71": 0.14814, + "72": 0.15208, + "73": 0.15012, + "74": 0.14617, + "75": 0.14599, + "76": 0.14537, + "77": 0.15575, + "78": 0.15044, + "79": 0.68569, + "80": 0.15145, + "81": 0.1455, + "82": 0.14662, + "83": 0.14886, + "84": 0.14582, + "85": 0.14802, + "86": 0.1466, + "87": 0.14632, + "88": 0.14515, + "89": 0.14994, + "90": 0.1445, + "91": 0.14773, + "92": 0.14812, + "93": 0.14796, + "94": 0.51237, + "95": 0.15138, + "96": 0.15025, + "97": 0.14525, + "98": 0.1449, + "99": 0.1508, + "100": 0.14531 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..fe8d3f78926 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.80397, + "2": 10.81064, + "3": 10.79812, + "4": 10.78677, + "5": 10.82981, + "6": 10.85273, + "7": 10.80976, + "8": 10.80152, + "9": 10.82476, + "10": 10.78235, + "11": 10.83837, + "12": 10.84645, + "13": 10.86121, + "14": 10.86494, + "15": 10.83809, + "16": 10.8346, + "17": 10.8121, + "18": 10.84437, + "19": 10.83592, + "20": 10.81732, + "21": 10.83519, + "22": 10.76256, + "23": 10.85511, + "24": 10.80666, + "25": 10.80025, + "26": 10.81426, + "27": 10.82995, + "28": 10.855, + "29": 10.86565, + "30": 10.79384, + "31": 10.74771, + "32": 10.84943, + "33": 10.83771, + "34": 10.80572, + "35": 10.80265, + "36": 10.79622, + "37": 10.82514, + "38": 10.79237, + "39": 10.84811, + "40": 10.77883, + "41": 10.79922, + "42": 10.81563, + "43": 10.74376, + "44": 10.76683, + "45": 10.76467, + "46": 10.77697, + "47": 10.79973, + "48": 10.77586, + "49": 10.72215, + "50": 10.78584, + "51": 10.78731, + "52": 10.7657, + "53": 10.81241, + "54": 10.79761, + "55": 10.80688, + "56": 10.75611, + "57": 10.71341, + "58": 10.78104, + "59": 10.7507, + "60": 10.72941, + "61": 10.76448, + "62": 10.8119, + "63": 10.69242, + "64": 10.76661, + "65": 10.62474, + "66": 10.75342, + "67": 10.69134, + "68": 10.77079, + "69": 10.76029, + "70": 10.76451, + "71": 10.73531, + "72": 10.72951, + "73": 10.7174, + "74": 10.57782, + "75": 10.68245, + "76": 10.61342, + "77": 10.80749, + "78": 10.7321, + "79": 10.66078, + "80": 10.68008, + "81": 10.69796, + "82": 10.72301, + "83": 10.6413, + "84": 10.6619, + "85": 10.70249, + "86": 10.58035, + "87": 10.69015, + "88": 10.73441, + "89": 10.67777, + "90": 10.74269, + "91": 10.62186, + "92": 10.63964, + "93": 10.56627, + "94": 10.49913, + "95": 10.65738, + "96": 10.65873, + "97": 10.57872, + "98": 10.6722, + "99": 10.4802, + "100": 10.59334 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1158.0, + "2": 1181.0, + "3": 1248.0, + "4": 1219.0, + "5": 1372.0, + "6": 1529.0, + "7": 1362.0, + "8": 1179.0, + "9": 1276.0, + "10": 1234.0, + "11": 1281.0, + "12": 1249.0, + "13": 1386.0, + "14": 1213.0, + "15": 1215.0, + "16": 1299.0, + "17": 1242.0, + "18": 1233.0, + "19": 1167.0, + "20": 1392.0, + "21": 1264.0, + "22": 1289.0, + "23": 1336.0, + "24": 1168.0, + "25": 1170.0, + "26": 1207.0, + "27": 1192.0, + "28": 1327.0, + "29": 1354.0, + "30": 1250.0, + "31": 1110.0, + "32": 1331.0, + "33": 1340.0, + "34": 1250.0, + "35": 1105.0, + "36": 1138.0, + "37": 1265.0, + "38": 1375.0, + "39": 1243.0, + "40": 1306.0, + "41": 1154.0, + "42": 1251.0, + "43": 1122.0, + "44": 1139.0, + "45": 1122.0, + "46": 1203.0, + "47": 1405.0, + "48": 1282.0, + "49": 1167.0, + "50": 1166.0, + "51": 1249.0, + "52": 1320.0, + "53": 1340.0, + "54": 1232.0, + "55": 1103.0, + "56": 1275.0, + "57": 1194.0, + "58": 1259.0, + "59": 1283.0, + "60": 1265.0, + "61": 1124.0, + "62": 1349.0, + "63": 1132.0, + "64": 1272.0, + "65": 1017.0, + "66": 1174.0, + "67": 1242.0, + "68": 1291.0, + "69": 1295.0, + "70": 1143.0, + "71": 1148.0, + "72": 1266.0, + "73": 1199.0, + "74": 1133.0, + "75": 1346.0, + "76": 1224.0, + "77": 1329.0, + "78": 1256.0, + "79": 997.0, + "80": 1093.0, + "81": 1204.0, + "82": 1213.0, + "83": 1128.0, + "84": 1228.0, + "85": 1316.0, + "86": 1101.0, + "87": 1278.0, + "88": 1286.0, + "89": 1163.0, + "90": 1415.0, + "91": 1248.0, + "92": 1137.0, + "93": 912.0, + "94": 985.0, + "95": 1097.0, + "96": 1087.0, + "97": 1098.0, + "98": 1170.0, + "99": 1047.0, + "100": 1205.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1095588352.0, + "2": 1095600640.0, + "3": 1095576576.0, + "4": 1095606272.0, + "5": 1095514624.0, + "6": 1095542272.0, + "7": 1095563776.0, + "8": 1095580160.0, + "9": 1095585792.0, + "10": 1095554048.0, + "11": 1095538176.0, + "12": 1095523328.0, + "13": 1095589888.0, + "14": 1095519744.0, + "15": 1095557120.0, + "16": 1095548928.0, + "17": 1095531008.0, + "18": 1095528448.0, + "19": 1095549440.0, + "20": 1095504384.0, + "21": 1095561728.0, + "22": 1095583232.0, + "23": 1095534592.0, + "24": 1095604736.0, + "25": 1095577088.0, + "26": 1095565824.0, + "27": 1095591424.0, + "28": 1095485952.0, + "29": 1095502848.0, + "30": 1095552512.0, + "31": 1095588352.0, + "32": 1095491072.0, + "33": 1095547392.0, + "34": 1095568384.0, + "35": 1095527424.0, + "36": 1095533568.0, + "37": 1095482880.0, + "38": 1095552000.0, + "39": 1095532544.0, + "40": 1095567360.0, + "41": 1095537152.0, + "42": 1095543296.0, + "43": 1095581184.0, + "44": 1095620096.0, + "45": 1095569408.0, + "46": 1095584768.0, + "47": 1095573504.0, + "48": 1095577088.0, + "49": 1095530496.0, + "50": 1095540736.0, + "51": 1095570944.0, + "52": 1095538176.0, + "53": 1095597568.0, + "54": 1095536640.0, + "55": 1095517184.0, + "56": 1095566848.0, + "57": 1095645696.0, + "58": 1095634944.0, + "59": 1095548928.0, + "60": 1095562752.0, + "61": 1095553536.0, + "62": 1095572480.0, + "63": 1095573504.0, + "64": 1095550464.0, + "65": 1095578112.0, + "66": 1095531008.0, + "67": 1095568896.0, + "68": 1095566848.0, + "69": 1095527424.0, + "70": 1095532032.0, + "71": 1095520768.0, + "72": 1095548928.0, + "73": 1095569920.0, + "74": 1095596032.0, + "75": 1095538688.0, + "76": 1095584768.0, + "77": 1095507968.0, + "78": 1095514624.0, + "79": 1095515648.0, + "80": 1095551488.0, + "81": 1095513600.0, + "82": 1095498240.0, + "83": 1095558656.0, + "84": 1095569408.0, + "85": 1095576064.0, + "86": 1095590400.0, + "87": 1095523840.0, + "88": 1095517696.0, + "89": 1095539712.0, + "90": 1095528960.0, + "91": 1095550976.0, + "92": 1095561216.0, + "93": 1095579136.0, + "94": 1095564288.0, + "95": 1095510528.0, + "96": 1095502336.0, + "97": 1095537152.0, + "98": 1095496192.0, + "99": 1095577600.0, + "100": 1095598592.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3125957632.0, + "2": 3477050368.0, + "3": 3477050368.0, + "4": 3481636352.0, + "5": 3481636352.0, + "6": 3481636352.0, + "7": 3481636352.0, + "8": 3481636352.0, + "9": 3481636352.0, + "10": 3481636352.0, + "11": 3481636352.0, + "12": 3481636352.0, + "13": 3481636352.0, + "14": 3481636352.0, + "15": 3481636352.0, + "16": 3481636352.0, + "17": 3481636352.0, + "18": 3481636352.0, + "19": 3481636352.0, + "20": 3481636352.0, + "21": 3481636352.0, + "22": 3481636352.0, + "23": 3481636352.0, + "24": 3482527744.0, + "25": 3482527744.0, + "26": 3482527744.0, + "27": 3482527744.0, + "28": 3482527744.0, + "29": 3482527744.0, + "30": 3482527744.0, + "31": 3482527744.0, + "32": 3482527744.0, + "33": 3482527744.0, + "34": 3482527744.0, + "35": 3482527744.0, + "36": 3482527744.0, + "37": 3482527744.0, + "38": 3482527744.0, + "39": 3482527744.0, + "40": 3482527744.0, + "41": 3482527744.0, + "42": 3482527744.0, + "43": 3482527744.0, + "44": 3495770112.0, + "45": 3495770112.0, + "46": 3495770112.0, + "47": 3495770112.0, + "48": 3495770112.0, + "49": 3495770112.0, + "50": 3495770112.0, + "51": 3495770112.0, + "52": 3495770112.0, + "53": 3495770112.0, + "54": 3495770112.0, + "55": 3495770112.0, + "56": 3495770112.0, + "57": 3505988608.0, + "58": 3505988608.0, + "59": 3505988608.0, + "60": 3505988608.0, + "61": 3505988608.0, + "62": 3505988608.0, + "63": 3505988608.0, + "64": 3505988608.0, + "65": 3505988608.0, + "66": 3505988608.0, + "67": 3505988608.0, + "68": 3505988608.0, + "69": 3505988608.0, + "70": 3505988608.0, + "71": 3505988608.0, + "72": 3505988608.0, + "73": 3505988608.0, + "74": 3505988608.0, + "75": 3505988608.0, + "76": 3505988608.0, + "77": 3505988608.0, + "78": 3505988608.0, + "79": 3505988608.0, + "80": 3505988608.0, + "81": 3505988608.0, + "82": 3505988608.0, + "83": 3505988608.0, + "84": 3505988608.0, + "85": 3505988608.0, + "86": 3505988608.0, + "87": 3505988608.0, + "88": 3505988608.0, + "89": 3505988608.0, + "90": 3505988608.0, + "91": 3505988608.0, + "92": 3505988608.0, + "93": 3505988608.0, + "94": 3505988608.0, + "95": 3505988608.0, + "96": 3505988608.0, + "97": 3505988608.0, + "98": 3505988608.0, + "99": 3505988608.0, + "100": 3505988608.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 4.71174, + "3": 0.47502, + "4": 0.44931, + "5": 0.44277, + "6": 0.44844, + "7": 0.45785, + "8": 0.44209, + "9": 0.43757, + "10": 0.42772, + "11": 0.44315, + "12": 0.42725, + "13": 0.42666, + "14": 0.41928, + "15": 0.42831, + "16": 0.42799, + "17": 0.42051, + "18": 0.41469, + "19": 0.41876, + "20": 0.41842, + "21": 0.43095, + "22": 0.41003, + "23": 0.41066, + "24": 0.41091, + "25": 0.40849, + "26": 0.4098, + "27": 0.41447, + "28": 0.4098, + "29": 0.40395, + "30": 0.41016, + "31": 0.41347, + "32": 0.40916, + "33": 0.41299, + "34": 0.40596, + "35": 0.40696, + "36": 0.40868, + "37": 0.40718, + "38": 0.40736, + "39": 0.40604, + "40": 0.40127, + "41": 0.4, + "42": 0.40197, + "43": 0.40902, + "44": 0.40712, + "45": 0.4098, + "46": 0.40168, + "47": 0.40487, + "48": 0.40622, + "49": 0.4089, + "50": 0.40406, + "51": 0.41118, + "52": 0.40412, + "53": 0.40027, + "54": 0.40192, + "55": 0.39782, + "56": 0.39731, + "57": 0.39836, + "58": 0.40128, + "59": 0.39958, + "60": 0.39863, + "61": 0.78712, + "62": 0.39887, + "63": 0.39967, + "64": 0.40024, + "65": 0.39891, + "66": 0.40058, + "67": 0.80982, + "68": 0.39889, + "69": 0.39895, + "70": 0.40201, + "71": 0.39871, + "72": 0.39819, + "73": 0.40638, + "74": 0.40241, + "75": 0.39867, + "76": 0.40192, + "77": 0.4032, + "78": 0.39871, + "79": 0.96252, + "80": 0.39811, + "81": 0.40176, + "82": 0.39856, + "83": 0.40217, + "84": 0.3966, + "85": 0.40212, + "86": 0.40144, + "87": 0.39779, + "88": 0.3989, + "89": 0.39982, + "90": 0.40291, + "91": 0.40052, + "92": 0.39772, + "93": 0.40147, + "94": 0.40072, + "95": 0.40007, + "96": 0.40232, + "97": 0.40777, + "98": 0.4002, + "99": 0.39995, + "100": 0.39879 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..64e256f8b57 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_gb200.json @@ -0,0 +1,344 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.02499, + "2": 11.05412, + "3": 10.03918, + "4": 9.80215, + "5": 13.60005, + "6": 8.54454, + "7": 9.77444, + "8": 8.35233, + "9": 7.88788, + "10": 7.14039, + "11": 9.06955, + "12": 9.20099, + "13": 8.15607, + "14": 8.36221, + "15": 8.43013, + "16": 8.48001, + "17": 8.52462, + "18": 7.90076, + "19": 8.35376, + "20": 7.90482, + "21": 8.17608, + "22": 7.55176, + "23": 8.27889, + "24": 7.65732, + "25": 8.43063, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 47165208.0, + "2": 46898016.0, + "3": 134483200.0, + "4": 98911144.0, + "5": 405652352.0, + "6": 510869088.0, + "7": 779007104.0, + "8": 538338816.0, + "9": 365348032.0, + "10": 620817088.0, + "11": 502895552.0, + "12": 572091776.0, + "13": 714972800.0, + "14": 748130816.0, + "15": 709938432.0, + "16": 691583488.0, + "17": 963168256.0, + "18": 953453952.0, + "19": 713408000.0, + "20": 919014656.0, + "21": 899637952.0, + "22": 688944512.0, + "23": 856034560.0, + "24": 858768064.0, + "25": 818025472.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5283603968.0, + "2": 5283808768.0, + "3": 5283504640.0, + "4": 5283707392.0, + "5": 5283910144.0, + "6": 5284112896.0, + "7": 5284315648.0, + "8": 5284518400.0, + "9": 5284721152.0, + "10": 5284923904.0, + "11": 5285126656.0, + "12": 5285329408.0, + "13": 5285532160.0, + "14": 5285734912.0, + "15": 5285937664.0, + "16": 5286140416.0, + "17": 5286343168.0, + "18": 5286545920.0, + "19": 5286748672.0, + "20": 5286951424.0, + "21": 5287154176.0, + "22": 5287356928.0, + "23": 5287559680.0, + "24": 5287762432.0, + "25": 5287965184.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5283606528.0, + "2": 8265524736.0, + "3": 8265524736.0, + "4": 8265524736.0, + "5": 8265524736.0, + "6": 8265524736.0, + "7": 8265524736.0, + "8": 8265524736.0, + "9": 8265524736.0, + "10": 8276976128.0, + "11": 8276976128.0, + "12": 8276976128.0, + "13": 8276976128.0, + "14": 8276976128.0, + "15": 8276976128.0, + "16": 8276976128.0, + "17": 8276976128.0, + "18": 8276976128.0, + "19": 8276976128.0, + "20": 8276976128.0, + "21": 8276976128.0, + "22": 8285769216.0, + "23": 8285769216.0, + "24": 8285769216.0, + "25": 8285769216.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.03308, + "2": 11.06518, + "3": 10.81759, + "4": 10.53818, + "5": 10.71366, + "6": 8.61304, + "7": 10.1377, + "8": 8.29237, + "9": 7.71086, + "10": 6.91516, + "11": 9.19783, + "12": 9.26769, + "13": 8.06484, + "14": 8.2784, + "15": 8.36908, + "16": 8.41495, + "17": 8.38655, + "18": 7.69044, + "19": 8.28621, + "20": 7.79896, + "21": 8.09324, + "22": 7.49223, + "23": 8.14261, + "24": 7.5863, + "25": 8.37107, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 25.92832, + "3": 2.8263, + "4": 1.73655, + "5": 2.05123, + "6": 2.39425, + "7": 2.15639, + "8": 0.92138, + "9": 0.94247, + "10": 1.64642, + "11": 0.96645, + "12": 0.91866, + "13": 0.94198, + "14": 0.9255, + "15": 0.95577, + "16": 0.92425, + "17": 0.94137, + "18": 0.93111, + "19": 0.89952, + "20": 0.90021, + "21": 0.91289, + "22": 0.93437, + "23": 0.96363, + "24": 1.00045, + "25": 0.96782, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..dc46db36c72 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_gb200.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.0146, + "2": 11.04991, + "3": 10.14357, + "4": 9.67761, + "5": 9.94473, + "6": 9.95632, + "7": 9.92367, + "8": 8.83165, + "9": 8.42103, + "10": 7.83364, + "11": 10.81778, + "12": 10.35014, + "13": 8.66833, + "14": 9.13058, + "15": 9.2484, + "16": 9.32149, + "17": 9.20232, + "18": 8.73719, + "19": 9.32726, + "20": 8.88552, + "21": 9.10111, + "22": 8.53259, + "23": 8.96918, + "24": 8.67428, + "25": 9.1617, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 47167824.0, + "2": 46900664.0, + "3": 96750344.0, + "4": 246765024.0, + "5": 817718912.0, + "6": 592653248.0, + "7": 1018015232.0, + "8": 657925824.0, + "9": 711406848.0, + "10": 576785472.0, + "11": 704207488.0, + "12": 619264576.0, + "13": 718118144.0, + "14": 656897024.0, + "15": 621830912.0, + "16": 729345984.0, + "17": 831063744.0, + "18": 1025804096.0, + "19": 832938368.0, + "20": 1003945088.0, + "21": 830430208.0, + "22": 846188736.0, + "23": 1035339456.0, + "24": 1003472384.0, + "25": 1019352320.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4313446912.0, + "2": 4313448448.0, + "3": 4313448448.0, + "4": 4313448448.0, + "5": 4313448448.0, + "6": 4313448448.0, + "7": 4313448448.0, + "8": 4313448448.0, + "9": 4313448448.0, + "10": 4313448448.0, + "11": 4313448448.0, + "12": 4313448448.0, + "13": 4313448448.0, + "14": 4313448448.0, + "15": 4313448448.0, + "16": 4313448448.0, + "17": 4313448448.0, + "18": 4313448448.0, + "19": 4313448448.0, + "20": 4313448448.0, + "21": 4313448448.0, + "22": 4313448448.0, + "23": 4313448448.0, + "24": 4313448448.0, + "25": 4313448448.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4313449472.0, + "2": 7058323968.0, + "3": 7093507072.0, + "4": 7103545856.0, + "5": 7103545856.0, + "6": 7103545856.0, + "7": 7103545856.0, + "8": 7103545856.0, + "9": 7103545856.0, + "10": 7103545856.0, + "11": 7105847296.0, + "12": 7107386368.0, + "13": 7107386368.0, + "14": 7107386368.0, + "15": 7107386368.0, + "16": 7107386368.0, + "17": 7107386368.0, + "18": 7107386368.0, + "19": 7107386368.0, + "20": 7107386368.0, + "21": 7107386368.0, + "22": 7108604416.0, + "23": 7108922368.0, + "24": 7109204992.0, + "25": 7109204992.0, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": "nan", + "2": 26.62047, + "3": 1.74298, + "4": 2.19744, + "5": 2.54081, + "6": 0.94207, + "7": 2.41564, + "8": 0.89613, + "9": 0.88788, + "10": 0.90363, + "11": 2.30063, + "12": 0.89466, + "13": 0.87273, + "14": 2.31557, + "15": 0.91663, + "16": 0.87731, + "17": 0.89596, + "18": 0.87486, + "19": 0.87795, + "20": 0.87855, + "21": 0.88064, + "22": 0.88881, + "23": 0.88358, + "24": 0.88347, + "25": 0.88411, + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..a35a7574e59 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81233, + "2": 10.82416, + "3": 10.81841, + "4": 10.81357, + "5": 10.85116, + "6": 10.85502, + "7": 10.84363, + "8": 10.83621, + "9": 10.84178, + "10": 10.77391, + "11": 10.86217, + "12": 10.84672, + "13": 10.85692, + "14": 10.8614, + "15": 10.80709, + "16": 10.78544, + "17": 10.7701, + "18": 10.79072, + "19": 10.78529, + "20": 10.71496, + "21": 10.67362, + "22": 10.5386, + "23": 10.69608, + "24": 10.58118, + "25": 10.52212, + "26": 10.58665, + "27": 10.60344, + "28": 10.5676, + "29": 10.5868, + "30": 10.36177, + "31": 10.09661, + "32": 10.45911, + "33": 10.45926, + "34": 10.21524, + "35": 10.2617, + "36": 10.22327, + "37": 10.35631, + "38": 10.20637, + "39": 10.40825, + "40": 10.08881, + "41": 10.13871, + "42": 10.22236, + "43": 9.82978, + "44": 9.96931, + "45": 9.83925, + "46": 9.81008, + "47": 10.16408, + "48": 9.84608, + "49": 9.53674, + "50": 9.91754, + "51": 9.86341, + "52": 9.74862, + "53": 10.08034, + "54": 9.96286, + "55": 9.89221, + "56": 9.64295, + "57": 9.48196, + "58": 9.85327, + "59": 9.58985, + "60": 9.5157, + "61": 9.70142, + "62": 10.01153, + "63": 9.40557, + "64": 9.78559, + "65": 8.96047, + "66": 9.72678, + "67": 9.38244, + "68": 9.79903, + "69": 9.81114, + "70": 9.74788, + "71": 9.6452, + "72": 9.6027, + "73": 9.51692, + "74": 8.95583, + "75": 9.43449, + "76": 9.10005, + "77": 10.07816, + "78": 9.72912, + "79": 9.39357, + "80": 9.41584, + "81": 9.49174, + "82": 9.71087, + "83": 9.32591, + "84": 9.42272, + "85": 9.62054, + "86": 9.08096, + "87": 9.59797, + "88": 9.7551, + "89": 9.6096, + "90": 9.83264, + "91": 9.34163, + "92": 9.3578, + "93": 9.09025, + "94": 8.83205, + "95": 9.52868, + "96": 9.5278, + "97": 9.30277, + "98": 9.66393, + "99": 8.89773, + "100": 9.404 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 5532.0, + "2": 5934.0, + "3": 5812.0, + "4": 5817.0, + "5": 6435.0, + "6": 6641.0, + "7": 5880.0, + "8": 5900.0, + "9": 6317.0, + "10": 5314.0, + "11": 6659.0, + "12": 6393.0, + "13": 6585.0, + "14": 6649.0, + "15": 6237.0, + "16": 6606.0, + "17": 6232.0, + "18": 6059.0, + "19": 6380.0, + "20": 5723.0, + "21": 6197.0, + "22": 5714.0, + "23": 6527.0, + "24": 5948.0, + "25": 5822.0, + "26": 6271.0, + "27": 6493.0, + "28": 6789.0, + "29": 6971.0, + "30": 6252.0, + "31": 5836.0, + "32": 6830.0, + "33": 7155.0, + "34": 6428.0, + "35": 6909.0, + "36": 6559.0, + "37": 7582.0, + "38": 7325.0, + "39": 8189.0, + "40": 7156.0, + "41": 7113.0, + "42": 7783.0, + "43": 7236.0, + "44": 6958.0, + "45": 7093.0, + "46": 7385.0, + "47": 7634.0, + "48": 7916.0, + "49": 7565.0, + "50": 7795.0, + "51": 7967.0, + "52": 7869.0, + "53": 9001.0, + "54": 8408.0, + "55": 7734.0, + "56": 8108.0, + "57": 7339.0, + "58": 8677.0, + "59": 8299.0, + "60": 7790.0, + "61": 8347.0, + "62": 8345.0, + "63": 7835.0, + "64": 8861.0, + "65": 8293.0, + "66": 9180.0, + "67": 8276.0, + "68": 8251.0, + "69": 8666.0, + "70": 9836.0, + "71": 9020.0, + "72": 8503.0, + "73": 8996.0, + "74": 6967.0, + "75": 7749.0, + "76": 8534.0, + "77": 10688.0, + "78": 48163.0, + "79": 9603.0, + "80": 9991.0, + "81": 55995.0, + "82": 9533.0, + "83": 65535.0, + "84": 9876.0, + "85": 15848.0, + "86": 8732.0, + "87": 10574.0, + "88": 12165.0, + "89": 9808.0, + "90": 9644.0, + "91": 8584.0, + "92": 9300.0, + "93": 8081.0, + "94": 9101.0, + "95": 9919.0, + "96": 9755.0, + "97": 11113.0, + "98": 10522.0, + "99": 8739.0, + "100": 9616.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 628064256.0, + "2": 628065280.0, + "3": 628065280.0, + "4": 628065280.0, + "5": 628065280.0, + "6": 628065280.0, + "7": 628065280.0, + "8": 628065280.0, + "9": 628065280.0, + "10": 628065280.0, + "11": 628065280.0, + "12": 628065280.0, + "13": 628065280.0, + "14": 628065280.0, + "15": 628065280.0, + "16": 628065280.0, + "17": 628065280.0, + "18": 628065280.0, + "19": 628065280.0, + "20": 628065280.0, + "21": 628065280.0, + "22": 628065280.0, + "23": 628065280.0, + "24": 628065280.0, + "25": 628065280.0, + "26": 628065280.0, + "27": 628065280.0, + "28": 628065280.0, + "29": 628065280.0, + "30": 628065280.0, + "31": 628065280.0, + "32": 628065280.0, + "33": 628065280.0, + "34": 628065280.0, + "35": 628065280.0, + "36": 628065280.0, + "37": 628065280.0, + "38": 628065280.0, + "39": 628065280.0, + "40": 628065280.0, + "41": 628065280.0, + "42": 628065280.0, + "43": 628065280.0, + "44": 628065280.0, + "45": 628065280.0, + "46": 628065280.0, + "47": 628065280.0, + "48": 628065280.0, + "49": 628065280.0, + "50": 628065280.0, + "51": 628065280.0, + "52": 628065280.0, + "53": 628065280.0, + "54": 628065280.0, + "55": 628065280.0, + "56": 628065280.0, + "57": 628065280.0, + "58": 628065280.0, + "59": 628065280.0, + "60": 628065280.0, + "61": 628065280.0, + "62": 628065280.0, + "63": 628065280.0, + "64": 628065280.0, + "65": 628065280.0, + "66": 628065280.0, + "67": 628065280.0, + "68": 628065280.0, + "69": 628065280.0, + "70": 628065280.0, + "71": 628065280.0, + "72": 628065280.0, + "73": 628065280.0, + "74": 628065280.0, + "75": 628065280.0, + "76": 628065280.0, + "77": 628065280.0, + "78": 628065280.0, + "79": 628065280.0, + "80": 628065280.0, + "81": 628065280.0, + "82": 628065280.0, + "83": 628065280.0, + "84": 628065280.0, + "85": 628065280.0, + "86": 628065280.0, + "87": 628065280.0, + "88": 628065280.0, + "89": 628065280.0, + "90": 628065280.0, + "91": 628065280.0, + "92": 628065280.0, + "93": 628065280.0, + "94": 628065280.0, + "95": 628065280.0, + "96": 628065280.0, + "97": 628065280.0, + "98": 628065280.0, + "99": 628065280.0, + "100": 628065280.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 982504960.0, + "2": 1156255744.0, + "3": 1156255744.0, + "4": 1156255744.0, + "5": 1156255744.0, + "6": 1156255744.0, + "7": 1156255744.0, + "8": 1156255744.0, + "9": 1156255744.0, + "10": 1156255744.0, + "11": 1156255744.0, + "12": 1156255744.0, + "13": 1156255744.0, + "14": 1156255744.0, + "15": 1156255744.0, + "16": 1156255744.0, + "17": 1156255744.0, + "18": 1156255744.0, + "19": 1156255744.0, + "20": 1156255744.0, + "21": 1156255744.0, + "22": 1156255744.0, + "23": 1156255744.0, + "24": 1156255744.0, + "25": 1156255744.0, + "26": 1156255744.0, + "27": 1157233664.0, + "28": 1157233664.0, + "29": 1157233664.0, + "30": 1157233664.0, + "31": 1157233664.0, + "32": 1157233664.0, + "33": 1157233664.0, + "34": 1157233664.0, + "35": 1157233664.0, + "36": 1157233664.0, + "37": 1157233664.0, + "38": 1157233664.0, + "39": 1157233664.0, + "40": 1157233664.0, + "41": 1158865408.0, + "42": 1158865408.0, + "43": 1158865408.0, + "44": 1158865408.0, + "45": 1158865408.0, + "46": 1158865408.0, + "47": 1158865408.0, + "48": 1158865408.0, + "49": 1158865408.0, + "50": 1158865408.0, + "51": 1158865408.0, + "52": 1158865408.0, + "53": 1158865408.0, + "54": 1158865408.0, + "55": 1159034368.0, + "56": 1159063040.0, + "57": 1159542784.0, + "58": 1159542784.0, + "59": 1159542784.0, + "60": 1159542784.0, + "61": 1165075456.0, + "62": 1165075456.0, + "63": 1165075456.0, + "64": 1165075456.0, + "65": 1165075456.0, + "66": 1165075456.0, + "67": 1165075456.0, + "68": 1165075456.0, + "69": 1165075456.0, + "70": 1165075456.0, + "71": 1165075456.0, + "72": 1165075456.0, + "73": 1165075456.0, + "74": 1165075456.0, + "75": 1165075456.0, + "76": 1166216192.0, + "77": 1166216192.0, + "78": 1166216192.0, + "79": 1166216192.0, + "80": 1166216192.0, + "81": 1166216192.0, + "82": 1166216192.0, + "83": 1166639104.0, + "84": 1166639104.0, + "85": 1166639104.0, + "86": 1166639104.0, + "87": 1166639104.0, + "88": 1166639104.0, + "89": 1166639104.0, + "90": 1166639104.0, + "91": 1166639104.0, + "92": 1166639104.0, + "93": 1166639104.0, + "94": 1166639104.0, + "95": 1166639104.0, + "96": 1166639104.0, + "97": 1166639104.0, + "98": 1166639104.0, + "99": 1166639104.0, + "100": 1166639104.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 7.77547, + "3": 1.01252, + "4": 1.00639, + "5": 0.9897, + "6": 0.99553, + "7": 0.99796, + "8": 1.00873, + "9": 0.99009, + "10": 0.99264, + "11": 0.98765, + "12": 0.99024, + "13": 0.98319, + "14": 0.98552, + "15": 0.99368, + "16": 0.98342, + "17": 0.97729, + "18": 0.97272, + "19": 0.97308, + "20": 0.96906, + "21": 0.9751, + "22": 0.97375, + "23": 0.97447, + "24": 0.98494, + "25": 0.9779, + "26": 1.30939, + "27": 0.9766, + "28": 0.9856, + "29": 0.99223, + "30": 1.27178, + "31": 0.98025, + "32": 1.22425, + "33": 1.27653, + "34": 0.99358, + "35": 1.00171, + "36": 1.25408, + "37": 1.60005, + "38": 1.00572, + "39": 0.98676, + "40": 0.97218, + "41": 1.30266, + "42": 1.29066, + "43": 0.99057, + "44": 0.98517, + "45": 0.97968, + "46": 0.97289, + "47": 0.98145, + "48": 0.9804, + "49": 0.98022, + "50": 0.97431, + "51": 0.97593, + "52": 0.97255, + "53": 0.97424, + "54": 0.97043, + "55": 0.96887, + "56": 0.97492, + "57": 0.97623, + "58": 0.97423, + "59": 0.98879, + "60": 0.97992, + "61": 0.97895, + "62": 0.98829, + "63": 0.98719, + "64": 0.98651, + "65": 0.97852, + "66": 0.98045, + "67": 0.97825, + "68": 0.9795, + "69": 0.97812, + "70": 0.96297, + "71": 0.96718, + "72": 0.98343, + "73": 0.978, + "74": 0.99341, + "75": 0.97768, + "76": 0.97508, + "77": 0.97891, + "78": 0.9739, + "79": 0.96825, + "80": 0.96595, + "81": 0.95551, + "82": 0.97223, + "83": 0.9633, + "84": 0.96539, + "85": 0.97065, + "86": 0.97198, + "87": 0.97978, + "88": 0.98268, + "89": 0.99894, + "90": 1.00246, + "91": 0.98763, + "92": 0.98552, + "93": 0.99698, + "94": 0.99827, + "95": 0.99936, + "96": 0.99295, + "97": 0.99144, + "98": 0.99227, + "99": 0.98859, + "100": 0.99158 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json new file mode 100644 index 00000000000..8710366a4a2 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json @@ -0,0 +1,644 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.94839, + "2": 10.94024, + "3": 10.95902, + "4": 10.9592, + "5": 10.93942, + "6": 10.95284, + "7": 10.95227, + "8": 10.93987, + "9": 10.94518, + "10": 10.94146, + "11": 10.94366, + "12": 10.93351, + "13": 10.92937, + "14": 10.93117, + "15": 10.87714, + "16": 10.88218, + "17": 10.87388, + "18": 10.86829, + "19": 10.86292, + "20": 10.78627, + "21": 10.73278, + "22": 10.62202, + "23": 10.72355, + "24": 10.61784, + "25": 10.54739, + "26": 10.64163, + "27": 10.63354, + "28": 10.59007, + "29": 10.59937, + "30": 10.36921, + "31": 10.1175, + "32": 10.457, + "33": 10.45238, + "34": 10.18943, + "35": 10.24409, + "36": 10.20779, + "37": 10.32099, + "38": 10.17141, + "39": 10.39579, + "40": 10.03318, + "41": 10.08573, + "42": 10.17487, + "43": 9.7274, + "44": 9.88257, + "45": 9.73978, + "46": 9.72104, + "47": 10.08354, + "48": 9.75251, + "49": 9.39373, + "50": 9.83765, + "51": 9.76236, + "52": 9.65444, + "53": 10.01594, + "54": 9.86969, + "55": 9.79645, + "56": 9.53492, + "57": 9.365, + "58": 9.75243, + "59": 9.4751, + "60": 9.40362, + "61": 9.59124, + "62": 9.91012, + "63": 9.24082, + "64": 9.67728, + "65": 8.79731, + "66": 9.60544, + "67": 9.24212, + "68": 9.70475, + "69": 9.70741, + "70": 9.65988, + "71": 9.50626, + "72": 9.45834, + "73": 9.38692, + "74": 8.79461, + "75": 9.32175, + "76": 8.92857, + "77": 9.99456, + "78": 9.63104, + "79": 9.26692, + "80": 9.29144, + "81": 9.37768, + "82": 9.60984, + "83": 9.21108, + "84": 9.33667, + "85": 9.52726, + "86": 8.94539, + "87": 9.49937, + "88": 9.67766, + "89": 9.49525, + "90": 9.7509, + "91": 9.22918, + "92": 9.25394, + "93": 8.96194, + "94": 8.69021, + "95": 9.43531, + "96": 9.39886, + "97": 9.19199, + "98": 9.57248, + "99": 8.75688, + "100": 9.2924 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22750392.0, + "2": 22953048.0, + "3": 22604332.0, + "4": 23266194.0, + "5": 22735592.0, + "6": 23061740.0, + "7": 22793278.0, + "8": 22960884.0, + "9": 22865532.0, + "10": 22950250.0, + "11": 22499592.0, + "12": 22455936.0, + "13": 22948014.0, + "14": 22384528.0, + "15": 22846172.0, + "16": 22856720.0, + "17": 22836352.0, + "18": 22590198.0, + "19": 22627006.0, + "20": 22712338.0, + "21": 22762590.0, + "22": 22816896.0, + "23": 22545280.0, + "24": 22794372.0, + "25": 22841964.0, + "26": 22549700.0, + "27": 22464724.0, + "28": 22453634.0, + "29": 22534620.0, + "30": 22636106.0, + "31": 22989484.0, + "32": 22593994.0, + "33": 22565948.0, + "34": 22855396.0, + "35": 22813606.0, + "36": 22595412.0, + "37": 22499234.0, + "38": 22926180.0, + "39": 22825052.0, + "40": 22675880.0, + "41": 22671624.0, + "42": 22682188.0, + "43": 23015228.0, + "44": 22766040.0, + "45": 22679588.0, + "46": 22915144.0, + "47": 22642744.0, + "48": 24003236.0, + "49": 23786618.0, + "50": 22931756.0, + "51": 23866290.0, + "52": 23807188.0, + "53": 24007482.0, + "54": 23916892.0, + "55": 23571308.0, + "56": 23954192.0, + "57": 24211600.0, + "58": 23914524.0, + "59": 23771900.0, + "60": 23813638.0, + "61": 23795512.0, + "62": 23739412.0, + "63": 23917700.0, + "64": 23895780.0, + "65": 24147262.0, + "66": 23794750.0, + "67": 23983810.0, + "68": 23674060.0, + "69": 23647770.0, + "70": 23907338.0, + "71": 23818256.0, + "72": 23723392.0, + "73": 22754048.0, + "74": 25181258.0, + "75": 24144968.0, + "76": 23976372.0, + "77": 22260516.0, + "78": 23862138.0, + "79": 23806872.0, + "80": 23773052.0, + "81": 25020468.0, + "82": 22812998.0, + "83": 23911992.0, + "84": 25144028.0, + "85": 22725432.0, + "86": 24205484.0, + "87": 24851672.0, + "88": 23700260.0, + "89": 22505492.0, + "90": 24062928.0, + "91": 22790310.0, + "92": 24923596.0, + "93": 23722104.0, + "94": 23993086.0, + "95": 24140048.0, + "96": 23909352.0, + "97": 23668280.0, + "98": 23832272.0, + "99": 23985032.0, + "100": 24101560.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 810140160.0, + "2": 804531200.0, + "3": 804531200.0, + "4": 934860800.0, + "5": 934860800.0, + "6": 934860800.0, + "7": 934860800.0, + "8": 934860800.0, + "9": 938611712.0, + "10": 938104832.0, + "11": 938379264.0, + "12": 934860800.0, + "13": 934860800.0, + "14": 934860800.0, + "15": 934860800.0, + "16": 942249984.0, + "17": 941443072.0, + "18": 937990144.0, + "19": 937548800.0, + "20": 937498624.0, + "21": 934860800.0, + "22": 934860800.0, + "23": 941533184.0, + "24": 942114816.0, + "25": 942398464.0, + "26": 934860800.0, + "27": 934860800.0, + "28": 934860800.0, + "29": 934860800.0, + "30": 934860800.0, + "31": 934860800.0, + "32": 934860800.0, + "33": 934860800.0, + "34": 941477888.0, + "35": 934860800.0, + "36": 934860800.0, + "37": 934860800.0, + "38": 934860800.0, + "39": 934860800.0, + "40": 934860800.0, + "41": 940742656.0, + "42": 940742656.0, + "43": 940742656.0, + "44": 940968960.0, + "45": 941581312.0, + "46": 934860800.0, + "47": 934860800.0, + "48": 940742656.0, + "49": 934860800.0, + "50": 934860800.0, + "51": 934860800.0, + "52": 940742656.0, + "53": 937498624.0, + "54": 937498624.0, + "55": 937498624.0, + "56": 937498624.0, + "57": 938199040.0, + "58": 937498624.0, + "59": 937498624.0, + "60": 940742656.0, + "61": 934860800.0, + "62": 934860800.0, + "63": 934860800.0, + "64": 934860800.0, + "65": 934860800.0, + "66": 934860800.0, + "67": 934860800.0, + "68": 934860800.0, + "69": 934860800.0, + "70": 934860800.0, + "71": 934860800.0, + "72": 934860800.0, + "73": 934860800.0, + "74": 934860800.0, + "75": 934860800.0, + "76": 934860800.0, + "77": 934860800.0, + "78": 934860800.0, + "79": 938199040.0, + "80": 938199040.0, + "81": 937498624.0, + "82": 938061824.0, + "83": 938412032.0, + "84": 937498624.0, + "85": 938199040.0, + "86": 938445824.0, + "87": 937498624.0, + "88": 937498624.0, + "89": 934860800.0, + "90": 934860800.0, + "91": 934860800.0, + "92": 940742656.0, + "93": 940742656.0, + "94": 938104832.0, + "95": 941451264.0, + "96": 940742656.0, + "97": 941542400.0, + "98": 938104832.0, + "99": 940742656.0, + "100": 938104832.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1017976320.0, + "2": 1226964480.0, + "3": 1228012032.0, + "4": 1300063744.0, + "5": 1300063744.0, + "6": 1300223488.0, + "7": 1300891648.0, + "8": 1300891648.0, + "9": 1300891648.0, + "10": 1303292416.0, + "11": 1303292416.0, + "12": 1303292416.0, + "13": 1303292416.0, + "14": 1303292416.0, + "15": 1303292416.0, + "16": 1303292416.0, + "17": 1303292416.0, + "18": 1303292416.0, + "19": 1303292416.0, + "20": 1303292416.0, + "21": 1303292416.0, + "22": 1303292416.0, + "23": 1303292416.0, + "24": 1303292416.0, + "25": 1303292416.0, + "26": 1303292416.0, + "27": 1303292416.0, + "28": 1303292416.0, + "29": 1303292416.0, + "30": 1303292416.0, + "31": 1303292416.0, + "32": 1303292416.0, + "33": 1303292416.0, + "34": 1303292416.0, + "35": 1303292416.0, + "36": 1303292416.0, + "37": 1303292416.0, + "38": 1303292416.0, + "39": 1303292416.0, + "40": 1303292416.0, + "41": 1303292416.0, + "42": 1303292416.0, + "43": 1303292416.0, + "44": 1303292416.0, + "45": 1303292416.0, + "46": 1303292416.0, + "47": 1303292416.0, + "48": 1303292416.0, + "49": 1303292416.0, + "50": 1303292416.0, + "51": 1303292416.0, + "52": 1303292416.0, + "53": 1303292416.0, + "54": 1303292416.0, + "55": 1303292416.0, + "56": 1303292416.0, + "57": 1303292416.0, + "58": 1303292416.0, + "59": 1303292416.0, + "60": 1303292416.0, + "61": 1303292416.0, + "62": 1303292416.0, + "63": 1303292416.0, + "64": 1303292416.0, + "65": 1303292416.0, + "66": 1303292416.0, + "67": 1303292416.0, + "68": 1303292416.0, + "69": 1303292416.0, + "70": 1303292416.0, + "71": 1303292416.0, + "72": 1303292416.0, + "73": 1303292416.0, + "74": 1303292416.0, + "75": 1303292416.0, + "76": 1303292416.0, + "77": 1303292416.0, + "78": 1303292416.0, + "79": 1303292416.0, + "80": 1303292416.0, + "81": 1303292416.0, + "82": 1303292416.0, + "83": 1303292416.0, + "84": 1303292416.0, + "85": 1303292416.0, + "86": 1303292416.0, + "87": 1303292416.0, + "88": 1303292416.0, + "89": 1303292416.0, + "90": 1303292416.0, + "91": 1303292416.0, + "92": 1303292416.0, + "93": 1303292416.0, + "94": 1303292416.0, + "95": 1303292416.0, + "96": 1303292416.0, + "97": 1303292416.0, + "98": 1303292416.0, + "99": 1303292416.0, + "100": 1303292416.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.89507, + "2": 10.90521, + "3": 10.90879, + "4": 10.86406, + "5": 10.91765, + "6": 10.92332, + "7": 10.90072, + "8": 10.8906, + "9": 10.90544, + "10": 10.88636, + "11": 10.93328, + "12": 10.91582, + "13": 10.90917, + "14": 10.92294, + "15": 10.89802, + "16": 10.90337, + "17": 10.88446, + "18": 10.90526, + "19": 10.90011, + "20": 10.88775, + "21": 10.88103, + "22": 10.85514, + "23": 10.89267, + "24": 10.87352, + "25": 10.86182, + "26": 10.87152, + "27": 10.88847, + "28": 10.87872, + "29": 10.88744, + "30": 10.87297, + "31": 10.80177, + "32": 10.8732, + "33": 10.88219, + "34": 10.83823, + "35": 10.85291, + "36": 10.84901, + "37": 10.85873, + "38": 10.83148, + "39": 10.86289, + "40": 10.82147, + "41": 10.82913, + "42": 10.84798, + "43": 10.7908, + "44": 10.81431, + "45": 10.7879, + "46": 10.78018, + "47": 10.83142, + "48": 10.78854, + "49": 10.71024, + "50": 10.76861, + "51": 10.76331, + "52": 10.73945, + "53": 10.80126, + "54": 10.77704, + "55": 10.765, + "56": 10.71649, + "57": 10.67368, + "58": 10.75107, + "59": 10.69607, + "60": 10.66236, + "61": 10.69617, + "62": 10.77167, + "63": 10.6134, + "64": 10.70881, + "65": 10.49259, + "66": 10.66843, + "67": 10.58084, + "68": 10.68215, + "69": 10.68669, + "70": 10.67296, + "71": 10.64397, + "72": 10.60997, + "73": 10.56734, + "74": 10.38624, + "75": 10.53623, + "76": 10.40297, + "77": 10.75436, + "78": 10.62548, + "79": 10.47858, + "80": 10.47388, + "81": 10.5143, + "82": 10.58579, + "83": 10.43913, + "84": 10.45418, + "85": 10.55042, + "86": 10.27831, + "87": 10.51067, + "88": 10.60469, + "89": 10.5084, + "90": 10.60243, + "91": 10.38487, + "92": 10.38165, + "93": 10.23549, + "94": 10.07844, + "95": 10.42709, + "96": 10.44697, + "97": 10.31686, + "98": 10.4968, + "99": 10.04966, + "100": 10.32944 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": "nan", + "2": 19.93451, + "3": 2.31445, + "4": 5.28856, + "5": 1.09994, + "6": 1.09399, + "7": 1.09697, + "8": 1.09872, + "9": 1.17005, + "10": 1.10071, + "11": 1.0994, + "12": 1.08313, + "13": 1.09364, + "14": 1.09082, + "15": 1.09269, + "16": 1.08133, + "17": 1.08872, + "18": 1.09032, + "19": 1.10458, + "20": 1.10126, + "21": 1.09029, + "22": 1.19723, + "23": 1.36303, + "24": 1.39758, + "25": 1.40863, + "26": 1.40985, + "27": 1.40231, + "28": 1.42816, + "29": 1.37678, + "30": 1.40545, + "31": 1.40841, + "32": 1.40385, + "33": 1.39528, + "34": 1.4028, + "35": 1.41768, + "36": 1.40649, + "37": 1.41813, + "38": 1.40674, + "39": 1.38881, + "40": 1.40998, + "41": 1.37775, + "42": 1.39701, + "43": 1.3967, + "44": 1.40408, + "45": 1.40972, + "46": 1.41116, + "47": 1.40427, + "48": 1.38905, + "49": 1.42541, + "50": 1.40474, + "51": 1.40708, + "52": 1.37484, + "53": 1.38539, + "54": 1.39988, + "55": 1.39635, + "56": 1.41326, + "57": 1.40012, + "58": 1.40599, + "59": 1.41023, + "60": 1.4209, + "61": 1.41702, + "62": 1.40134, + "63": 1.40282, + "64": 1.40573, + "65": 1.41933, + "66": 1.40057, + "67": 1.41526, + "68": 1.40285, + "69": 1.41947, + "70": 1.37747, + "71": 1.41206, + "72": 1.39123, + "73": 1.42381, + "74": 1.40806, + "75": 1.40032, + "76": 1.41783, + "77": 1.39133, + "78": 1.41146, + "79": 1.42648, + "80": 1.40774, + "81": 1.40046, + "82": 1.39158, + "83": 1.4079, + "84": 1.40469, + "85": 1.39689, + "86": 1.41401, + "87": 1.40637, + "88": 1.40569, + "89": 1.45225, + "90": 1.39469, + "91": 1.39677, + "92": 1.39569, + "93": 1.38882, + "94": 1.40133, + "95": 1.41493, + "96": 1.40659, + "97": 1.39059, + "98": 1.40044, + "99": 1.41118, + "100": 1.39159 + } + } +} \ No newline at end of file diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py index 480b2dca8ae..394bda30a01 100644 --- a/tests/test_utils/python_scripts/recipe_parser.py +++ b/tests/test_utils/python_scripts/recipe_parser.py @@ -39,14 +39,25 @@ def resolve_cluster_config(cluster: str) -> str: def flatten_products(workload_manifest: dotdict) -> dotdict: """Flattens a nested dict of products""" - workload_manifest.products = [ - dict(**dict(zip(inp.keys(), values)), **{"test_case": product["test_case"][0]}) - for product in (workload_manifest.products or []) - if "products" in product - for inp in product["products"] - for values in itertools.product(*inp.values()) - ] + flattened_products = [] + products = workload_manifest.products or [] + for product in products: + if "products" not in product: + continue + + test_case = product["test_case"][0] + for param_dict in product["products"]: + # Generate all combinations of parameter values + param_combinations = itertools.product(*param_dict.values()) + + for value_combination in param_combinations: + # Map parameter names to their values + flattened = dict(zip(param_dict.keys(), value_combination)) + flattened["test_case"] = test_case + flattened_products.append(flattened) + + workload_manifest.products = flattened_products return workload_manifest @@ -223,14 +234,13 @@ def load_workloads( workloads: List[dotdict] = [] build_workloads: List = [] - for file in list(recipes_dir.glob("*.yaml")) + list(local_dir.glob("*.yaml")): + for file in list(recipes_dir.glob("**/*.yaml")) + list(local_dir.glob("**/*.yaml")): workloads += load_and_flatten(config_path=str(file)) if file.stem.startswith("_build"): build_workloads.append(load_config(config_path=str(file))) if scope: workloads = filter_by_scope(workload_manifests=workloads, scope=scope) - if workloads and environment: workloads = filter_by_environment(workload_manifests=workloads, environment=environment) diff --git a/tests/test_utils/recipes/gpt-gb200.yaml b/tests/test_utils/recipes/gb200/gpt.yaml similarity index 68% rename from tests/test_utils/recipes/gpt-gb200.yaml rename to tests/test_utils/recipes/gb200/gpt.yaml index 9c3786332c9..f387fbb9a13 100644 --- a/tests/test_utils/recipes/gpt-gb200.yaml +++ b/tests/test_utils/recipes/gb200/gpt.yaml @@ -37,7 +37,7 @@ spec: ls cd /opt/megatron-lm - NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g') + NAME=$(echo {test_case}_{environment} | sed 's/dgx_gb200/dgx_a100/g') export GPUS_PER_NODE={gpus} ARGUMENTS=( @@ -64,6 +64,9 @@ spec: exit $exit_code products: + ####################################################################### + # Nightly tests: Run DEV unless something is flaky # + ####################################################################### - test_case: [gpt3_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] @@ -78,7 +81,6 @@ products: products: - environment: [dev] scope: [nightly] - platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp1_pp4] products: - environment: [dev] @@ -88,7 +90,6 @@ products: products: - environment: [dev] scope: [nightly] - platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp4_pp1_resume_torch] products: - environment: [dev] @@ -99,258 +100,324 @@ products: - environment: [dev] scope: [nightly] platforms: [dgx_gb200] + ####################################################################### + # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # + # some very important tests. # + ####################################################################### - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_no_mmap_bin_files] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_dist_optimizer_fim_dataset] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_dist_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [mr, mr-github-broken] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp1_resume_torch_dist_uniform_full_recompute] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_cp4_a2a_p2p_nondeterministic] + # products: + # - environment: [dev] + # scope: [mr] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_interleaved_no_fusion] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_swiglu] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_resume_torch_dist_untie_embeddings_and_outputs] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_decoupled_lr] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_calculate_per_token_loss] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] # Hangs: #513 - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [nightly] - platforms: [dgx_gb200] - - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_gb200] - - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_untied] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_gb200] - - test_case: [gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap] - products: - - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_uneven_pipeline] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp2_account_for_embedding_loss_in_pipeline_split] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_multi_dist_optimizer_instances] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_calculate_per_token_loss_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cp2_etp4_calculate_per_token_loss_nondeterministic_dp_last] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cp2_nondeterministic] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_cross_entropy_loss_fusion] products: - environment: [dev] - scope: [nightly] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp1_gdn] + products: + - environment: [dev] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_mla] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_ddp_average_in_collective] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_defer_embedding_wgrad_compute] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist] products: - environment: [dev] - scope: [nightly] - platforms: [dgx_gb200] - - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_no_create_attention_mask_in_dataloader] - products: - - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp2_resume_torch_dist_reshard_1x4xNone] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_qk_layernorm_test_mode] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_tp2_pp2_uninstall_te] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp1_pp4_memory_speed] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_7b_tp4_pp1_memory_speed] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_fsdp_dtensor] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp2_pp1_modelopt_distill_resume] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] + ####################################################################### + # Super important mr, mr-github tests that run for DEV per mr, mr-github # + ####################################################################### - test_case: [gpt3_mcore_reruns_persistent_1] products: - environment: [dev] - scope: [nightly] + scope: [mr, mr-github-broken] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp1_pp4_vp1_dist_optimizer_overlap_grad_reduce_param_gather_overlap_optimizer] products: - environment: [dev] - scope: [nightly] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [nightly] + scope: [mr] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather] products: - environment: [dev] - scope: [nightly] + scope: [mr, mr-github, mr-github-slim] platforms: [dgx_gb200] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: - environment: [dev] - scope: [nightly] + scope: [mr] + platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp4_pp2_frozen_resume_torch_dist_reshard_8x1xNone] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp1_pp1_frozen_resume_torch_dist_dist_optimizer] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp1_pp4_frozen_resume_torch_dist_swiglu] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_cp2_nondeterministic] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_a100, dgx_gb200] + # - test_case: [gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_b200] + # - test_case: [gpt3_weekly_dgx_b200_mcore_tp4_cp2_current_scaling_native_fp8_tp_fsdp] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_b200] + # - test_case: [gpt3_weekly_dgx_b200_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_b200] + # - test_case: [gpt3_weekly_dgx_b200_mcore_tp4_cp2_mxfp8_tp_sp_cp] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_b200] + # - test_case: [gpt3_weekly_dgx_b200_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_b200] + - test_case: [gpt3_weekly_dgx_gb200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap] + products: + - environment: [dev] + scope: [weekly] + platforms: [dgx_gb200] + # - test_case: [gpt3_weekly_dgx_gb200_mcore_tp4_cp2_current_scaling_native_fp8_tp_fsdp] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_gb200] + # - test_case: [gpt3_weekly_dgx_gb200_mcore_tp4_cp2_current_scaling_native_fp8_tp_sp_cp_tp_overlap] + # products: + # - environment: [dev] + # scope: [weekly] + # platforms: [dgx_gb200] + - test_case: [gpt3_weekly_dgx_gb200_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap] + products: + - environment: [dev] + scope: [weekly] platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/moe.yaml b/tests/test_utils/recipes/gb200/moe.yaml new file mode 100644 index 00000000000..28ae2415aac --- /dev/null +++ b/tests/test_utils/recipes/gb200/moe.yaml @@ -0,0 +1,220 @@ +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: "{test_case}_{environment}_{platforms}" + model: moe + build: mcore-pyt-{environment} + nodes: 2 + gpus: 4 + n_repeat: 5 + platforms: dgx_gb200 + script_setup: | + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + # Checkout latest + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + + # Checkout backwards-ref + cd /opt + rm -rf /opt/megatron-lm-legacy; mkdir megatron-lm-legacy; cd megatron-lm-legacy + git init + git remote add origin $MCORE_REPO + git fetch origin $MCORE_BACKWARDS_COMMIT + git checkout $MCORE_BACKWARDS_COMMIT + git rev-parse HEAD + rm -rf megatron; cp -a /opt/megatron-lm/megatron ./ + script: |- + ls + cd /opt/megatron-lm + + NAME=$(echo {test_case}_{environment} | sed 's/dgx_gb200/dgx_a100/g') + export GPUS_PER_NODE={gpus} + + ARGUMENTS=( + "DATA_PATH=/mnt/artifacts" + "DATA_CACHE_PATH=/lustre/fsw/coreai_dlalgo_mcore/mcore_ci/data/$RUN_ID/cache/" + "OUTPUT_PATH={assets_dir}" + "TENSORBOARD_PATH={assets_dir}/tensorboard" + "CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints" + "CHECKPOINT_LOAD_PATH=/mnt/artifacts" + "TRAINING_SCRIPT_PATH=pretrain_gpt.py" + "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" + "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json" + "N_REPEAT={n_repeat}" + "ENABLE_LIGHTWEIGHT_MODE=${{ENABLE_LIGHTWEIGHT_MODE}}" + "RECORD_CHECKPOINTS=${{RECORD_CHECKPOINTS}}" + ) + + bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}} + +products: + ####################################################################### + # Nightly tests: Run both DEV and LTS unless something is flaky # + ####################################################################### + - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] + products: + - environment: [dev] + scope: [nightly] + platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] + # products: # non-determinism: #478 + # - environment: [dev, lts] + # scope: [nightly] + ####################################################################### + # Weekly tests: Run both DEV and LTS unless something is flaky # + ####################################################################### + ####################################################################### + # mr, mr-github tests: Mostly DEV on mr, mr-github, and LTS on nightly cadence, except for # + # some very important tests. # + ####################################################################### + - test_case: [gpt3_mcore_te_tp1_pp2_resume_torch_dist_reshard_2x1x4_te_8experts2parallel_dist_optimizer] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] # hang: #513 + - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph] + products: + - environment: [dev] + scope: [mr-broken] + platforms: [dgx_gb200] # hang: #513 + # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_selective_recompute_experimental] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] # hang: #513 + - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + # - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + # - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_gb200] + - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] + products: + - environment: [dev] + scope: [mr, mr-github, mr-slim] + platforms: [dgx_gb200] + - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] + products: + - environment: [dev] + scope: [mr, mr-github, mr-slim] + platforms: [dgx_gb200] + - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] + products: + - environment: [dev] + scope: [mr-broken] + platforms: [dgx_gb200] + ####################################################################### + # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # + ####################################################################### + # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_te_8experts2parallel_dist_optimizer] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + # - test_case: [gpt3_mcore_te_tp2_pp1_frozen_resume_torch_dist_te_8experts2parallel_groupedGEMM] + # products: + # - environment: [dev] + # scope: [mr] + # platforms: [dgx_gb200] + ########################### + # Merge train tests # + ########################### + - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer] + products: + - environment: [dev] + scope: [mr, mr-github, mr-github-slim] + platforms: [dgx_gb200] + - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed] + products: + - environment: [dev] + scope: [mr-broken] + platforms: [dgx_gb200] diff --git a/tests/test_utils/recipes/gb200/unit-tests.yaml b/tests/test_utils/recipes/gb200/unit-tests.yaml new file mode 100644 index 00000000000..0e8cb72916b --- /dev/null +++ b/tests/test_utils/recipes/gb200/unit-tests.yaml @@ -0,0 +1,153 @@ +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: "{test_case}_{environment}_{platforms}_{tag}" + model: unit-tests + nodes: 2 + build: mcore-pyt-{environment} + gpus: 4 + platforms: dgx_gb200 + script_setup: | + unset https_proxy + echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc + + # Checkout latest + cd /opt + rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm + git init + git remote add origin $MCORE_REPO + git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' + git fetch origin $MCORE_MR_COMMIT + git checkout $MCORE_MR_COMMIT + git rev-parse HEAD + + # Checkout backwards-ref + cd /opt + bash /opt/megatron-lm/.gitlab/scripts/fetch-legacy-suite.sh \ + --backwards-commit $MCORE_BACKWARDS_COMMIT \ + --repo $MCORE_REPO + + script: |- + ls + + TAG={tag} + ENVIRONMENT={environment} + BUCKET="{test_case}" + UNIT_TEST_REPEAT={n_repeat} + export GPUS_PER_NODE={gpus} + + if [[ "$TAG" == "latest" ]]; then + TEST_PATH="/opt/megatron-lm" + else + TEST_PATH="/opt/megatron-lm-legacy/" + fi + + + bash $TEST_PATH/tests/unit_tests/run_ci_test.sh \ + --tag $TAG \ + --environment $ENVIRONMENT \ + --bucket $BUCKET \ + --unit-test-repeat $UNIT_TEST_REPEAT \ + --log-dir {assets_dir}/logs/1/ + + ls -al + + cd $TEST_PATH + /opt/venv/bin/coverage xml + cp .coverage {assets_dir}/coverage_report + cp coverage.xml {assets_dir} + +products: + - test_case: [tests/unit_tests/test_model_configs.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/test_fp8_param.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/pipeline_parallel/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/models/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/data/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/dist_checkpointing/test_optimizer.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/dist_checkpointing/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/dist_checkpointing/models/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/dist_checkpointing/models/test_moe_experts.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/transformer/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/transformer/moe/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/distributed/megatron_fsdp/**/*.py] + products: + - environment: [dev] + tag: [latest] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] + - test_case: [tests/unit_tests/**/*.py] + products: + - environment: [dev] + tag: [latest, legacy] + scope: [unit-tests] + n_repeat: [1] + time_limit: [1800] diff --git a/tests/test_utils/recipes/bert.yaml b/tests/test_utils/recipes/h100/bert.yaml similarity index 100% rename from tests/test_utils/recipes/bert.yaml rename to tests/test_utils/recipes/h100/bert.yaml diff --git a/tests/test_utils/recipes/ckpt_converter.yaml b/tests/test_utils/recipes/h100/ckpt_converter.yaml similarity index 100% rename from tests/test_utils/recipes/ckpt_converter.yaml rename to tests/test_utils/recipes/h100/ckpt_converter.yaml diff --git a/tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference-cuda-graphs.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-dynamic-inference-cuda-graphs.yaml rename to tests/test_utils/recipes/h100/gpt-dynamic-inference-cuda-graphs.yaml diff --git a/tests/test_utils/recipes/gpt-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-dynamic-inference-with-coordinator.yaml rename to tests/test_utils/recipes/h100/gpt-dynamic-inference-with-coordinator.yaml diff --git a/tests/test_utils/recipes/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-dynamic-inference.yaml rename to tests/test_utils/recipes/h100/gpt-dynamic-inference.yaml diff --git a/tests/test_utils/recipes/gpt-grads.yaml b/tests/test_utils/recipes/h100/gpt-grads.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-grads.yaml rename to tests/test_utils/recipes/h100/gpt-grads.yaml diff --git a/tests/test_utils/recipes/gpt-grpo.yaml b/tests/test_utils/recipes/h100/gpt-grpo.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-grpo.yaml rename to tests/test_utils/recipes/h100/gpt-grpo.yaml diff --git a/tests/test_utils/recipes/gpt-nemo.yaml b/tests/test_utils/recipes/h100/gpt-nemo.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-nemo.yaml rename to tests/test_utils/recipes/h100/gpt-nemo.yaml diff --git a/tests/test_utils/recipes/gpt-static-inference.yaml b/tests/test_utils/recipes/h100/gpt-static-inference.yaml similarity index 100% rename from tests/test_utils/recipes/gpt-static-inference.yaml rename to tests/test_utils/recipes/h100/gpt-static-inference.yaml diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/h100/gpt.yaml similarity index 100% rename from tests/test_utils/recipes/gpt.yaml rename to tests/test_utils/recipes/h100/gpt.yaml diff --git a/tests/test_utils/recipes/mamba-dynamic-inference.yaml b/tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml similarity index 100% rename from tests/test_utils/recipes/mamba-dynamic-inference.yaml rename to tests/test_utils/recipes/h100/mamba-dynamic-inference.yaml diff --git a/tests/test_utils/recipes/mamba-static-inference.yaml b/tests/test_utils/recipes/h100/mamba-static-inference.yaml similarity index 100% rename from tests/test_utils/recipes/mamba-static-inference.yaml rename to tests/test_utils/recipes/h100/mamba-static-inference.yaml diff --git a/tests/test_utils/recipes/mamba.yaml b/tests/test_utils/recipes/h100/mamba.yaml similarity index 100% rename from tests/test_utils/recipes/mamba.yaml rename to tests/test_utils/recipes/h100/mamba.yaml diff --git a/tests/test_utils/recipes/mimo.yaml b/tests/test_utils/recipes/h100/mimo.yaml similarity index 100% rename from tests/test_utils/recipes/mimo.yaml rename to tests/test_utils/recipes/h100/mimo.yaml diff --git a/tests/test_utils/recipes/module_performance.yaml b/tests/test_utils/recipes/h100/module_performance.yaml similarity index 100% rename from tests/test_utils/recipes/module_performance.yaml rename to tests/test_utils/recipes/h100/module_performance.yaml diff --git a/tests/test_utils/recipes/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml similarity index 100% rename from tests/test_utils/recipes/moe-dynamic-inference-with-coordinator.yaml rename to tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml diff --git a/tests/test_utils/recipes/moe-dynamic-inference.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference.yaml similarity index 100% rename from tests/test_utils/recipes/moe-dynamic-inference.yaml rename to tests/test_utils/recipes/h100/moe-dynamic-inference.yaml diff --git a/tests/test_utils/recipes/moe-grpo.yaml b/tests/test_utils/recipes/h100/moe-grpo.yaml similarity index 100% rename from tests/test_utils/recipes/moe-grpo.yaml rename to tests/test_utils/recipes/h100/moe-grpo.yaml diff --git a/tests/test_utils/recipes/moe-static-inference.yaml b/tests/test_utils/recipes/h100/moe-static-inference.yaml similarity index 100% rename from tests/test_utils/recipes/moe-static-inference.yaml rename to tests/test_utils/recipes/h100/moe-static-inference.yaml diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/h100/moe.yaml similarity index 100% rename from tests/test_utils/recipes/moe.yaml rename to tests/test_utils/recipes/h100/moe.yaml diff --git a/tests/test_utils/recipes/multimodal-llava.yaml b/tests/test_utils/recipes/h100/multimodal-llava.yaml similarity index 100% rename from tests/test_utils/recipes/multimodal-llava.yaml rename to tests/test_utils/recipes/h100/multimodal-llava.yaml diff --git a/tests/test_utils/recipes/t5.yaml b/tests/test_utils/recipes/h100/t5.yaml similarity index 100% rename from tests/test_utils/recipes/t5.yaml rename to tests/test_utils/recipes/h100/t5.yaml diff --git a/tests/test_utils/recipes/unit-tests.yaml b/tests/test_utils/recipes/h100/unit-tests.yaml similarity index 100% rename from tests/test_utils/recipes/unit-tests.yaml rename to tests/test_utils/recipes/h100/unit-tests.yaml diff --git a/tests/unit_tests/find_test_cases.py b/tests/unit_tests/find_test_cases.py index 2e9f5515b7d..1445206cab5 100644 --- a/tests/unit_tests/find_test_cases.py +++ b/tests/unit_tests/find_test_cases.py @@ -50,7 +50,8 @@ def expand_pattern(pattern): def main(): BUCKET = sys.argv[1] - YAML_FILE = 'tests/test_utils/recipes/unit-tests.yaml' + GPU_TYPE = sys.argv[2] + YAML_FILE = f'tests/test_utils/recipes/{GPU_TYPE}/unit-tests.yaml' all_test_cases = get_test_cases(YAML_FILE) bucket_files = set(expand_pattern(BUCKET)) diff --git a/tests/unit_tests/run_ci_test.sh b/tests/unit_tests/run_ci_test.sh index 81dd3ae2a14..c65f197d2db 100755 --- a/tests/unit_tests/run_ci_test.sh +++ b/tests/unit_tests/run_ci_test.sh @@ -117,7 +117,7 @@ export BUCKET IGNORE_ARGS=() while IFS= read -r line; do [[ -n "$line" ]] && IGNORE_ARGS+=("$line") -done < <(python tests/unit_tests/find_test_cases.py "$BUCKET") +done < <(python tests/unit_tests/find_test_cases.py "$BUCKET" "h100") echo "------ARGUMENTS for SLURM ---" MASTER_ADDR=${MASTER_ADDR:-localhost}