From a021e4ed6ad4ff722527e80d8cdc4e901af6ed64 Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Thu, 2 Jul 2026 14:17:57 -0700 Subject: [PATCH 1/7] Update base image to nvcr.io/nvidia/pytorch:26.06-py3 and adjust golden values in functional tests for consistency. Add skip condition in unit tests for unsupported in-place operations in new PyTorch version. Signed-off-by: Ajay Balasa --- .gitlab/stages/01.build.yml | 4 +- docker/.ngc_version.dev | 2 +- .../golden_values_dev_dgx_gb200.json | 494 ++++----- .../golden_values_dev_dgx_gb200.json | 992 +++++++++--------- .../megatron_fsdp/test_mfsdp_fully_shard.py | 14 + 5 files changed, 760 insertions(+), 746 deletions(-) diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index f06add00fa2..b41f4635386 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -62,12 +62,12 @@ test:pre_build_image: - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev IMAGE_TYPE: dev - BASE_IMAGE: nvcr.io/nvidia/pytorch:26.04-py3 + BASE_IMAGE: nvcr.io/nvidia/pytorch:26.06-py3 PLATFORM: amd64 - IMAGE: CI_MCORE_DEV_IMAGE FILE: Dockerfile.ci.dev IMAGE_TYPE: dev - BASE_IMAGE: nvcr.io/nvidia/pytorch:26.04-py3 + BASE_IMAGE: nvcr.io/nvidia/pytorch:26.06-py3 PLATFORM: arm64 - IMAGE: UTILITY_IMAGE FILE: Dockerfile.linting diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev index 3356f1f0bca..e3cb6b663ab 100644 --- a/docker/.ngc_version.dev +++ b/docker/.ngc_version.dev @@ -1 +1 @@ -nvcr.io/nvidia/pytorch:26.04-py3 +nvcr.io/nvidia/pytorch:26.06-py3 diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json index b3109ea4e60..b7f3c858488 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.90135, "2": 10.93243, - "3": 10.64753, - "4": 10.41175, - "5": 10.4009, - "6": 10.36638, - "7": 10.11315, - "8": 9.90158, - "9": 9.96444, - "10": 9.51354, - "11": 10.16344, - "12": 9.86183, - "13": 9.87016, - "14": 9.90033, - "15": 9.50652, - "16": 9.45014, - "17": 9.25948, - "18": 9.3023, - "19": 9.20954, - "20": 8.97165, - "21": 9.00577, - "22": 8.60538, - "23": 9.0944, - "24": 8.68456, - "25": 8.50632, - "26": 8.73853, - "27": 8.82892, - "28": 8.95594, - "29": 8.94456, - "30": 8.49313, - "31": 7.95881, - "32": 8.67861, - "33": 8.74652, - "34": 8.26637, - "35": 8.35333, - "36": 8.26969, - "37": 8.50648, - "38": 8.26655, - "39": 8.63225, - "40": 8.22586, - "41": 8.27127, - "42": 8.42389, - "43": 8.00487, - "44": 8.11371, - "45": 7.99679, - "46": 8.08249, - "47": 8.3834, - "48": 8.09428, - "49": 7.72806, - "50": 8.19206 + "3": 10.64756, + "4": 10.41244, + "5": 10.40014, + "6": 10.36694, + "7": 10.11207, + "8": 9.90201, + "9": 9.96412, + "10": 9.51359, + "11": 10.16293, + "12": 9.86215, + "13": 9.86921, + "14": 9.89993, + "15": 9.50658, + "16": 9.45039, + "17": 9.25976, + "18": 9.30292, + "19": 9.21052, + "20": 8.97215, + "21": 9.00487, + "22": 8.60627, + "23": 9.09462, + "24": 8.6836, + "25": 8.50661, + "26": 8.73877, + "27": 8.82677, + "28": 8.95653, + "29": 8.94427, + "30": 8.49346, + "31": 7.96207, + "32": 8.67493, + "33": 8.74625, + "34": 8.26636, + "35": 8.35393, + "36": 8.26961, + "37": 8.50654, + "38": 8.2666, + "39": 8.63346, + "40": 8.22569, + "41": 8.27208, + "42": 8.42415, + "43": 8.00513, + "44": 8.11394, + "45": 7.99676, + "46": 8.08262, + "47": 8.38365, + "48": 8.09393, + "49": 7.72757, + "50": 8.19211 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 30095348.0, - "2": 30298064.0, - "3": 29950144.0, - "4": 30612180.0, - "5": 30082084.0, - "6": 30408536.0, - "7": 30139302.0, - "8": 30307558.0, - "9": 30212856.0, - "10": 30297188.0, - "11": 29846396.0, - "12": 29802856.0, - "13": 30294960.0, - "14": 29731220.0, - "15": 30193566.0, - "16": 30203640.0, - "17": 30183640.0, - "18": 29936598.0, - "19": 29974136.0, - "20": 30059000.0, - "21": 30109008.0, - "22": 30164864.0, - "23": 29892452.0, - "24": 30141604.0, - "25": 30189348.0, - "26": 29897064.0, - "27": 29812220.0, - "28": 29801036.0, - "29": 29881332.0, - "30": 29982066.0, - "31": 30335112.0, - "32": 29941254.0, - "33": 29912656.0, - "34": 30203268.0, - "35": 30161104.0, - "36": 29942994.0, - "37": 29846750.0, - "38": 30273652.0, - "39": 30172288.0, - "40": 30022886.0, - "41": 30018652.0, - "42": 30029662.0, - "43": 30361064.0, - "44": 30112770.0, - "45": 30026328.0, - "46": 30258596.0, - "47": 29989848.0, - "48": 30301248.0, - "49": 30085594.0, - "50": 30278828.0 + "1": 30095338.0, + "2": 30298062.0, + "3": 29950230.0, + "4": 30612176.0, + "5": 30082060.0, + "6": 30408554.0, + "7": 30139268.0, + "8": 30307444.0, + "9": 30213024.0, + "10": 30297334.0, + "11": 29846428.0, + "12": 29802896.0, + "13": 30294884.0, + "14": 29731016.0, + "15": 30193460.0, + "16": 30203566.0, + "17": 30183536.0, + "18": 29936624.0, + "19": 29974144.0, + "20": 30058912.0, + "21": 30108856.0, + "22": 30164796.0, + "23": 29892404.0, + "24": 30141712.0, + "25": 30189144.0, + "26": 29897320.0, + "27": 29812080.0, + "28": 29801180.0, + "29": 29881446.0, + "30": 29982216.0, + "31": 30335148.0, + "32": 29941518.0, + "33": 29913090.0, + "34": 30203164.0, + "35": 30160876.0, + "36": 29942982.0, + "37": 29846956.0, + "38": 30273396.0, + "39": 30172368.0, + "40": 30022812.0, + "41": 30018776.0, + "42": 30029456.0, + "43": 30361110.0, + "44": 30112740.0, + "45": 30026306.0, + "46": 30258980.0, + "47": 29989740.0, + "48": 30301392.0, + "49": 30085468.0, + "50": 30278812.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 800470016.0, - "2": 800471552.0, - "3": 800471552.0, - "4": 800471552.0, - "5": 800471552.0, - "6": 800471552.0, - "7": 800471552.0, - "8": 800471552.0, - "9": 800471552.0, - "10": 800471552.0, - "11": 800471552.0, - "12": 800471552.0, - "13": 800471552.0, - "14": 800471552.0, - "15": 800471552.0, - "16": 800471552.0, - "17": 800471552.0, - "18": 800471552.0, - "19": 800471552.0, - "20": 800471552.0, - "21": 800471552.0, - "22": 800471552.0, - "23": 800471552.0, - "24": 800471552.0, - "25": 800471552.0, - "26": 800471552.0, - "27": 800471552.0, - "28": 800471552.0, - "29": 800471552.0, - "30": 800471552.0, - "31": 800471552.0, - "32": 800471552.0, - "33": 800471552.0, - "34": 800471552.0, - "35": 800471552.0, - "36": 800471552.0, - "37": 800471552.0, - "38": 800471552.0, - "39": 800471552.0, - "40": 800471552.0, - "41": 800471552.0, - "42": 800471552.0, - "43": 800471552.0, - "44": 800471552.0, - "45": 800471552.0, - "46": 800471552.0, - "47": 800471552.0, - "48": 800471552.0, - "49": 800471552.0, - "50": 800471552.0 + "1": 808944128.0, + "2": 808945664.0, + "3": 808945664.0, + "4": 808945664.0, + "5": 808945664.0, + "6": 808945664.0, + "7": 808945664.0, + "8": 808945664.0, + "9": 808945664.0, + "10": 808945664.0, + "11": 808945664.0, + "12": 808945664.0, + "13": 808945664.0, + "14": 808945664.0, + "15": 808945664.0, + "16": 808945664.0, + "17": 808945664.0, + "18": 808945664.0, + "19": 808945664.0, + "20": 808945664.0, + "21": 808945664.0, + "22": 808945664.0, + "23": 808945664.0, + "24": 808945664.0, + "25": 808945664.0, + "26": 808945664.0, + "27": 808945664.0, + "28": 808945664.0, + "29": 808945664.0, + "30": 808945664.0, + "31": 808945664.0, + "32": 808945664.0, + "33": 808945664.0, + "34": 808945664.0, + "35": 808945664.0, + "36": 808945664.0, + "37": 808945664.0, + "38": 808945664.0, + "39": 808945664.0, + "40": 808945664.0, + "41": 808945664.0, + "42": 808945664.0, + "43": 808945664.0, + "44": 808945664.0, + "45": 808945664.0, + "46": 808945664.0, + "47": 808945664.0, + "48": 808945664.0, + "49": 808945664.0, + "50": 808945664.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2385787904.0, - "2": 2508961280.0, - "3": 2529843712.0, - "4": 2529843712.0, - "5": 2529843712.0, - "6": 2529843712.0, - "7": 2529843712.0, - "8": 2529843712.0, - "9": 2529843712.0, - "10": 2529843712.0, - "11": 2529843712.0, - "12": 2529843712.0, - "13": 2529843712.0, - "14": 2529843712.0, - "15": 2529843712.0, - "16": 2529843712.0, - "17": 2529843712.0, - "18": 2529843712.0, - "19": 2529843712.0, - "20": 2529843712.0, - "21": 2529843712.0, - "22": 2529843712.0, - "23": 2529843712.0, - "24": 2529843712.0, - "25": 2529843712.0, - "26": 2529843712.0, - "27": 2529843712.0, - "28": 2529843712.0, - "29": 2529843712.0, - "30": 2529843712.0, - "31": 2529843712.0, - "32": 2529843712.0, - "33": 2529843712.0, - "34": 2529843712.0, - "35": 2529843712.0, - "36": 2529843712.0, - "37": 2529843712.0, - "38": 2529843712.0, - "39": 2531715072.0, - "40": 2531715072.0, - "41": 2531715072.0, - "42": 2531715072.0, - "43": 2531715072.0, - "44": 2531715072.0, - "45": 2539506688.0, - "46": 2539506688.0, - "47": 2539506688.0, - "48": 2539506688.0, - "49": 2539506688.0, - "50": 2539506688.0 + "1": 2389920256.0, + "2": 2517091328.0, + "3": 2533976064.0, + "4": 2533976064.0, + "5": 2533976064.0, + "6": 2533976064.0, + "7": 2533976064.0, + "8": 2533976064.0, + "9": 2533976064.0, + "10": 2533976064.0, + "11": 2533976064.0, + "12": 2533976064.0, + "13": 2533976064.0, + "14": 2533976064.0, + "15": 2533976064.0, + "16": 2533976064.0, + "17": 2533976064.0, + "18": 2533976064.0, + "19": 2533976064.0, + "20": 2533976064.0, + "21": 2533976064.0, + "22": 2533976064.0, + "23": 2533976064.0, + "24": 2533976064.0, + "25": 2533976064.0, + "26": 2533976064.0, + "27": 2533976064.0, + "28": 2533976064.0, + "29": 2533976064.0, + "30": 2533976064.0, + "31": 2533976064.0, + "32": 2533976064.0, + "33": 2533976064.0, + "34": 2533976064.0, + "35": 2533976064.0, + "36": 2533976064.0, + "37": 2533976064.0, + "38": 2533976064.0, + "39": 2541403136.0, + "40": 2541403136.0, + "41": 2541403136.0, + "42": 2541403136.0, + "43": 2541403136.0, + "44": 2541403136.0, + "45": 2547948032.0, + "46": 2547948032.0, + "47": 2547948032.0, + "48": 2547948032.0, + "49": 2547948032.0, + "50": 2547948032.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 12.78616, - "3": 0.1263, - "4": 0.69944, - "5": 0.70311, - "6": 0.72094, - "7": 0.33756, - "8": 0.55932, - "9": 0.73242, - "10": 0.57684, - "11": 0.90862, - "12": 1.02775, - "13": 0.68326, - "14": 0.51122, - "15": 0.4671, - "16": 0.59291, - "17": 0.65569, - "18": 0.58508, - "19": 1.01652, - "20": 0.54363, - "21": 0.39484, - "22": 0.90435, - "23": 0.80357, - "24": 0.77363, - "25": 0.43172, - "26": 0.77249, - "27": 0.96313, - "28": 0.47261, - "29": 0.51333, - "30": 0.81788, - "31": 0.60892, - "32": 0.4855, - "33": 0.97321, - "34": 0.64729, - "35": 0.46514, - "36": 0.84552, - "37": 0.46938, - "38": 0.52313, - "39": 0.43514, - "40": 0.5703, - "41": 0.91146, - "42": 0.51738, - "43": 0.39267, - "44": 0.46113, - "45": 1.04646, - "46": 0.46625, - "47": 0.74725, - "48": 0.92515, - "49": 0.46575, - "50": 1.09022 + "2": 15.66901, + "3": 0.13957, + "4": 2.88309, + "5": 2.44054, + "6": 3.92555, + "7": 1.56638, + "8": 1.82741, + "9": 2.81256, + "10": 2.88303, + "11": 2.69557, + "12": 4.34562, + "13": 1.81868, + "14": 2.43537, + "15": 2.07544, + "16": 1.79247, + "17": 1.70001, + "18": 2.12003, + "19": 4.55465, + "20": 2.01432, + "21": 1.62555, + "22": 2.61272, + "23": 4.56883, + "24": 2.20191, + "25": 0.65233, + "26": 4.23847, + "27": 4.05559, + "28": 0.69792, + "29": 2.50736, + "30": 2.81427, + "31": 2.51059, + "32": 1.48739, + "33": 2.82868, + "34": 2.40251, + "35": 2.12338, + "36": 3.64245, + "37": 1.96823, + "38": 1.79352, + "39": 1.8208, + "40": 2.67235, + "41": 4.01385, + "42": 1.73888, + "43": 1.75892, + "44": 3.9522, + "45": 11.12529, + "46": 1.19683, + "47": 1.72126, + "48": 1.65029, + "49": 1.96077, + "50": 2.77822 } } } diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json index b441416be8b..59a3e07b6cf 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.93064, "2": 10.92264, - "3": 10.92453, - "4": 10.91706, - "5": 10.92766, - "6": 10.92069, - "7": 10.9166, - "8": 10.91808, - "9": 10.93792, - "10": 10.92356, - "11": 10.91567, - "12": 10.92593, - "13": 10.93441, - "14": 10.91528, - "15": 10.92879, - "16": 10.9189, - "17": 10.92699, - "18": 10.91334, - "19": 10.93068, - "20": 10.90002, - "21": 10.9404, - "22": 10.92589, - "23": 10.93025, - "24": 10.90535, - "25": 10.91492, - "26": 10.92587, - "27": 10.9279, - "28": 10.91992, - "29": 10.92593, - "30": 10.91378, - "31": 10.9109, - "32": 10.91882, - "33": 10.92004, - "34": 10.90075, - "35": 10.91879, - "36": 10.90792, - "37": 10.91301, - "38": 10.92359, - "39": 10.9122, - "40": 10.91099, - "41": 10.91319, + "3": 10.92465, + "4": 10.91715, + "5": 10.92717, + "6": 10.92055, + "7": 10.91609, + "8": 10.91785, + "9": 10.93786, + "10": 10.92372, + "11": 10.91571, + "12": 10.92553, + "13": 10.93439, + "14": 10.91548, + "15": 10.92876, + "16": 10.91936, + "17": 10.92729, + "18": 10.91319, + "19": 10.93054, + "20": 10.90032, + "21": 10.94064, + "22": 10.92551, + "23": 10.93042, + "24": 10.90582, + "25": 10.91511, + "26": 10.92613, + "27": 10.92773, + "28": 10.92039, + "29": 10.92588, + "30": 10.91343, + "31": 10.91057, + "32": 10.91866, + "33": 10.91981, + "34": 10.90077, + "35": 10.91932, + "36": 10.90795, + "37": 10.91252, + "38": 10.92363, + "39": 10.91252, + "40": 10.91054, + "41": 10.91301, "42": 10.90912, - "43": 10.90596, - "44": 10.90688, - "45": 10.89722, - "46": 10.90681, - "47": 10.90148, - "48": 10.8843, - "49": 10.90016, - "50": 10.89916, - "51": 10.89786, - "52": 10.90405, - "53": 10.90069, - "54": 10.88764, - "55": 10.88607, - "56": 10.88893, - "57": 10.88739, - "58": 10.88856, - "59": 10.88187, - "60": 10.87372, - "61": 10.87299, - "62": 10.86808, - "63": 10.87591, - "64": 10.85915, - "65": 10.86096, - "66": 10.85725, - "67": 10.86131, - "68": 10.86071, - "69": 10.85024, - "70": 10.86702, - "71": 10.84888, - "72": 10.83992, - "73": 10.84898, - "74": 10.839, - "75": 10.83634, - "76": 10.83445, - "77": 10.82965, - "78": 10.82317, - "79": 10.82842, - "80": 10.82931, - "81": 10.81726, - "82": 10.81715, - "83": 10.80494, - "84": 10.78359, - "85": 10.78354, - "86": 10.80074, - "87": 10.79543, - "88": 10.79476, - "89": 10.77699, - "90": 10.78295, - "91": 10.79036, - "92": 10.78271, - "93": 10.75468, - "94": 10.7609, - "95": 10.77286, - "96": 10.73628, - "97": 10.74115, - "98": 10.74693, - "99": 10.76285, - "100": 10.74247 + "43": 10.90589, + "44": 10.90719, + "45": 10.89729, + "46": 10.90722, + "47": 10.90182, + "48": 10.88483, + "49": 10.89993, + "50": 10.89889, + "51": 10.89766, + "52": 10.90433, + "53": 10.90112, + "54": 10.88726, + "55": 10.88627, + "56": 10.88895, + "57": 10.88726, + "58": 10.88928, + "59": 10.88247, + "60": 10.87351, + "61": 10.87295, + "62": 10.86802, + "63": 10.87643, + "64": 10.85969, + "65": 10.86057, + "66": 10.85704, + "67": 10.86137, + "68": 10.86025, + "69": 10.85031, + "70": 10.86688, + "71": 10.84897, + "72": 10.84054, + "73": 10.84845, + "74": 10.83854, + "75": 10.83595, + "76": 10.83467, + "77": 10.82929, + "78": 10.82346, + "79": 10.8292, + "80": 10.82953, + "81": 10.81765, + "82": 10.81677, + "83": 10.80473, + "84": 10.78345, + "85": 10.78324, + "86": 10.80056, + "87": 10.79516, + "88": 10.79549, + "89": 10.7764, + "90": 10.78266, + "91": 10.78961, + "92": 10.78345, + "93": 10.75503, + "94": 10.7601, + "95": 10.77322, + "96": 10.73553, + "97": 10.7411, + "98": 10.74753, + "99": 10.76289, + "100": 10.74232 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 49180.0, - "2": 50476.0, - "3": 49152.0, - "4": 49093.0, - "5": 49913.0, - "6": 49695.0, - "7": 51527.0, - "8": 49236.0, - "9": 50963.0, - "10": 51988.0, - "11": 50179.0, - "12": 48660.0, - "13": 52142.0, - "14": 50135.0, - "15": 47947.0, - "16": 49733.0, - "17": 49715.0, - "18": 51893.0, - "19": 50900.0, - "20": 52897.0, - "21": 50249.0, - "22": 49494.0, - "23": 53440.0, - "24": 51476.0, - "25": 48836.0, - "26": 52057.0, - "27": 51764.0, - "28": 48472.0, - "29": 50091.0, - "30": 51160.0, - "31": 52934.0, - "32": 49271.0, - "33": 52492.0, - "34": 49785.0, - "35": 51389.0, - "36": 50759.0, - "37": 49447.0, - "38": 50917.0, - "39": 50977.0, - "40": 51303.0, - "41": 47884.0, - "42": 50199.0, - "43": 51264.0, - "44": 48123.0, - "45": 54319.0, - "46": 47721.0, - "47": 50709.0, - "48": 50505.0, - "49": 54725.0, - "50": 50759.0, - "51": 48818.0, - "52": 50484.0, - "53": 49662.0, - "54": 49308.0, - "55": 49348.0, - "56": 48679.0, - "57": 48551.0, - "58": 52274.0, - "59": 49618.0, - "60": 49684.0, - "61": 47669.0, - "62": 50900.0, - "63": 48419.0, - "64": 55920.0, - "65": 51019.0, - "66": 49471.0, - "67": 51576.0, - "68": 49653.0, - "69": 54382.0, - "70": 50016.0, - "71": 49561.0, - "72": 51901.0, - "73": 52536.0, - "74": 49702.0, - "75": 50683.0, - "76": 51554.0, - "77": 49757.0, - "78": 49608.0, - "79": 49124.0, - "80": 52318.0, - "81": 49552.0, - "82": 46737.0, - "83": 52475.0, - "84": 48461.0, - "85": 50083.0, - "86": 49539.0, - "87": 46565.0, - "88": 48397.0, - "89": 48861.0, - "90": 52531.0, - "91": 50128.0, - "92": 50276.0, - "93": 50731.0, - "94": 51663.0, - "95": 48703.0, - "96": 49914.0, - "97": 49912.0, - "98": 48976.0, - "99": 48894.0, - "100": 47024.0 + "1": 49235.0, + "2": 50357.0, + "3": 49546.0, + "4": 48846.0, + "5": 50379.0, + "6": 50110.0, + "7": 51587.0, + "8": 49485.0, + "9": 51327.0, + "10": 52143.0, + "11": 50118.0, + "12": 48318.0, + "13": 52021.0, + "14": 49866.0, + "15": 48240.0, + "16": 49389.0, + "17": 49518.0, + "18": 51956.0, + "19": 51177.0, + "20": 52168.0, + "21": 50261.0, + "22": 49713.0, + "23": 53249.0, + "24": 51193.0, + "25": 49064.0, + "26": 52264.0, + "27": 51694.0, + "28": 48600.0, + "29": 50031.0, + "30": 51416.0, + "31": 53317.0, + "32": 49297.0, + "33": 52659.0, + "34": 50539.0, + "35": 51219.0, + "36": 51109.0, + "37": 49480.0, + "38": 51056.0, + "39": 50242.0, + "40": 50913.0, + "41": 48081.0, + "42": 50313.0, + "43": 50803.0, + "44": 47790.0, + "45": 54402.0, + "46": 48568.0, + "47": 50322.0, + "48": 50374.0, + "49": 54562.0, + "50": 50805.0, + "51": 49176.0, + "52": 50148.0, + "53": 49566.0, + "54": 49257.0, + "55": 49442.0, + "56": 48208.0, + "57": 48313.0, + "58": 53025.0, + "59": 49904.0, + "60": 49229.0, + "61": 47194.0, + "62": 50834.0, + "63": 48598.0, + "64": 56290.0, + "65": 51257.0, + "66": 48942.0, + "67": 51795.0, + "68": 49731.0, + "69": 54766.0, + "70": 49990.0, + "71": 49335.0, + "72": 51986.0, + "73": 53024.0, + "74": 49671.0, + "75": 50182.0, + "76": 51257.0, + "77": 50192.0, + "78": 49595.0, + "79": 48606.0, + "80": 52319.0, + "81": 50440.0, + "82": 46773.0, + "83": 52693.0, + "84": 48845.0, + "85": 50043.0, + "86": 49294.0, + "87": 46237.0, + "88": 48662.0, + "89": 48597.0, + "90": 51859.0, + "91": 50314.0, + "92": 49975.0, + "93": 51092.0, + "94": 52031.0, + "95": 48939.0, + "96": 49841.0, + "97": 49971.0, + "98": 48580.0, + "99": 48968.0, + "100": 47509.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1257369088.0, - "2": 1257388032.0, - "3": 1257372160.0, - "4": 1257370112.0, - "5": 1257382400.0, - "6": 1257369600.0, - "7": 1257363456.0, - "8": 1257391104.0, - "9": 1257341440.0, - "10": 1257339904.0, - "11": 1257326080.0, - "12": 1257356288.0, - "13": 1257354240.0, - "14": 1257347584.0, - "15": 1257373184.0, - "16": 1257364480.0, - "17": 1257375232.0, - "18": 1257352704.0, - "19": 1257348608.0, - "20": 1257361408.0, - "21": 1257383424.0, - "22": 1257318400.0, - "23": 1257361920.0, - "24": 1257377280.0, - "25": 1257337344.0, - "26": 1257374208.0, - "27": 1257379840.0, - "28": 1257345024.0, - "29": 1257360896.0, - "30": 1257371136.0, - "31": 1257333760.0, - "32": 1257354752.0, - "33": 1257340928.0, - "34": 1257382400.0, - "35": 1257311232.0, - "36": 1257384960.0, - "37": 1257352704.0, - "38": 1257378304.0, - "39": 1257342464.0, - "40": 1257394176.0, - "41": 1257356800.0, - "42": 1257336832.0, - "43": 1257338368.0, - "44": 1257350656.0, - "45": 1257364480.0, - "46": 1257312256.0, - "47": 1257336832.0, - "48": 1257326592.0, - "49": 1257357312.0, - "50": 1257341440.0, - "51": 1257309696.0, - "52": 1257357312.0, - "53": 1257314304.0, - "54": 1257373184.0, - "55": 1257322496.0, - "56": 1257371136.0, - "57": 1257340928.0, - "58": 1257348608.0, - "59": 1257349120.0, - "60": 1257313792.0, - "61": 1257295360.0, - "62": 1257327104.0, - "63": 1257333248.0, - "64": 1257348608.0, - "65": 1257353216.0, - "66": 1257275392.0, - "67": 1257360384.0, - "68": 1257307136.0, - "69": 1257333760.0, - "70": 1257299968.0, - "71": 1257321984.0, - "72": 1257320448.0, - "73": 1257316864.0, - "74": 1257284608.0, - "75": 1257309696.0, - "76": 1257312256.0, - "77": 1257349632.0, - "78": 1257344512.0, - "79": 1257313280.0, - "80": 1257257984.0, - "81": 1257302528.0, - "82": 1257277952.0, - "83": 1257269248.0, - "84": 1257312768.0, - "85": 1257284096.0, - "86": 1257276928.0, - "87": 1257267712.0, - "88": 1257282560.0, - "89": 1257268736.0, - "90": 1257306112.0, - "91": 1257267712.0, - "92": 1257269760.0, - "93": 1257285120.0, - "94": 1257268736.0, - "95": 1257267712.0, - "96": 1257271808.0, - "97": 1257264640.0, - "98": 1257313792.0, - "99": 1257251328.0, - "100": 1257255424.0 + "1": 1263890432.0, + "2": 1263909376.0, + "3": 1263893504.0, + "4": 1263891968.0, + "5": 1263902720.0, + "6": 1263890432.0, + "7": 1263884800.0, + "8": 1263913984.0, + "9": 1263862272.0, + "10": 1263862272.0, + "11": 1263847424.0, + "12": 1263876608.0, + "13": 1263877632.0, + "14": 1263867392.0, + "15": 1263895040.0, + "16": 1263884800.0, + "17": 1263926784.0, + "18": 1263874560.0, + "19": 1263869952.0, + "20": 1263884288.0, + "21": 1263904256.0, + "22": 1263838720.0, + "23": 1263882752.0, + "24": 1263896576.0, + "25": 1263860736.0, + "26": 1263897600.0, + "27": 1263898112.0, + "28": 1263863808.0, + "29": 1263882240.0, + "30": 1263892480.0, + "31": 1263855104.0, + "32": 1263875584.0, + "33": 1263861248.0, + "34": 1263904768.0, + "35": 1263832576.0, + "36": 1263906304.0, + "37": 1263875072.0, + "38": 1263901184.0, + "39": 1263863296.0, + "40": 1263912448.0, + "41": 1263879168.0, + "42": 1263856640.0, + "43": 1263859200.0, + "44": 1263870464.0, + "45": 1263886848.0, + "46": 1263834624.0, + "47": 1263858688.0, + "48": 1263847936.0, + "49": 1263878656.0, + "50": 1263862784.0, + "51": 1263832576.0, + "52": 1263879168.0, + "53": 1263835648.0, + "54": 1263896576.0, + "55": 1263842816.0, + "56": 1263890432.0, + "57": 1263863296.0, + "58": 1263868928.0, + "59": 1263869952.0, + "60": 1263834112.0, + "61": 1263816192.0, + "62": 1263848960.0, + "63": 1263854592.0, + "64": 1263868928.0, + "65": 1263874560.0, + "66": 1263795712.0, + "67": 1263881216.0, + "68": 1263827456.0, + "69": 1263854080.0, + "70": 1263822336.0, + "71": 1263841792.0, + "72": 1263840256.0, + "73": 1263839744.0, + "74": 1263805440.0, + "75": 1263831552.0, + "76": 1263835136.0, + "77": 1263869952.0, + "78": 1263865344.0, + "79": 1263834112.0, + "80": 1263779840.0, + "81": 1263824384.0, + "82": 1263798272.0, + "83": 1263790592.0, + "84": 1263834112.0, + "85": 1263806464.0, + "86": 1263798272.0, + "87": 1263790592.0, + "88": 1263804928.0, + "89": 1263788544.0, + "90": 1263829504.0, + "91": 1263791104.0, + "92": 1263791616.0, + "93": 1263809024.0, + "94": 1263790080.0, + "95": 1263788032.0, + "96": 1263789568.0, + "97": 1263785472.0, + "98": 1263834112.0, + "99": 1263773696.0, + "100": 1263777280.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3479515648.0, - "2": 3875144192.0, - "3": 3875144192.0, - "4": 3875144192.0, - "5": 3875144192.0, - "6": 3875144192.0, - "7": 3875144192.0, - "8": 3875144192.0, - "9": 3875144192.0, - "10": 3875144192.0, - "11": 3875144192.0, - "12": 3875144192.0, - "13": 3875144192.0, - "14": 3875144192.0, - "15": 3875144192.0, - "16": 3875144192.0, - "17": 3875144192.0, - "18": 3875144192.0, - "19": 3875144192.0, - "20": 3875144192.0, - "21": 3875144192.0, - "22": 3875144192.0, - "23": 3875144192.0, - "24": 3875144192.0, - "25": 3875144192.0, - "26": 3875144192.0, - "27": 3875144192.0, - "28": 3875144192.0, - "29": 3875144192.0, - "30": 3875144192.0, - "31": 3875144192.0, - "32": 3875144192.0, - "33": 3875144192.0, - "34": 3875144192.0, - "35": 3875144192.0, - "36": 3875144192.0, - "37": 3875144192.0, - "38": 3875144192.0, - "39": 3875144192.0, - "40": 3875144192.0, - "41": 3875144192.0, - "42": 3875144192.0, - "43": 3875144192.0, - "44": 3875144192.0, - "45": 3875144192.0, - "46": 3875144192.0, - "47": 3875144192.0, - "48": 3875144192.0, - "49": 3875144192.0, - "50": 3875144192.0, - "51": 3875144192.0, - "52": 3875144192.0, - "53": 3875144192.0, - "54": 3875144192.0, - "55": 3875144192.0, - "56": 3875144192.0, - "57": 3875144192.0, - "58": 3875144192.0, - "59": 3875144192.0, - "60": 3875144192.0, - "61": 3875144192.0, - "62": 3875144192.0, - "63": 3875144192.0, - "64": 3875144192.0, - "65": 3875144192.0, - "66": 3875144192.0, - "67": 3875144192.0, - "68": 3875144192.0, - "69": 3875144192.0, - "70": 3875144192.0, - "71": 3875144192.0, - "72": 3875144192.0, - "73": 3875144192.0, - "74": 3875144192.0, - "75": 3875144192.0, - "76": 3875144192.0, - "77": 3875144192.0, - "78": 3875144192.0, - "79": 3875144192.0, - "80": 3875144192.0, - "81": 3875144192.0, - "82": 3875144192.0, - "83": 3875144192.0, - "84": 3875144192.0, - "85": 3875144192.0, - "86": 3875144192.0, - "87": 3875144192.0, - "88": 3875144192.0, - "89": 3875144192.0, - "90": 3875144192.0, - "91": 3875144192.0, - "92": 3875144192.0, - "93": 3875144192.0, - "94": 3875144192.0, - "95": 3875144192.0, - "96": 3875144192.0, - "97": 3875144192.0, - "98": 3875144192.0, - "99": 3875144192.0, - "100": 3875144192.0 + "1": 3484759552.0, + "2": 3873047552.0, + "3": 3873047552.0, + "4": 3873047552.0, + "5": 3873047552.0, + "6": 3873047552.0, + "7": 3873047552.0, + "8": 3873047552.0, + "9": 3873047552.0, + "10": 3873047552.0, + "11": 3873047552.0, + "12": 3873047552.0, + "13": 3873047552.0, + "14": 3873047552.0, + "15": 3873047552.0, + "16": 3873047552.0, + "17": 3873047552.0, + "18": 3873047552.0, + "19": 3873047552.0, + "20": 3873047552.0, + "21": 3873047552.0, + "22": 3873047552.0, + "23": 3873047552.0, + "24": 3873047552.0, + "25": 3873047552.0, + "26": 3873047552.0, + "27": 3873047552.0, + "28": 3873047552.0, + "29": 3873047552.0, + "30": 3873047552.0, + "31": 3873047552.0, + "32": 3873047552.0, + "33": 3873047552.0, + "34": 3873047552.0, + "35": 3873047552.0, + "36": 3873047552.0, + "37": 3873047552.0, + "38": 3873047552.0, + "39": 3873047552.0, + "40": 3873047552.0, + "41": 3873047552.0, + "42": 3873047552.0, + "43": 3873047552.0, + "44": 3873047552.0, + "45": 3873047552.0, + "46": 3873047552.0, + "47": 3873047552.0, + "48": 3873047552.0, + "49": 3873047552.0, + "50": 3873047552.0, + "51": 3873047552.0, + "52": 3873047552.0, + "53": 3873047552.0, + "54": 3873047552.0, + "55": 3873047552.0, + "56": 3873047552.0, + "57": 3873047552.0, + "58": 3873047552.0, + "59": 3873047552.0, + "60": 3873047552.0, + "61": 3873047552.0, + "62": 3873047552.0, + "63": 3873047552.0, + "64": 3873047552.0, + "65": 3873047552.0, + "66": 3873047552.0, + "67": 3873047552.0, + "68": 3873047552.0, + "69": 3873047552.0, + "70": 3873047552.0, + "71": 3873047552.0, + "72": 3873047552.0, + "73": 3873047552.0, + "74": 3873047552.0, + "75": 3873047552.0, + "76": 3873047552.0, + "77": 3873047552.0, + "78": 3873047552.0, + "79": 3873047552.0, + "80": 3873047552.0, + "81": 3873047552.0, + "82": 3873047552.0, + "83": 3873047552.0, + "84": 3873047552.0, + "85": 3873047552.0, + "86": 3873047552.0, + "87": 3873047552.0, + "88": 3873047552.0, + "89": 3873047552.0, + "90": 3873047552.0, + "91": 3873047552.0, + "92": 3873047552.0, + "93": 3873047552.0, + "94": 3873047552.0, + "95": 3873047552.0, + "96": 3873047552.0, + "97": 3873047552.0, + "98": 3873047552.0, + "99": 3873047552.0, + "100": 3873047552.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.06008, - "3": 0.62212, - "4": 0.59439, - "5": 0.59583, - "6": 0.60184, - "7": 0.68944, - "8": 0.64314, - "9": 0.6749, - "10": 0.67461, - "11": 1.42804, - "12": 0.58165, - "13": 0.59318, - "14": 0.79281, - "15": 0.58563, - "16": 0.70911, - "17": 0.58841, - "18": 0.93635, - "19": 0.58466, - "20": 0.57567, - "21": 0.75501, - "22": 0.7513, - "23": 0.57466, - "24": 0.59062, - "25": 0.57249, - "26": 0.57164, - "27": 0.57991, - "28": 0.58345, - "29": 0.58425, - "30": 0.57143, - "31": 0.57751, - "32": 0.63679, - "33": 0.57343, - "34": 0.692, - "35": 0.576, - "36": 0.57893, - "37": 0.57805, - "38": 0.57591, - "39": 0.56481, - "40": 0.57011, - "41": 0.58034, - "42": 0.5834, - "43": 0.58412, - "44": 0.57894, - "45": 0.64352, - "46": 0.5782, - "47": 0.57996, - "48": 0.83465, - "49": 0.78578, - "50": 0.63159, - "51": 0.64775, - "52": 0.60094, - "53": 0.57927, - "54": 0.57978, - "55": 0.57904, - "56": 0.75419, - "57": 0.5724, - "58": 0.58029, - "59": 0.61934, - "60": 0.58073, - "61": 0.57339, - "62": 0.83927, - "63": 0.60116, - "64": 1.07001, - "65": 0.57078, - "66": 0.69353, - "67": 0.64965, - "68": 0.57507, - "69": 0.58624, - "70": 0.57896, - "71": 0.57264, - "72": 0.58547, - "73": 0.58352, - "74": 0.57325, - "75": 0.57046, - "76": 0.57555, - "77": 0.57548, - "78": 1.24371, - "79": 0.58474, - "80": 0.57365, - "81": 0.61678, - "82": 0.65707, - "83": 0.85963, - "84": 0.57211, - "85": 0.58096, - "86": 0.57084, - "87": 0.89269, - "88": 0.8608, - "89": 0.75853, - "90": 0.57419, - "91": 0.57351, - "92": 0.9021, - "93": 0.57971, - "94": 0.58609, - "95": 0.91409, - "96": 0.59088, - "97": 0.6732, - "98": 0.7339, - "99": 1.99078, - "100": 0.57381 + "2": 6.88765, + "3": 0.57328, + "4": 2.25702, + "5": 1.13142, + "6": 1.86123, + "7": 2.68816, + "8": 2.47495, + "9": 0.54058, + "10": 1.14232, + "11": 2.64069, + "12": 0.72483, + "13": 1.03045, + "14": 1.74573, + "15": 1.55469, + "16": 1.28271, + "17": 1.62938, + "18": 2.0367, + "19": 0.97262, + "20": 1.08142, + "21": 1.58831, + "22": 1.94251, + "23": 2.2163, + "24": 0.66754, + "25": 2.17032, + "26": 0.7232, + "27": 2.06189, + "28": 1.09781, + "29": 0.74173, + "30": 0.54668, + "31": 1.17978, + "32": 2.08114, + "33": 1.72779, + "34": 0.54942, + "35": 2.08125, + "36": 1.01405, + "37": 1.34409, + "38": 0.541, + "39": 1.39814, + "40": 1.22124, + "41": 0.92108, + "42": 1.9967, + "43": 1.29779, + "44": 1.42978, + "45": 1.10149, + "46": 1.33998, + "47": 0.7521, + "48": 1.49875, + "49": 0.77721, + "50": 0.82017, + "51": 0.83749, + "52": 1.46796, + "53": 1.44614, + "54": 1.55814, + "55": 1.05888, + "56": 1.32449, + "57": 1.12673, + "58": 1.5076, + "59": 1.03704, + "60": 0.71864, + "61": 1.55561, + "62": 1.60835, + "63": 0.55837, + "64": 1.66579, + "65": 0.69444, + "66": 1.06912, + "67": 0.98096, + "68": 0.54909, + "69": 1.22388, + "70": 1.89153, + "71": 0.53181, + "72": 0.53816, + "73": 1.2005, + "74": 0.85398, + "75": 1.23409, + "76": 1.51336, + "77": 0.95957, + "78": 2.62906, + "79": 1.01729, + "80": 0.6891, + "81": 1.19803, + "82": 1.63101, + "83": 0.53359, + "84": 2.52747, + "85": 0.53905, + "86": 1.299, + "87": 2.50317, + "88": 0.53214, + "89": 2.00396, + "90": 1.14012, + "91": 1.27655, + "92": 1.88529, + "93": 0.53833, + "94": 1.05065, + "95": 2.18666, + "96": 2.00006, + "97": 1.00642, + "98": 1.17954, + "99": 1.95542, + "100": 1.25069 } } } \ No newline at end of file diff --git a/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_fully_shard.py b/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_fully_shard.py index 88354fbb282..2bc198695b0 100644 --- a/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_fully_shard.py +++ b/tests/unit_tests/distributed/megatron_fsdp/test_mfsdp_fully_shard.py @@ -322,6 +322,20 @@ def test_fully_shard( "Meta device initialization (init_model_with_meta_device=True) is not " "supported or necessary for the 'no_shard' / 0 sharding strategy." ) + elif dp_shard_strategy == NO_SHARD and dp_outer_strategy == NO_SHARD: + # When both inner and outer DP are unsharded, the optimizer state is a + # fully-replicated DTensor. Starting with the PyTorch shipped in + # nvcr.io/nvidia/pytorch:26.06-py3, the Adam step's in-place + # `aten.lerp.Scalar` on a Replicate() DTensor raises + # "in-place operations that require placement changes are not supported". + # This is a PyTorch DTensor behavior change, not a Megatron-FSDP + # regression; skip until Megatron-FSDP's NO_SHARD optimizer path avoids + # the in-place op. See https://github.com/NVIDIA/Megatron-LM/issues/4611. + pytest.skip( + "Fully-unsharded ('no_shard'/'no_shard') optimizer step uses an in-place " + "lerp on a Replicate() DTensor, which is unsupported by the DTensor " + "dispatcher in PyTorch 26.06+." + ) elif dp_outer_strategy == OPTIM and dp_shard_strategy != OPTIM_GRADS_PARAMS: # TODO(@shjwudp, @cspades): Requires various modifications to support. pytest.skip( From 3038a4e32f219c89fb419f208e13da2584b9da17 Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Sat, 4 Jul 2026 17:28:05 -0700 Subject: [PATCH 2/7] Update golden values in functional tests for various models to ensure consistency across test cases. Adjustments made to numerical values in JSON files for improved accuracy in results. Relative diff compare_golden_values_kl.py covered 5 files x 5 metrics = 25 rows. | metric | n | median |avg_rel_diff| | max |avg_rel_diff| | | --- | ---: | ---: | ---: | | iteration-time | 5 | 0.654051 | 0.82148 | | lm loss | 5 | 6.705e-06 | 8.006e-06 | | mem-allocated-bytes | 5 | 3.632e-07 | 0.046965 | | mem-max-allocated-bytes | 5 | 1.185e-04 | 0.013356 | | num-zeros | 5 | 4.082e-04 | 0.084901 | Signed-off-by: Ajay Balasa --- .../golden_values_dev_dgx_gb200.json | 966 ++++++++--------- .../golden_values_dev_dgx_gb200.json | 962 ++++++++--------- .../golden_values_dev_dgx_gb200.json | 480 ++++----- .../golden_values_dev_dgx_gb200.json | 480 ++++----- .../golden_values_dev_dgx_gb200.json | 992 +++++++++--------- 5 files changed, 1940 insertions(+), 1940 deletions(-) diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json index 78c53757a24..220d73d17a7 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear_1node/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.92752, "2": 10.92554, "3": 10.93114, - "4": 10.93727, - "5": 10.93015, - "6": 10.92346, - "7": 10.93864, - "8": 10.92777, - "9": 10.93074, - "10": 10.92801, - "11": 10.91409, + "4": 10.93726, + "5": 10.93014, + "6": 10.92348, + "7": 10.93867, + "8": 10.92776, + "9": 10.93077, + "10": 10.92799, + "11": 10.91407, "12": 10.92299, - "13": 10.92006, - "14": 10.90618, + "13": 10.92008, + "14": 10.90616, "15": 10.88338, - "16": 10.87953, - "17": 10.88861, - "18": 10.86454, + "16": 10.87952, + "17": 10.88864, + "18": 10.86456, "19": 10.87347, - "20": 10.78318, - "21": 10.78551, - "22": 10.76768, - "23": 10.76394, + "20": 10.78316, + "21": 10.78552, + "22": 10.76771, + "23": 10.76393, "24": 10.72672, - "25": 10.734, - "26": 10.71269, - "27": 10.6757, - "28": 10.60128, - "29": 10.5823, - "30": 10.55707, - "31": 10.56165, - "32": 10.53484, - "33": 10.50353, - "34": 10.46529, - "35": 10.47894, - "36": 10.45605, - "37": 10.41812, - "38": 10.41677, - "39": 10.38189, - "40": 10.37623, - "41": 10.34336, - "42": 10.33171, - "43": 10.30768, - "44": 10.27724, - "45": 10.29884, - "46": 10.2598, - "47": 10.23664, - "48": 10.1938, - "49": 10.18553, - "50": 10.19738, - "51": 10.1937, - "52": 10.14694, - "53": 10.1499, - "54": 10.11736, - "55": 10.08481, - "56": 10.12206, - "57": 10.10473, + "25": 10.73395, + "26": 10.71271, + "27": 10.67569, + "28": 10.60125, + "29": 10.58228, + "30": 10.55708, + "31": 10.56163, + "32": 10.53486, + "33": 10.50354, + "34": 10.4653, + "35": 10.47893, + "36": 10.4561, + "37": 10.41813, + "38": 10.41673, + "39": 10.38188, + "40": 10.37622, + "41": 10.34337, + "42": 10.33167, + "43": 10.3077, + "44": 10.27723, + "45": 10.29881, + "46": 10.25982, + "47": 10.23666, + "48": 10.19383, + "49": 10.1855, + "50": 10.19733, + "51": 10.19368, + "52": 10.14693, + "53": 10.14989, + "54": 10.11739, + "55": 10.08479, + "56": 10.12208, + "57": 10.10471, "58": 10.11908, - "59": 10.06432, - "60": 10.08092, + "59": 10.0643, + "60": 10.0809, "61": 10.03845, - "62": 10.00515, - "63": 10.07878, - "64": 10.03206, + "62": 10.00519, + "63": 10.07876, + "64": 10.0321, "65": 10.00349, - "66": 10.03495, + "66": 10.03497, "67": 10.00575, - "68": 9.97033, - "69": 9.99157, - "70": 9.97869, - "71": 9.99986, - "72": 9.97535, - "73": 9.96568, - "74": 9.95986, - "75": 9.92549, + "68": 9.9703, + "69": 9.99159, + "70": 9.97867, + "71": 9.99985, + "72": 9.97531, + "73": 9.96569, + "74": 9.95989, + "75": 9.92547, "76": 9.96565, - "77": 9.95606, - "78": 9.9045, - "79": 9.91133, + "77": 9.95608, + "78": 9.90451, + "79": 9.91134, "80": 9.92487, - "81": 9.94438, - "82": 9.88871, - "83": 9.85105, - "84": 9.78905, - "85": 9.77533, - "86": 9.88023, - "87": 9.91309, - "88": 9.88646, - "89": 9.8237, - "90": 9.81331, + "81": 9.94437, + "82": 9.88872, + "83": 9.85103, + "84": 9.78907, + "85": 9.7753, + "86": 9.88017, + "87": 9.91311, + "88": 9.88647, + "89": 9.82373, + "90": 9.81329, "91": 9.82141, - "92": 9.81015, - "93": 9.75217, - "94": 9.82575, - "95": 9.8191, - "96": 9.80552, + "92": 9.81013, + "93": 9.75216, + "94": 9.82577, + "95": 9.81914, + "96": 9.80555, "97": 9.74366, - "98": 9.7725, + "98": 9.77253, "99": 9.8167, - "100": 9.71129 + "100": 9.71128 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1845.0, - "2": 1697.0, - "3": 1692.0, - "4": 1714.0, - "5": 1786.0, - "6": 1732.0, - "7": 1952.0, - "8": 1761.0, - "9": 1829.0, - "10": 1835.0, - "11": 1692.0, - "12": 1719.0, - "13": 1771.0, - "14": 1925.0, - "15": 1565.0, - "16": 1676.0, - "17": 1839.0, - "18": 1831.0, - "19": 1657.0, - "20": 1646.0, - "21": 1758.0, - "22": 1764.0, - "23": 1679.0, - "24": 1796.0, - "25": 1803.0, - "26": 1861.0, - "27": 1680.0, - "28": 1836.0, - "29": 1974.0, - "30": 1870.0, - "31": 1991.0, - "32": 1855.0, - "33": 1929.0, - "34": 1979.0, - "35": 2026.0, - "36": 1997.0, - "37": 2154.0, - "38": 2182.0, - "39": 2162.0, - "40": 2249.0, - "41": 2357.0, - "42": 1947.0, - "43": 2334.0, - "44": 2145.0, - "45": 2601.0, - "46": 2416.0, - "47": 2528.0, - "48": 2505.0, - "49": 2807.0, - "50": 2606.0, - "51": 2529.0, - "52": 2706.0, - "53": 2546.0, - "54": 2820.0, - "55": 2723.0, - "56": 2570.0, - "57": 2215.0, - "58": 3573.0, - "59": 2904.0, - "60": 2916.0, - "61": 2765.0, - "62": 3181.0, - "63": 3220.0, - "64": 3594.0, - "65": 2595.0, - "66": 3023.0, - "67": 3840.0, - "68": 3470.0, - "69": 2927.0, - "70": 3210.0, - "71": 3080.0, - "72": 2938.0, - "73": 3455.0, - "74": 3146.0, - "75": 3283.0, - "76": 3170.0, - "77": 3808.0, - "78": 3264.0, - "79": 3266.0, - "80": 3117.0, - "81": 3423.0, - "82": 3087.0, - "83": 3214.0, - "84": 2912.0, - "85": 2742.0, - "86": 3145.0, - "87": 2981.0, - "88": 3157.0, - "89": 2922.0, - "90": 3525.0, - "91": 3112.0, - "92": 3039.0, - "93": 3001.0, - "94": 3145.0, - "95": 3363.0, - "96": 3581.0, - "97": 3603.0, - "98": 3403.0, - "99": 3370.0, - "100": 3303.0 + "1": 1757.0, + "2": 1713.0, + "3": 1727.0, + "4": 1741.0, + "5": 1736.0, + "6": 1776.0, + "7": 1991.0, + "8": 1709.0, + "9": 1880.0, + "10": 1852.0, + "11": 1682.0, + "12": 1692.0, + "13": 1720.0, + "14": 1889.0, + "15": 1686.0, + "16": 1636.0, + "17": 1832.0, + "18": 1873.0, + "19": 1775.0, + "20": 1698.0, + "21": 1811.0, + "22": 1759.0, + "23": 1700.0, + "24": 1725.0, + "25": 1713.0, + "26": 1792.0, + "27": 1737.0, + "28": 1849.0, + "29": 1909.0, + "30": 1843.0, + "31": 2003.0, + "32": 1862.0, + "33": 1952.0, + "34": 1907.0, + "35": 1954.0, + "36": 1909.0, + "37": 2152.0, + "38": 2093.0, + "39": 2176.0, + "40": 2303.0, + "41": 2334.0, + "42": 1985.0, + "43": 2487.0, + "44": 2141.0, + "45": 2576.0, + "46": 2271.0, + "47": 2505.0, + "48": 2487.0, + "49": 2902.0, + "50": 2474.0, + "51": 2526.0, + "52": 2686.0, + "53": 2568.0, + "54": 2739.0, + "55": 2688.0, + "56": 2701.0, + "57": 2210.0, + "58": 3611.0, + "59": 2837.0, + "60": 2919.0, + "61": 2693.0, + "62": 3263.0, + "63": 3288.0, + "64": 3723.0, + "65": 2534.0, + "66": 3038.0, + "67": 3791.0, + "68": 3544.0, + "69": 2892.0, + "70": 3252.0, + "71": 3171.0, + "72": 2962.0, + "73": 3366.0, + "74": 3140.0, + "75": 3171.0, + "76": 3067.0, + "77": 3755.0, + "78": 3225.0, + "79": 3337.0, + "80": 3084.0, + "81": 3432.0, + "82": 3196.0, + "83": 3284.0, + "84": 2909.0, + "85": 2650.0, + "86": 3153.0, + "87": 3032.0, + "88": 3080.0, + "89": 3027.0, + "90": 3612.0, + "91": 3136.0, + "92": 2954.0, + "93": 3027.0, + "94": 3170.0, + "95": 3372.0, + "96": 3572.0, + "97": 3713.0, + "98": 3350.0, + "99": 3371.0, + "100": 3373.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 745146880.0, - "2": 745146880.0, - "3": 745146880.0, - "4": 745146880.0, - "5": 745146880.0, - "6": 745146880.0, - "7": 745146880.0, - "8": 745146880.0, - "9": 745146880.0, - "10": 745146880.0, - "11": 745146880.0, - "12": 745146880.0, - "13": 745146880.0, - "14": 745146880.0, - "15": 745146880.0, - "16": 745146880.0, - "17": 745146880.0, - "18": 745146880.0, - "19": 745146880.0, - "20": 745146880.0, - "21": 745146880.0, - "22": 745146880.0, - "23": 745146880.0, - "24": 745146880.0, - "25": 745146880.0, - "26": 745146880.0, - "27": 745146880.0, - "28": 745146880.0, - "29": 745146880.0, - "30": 745146880.0, - "31": 745146880.0, - "32": 745146880.0, - "33": 745146880.0, - "34": 745146880.0, - "35": 745146880.0, - "36": 745146880.0, - "37": 745146880.0, - "38": 745146880.0, - "39": 745146880.0, - "40": 745146880.0, - "41": 745146880.0, - "42": 745146880.0, - "43": 745146880.0, - "44": 745146880.0, - "45": 745146880.0, - "46": 745146880.0, - "47": 745146880.0, - "48": 745146880.0, - "49": 745146880.0, - "50": 745146880.0, - "51": 745146880.0, - "52": 745146880.0, - "53": 745146880.0, - "54": 745146880.0, - "55": 745146880.0, - "56": 745146880.0, - "57": 745146880.0, - "58": 745146880.0, - "59": 745146880.0, - "60": 745146880.0, - "61": 745146880.0, - "62": 745146880.0, - "63": 745146880.0, - "64": 745146880.0, - "65": 745146880.0, - "66": 745146880.0, - "67": 745146880.0, - "68": 745146880.0, - "69": 745146880.0, - "70": 745146880.0, - "71": 745146880.0, - "72": 745146880.0, - "73": 745146880.0, - "74": 745146880.0, - "75": 745146880.0, - "76": 745146880.0, - "77": 745146880.0, - "78": 745146880.0, - "79": 745146880.0, - "80": 745146880.0, - "81": 745146880.0, - "82": 745146880.0, - "83": 745146880.0, - "84": 745146880.0, - "85": 745146880.0, - "86": 745146880.0, - "87": 745146880.0, - "88": 745146880.0, - "89": 745146880.0, - "90": 745146880.0, - "91": 745146880.0, - "92": 745146880.0, - "93": 745146880.0, - "94": 745146880.0, - "95": 745146880.0, - "96": 745146880.0, - "97": 745146880.0, - "98": 745146880.0, - "99": 745146880.0, - "100": 745146880.0 + "1": 780142592.0, + "2": 780142592.0, + "3": 780142592.0, + "4": 780142592.0, + "5": 780142592.0, + "6": 780142592.0, + "7": 780142592.0, + "8": 780142592.0, + "9": 780142592.0, + "10": 780142592.0, + "11": 780142592.0, + "12": 780142592.0, + "13": 780142592.0, + "14": 780142592.0, + "15": 780142592.0, + "16": 780142592.0, + "17": 780142592.0, + "18": 780142592.0, + "19": 780142592.0, + "20": 780142592.0, + "21": 780142592.0, + "22": 780142592.0, + "23": 780142592.0, + "24": 780142592.0, + "25": 780142592.0, + "26": 780142592.0, + "27": 780142592.0, + "28": 780142592.0, + "29": 780142592.0, + "30": 780142592.0, + "31": 780142592.0, + "32": 780142592.0, + "33": 780142592.0, + "34": 780142592.0, + "35": 780142592.0, + "36": 780142592.0, + "37": 780142592.0, + "38": 780142592.0, + "39": 780142592.0, + "40": 780142592.0, + "41": 780142592.0, + "42": 780142592.0, + "43": 780142592.0, + "44": 780142592.0, + "45": 780142592.0, + "46": 780142592.0, + "47": 780142592.0, + "48": 780142592.0, + "49": 780142592.0, + "50": 780142592.0, + "51": 780142592.0, + "52": 780142592.0, + "53": 780142592.0, + "54": 780142592.0, + "55": 780142592.0, + "56": 780142592.0, + "57": 780142592.0, + "58": 780142592.0, + "59": 780142592.0, + "60": 780142592.0, + "61": 780142592.0, + "62": 780142592.0, + "63": 780142592.0, + "64": 780142592.0, + "65": 780142592.0, + "66": 780142592.0, + "67": 780142592.0, + "68": 780142592.0, + "69": 780142592.0, + "70": 780142592.0, + "71": 780142592.0, + "72": 780142592.0, + "73": 780142592.0, + "74": 780142592.0, + "75": 780142592.0, + "76": 780142592.0, + "77": 780142592.0, + "78": 780142592.0, + "79": 780142592.0, + "80": 780142592.0, + "81": 780142592.0, + "82": 780142592.0, + "83": 780142592.0, + "84": 780142592.0, + "85": 780142592.0, + "86": 780142592.0, + "87": 780142592.0, + "88": 780142592.0, + "89": 780142592.0, + "90": 780142592.0, + "91": 780142592.0, + "92": 780142592.0, + "93": 780142592.0, + "94": 780142592.0, + "95": 780142592.0, + "96": 780142592.0, + "97": 780142592.0, + "98": 780142592.0, + "99": 780142592.0, + "100": 780142592.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1938739200.0, - "2": 2222436352.0, - "3": 2222436352.0, - "4": 2222436352.0, - "5": 2222436352.0, - "6": 2222436352.0, - "7": 2222436352.0, - "8": 2222436352.0, - "9": 2222436352.0, - "10": 2222436352.0, - "11": 2222436352.0, - "12": 2222436352.0, - "13": 2222436352.0, - "14": 2222436352.0, - "15": 2222436352.0, - "16": 2222436352.0, - "17": 2222436352.0, - "18": 2222436352.0, - "19": 2222436352.0, - "20": 2222436352.0, - "21": 2222436352.0, - "22": 2222436352.0, - "23": 2222436352.0, - "24": 2222436352.0, - "25": 2222436352.0, - "26": 2222436352.0, - "27": 2222436352.0, - "28": 2222436352.0, - "29": 2222436352.0, - "30": 2222436352.0, - "31": 2222436352.0, - "32": 2222436352.0, - "33": 2222436352.0, - "34": 2222436352.0, - "35": 2222436352.0, - "36": 2222436352.0, - "37": 2222436352.0, - "38": 2222436352.0, - "39": 2222436352.0, - "40": 2222436352.0, - "41": 2222436352.0, - "42": 2222436352.0, - "43": 2222436352.0, - "44": 2222436352.0, - "45": 2222436352.0, - "46": 2222436352.0, - "47": 2222436352.0, - "48": 2222436352.0, - "49": 2222436352.0, - "50": 2222436352.0, - "51": 2222436352.0, - "52": 2222436352.0, - "53": 2222436352.0, - "54": 2222436352.0, - "55": 2222436352.0, - "56": 2222436352.0, - "57": 2222436352.0, - "58": 2222436352.0, - "59": 2222436352.0, - "60": 2222436352.0, - "61": 2222436352.0, - "62": 2222436352.0, - "63": 2222436352.0, - "64": 2222436352.0, - "65": 2222436352.0, - "66": 2222436352.0, - "67": 2222436352.0, - "68": 2222436352.0, - "69": 2222436352.0, - "70": 2222436352.0, - "71": 2222436352.0, - "72": 2222436352.0, - "73": 2222436352.0, - "74": 2222436352.0, - "75": 2222436352.0, - "76": 2222436352.0, - "77": 2222436352.0, - "78": 2222436352.0, - "79": 2222436352.0, - "80": 2222436352.0, - "81": 2222436352.0, - "82": 2222436352.0, - "83": 2222436352.0, - "84": 2222436352.0, - "85": 2222436352.0, - "86": 2222436352.0, - "87": 2222436352.0, - "88": 2222436352.0, - "89": 2222436352.0, - "90": 2222436352.0, - "91": 2222436352.0, - "92": 2222436352.0, - "93": 2222436352.0, - "94": 2222436352.0, - "95": 2222436352.0, - "96": 2222436352.0, - "97": 2222436352.0, - "98": 2222436352.0, - "99": 2222436352.0, - "100": 2222436352.0 + "1": 1968377344.0, + "2": 2252075520.0, + "3": 2252075520.0, + "4": 2252075520.0, + "5": 2252075520.0, + "6": 2252075520.0, + "7": 2252075520.0, + "8": 2252075520.0, + "9": 2252075520.0, + "10": 2252075520.0, + "11": 2252075520.0, + "12": 2252075520.0, + "13": 2252075520.0, + "14": 2252075520.0, + "15": 2252075520.0, + "16": 2252075520.0, + "17": 2252075520.0, + "18": 2252075520.0, + "19": 2252075520.0, + "20": 2252075520.0, + "21": 2252075520.0, + "22": 2252075520.0, + "23": 2252075520.0, + "24": 2252075520.0, + "25": 2252075520.0, + "26": 2252075520.0, + "27": 2252075520.0, + "28": 2252075520.0, + "29": 2252075520.0, + "30": 2252075520.0, + "31": 2252075520.0, + "32": 2252075520.0, + "33": 2252075520.0, + "34": 2252075520.0, + "35": 2252075520.0, + "36": 2252075520.0, + "37": 2252075520.0, + "38": 2252075520.0, + "39": 2252075520.0, + "40": 2252075520.0, + "41": 2252075520.0, + "42": 2252075520.0, + "43": 2252075520.0, + "44": 2252075520.0, + "45": 2252075520.0, + "46": 2252075520.0, + "47": 2252075520.0, + "48": 2252075520.0, + "49": 2252075520.0, + "50": 2252075520.0, + "51": 2252075520.0, + "52": 2252075520.0, + "53": 2252075520.0, + "54": 2252075520.0, + "55": 2252075520.0, + "56": 2252075520.0, + "57": 2252075520.0, + "58": 2252075520.0, + "59": 2252075520.0, + "60": 2252075520.0, + "61": 2252075520.0, + "62": 2252075520.0, + "63": 2252075520.0, + "64": 2252075520.0, + "65": 2252075520.0, + "66": 2252075520.0, + "67": 2252075520.0, + "68": 2252075520.0, + "69": 2252075520.0, + "70": 2252075520.0, + "71": 2252075520.0, + "72": 2252075520.0, + "73": 2252075520.0, + "74": 2252075520.0, + "75": 2252075520.0, + "76": 2252075520.0, + "77": 2252075520.0, + "78": 2252075520.0, + "79": 2252075520.0, + "80": 2252075520.0, + "81": 2252075520.0, + "82": 2252075520.0, + "83": 2252075520.0, + "84": 2252075520.0, + "85": 2252075520.0, + "86": 2252075520.0, + "87": 2252075520.0, + "88": 2252075520.0, + "89": 2252075520.0, + "90": 2252075520.0, + "91": 2252075520.0, + "92": 2252075520.0, + "93": 2252075520.0, + "94": 2252075520.0, + "95": 2252075520.0, + "96": 2252075520.0, + "97": 2252075520.0, + "98": 2252075520.0, + "99": 2252075520.0, + "100": 2252075520.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.04703, - "3": 1.99199, - "4": 2.09518, - "5": 1.58663, - "6": 1.99113, - "7": 1.45759, - "8": 2.33841, - "9": 2.36279, - "10": 1.87261, - "11": 2.13869, - "12": 1.92109, - "13": 2.10956, - "14": 1.98257, - "15": 1.303, - "16": 1.90815, - "17": 2.29532, - "18": 1.93725, - "19": 2.12302, - "20": 1.87664, - "21": 2.24503, - "22": 1.90587, - "23": 2.25722, - "24": 1.89072, - "25": 2.10088, - "26": 2.5103, - "27": 1.52903, - "28": 1.51566, - "29": 1.19658, - "30": 1.576, - "31": 1.41349, - "32": 1.98279, - "33": 1.38672, - "34": 1.30376, - "35": 1.8636, - "36": 1.46009, - "37": 1.26068, - "38": 1.20989, - "39": 1.73631, - "40": 1.71229, - "41": 1.51323, - "42": 2.23144, - "43": 1.66929, - "44": 1.82524, - "45": 2.29129, - "46": 1.77141, - "47": 1.69348, - "48": 1.85769, - "49": 1.6205, - "50": 2.04835, - "51": 1.0882, - "52": 2.28317, - "53": 1.97761, - "54": 2.18908, - "55": 1.78584, - "56": 2.49115, - "57": 1.31026, - "58": 1.92964, - "59": 1.86414, - "60": 1.53923, - "61": 2.43823, - "62": 2.09221, - "63": 1.48294, - "64": 1.8834, - "65": 1.48038, - "66": 1.90369, - "67": 1.52439, - "68": 1.38547, - "69": 1.9442, - "70": 1.60013, - "71": 2.24078, - "72": 1.65969, - "73": 1.48586, - "74": 1.05832, - "75": 1.53395, - "76": 1.99654, - "77": 1.41558, - "78": 1.44104, - "79": 2.14073, - "80": 1.64252, - "81": 2.02058, - "82": 1.9555, - "83": 1.87248, - "84": 1.2153, - "85": 1.64283, - "86": 1.27483, - "87": 1.7866, - "88": 1.41131, - "89": 1.96302, - "90": 1.6895, - "91": 1.77288, - "92": 2.23073, - "93": 1.75114, - "94": 1.90935, - "95": 2.10873, - "96": 2.15486, - "97": 1.76351, - "98": 2.20924, - "99": 2.45275, - "100": 1.71934 + "2": 8.05588, + "3": 4.21644, + "4": 4.55821, + "5": 3.08896, + "6": 2.41911, + "7": 3.38987, + "8": 4.41363, + "9": 6.81505, + "10": 3.7205, + "11": 2.34173, + "12": 2.44491, + "13": 3.507, + "14": 3.1366, + "15": 2.33188, + "16": 7.74296, + "17": 6.76186, + "18": 3.44591, + "19": 2.74359, + "20": 3.71247, + "21": 2.21452, + "22": 2.59615, + "23": 3.44578, + "24": 2.36901, + "25": 3.67179, + "26": 3.35024, + "27": 2.04771, + "28": 2.68441, + "29": 2.15284, + "30": 2.56971, + "31": 2.33302, + "32": 3.8958, + "33": 2.13306, + "34": 2.74413, + "35": 3.58358, + "36": 3.55499, + "37": 3.04934, + "38": 2.86843, + "39": 3.5915, + "40": 3.4157, + "41": 5.14127, + "42": 5.53485, + "43": 2.577, + "44": 2.721, + "45": 2.97152, + "46": 2.42769, + "47": 2.96614, + "48": 2.4142, + "49": 2.17439, + "50": 2.76823, + "51": 2.37932, + "52": 2.83108, + "53": 2.67773, + "54": 2.77075, + "55": 2.39154, + "56": 3.75464, + "57": 1.95096, + "58": 2.72758, + "59": 2.55399, + "60": 1.90578, + "61": 2.62151, + "62": 2.45851, + "63": 2.12262, + "64": 2.20327, + "65": 2.58017, + "66": 2.49514, + "67": 2.88672, + "68": 2.23855, + "69": 3.04805, + "70": 2.87361, + "71": 3.71101, + "72": 2.31477, + "73": 2.80397, + "74": 1.63837, + "75": 2.23089, + "76": 2.00171, + "77": 2.82095, + "78": 3.96434, + "79": 4.06079, + "80": 2.37549, + "81": 2.70248, + "82": 2.35271, + "83": 2.57876, + "84": 2.36207, + "85": 2.47027, + "86": 2.27381, + "87": 3.32434, + "88": 2.85031, + "89": 2.27918, + "90": 2.40731, + "91": 2.04199, + "92": 2.38299, + "93": 1.73456, + "94": 2.41345, + "95": 2.1104, + "96": 2.20772, + "97": 2.31337, + "98": 1.99064, + "99": 5.13815, + "100": 1.69242 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index b267dfdabd4..96d99499695 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -6,103 +6,103 @@ "values": { "1": 10.91078, "2": 10.89671, - "3": 10.9068, - "4": 10.90309, - "5": 10.91231, - "6": 10.90619, - "7": 10.89729, - "8": 10.90225, - "9": 10.89974, - "10": 10.90271, - "11": 10.89174, - "12": 10.89972, - "13": 10.88445, - "14": 10.88017, - "15": 10.8721, - "16": 10.86401, - "17": 10.86921, - "18": 10.85741, - "19": 10.8598, - "20": 10.81848, - "21": 10.80338, - "22": 10.78816, - "23": 10.77549, - "24": 10.74977, - "25": 10.75623, - "26": 10.74088, - "27": 10.72474, - "28": 10.6708, - "29": 10.63634, - "30": 10.61286, - "31": 10.60565, - "32": 10.58807, - "33": 10.56535, - "34": 10.53443, - "35": 10.53086, - "36": 10.5136, - "37": 10.47664, - "38": 10.48988, - "39": 10.44993, - "40": 10.43435, - "41": 10.41501, - "42": 10.40213, - "43": 10.38757, - "44": 10.34276, - "45": 10.36178, - "46": 10.32389, - "47": 10.30661, - "48": 10.27197, - "49": 10.26302, - "50": 10.26171, - "51": 10.25787, - "52": 10.2153, - "53": 10.22533, - "54": 10.17887, - "55": 10.16493, - "56": 10.17821, - "57": 10.1693, - "58": 10.17732, - "59": 10.13049, - "60": 10.14409, - "61": 10.09377, - "62": 10.07236, - "63": 10.1274, - "64": 10.09298, - "65": 10.07117, - "66": 10.0824, - "67": 10.06131, - "68": 10.02633, - "69": 10.04916, - "70": 10.02744, - "71": 10.04579, - "72": 10.03071, - "73": 10.02627, - "74": 10.00693, - "75": 9.9848, - "76": 10.00692, - "77": 10.00432, - "78": 9.95648, - "79": 9.96782, - "80": 9.98203, + "3": 10.90645, + "4": 10.90371, + "5": 10.91246, + "6": 10.9056, + "7": 10.89785, + "8": 10.90079, + "9": 10.90027, + "10": 10.90302, + "11": 10.89178, + "12": 10.89887, + "13": 10.88521, + "14": 10.87977, + "15": 10.87212, + "16": 10.86418, + "17": 10.86934, + "18": 10.85765, + "19": 10.85938, + "20": 10.81825, + "21": 10.80298, + "22": 10.7886, + "23": 10.77627, + "24": 10.74869, + "25": 10.75637, + "26": 10.74006, + "27": 10.72449, + "28": 10.67103, + "29": 10.63572, + "30": 10.6135, + "31": 10.60514, + "32": 10.58801, + "33": 10.56466, + "34": 10.53483, + "35": 10.53048, + "36": 10.51401, + "37": 10.47609, + "38": 10.48981, + "39": 10.44944, + "40": 10.4336, + "41": 10.41549, + "42": 10.40188, + "43": 10.38735, + "44": 10.34288, + "45": 10.36216, + "46": 10.32327, + "47": 10.30658, + "48": 10.27212, + "49": 10.26304, + "50": 10.26061, + "51": 10.25771, + "52": 10.21491, + "53": 10.22516, + "54": 10.17857, + "55": 10.1647, + "56": 10.1781, + "57": 10.16927, + "58": 10.17745, + "59": 10.13063, + "60": 10.1437, + "61": 10.09378, + "62": 10.07256, + "63": 10.12717, + "64": 10.09321, + "65": 10.07113, + "66": 10.08268, + "67": 10.06139, + "68": 10.02612, + "69": 10.04941, + "70": 10.02746, + "71": 10.04596, + "72": 10.03028, + "73": 10.02638, + "74": 10.00717, + "75": 9.98505, + "76": 10.00668, + "77": 10.00418, + "78": 9.95608, + "79": 9.96793, + "80": 9.98219, "81": 10.01091, - "82": 9.94328, - "83": 9.92101, - "84": 9.84991, - "85": 9.84859, - "86": 9.93522, - "87": 9.96824, - "88": 9.93385, - "89": 9.87306, - "90": 9.87695, - "91": 9.89293, - "92": 9.87527, - "93": 9.81105, - "94": 9.8906, - "95": 9.87358, - "96": 9.85291, - "97": 9.79664, - "98": 9.83443, - "99": 9.87124, + "82": 9.94319, + "83": 9.92058, + "84": 9.85007, + "85": 9.84854, + "86": 9.93503, + "87": 9.96834, + "88": 9.934, + "89": 9.87352, + "90": 9.87673, + "91": 9.89283, + "92": 9.87516, + "93": 9.81115, + "94": 9.89047, + "95": 9.8736, + "96": 9.853, + "97": 9.79663, + "98": 9.8342, + "99": 9.87127, "100": 9.76382 } }, @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36671.0, - "2": 37197.0, - "3": 36559.0, - "4": 36852.0, - "5": 36703.0, - "6": 35840.0, - "7": 36983.0, - "8": 36383.0, - "9": 36544.0, - "10": 37416.0, - "11": 36558.0, - "12": 35768.0, - "13": 36071.0, - "14": 36725.0, - "15": 36398.0, - "16": 35673.0, - "17": 36265.0, - "18": 36819.0, - "19": 36835.0, - "20": 36479.0, - "21": 36112.0, - "22": 36404.0, - "23": 36252.0, - "24": 36731.0, - "25": 35963.0, - "26": 36050.0, - "27": 36455.0, - "28": 35396.0, - "29": 35673.0, - "30": 35389.0, - "31": 36231.0, - "32": 36304.0, - "33": 35334.0, - "34": 35822.0, - "35": 36789.0, - "36": 35934.0, - "37": 35999.0, - "38": 36004.0, - "39": 36913.0, - "40": 37742.0, - "41": 35768.0, - "42": 35680.0, - "43": 37957.0, - "44": 36315.0, - "45": 39567.0, - "46": 38316.0, - "47": 38689.0, - "48": 39256.0, - "49": 42420.0, - "50": 39683.0, - "51": 39232.0, - "52": 41791.0, - "53": 40285.0, - "54": 41889.0, - "55": 39269.0, - "56": 41161.0, - "57": 36797.0, - "58": 46655.0, - "59": 42337.0, - "60": 42800.0, - "61": 42084.0, - "62": 45422.0, - "63": 44419.0, - "64": 49741.0, - "65": 42303.0, - "66": 45995.0, - "67": 49650.0, - "68": 46704.0, - "69": 45620.0, - "70": 46489.0, - "71": 46470.0, - "72": 45280.0, - "73": 47307.0, - "74": 45593.0, - "75": 48865.0, - "76": 47482.0, - "77": 48068.0, - "78": 48859.0, - "79": 47823.0, - "80": 45845.0, - "81": 55013.0, - "82": 41851.0, - "83": 46834.0, - "84": 45934.0, - "85": 46257.0, - "86": 45740.0, - "87": 36471.0, - "88": 45175.0, - "89": 48591.0, - "90": 57540.0, - "91": 38958.0, - "92": 49460.0, - "93": 44750.0, - "94": 41945.0, - "95": 45537.0, - "96": 49807.0, - "97": 46836.0, - "98": 49860.0, - "99": 45566.0, - "100": 46454.0 + "1": 36596.0, + "2": 36979.0, + "3": 36564.0, + "4": 36801.0, + "5": 36525.0, + "6": 35740.0, + "7": 37232.0, + "8": 36034.0, + "9": 36648.0, + "10": 37496.0, + "11": 36778.0, + "12": 35994.0, + "13": 36882.0, + "14": 36632.0, + "15": 36357.0, + "16": 35869.0, + "17": 36364.0, + "18": 36416.0, + "19": 36759.0, + "20": 36646.0, + "21": 36213.0, + "22": 35949.0, + "23": 36268.0, + "24": 36708.0, + "25": 35554.0, + "26": 36686.0, + "27": 36263.0, + "28": 35122.0, + "29": 35630.0, + "30": 35289.0, + "31": 36577.0, + "32": 36319.0, + "33": 35646.0, + "34": 36017.0, + "35": 36536.0, + "36": 35129.0, + "37": 36416.0, + "38": 35453.0, + "39": 37045.0, + "40": 37634.0, + "41": 36234.0, + "42": 35327.0, + "43": 38077.0, + "44": 36774.0, + "45": 39644.0, + "46": 38221.0, + "47": 38327.0, + "48": 39096.0, + "49": 42330.0, + "50": 39482.0, + "51": 39096.0, + "52": 41798.0, + "53": 40389.0, + "54": 42186.0, + "55": 39240.0, + "56": 41210.0, + "57": 36989.0, + "58": 46710.0, + "59": 42634.0, + "60": 42403.0, + "61": 42153.0, + "62": 45736.0, + "63": 44073.0, + "64": 49431.0, + "65": 42294.0, + "66": 45813.0, + "67": 49959.0, + "68": 46603.0, + "69": 46182.0, + "70": 46806.0, + "71": 46331.0, + "72": 45365.0, + "73": 47697.0, + "74": 45931.0, + "75": 48790.0, + "76": 47449.0, + "77": 48250.0, + "78": 48808.0, + "79": 47595.0, + "80": 45910.0, + "81": 54932.0, + "82": 42219.0, + "83": 47223.0, + "84": 45713.0, + "85": 46322.0, + "86": 45732.0, + "87": 36767.0, + "88": 45526.0, + "89": 48634.0, + "90": 57509.0, + "91": 39051.0, + "92": 49380.0, + "93": 45014.0, + "94": 42375.0, + "95": 45838.0, + "96": 50134.0, + "97": 46863.0, + "98": 49323.0, + "99": 45944.0, + "100": 46285.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254510592.0, - "2": 1254503936.0, - "3": 1254510080.0, - "4": 1254507520.0, - "5": 1254503936.0, - "6": 1254504960.0, - "7": 1254507008.0, - "8": 1254505984.0, - "9": 1254510592.0, - "10": 1254508032.0, - "11": 1254504960.0, - "12": 1254507520.0, - "13": 1254508032.0, - "14": 1254506496.0, - "15": 1254507008.0, - "16": 1254508032.0, - "17": 1254507008.0, - "18": 1254506496.0, - "19": 1254505472.0, - "20": 1254509568.0, - "21": 1254505984.0, - "22": 1254506496.0, + "1": 1254510080.0, + "2": 1254503424.0, + "3": 1254509568.0, + "4": 1254507008.0, + "5": 1254502400.0, + "6": 1254504448.0, + "7": 1254505984.0, + "8": 1254505472.0, + "9": 1254509568.0, + "10": 1254507520.0, + "11": 1254503424.0, + "12": 1254506496.0, + "13": 1254507008.0, + "14": 1254507008.0, + "15": 1254505472.0, + "16": 1254507520.0, + "17": 1254505984.0, + "18": 1254505984.0, + "19": 1254505984.0, + "20": 1254509056.0, + "21": 1254506496.0, + "22": 1254505472.0, "23": 1254505472.0, - "24": 1254503424.0, - "25": 1254508032.0, - "26": 1254506496.0, - "27": 1254508032.0, - "28": 1254506496.0, - "29": 1254508032.0, - "30": 1254508544.0, - "31": 1254506496.0, - "32": 1254504448.0, - "33": 1254506496.0, - "34": 1254506496.0, - "35": 1254504448.0, - "36": 1254504960.0, - "37": 1254508544.0, - "38": 1254507520.0, - "39": 1254505472.0, - "40": 1254507008.0, - "41": 1254506496.0, - "42": 1254504960.0, - "43": 1254506496.0, - "44": 1254504448.0, - "45": 1254504448.0, - "46": 1254505984.0, - "47": 1254506496.0, + "24": 1254501888.0, + "25": 1254505984.0, + "26": 1254505984.0, + "27": 1254507520.0, + "28": 1254505984.0, + "29": 1254507008.0, + "30": 1254507520.0, + "31": 1254508032.0, + "32": 1254503936.0, + "33": 1254505984.0, + "34": 1254505472.0, + "35": 1254503936.0, + "36": 1254504448.0, + "37": 1254507008.0, + "38": 1254508032.0, + "39": 1254503424.0, + "40": 1254505984.0, + "41": 1254505472.0, + "42": 1254505472.0, + "43": 1254507008.0, + "44": 1254503936.0, + "45": 1254503936.0, + "46": 1254506496.0, + "47": 1254504960.0, "48": 1254507008.0, - "49": 1254507520.0, - "50": 1254504448.0, - "51": 1254503424.0, - "52": 1254506496.0, - "53": 1254504448.0, - "54": 1254504448.0, - "55": 1254506496.0, - "56": 1254504448.0, - "57": 1254507008.0, - "58": 1254507008.0, - "59": 1254504960.0, - "60": 1254505984.0, - "61": 1254502912.0, - "62": 1254505984.0, - "63": 1254505472.0, - "64": 1254505984.0, - "65": 1254505984.0, - "66": 1254507008.0, + "49": 1254506496.0, + "50": 1254504960.0, + "51": 1254500864.0, + "52": 1254505472.0, + "53": 1254503424.0, + "54": 1254503936.0, + "55": 1254504448.0, + "56": 1254504960.0, + "57": 1254505472.0, + "58": 1254508032.0, + "59": 1254504448.0, + "60": 1254506496.0, + "61": 1254502400.0, + "62": 1254505472.0, + "63": 1254505984.0, + "64": 1254506496.0, + "65": 1254505472.0, + "66": 1254505984.0, "67": 1254503936.0, - "68": 1254504448.0, - "69": 1254505472.0, - "70": 1254506496.0, - "71": 1254502400.0, - "72": 1254503936.0, + "68": 1254504960.0, + "69": 1254504960.0, + "70": 1254505984.0, + "71": 1254503424.0, + "72": 1254504448.0, "73": 1254502400.0, - "74": 1254505984.0, + "74": 1254505472.0, "75": 1254505984.0, - "76": 1254505472.0, - "77": 1254505472.0, + "76": 1254504960.0, + "77": 1254503936.0, "78": 1254501376.0, - "79": 1254504960.0, + "79": 1254504448.0, "80": 1254505984.0, - "81": 1254504960.0, - "82": 1254505472.0, - "83": 1254504448.0, - "84": 1254505984.0, + "81": 1254505472.0, + "82": 1254505984.0, + "83": 1254502400.0, + "84": 1254506496.0, "85": 1254503424.0, - "86": 1254505984.0, - "87": 1254502912.0, + "86": 1254504960.0, + "87": 1254502400.0, "88": 1254501376.0, "89": 1254508032.0, - "90": 1254507008.0, + "90": 1254507520.0, "91": 1254504448.0, - "92": 1254505984.0, - "93": 1254507008.0, + "92": 1254506496.0, + "93": 1254505984.0, "94": 1254506496.0, - "95": 1254513664.0, + "95": 1254512640.0, "96": 1254503936.0, - "97": 1254509056.0, - "98": 1254508032.0, + "97": 1254508544.0, + "98": 1254507008.0, "99": 1254508032.0, - "100": 1254509056.0 + "100": 1254508544.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2065977344.0, - "2": 2545691648.0, - "3": 2546333696.0, - "4": 2546333696.0, - "5": 2546333696.0, - "6": 2546333696.0, - "7": 2546546688.0, - "8": 2546546688.0, - "9": 2546546688.0, - "10": 2546546688.0, - "11": 2546546688.0, - "12": 2546546688.0, - "13": 2546546688.0, - "14": 2546546688.0, - "15": 2546546688.0, - "16": 2546546688.0, - "17": 2546546688.0, - "18": 2546546688.0, - "19": 2546546688.0, - "20": 2546546688.0, - "21": 2546546688.0, - "22": 2546546688.0, - "23": 2546546688.0, - "24": 2546625024.0, - "25": 2546625024.0, - "26": 2546625024.0, - "27": 2546625024.0, - "28": 2546625024.0, - "29": 2546625024.0, - "30": 2546625024.0, - "31": 2546924544.0, - "32": 2546924544.0, - "33": 2546924544.0, - "34": 2546924544.0, - "35": 2546924544.0, - "36": 2546924544.0, - "37": 2546924544.0, - "38": 2546924544.0, - "39": 2546924544.0, - "40": 2546924544.0, - "41": 2546924544.0, - "42": 2546924544.0, - "43": 2546924544.0, - "44": 2546924544.0, - "45": 2546924544.0, - "46": 2546924544.0, - "47": 2546924544.0, - "48": 2546924544.0, - "49": 2546924544.0, - "50": 2546924544.0, - "51": 2546924544.0, - "52": 2546924544.0, - "53": 2546924544.0, - "54": 2546924544.0, - "55": 2546924544.0, - "56": 2546924544.0, - "57": 2546924544.0, - "58": 2546924544.0, - "59": 2546924544.0, - "60": 2546924544.0, - "61": 2546924544.0, - "62": 2546924544.0, - "63": 2546924544.0, - "64": 2546924544.0, - "65": 2546924544.0, - "66": 2546924544.0, - "67": 2546924544.0, - "68": 2546924544.0, - "69": 2546924544.0, - "70": 2546924544.0, - "71": 2546924544.0, - "72": 2546924544.0, - "73": 2546924544.0, - "74": 2546924544.0, - "75": 2546924544.0, - "76": 2546924544.0, - "77": 2546924544.0, - "78": 2546924544.0, - "79": 2546924544.0, - "80": 2546924544.0, - "81": 2546924544.0, - "82": 2546924544.0, - "83": 2546924544.0, - "84": 2546924544.0, - "85": 2548597248.0, - "86": 2548597248.0, - "87": 2548597248.0, - "88": 2548597248.0, - "89": 2548597248.0, - "90": 2548597248.0, - "91": 2548597248.0, - "92": 2548597248.0, - "93": 2550968832.0, - "94": 2550968832.0, - "95": 2550968832.0, - "96": 2550968832.0, - "97": 2551127552.0, - "98": 2551127552.0, - "99": 2551127552.0, - "100": 2552623616.0 + "1": 2067738624.0, + "2": 2544674304.0, + "3": 2546890240.0, + "4": 2546890240.0, + "5": 2546890240.0, + "6": 2546890240.0, + "7": 2546890240.0, + "8": 2546890240.0, + "9": 2546890240.0, + "10": 2546890240.0, + "11": 2546890240.0, + "12": 2546890240.0, + "13": 2546890240.0, + "14": 2546890240.0, + "15": 2546890240.0, + "16": 2546890240.0, + "17": 2546890240.0, + "18": 2546890240.0, + "19": 2546890240.0, + "20": 2546890240.0, + "21": 2546890240.0, + "22": 2546890240.0, + "23": 2546890240.0, + "24": 2546890240.0, + "25": 2546890240.0, + "26": 2546890240.0, + "27": 2546890240.0, + "28": 2546890240.0, + "29": 2546890240.0, + "30": 2546890240.0, + "31": 2546890240.0, + "32": 2546890240.0, + "33": 2546890240.0, + "34": 2546890240.0, + "35": 2546890240.0, + "36": 2546890240.0, + "37": 2546890240.0, + "38": 2546890240.0, + "39": 2546890240.0, + "40": 2546890240.0, + "41": 2546890240.0, + "42": 2546890240.0, + "43": 2546890240.0, + "44": 2546890240.0, + "45": 2546890240.0, + "46": 2546890240.0, + "47": 2546890240.0, + "48": 2546890240.0, + "49": 2546890240.0, + "50": 2546890240.0, + "51": 2546890240.0, + "52": 2546890240.0, + "53": 2546890240.0, + "54": 2546890240.0, + "55": 2546890240.0, + "56": 2546890240.0, + "57": 2546890240.0, + "58": 2546890240.0, + "59": 2546890240.0, + "60": 2546890240.0, + "61": 2546890240.0, + "62": 2546890240.0, + "63": 2546890240.0, + "64": 2546890240.0, + "65": 2546890240.0, + "66": 2546890240.0, + "67": 2546890240.0, + "68": 2546890240.0, + "69": 2546890240.0, + "70": 2546890240.0, + "71": 2546890240.0, + "72": 2546890240.0, + "73": 2546890240.0, + "74": 2546890240.0, + "75": 2546890240.0, + "76": 2546890240.0, + "77": 2546890240.0, + "78": 2546890240.0, + "79": 2546890240.0, + "80": 2546890240.0, + "81": 2546890240.0, + "82": 2546890240.0, + "83": 2546890240.0, + "84": 2546890240.0, + "85": 2548571136.0, + "86": 2548571136.0, + "87": 2548571136.0, + "88": 2548571136.0, + "89": 2548571136.0, + "90": 2548571136.0, + "91": 2548571136.0, + "92": 2548571136.0, + "93": 2550839296.0, + "94": 2550839296.0, + "95": 2551439872.0, + "96": 2551439872.0, + "97": 2551439872.0, + "98": 2551439872.0, + "99": 2551439872.0, + "100": 2552939008.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.16676, - "3": 1.33849, - "4": 1.23335, - "5": 0.73686, - "6": 1.28572, - "7": 1.09349, - "8": 1.3093, - "9": 1.3909, - "10": 0.97654, - "11": 1.23723, - "12": 1.25701, - "13": 1.28286, - "14": 1.73245, - "15": 1.27684, - "16": 1.53975, - "17": 1.44709, - "18": 1.28735, - "19": 1.54823, - "20": 1.23997, - "21": 1.2885, - "22": 1.20702, - "23": 1.5628, - "24": 1.82853, - "25": 0.70485, - "26": 1.41088, - "27": 1.24459, - "28": 1.68268, - "29": 1.09092, - "30": 0.87799, - "31": 1.1021, - "32": 2.05649, - "33": 1.57849, - "34": 0.99956, - "35": 1.70846, - "36": 0.97953, - "37": 1.04274, - "38": 0.76389, - "39": 1.31153, - "40": 1.44917, - "41": 1.15798, - "42": 1.23798, - "43": 1.27748, - "44": 1.53042, - "45": 1.01719, - "46": 1.00468, - "47": 1.28352, - "48": 1.23432, - "49": 1.03827, - "50": 1.43566, - "51": 0.71174, - "52": 1.01744, - "53": 1.40236, - "54": 1.47379, - "55": 1.34089, - "56": 1.23548, - "57": 0.97281, - "58": 1.32399, - "59": 0.92588, - "60": 0.98813, - "61": 1.31305, - "62": 1.31408, - "63": 1.10625, - "64": 1.41175, - "65": 0.83015, - "66": 1.30029, - "67": 1.34831, - "68": 1.04914, - "69": 1.60422, - "70": 0.99852, - "71": 1.30909, - "72": 1.20603, - "73": 1.12393, - "74": 0.74197, - "75": 1.15464, - "76": 1.13503, - "77": 1.3649, - "78": 1.33475, - "79": 1.93519, - "80": 1.06877, - "81": 1.16103, - "82": 1.52244, - "83": 1.22539, - "84": 1.12373, - "85": 0.81898, - "86": 0.99191, - "87": 1.92012, - "88": 1.06987, - "89": 0.98033, - "90": 1.55702, - "91": 1.03794, - "92": 1.76067, - "93": 1.0433, - "94": 1.29033, - "95": 1.17477, - "96": 1.20314, - "97": 1.0363, - "98": 1.44299, - "99": 1.54793, - "100": 1.20692 + "2": 6.57628, + "3": 2.09232, + "4": 1.87829, + "5": 1.12956, + "6": 2.00633, + "7": 1.77878, + "8": 2.12268, + "9": 2.41808, + "10": 3.94194, + "11": 3.14346, + "12": 3.60486, + "13": 0.8081, + "14": 2.60088, + "15": 2.4397, + "16": 4.04462, + "17": 5.58445, + "18": 2.10705, + "19": 2.99099, + "20": 3.23982, + "21": 1.19846, + "22": 2.07513, + "23": 3.68579, + "24": 4.98872, + "25": 0.88284, + "26": 2.24275, + "27": 2.8425, + "28": 3.96256, + "29": 3.40862, + "30": 1.64296, + "31": 1.84733, + "32": 3.64057, + "33": 2.59148, + "34": 2.43476, + "35": 2.59647, + "36": 2.28415, + "37": 2.13204, + "38": 1.78143, + "39": 3.84551, + "40": 3.97142, + "41": 2.44742, + "42": 3.26594, + "43": 1.74238, + "44": 3.38833, + "45": 1.97133, + "46": 1.35878, + "47": 2.5629, + "48": 2.23438, + "49": 2.01344, + "50": 1.56034, + "51": 0.84584, + "52": 1.77699, + "53": 2.61358, + "54": 2.50934, + "55": 3.05826, + "56": 2.41218, + "57": 1.9358, + "58": 2.27437, + "59": 2.13988, + "60": 2.26294, + "61": 2.28633, + "62": 3.06686, + "63": 1.06079, + "64": 2.40051, + "65": 1.52436, + "66": 2.08985, + "67": 1.79397, + "68": 1.71038, + "69": 2.50156, + "70": 2.60223, + "71": 1.64128, + "72": 1.17394, + "73": 2.08923, + "74": 0.86467, + "75": 1.42077, + "76": 1.81515, + "77": 1.99924, + "78": 2.27015, + "79": 3.67951, + "80": 2.74933, + "81": 1.58398, + "82": 1.74651, + "83": 1.50768, + "84": 1.26748, + "85": 1.61501, + "86": 2.03069, + "87": 2.22094, + "88": 1.47105, + "89": 1.72448, + "90": 1.6317, + "91": 1.92281, + "92": 1.9365, + "93": 1.71739, + "94": 2.06196, + "95": 1.37436, + "96": 2.10712, + "97": 1.94998, + "98": 1.77425, + "99": 2.22015, + "100": 2.03185 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json index ae71d653845..7c9dfc9c650 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92337, "2": 10.91184, - "3": 10.9245, - "4": 10.91413, - "5": 10.91791, - "6": 10.90788, - "7": 10.90285, - "8": 10.9094, - "9": 10.92264, - "10": 10.91325, - "11": 10.90217, - "12": 10.91139, - "13": 10.88971, - "14": 10.88954, - "15": 10.88501, - "16": 10.87598, - "17": 10.87426, - "18": 10.8634, - "19": 10.87212, - "20": 10.82076, - "21": 10.80748, - "22": 10.79544, - "23": 10.79148, - "24": 10.75643, - "25": 10.76142, - "26": 10.75154, - "27": 10.73598, - "28": 10.67469, - "29": 10.64016, - "30": 10.62245, - "31": 10.61276, - "32": 10.59683, - "33": 10.57641, - "34": 10.53937, - "35": 10.53846, - "36": 10.52255, - "37": 10.48675, - "38": 10.49927, - "39": 10.45887, - "40": 10.43983, - "41": 10.42774, - "42": 10.41415, - "43": 10.3903, - "44": 10.35238, - "45": 10.36987, - "46": 10.33008, - "47": 10.31244, - "48": 10.27782, - "49": 10.27035, - "50": 10.27145 + "3": 10.92435, + "4": 10.91391, + "5": 10.91783, + "6": 10.90822, + "7": 10.90214, + "8": 10.90968, + "9": 10.92327, + "10": 10.91324, + "11": 10.90152, + "12": 10.91168, + "13": 10.88952, + "14": 10.88978, + "15": 10.88462, + "16": 10.87656, + "17": 10.87413, + "18": 10.8627, + "19": 10.87155, + "20": 10.8213, + "21": 10.8075, + "22": 10.79454, + "23": 10.79087, + "24": 10.75698, + "25": 10.76203, + "26": 10.75144, + "27": 10.73619, + "28": 10.67458, + "29": 10.63986, + "30": 10.62269, + "31": 10.6127, + "32": 10.59666, + "33": 10.57595, + "34": 10.53962, + "35": 10.53801, + "36": 10.52144, + "37": 10.48719, + "38": 10.49901, + "39": 10.45863, + "40": 10.43984, + "41": 10.42707, + "42": 10.41355, + "43": 10.38977, + "44": 10.35254, + "45": 10.36992, + "46": 10.32982, + "47": 10.31281, + "48": 10.27777, + "49": 10.27075, + "50": 10.27166 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18811.0, - "2": 19165.0, - "3": 19169.0, - "4": 18794.0, - "5": 18806.0, - "6": 18547.0, - "7": 18947.0, - "8": 18963.0, - "9": 18914.0, - "10": 19344.0, - "11": 18881.0, - "12": 18329.0, - "13": 18937.0, - "14": 18987.0, - "15": 18708.0, - "16": 18957.0, - "17": 19271.0, - "18": 18855.0, - "19": 18843.0, - "20": 19091.0, - "21": 18628.0, - "22": 18905.0, - "23": 18905.0, - "24": 19000.0, - "25": 18306.0, - "26": 18509.0, - "27": 18924.0, - "28": 18259.0, - "29": 18452.0, - "30": 18270.0, - "31": 18685.0, - "32": 18898.0, - "33": 18250.0, - "34": 18394.0, - "35": 18736.0, - "36": 18582.0, - "37": 18806.0, - "38": 18396.0, - "39": 19248.0, - "40": 19332.0, - "41": 18568.0, - "42": 18144.0, - "43": 19243.0, - "44": 18642.0, - "45": 20136.0, - "46": 19403.0, - "47": 19891.0, - "48": 20076.0, - "49": 21584.0, - "50": 20152.0 + "1": 18852.0, + "2": 19201.0, + "3": 18829.0, + "4": 18799.0, + "5": 18661.0, + "6": 18606.0, + "7": 19036.0, + "8": 18992.0, + "9": 19205.0, + "10": 19409.0, + "11": 18725.0, + "12": 18412.0, + "13": 19021.0, + "14": 19235.0, + "15": 18796.0, + "16": 18661.0, + "17": 18978.0, + "18": 18747.0, + "19": 18689.0, + "20": 18999.0, + "21": 18460.0, + "22": 18943.0, + "23": 19110.0, + "24": 18724.0, + "25": 18209.0, + "26": 18944.0, + "27": 18769.0, + "28": 18225.0, + "29": 18553.0, + "30": 18154.0, + "31": 18660.0, + "32": 19011.0, + "33": 18551.0, + "34": 18477.0, + "35": 18870.0, + "36": 18407.0, + "37": 18996.0, + "38": 18270.0, + "39": 19069.0, + "40": 19467.0, + "41": 18520.0, + "42": 18295.0, + "43": 19123.0, + "44": 18827.0, + "45": 20268.0, + "46": 19722.0, + "47": 19576.0, + "48": 19950.0, + "49": 21813.0, + "50": 20205.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388729856.0, - "2": 1388730368.0, - "3": 1388729856.0, + "1": 1388729344.0, + "2": 1388729856.0, + "3": 1388729344.0, "4": 1388727808.0, "5": 1388728832.0, - "6": 1388730368.0, - "7": 1388731904.0, - "8": 1388729856.0, - "9": 1388731904.0, + "6": 1388729856.0, + "7": 1388731392.0, + "8": 1388730880.0, + "9": 1388731392.0, "10": 1388731392.0, - "11": 1388728832.0, - "12": 1388728832.0, - "13": 1388730880.0, - "14": 1388728320.0, - "15": 1388731904.0, - "16": 1388728320.0, - "17": 1388730880.0, - "18": 1388730368.0, - "19": 1388729856.0, - "20": 1388730368.0, - "21": 1388732928.0, - "22": 1388729344.0, + "11": 1388729344.0, + "12": 1388728320.0, + "13": 1388732928.0, + "14": 1388727808.0, + "15": 1388730880.0, + "16": 1388728832.0, + "17": 1388729856.0, + "18": 1388729856.0, + "19": 1388730880.0, + "20": 1388731904.0, + "21": 1388729856.0, + "22": 1388730368.0, "23": 1388731392.0, - "24": 1388729344.0, - "25": 1388732928.0, - "26": 1388732928.0, - "27": 1388729344.0, - "28": 1388730368.0, - "29": 1388731392.0, - "30": 1388733440.0, - "31": 1388732416.0, - "32": 1388729344.0, - "33": 1388732416.0, - "34": 1388734464.0, + "24": 1388730880.0, + "25": 1388732416.0, + "26": 1388732416.0, + "27": 1388731904.0, + "28": 1388728832.0, + "29": 1388730880.0, + "30": 1388732928.0, + "31": 1388731392.0, + "32": 1388728832.0, + "33": 1388731904.0, + "34": 1388733952.0, "35": 1388731392.0, "36": 1388730368.0, - "37": 1388731904.0, + "37": 1388731392.0, "38": 1388730368.0, - "39": 1388733440.0, - "40": 1388734464.0, - "41": 1388730368.0, - "42": 1388732928.0, - "43": 1388731392.0, - "44": 1388729856.0, - "45": 1388732928.0, - "46": 1388731904.0, - "47": 1388733952.0, - "48": 1388731904.0, - "49": 1388736512.0, - "50": 1388731904.0 + "39": 1388731904.0, + "40": 1388731904.0, + "41": 1388729344.0, + "42": 1388732416.0, + "43": 1388731904.0, + "44": 1388729344.0, + "45": 1388731904.0, + "46": 1388732416.0, + "47": 1388733440.0, + "48": 1388731392.0, + "49": 1388735488.0, + "50": 1388730368.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179662336.0, - "2": 3662152704.0, - "3": 3662152704.0, - "4": 3662152704.0, - "5": 3662152704.0, - "6": 3662152704.0, - "7": 3662152704.0, - "8": 3662152704.0, - "9": 3662152704.0, - "10": 3663182336.0, - "11": 3663182336.0, - "12": 3663182336.0, - "13": 3663182336.0, - "14": 3663182336.0, - "15": 3663182336.0, - "16": 3663182336.0, - "17": 3663182336.0, - "18": 3663182336.0, - "19": 3663182336.0, - "20": 3663182336.0, - "21": 3663182336.0, - "22": 3663182336.0, - "23": 3663182336.0, - "24": 3663182336.0, - "25": 3663182336.0, - "26": 3663353344.0, - "27": 3663353344.0, - "28": 3663353344.0, - "29": 3663353344.0, - "30": 3663353344.0, - "31": 3663353344.0, - "32": 3663353344.0, - "33": 3663353344.0, - "34": 3663353344.0, - "35": 3663353344.0, - "36": 3663353344.0, - "37": 3663353344.0, - "38": 3663353344.0, - "39": 3664240128.0, - "40": 3664612864.0, - "41": 3664612864.0, - "42": 3664612864.0, - "43": 3664612864.0, - "44": 3664748544.0, - "45": 3664748544.0, - "46": 3664803840.0, - "47": 3664803840.0, - "48": 3664803840.0, - "49": 3664803840.0, - "50": 3664803840.0 + "1": 3179891200.0, + "2": 3662152192.0, + "3": 3662164480.0, + "4": 3663250944.0, + "5": 3663250944.0, + "6": 3663250944.0, + "7": 3663250944.0, + "8": 3663250944.0, + "9": 3663250944.0, + "10": 3663250944.0, + "11": 3663250944.0, + "12": 3663250944.0, + "13": 3663250944.0, + "14": 3663250944.0, + "15": 3663250944.0, + "16": 3663250944.0, + "17": 3663250944.0, + "18": 3663250944.0, + "19": 3663250944.0, + "20": 3663250944.0, + "21": 3663250944.0, + "22": 3663250944.0, + "23": 3663250944.0, + "24": 3663250944.0, + "25": 3663250944.0, + "26": 3663250944.0, + "27": 3663250944.0, + "28": 3663250944.0, + "29": 3663250944.0, + "30": 3663250944.0, + "31": 3663250944.0, + "32": 3663250944.0, + "33": 3663250944.0, + "34": 3663250944.0, + "35": 3663897600.0, + "36": 3663897600.0, + "37": 3663897600.0, + "38": 3663897600.0, + "39": 3663897600.0, + "40": 3663897600.0, + "41": 3663897600.0, + "42": 3664121344.0, + "43": 3664121344.0, + "44": 3664593408.0, + "45": 3664593408.0, + "46": 3664593408.0, + "47": 3664593408.0, + "48": 3664593408.0, + "49": 3664994816.0, + "50": 3664994816.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.63543, - "3": 1.59168, - "4": 1.22993, - "5": 0.87487, - "6": 1.40923, - "7": 1.36534, - "8": 1.60732, - "9": 1.88024, - "10": 0.81614, - "11": 1.37433, - "12": 1.27737, - "13": 1.31882, - "14": 1.68779, - "15": 0.93178, - "16": 1.1834, - "17": 1.45294, - "18": 1.23079, - "19": 1.5691, - "20": 1.23071, - "21": 1.06665, - "22": 1.25446, - "23": 1.39808, - "24": 1.89063, - "25": 0.76182, - "26": 1.40748, - "27": 1.30504, - "28": 1.4192, - "29": 1.06447, - "30": 1.14242, - "31": 0.89482, - "32": 2.12694, - "33": 1.53717, - "34": 0.5599, - "35": 1.65299, - "36": 1.68494, - "37": 1.5207, - "38": 1.06257, - "39": 1.17441, - "40": 1.86046, - "41": 1.29818, - "42": 1.5381, - "43": 1.47386, - "44": 1.76445, - "45": 0.79074, - "46": 1.15751, - "47": 1.47016, - "48": 1.53904, - "49": 1.12239, - "50": 1.5319 + "2": 6.85618, + "3": 2.07992, + "4": 1.87215, + "5": 1.48011, + "6": 2.02399, + "7": 2.13514, + "8": 2.16634, + "9": 1.77359, + "10": 1.31073, + "11": 1.64395, + "12": 1.59434, + "13": 1.52469, + "14": 1.92437, + "15": 1.76445, + "16": 1.88589, + "17": 2.05598, + "18": 1.86971, + "19": 2.0144, + "20": 1.80886, + "21": 1.87488, + "22": 1.82297, + "23": 2.50818, + "24": 2.75346, + "25": 0.74091, + "26": 2.16027, + "27": 2.04848, + "28": 1.31746, + "29": 1.64889, + "30": 1.25634, + "31": 1.91444, + "32": 3.0018, + "33": 2.02046, + "34": 0.90031, + "35": 2.58547, + "36": 2.4573, + "37": 2.09483, + "38": 0.8511, + "39": 1.81657, + "40": 2.84934, + "41": 1.28469, + "42": 2.64455, + "43": 1.75379, + "44": 2.06723, + "45": 1.55995, + "46": 1.02072, + "47": 2.05419, + "48": 2.10853, + "49": 1.68931, + "50": 1.50254 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json index b9641ae98c4..4e1261dee6a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM_1node/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92337, "2": 10.91184, - "3": 10.9245, - "4": 10.91413, - "5": 10.91791, - "6": 10.90788, - "7": 10.90285, - "8": 10.9094, - "9": 10.92264, - "10": 10.91325, - "11": 10.90217, - "12": 10.91139, - "13": 10.88971, - "14": 10.88954, - "15": 10.88501, - "16": 10.87598, - "17": 10.87426, - "18": 10.8634, - "19": 10.87212, - "20": 10.82076, - "21": 10.80748, - "22": 10.79544, - "23": 10.79148, - "24": 10.75643, - "25": 10.76142, - "26": 10.75154, - "27": 10.73598, - "28": 10.67469, - "29": 10.64016, - "30": 10.62245, - "31": 10.61276, - "32": 10.59683, - "33": 10.57641, - "34": 10.53937, - "35": 10.53846, - "36": 10.52255, - "37": 10.48675, - "38": 10.49927, - "39": 10.45887, - "40": 10.43983, - "41": 10.42774, - "42": 10.41415, - "43": 10.3903, - "44": 10.35238, - "45": 10.36987, - "46": 10.33008, - "47": 10.31244, - "48": 10.27782, - "49": 10.27035, - "50": 10.27145 + "3": 10.92435, + "4": 10.91391, + "5": 10.91783, + "6": 10.90822, + "7": 10.90214, + "8": 10.90968, + "9": 10.92327, + "10": 10.91324, + "11": 10.90152, + "12": 10.91168, + "13": 10.88952, + "14": 10.88978, + "15": 10.88462, + "16": 10.87656, + "17": 10.87413, + "18": 10.8627, + "19": 10.87155, + "20": 10.8213, + "21": 10.8075, + "22": 10.79454, + "23": 10.79087, + "24": 10.75698, + "25": 10.76203, + "26": 10.75144, + "27": 10.73619, + "28": 10.67458, + "29": 10.63986, + "30": 10.62269, + "31": 10.6127, + "32": 10.59666, + "33": 10.57595, + "34": 10.53962, + "35": 10.53801, + "36": 10.52144, + "37": 10.48719, + "38": 10.49901, + "39": 10.45863, + "40": 10.43984, + "41": 10.42707, + "42": 10.41355, + "43": 10.38977, + "44": 10.35254, + "45": 10.36992, + "46": 10.32982, + "47": 10.31281, + "48": 10.27777, + "49": 10.27075, + "50": 10.27166 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18811.0, - "2": 19165.0, - "3": 19169.0, - "4": 18794.0, - "5": 18806.0, - "6": 18547.0, - "7": 18947.0, - "8": 18963.0, - "9": 18914.0, - "10": 19344.0, - "11": 18881.0, - "12": 18329.0, - "13": 18937.0, - "14": 18987.0, - "15": 18708.0, - "16": 18957.0, - "17": 19271.0, - "18": 18855.0, - "19": 18843.0, - "20": 19091.0, - "21": 18628.0, - "22": 18905.0, - "23": 18905.0, - "24": 19000.0, - "25": 18306.0, - "26": 18509.0, - "27": 18924.0, - "28": 18259.0, - "29": 18452.0, - "30": 18270.0, - "31": 18685.0, - "32": 18898.0, - "33": 18250.0, - "34": 18394.0, - "35": 18736.0, - "36": 18582.0, - "37": 18806.0, - "38": 18396.0, - "39": 19248.0, - "40": 19332.0, - "41": 18568.0, - "42": 18144.0, - "43": 19243.0, - "44": 18642.0, - "45": 20136.0, - "46": 19403.0, - "47": 19891.0, - "48": 20076.0, - "49": 21584.0, - "50": 20152.0 + "1": 18852.0, + "2": 19201.0, + "3": 18829.0, + "4": 18799.0, + "5": 18661.0, + "6": 18606.0, + "7": 19036.0, + "8": 18992.0, + "9": 19205.0, + "10": 19409.0, + "11": 18725.0, + "12": 18412.0, + "13": 19021.0, + "14": 19235.0, + "15": 18796.0, + "16": 18661.0, + "17": 18978.0, + "18": 18747.0, + "19": 18689.0, + "20": 18999.0, + "21": 18460.0, + "22": 18943.0, + "23": 19110.0, + "24": 18724.0, + "25": 18209.0, + "26": 18944.0, + "27": 18769.0, + "28": 18225.0, + "29": 18553.0, + "30": 18154.0, + "31": 18660.0, + "32": 19011.0, + "33": 18551.0, + "34": 18477.0, + "35": 18870.0, + "36": 18407.0, + "37": 18996.0, + "38": 18270.0, + "39": 19069.0, + "40": 19467.0, + "41": 18520.0, + "42": 18295.0, + "43": 19123.0, + "44": 18827.0, + "45": 20268.0, + "46": 19722.0, + "47": 19576.0, + "48": 19950.0, + "49": 21813.0, + "50": 20205.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1388729856.0, - "2": 1388730368.0, - "3": 1388729856.0, + "1": 1388729344.0, + "2": 1388729856.0, + "3": 1388729344.0, "4": 1388727808.0, "5": 1388728832.0, - "6": 1388730368.0, - "7": 1388731904.0, - "8": 1388729856.0, - "9": 1388731904.0, + "6": 1388729856.0, + "7": 1388731392.0, + "8": 1388730880.0, + "9": 1388731392.0, "10": 1388731392.0, - "11": 1388728832.0, - "12": 1388728832.0, - "13": 1388730880.0, - "14": 1388728320.0, - "15": 1388731904.0, - "16": 1388728320.0, - "17": 1388730880.0, - "18": 1388730368.0, - "19": 1388729856.0, - "20": 1388730368.0, - "21": 1388732928.0, - "22": 1388729344.0, + "11": 1388729344.0, + "12": 1388728320.0, + "13": 1388732928.0, + "14": 1388727808.0, + "15": 1388730880.0, + "16": 1388728832.0, + "17": 1388729856.0, + "18": 1388729856.0, + "19": 1388730880.0, + "20": 1388731904.0, + "21": 1388729856.0, + "22": 1388730368.0, "23": 1388731392.0, - "24": 1388729344.0, - "25": 1388732928.0, - "26": 1388732928.0, - "27": 1388729344.0, - "28": 1388730368.0, - "29": 1388731392.0, - "30": 1388733440.0, - "31": 1388732416.0, - "32": 1388729344.0, - "33": 1388732416.0, - "34": 1388734464.0, + "24": 1388730880.0, + "25": 1388732416.0, + "26": 1388732416.0, + "27": 1388731904.0, + "28": 1388728832.0, + "29": 1388730880.0, + "30": 1388732928.0, + "31": 1388731392.0, + "32": 1388728832.0, + "33": 1388731904.0, + "34": 1388733952.0, "35": 1388731392.0, "36": 1388730368.0, - "37": 1388731904.0, + "37": 1388731392.0, "38": 1388730368.0, - "39": 1388733440.0, - "40": 1388734464.0, - "41": 1388730368.0, - "42": 1388732928.0, - "43": 1388731392.0, - "44": 1388729856.0, - "45": 1388732928.0, - "46": 1388731904.0, - "47": 1388733952.0, - "48": 1388731904.0, - "49": 1388736512.0, - "50": 1388731904.0 + "39": 1388731904.0, + "40": 1388731904.0, + "41": 1388729344.0, + "42": 1388732416.0, + "43": 1388731904.0, + "44": 1388729344.0, + "45": 1388731904.0, + "46": 1388732416.0, + "47": 1388733440.0, + "48": 1388731392.0, + "49": 1388735488.0, + "50": 1388730368.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3179598848.0, - "2": 3662152704.0, - "3": 3662152704.0, - "4": 3662152704.0, - "5": 3662152704.0, - "6": 3662152704.0, - "7": 3662152704.0, - "8": 3662152704.0, - "9": 3662152704.0, - "10": 3662260736.0, - "11": 3662260736.0, - "12": 3662260736.0, - "13": 3662260736.0, - "14": 3662260736.0, - "15": 3662260736.0, - "16": 3662260736.0, - "17": 3662260736.0, - "18": 3662260736.0, - "19": 3662745088.0, - "20": 3662745088.0, - "21": 3662745088.0, - "22": 3662745088.0, - "23": 3662745088.0, - "24": 3662745088.0, - "25": 3662901248.0, - "26": 3663353344.0, - "27": 3663353344.0, - "28": 3663353344.0, - "29": 3663353344.0, - "30": 3663353344.0, - "31": 3663353344.0, - "32": 3663353344.0, - "33": 3663353344.0, - "34": 3663353344.0, - "35": 3663353344.0, - "36": 3663353344.0, - "37": 3663353344.0, - "38": 3663353344.0, - "39": 3664240128.0, - "40": 3664612864.0, - "41": 3664612864.0, - "42": 3664612864.0, - "43": 3664612864.0, - "44": 3664748544.0, - "45": 3664748544.0, - "46": 3664803840.0, - "47": 3664803840.0, - "48": 3664803840.0, - "49": 3664803840.0, - "50": 3664803840.0 + "1": 3179891200.0, + "2": 3662227968.0, + "3": 3662227968.0, + "4": 3663250944.0, + "5": 3663250944.0, + "6": 3663250944.0, + "7": 3663250944.0, + "8": 3663250944.0, + "9": 3663250944.0, + "10": 3663250944.0, + "11": 3663250944.0, + "12": 3663250944.0, + "13": 3663250944.0, + "14": 3663250944.0, + "15": 3663250944.0, + "16": 3663250944.0, + "17": 3663250944.0, + "18": 3663250944.0, + "19": 3663250944.0, + "20": 3663250944.0, + "21": 3663250944.0, + "22": 3663250944.0, + "23": 3663250944.0, + "24": 3663250944.0, + "25": 3663250944.0, + "26": 3663250944.0, + "27": 3663250944.0, + "28": 3663250944.0, + "29": 3663250944.0, + "30": 3663250944.0, + "31": 3663250944.0, + "32": 3663250944.0, + "33": 3663250944.0, + "34": 3664421888.0, + "35": 3664421888.0, + "36": 3664421888.0, + "37": 3664421888.0, + "38": 3664421888.0, + "39": 3664421888.0, + "40": 3664421888.0, + "41": 3664421888.0, + "42": 3664421888.0, + "43": 3664421888.0, + "44": 3664535552.0, + "45": 3664535552.0, + "46": 3664535552.0, + "47": 3664535552.0, + "48": 3664535552.0, + "49": 3664612864.0, + "50": 3664612864.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.08415, - "3": 1.38326, - "4": 1.12865, - "5": 0.75816, - "6": 1.25852, - "7": 0.96185, - "8": 1.65183, - "9": 1.44873, - "10": 1.00039, - "11": 1.10691, - "12": 1.21729, - "13": 0.86669, - "14": 1.5195, - "15": 0.8088, - "16": 1.1441, - "17": 1.11777, - "18": 1.13236, - "19": 1.23059, - "20": 1.05827, - "21": 1.06481, - "22": 1.03062, - "23": 1.28263, - "24": 1.59248, - "25": 0.58856, - "26": 1.11112, - "27": 1.12276, - "28": 1.01627, - "29": 1.06509, - "30": 0.89904, - "31": 0.90921, - "32": 2.09652, - "33": 1.17838, - "34": 0.58787, - "35": 1.41686, - "36": 1.26506, - "37": 1.03891, - "38": 0.68737, - "39": 1.13036, - "40": 1.45597, - "41": 1.04997, - "42": 1.32906, - "43": 1.12204, - "44": 1.39831, - "45": 0.65378, - "46": 1.13893, - "47": 1.10165, - "48": 1.09415, - "49": 0.95128, - "50": 1.20935 + "2": 6.76257, + "3": 2.41181, + "4": 2.14671, + "5": 1.68613, + "6": 2.35596, + "7": 1.85689, + "8": 3.55826, + "9": 2.10313, + "10": 1.69174, + "11": 2.07594, + "12": 2.11237, + "13": 2.0346, + "14": 2.34804, + "15": 1.56257, + "16": 1.96304, + "17": 2.0887, + "18": 2.24792, + "19": 1.71276, + "20": 1.64151, + "21": 2.03443, + "22": 1.86292, + "23": 1.95656, + "24": 2.17465, + "25": 2.28198, + "26": 1.89479, + "27": 1.24118, + "28": 2.07544, + "29": 1.97422, + "30": 1.20941, + "31": 1.60534, + "32": 2.43502, + "33": 2.08637, + "34": 1.84735, + "35": 2.40386, + "36": 1.40304, + "37": 2.14664, + "38": 0.89545, + "39": 2.42476, + "40": 2.25422, + "41": 1.29899, + "42": 2.6352, + "43": 1.42406, + "44": 2.03708, + "45": 2.10233, + "46": 0.7566, + "47": 1.72618, + "48": 1.51965, + "49": 1.43953, + "50": 1.66225 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json index 30c08fdc1d1..03672a09acd 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer_1node/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.92169, "2": 10.90762, - "3": 10.90909, - "4": 10.91955, - "5": 10.90573, - "6": 10.91571, - "7": 10.92287, - "8": 10.90904, - "9": 10.91203, - "10": 10.90702, - "11": 10.88873, - "12": 10.90475, - "13": 10.89504, - "14": 10.89573, - "15": 10.87518, - "16": 10.8614, - "17": 10.86441, - "18": 10.85053, - "19": 10.85499, - "20": 10.78455, - "21": 10.7775, - "22": 10.75611, - "23": 10.74483, - "24": 10.71735, - "25": 10.71547, - "26": 10.71001, - "27": 10.67234, - "28": 10.60468, - "29": 10.57052, - "30": 10.54195, - "31": 10.54485, - "32": 10.53174, - "33": 10.49752, - "34": 10.47464, - "35": 10.47083, - "36": 10.44096, - "37": 10.41642, - "38": 10.41727, - "39": 10.38507, - "40": 10.36831, - "41": 10.34679, - "42": 10.33436, - "43": 10.31332, - "44": 10.28897, - "45": 10.29332, - "46": 10.26565, - "47": 10.24514, - "48": 10.19831, - "49": 10.19477, - "50": 10.1961, - "51": 10.20232, - "52": 10.15459, - "53": 10.16262, - "54": 10.12729, - "55": 10.09552, - "56": 10.12465, - "57": 10.11344, - "58": 10.12412, - "59": 10.07409, - "60": 10.09369, - "61": 10.04308, - "62": 10.0163, - "63": 10.08727, - "64": 10.03778, - "65": 10.01755, - "66": 10.04058, - "67": 10.01819, - "68": 9.9801, - "69": 10.00352, - "70": 9.98393, - "71": 10.01032, - "72": 9.99173, - "73": 9.98076, - "74": 9.9623, - "75": 9.94249, - "76": 9.97148, - "77": 9.96097, - "78": 9.91558, - "79": 9.92241, - "80": 9.93107, - "81": 9.95879, - "82": 9.89539, + "3": 10.90914, + "4": 10.91959, + "5": 10.90569, + "6": 10.91601, + "7": 10.92268, + "8": 10.90935, + "9": 10.91155, + "10": 10.90713, + "11": 10.88867, + "12": 10.90462, + "13": 10.89519, + "14": 10.89552, + "15": 10.8746, + "16": 10.86175, + "17": 10.86443, + "18": 10.85034, + "19": 10.85454, + "20": 10.78432, + "21": 10.77769, + "22": 10.75615, + "23": 10.74474, + "24": 10.71765, + "25": 10.71594, + "26": 10.70996, + "27": 10.67222, + "28": 10.60454, + "29": 10.57083, + "30": 10.54221, + "31": 10.54476, + "32": 10.53155, + "33": 10.49746, + "34": 10.4747, + "35": 10.47049, + "36": 10.44085, + "37": 10.41646, + "38": 10.41713, + "39": 10.38512, + "40": 10.36837, + "41": 10.34662, + "42": 10.3344, + "43": 10.31362, + "44": 10.28901, + "45": 10.29313, + "46": 10.26547, + "47": 10.24518, + "48": 10.1985, + "49": 10.19468, + "50": 10.19581, + "51": 10.20229, + "52": 10.15471, + "53": 10.16268, + "54": 10.12733, + "55": 10.09567, + "56": 10.12483, + "57": 10.11356, + "58": 10.12404, + "59": 10.07408, + "60": 10.09371, + "61": 10.04305, + "62": 10.01618, + "63": 10.08733, + "64": 10.03798, + "65": 10.01758, + "66": 10.04082, + "67": 10.0182, + "68": 9.98006, + "69": 10.00358, + "70": 9.98391, + "71": 10.0103, + "72": 9.99161, + "73": 9.98074, + "74": 9.96226, + "75": 9.94239, + "76": 9.97139, + "77": 9.96076, + "78": 9.91543, + "79": 9.92224, + "80": 9.93093, + "81": 9.95886, + "82": 9.8954, "83": 9.86306, - "84": 9.79603, - "85": 9.78813, - "86": 9.88181, - "87": 9.90776, - "88": 9.88258, - "89": 9.82999, - "90": 9.82224, - "91": 9.82944, - "92": 9.82018, - "93": 9.76029, - "94": 9.83143, - "95": 9.82804, - "96": 9.80736, - "97": 9.75384, - "98": 9.78406, - "99": 9.82607, - "100": 9.7184 + "84": 9.79599, + "85": 9.78812, + "86": 9.88186, + "87": 9.90774, + "88": 9.88268, + "89": 9.82997, + "90": 9.8221, + "91": 9.82939, + "92": 9.82008, + "93": 9.76022, + "94": 9.83152, + "95": 9.82801, + "96": 9.80754, + "97": 9.75369, + "98": 9.78408, + "99": 9.826, + "100": 9.71837 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2468.0, - "2": 2401.0, - "3": 2493.0, - "4": 2505.0, - "5": 2492.0, - "6": 2441.0, - "7": 2647.0, - "8": 2461.0, - "9": 2625.0, - "10": 2517.0, - "11": 2523.0, - "12": 2365.0, - "13": 2673.0, - "14": 2542.0, - "15": 2403.0, - "16": 2464.0, - "17": 2630.0, - "18": 2450.0, - "19": 2642.0, - "20": 2426.0, - "21": 2654.0, - "22": 2624.0, - "23": 2472.0, - "24": 2570.0, - "25": 2385.0, - "26": 2633.0, - "27": 2532.0, - "28": 2541.0, - "29": 2652.0, - "30": 2563.0, - "31": 2722.0, - "32": 2789.0, - "33": 3973.0, - "34": 2782.0, - "35": 527627.0, - "36": 527593.0, - "37": 25917.0, - "38": 527689.0, - "39": 527839.0, - "40": 527891.0, - "41": 31727.0, - "42": 527517.0, - "43": 528077.0, - "44": 527852.0, - "45": 30413.0, - "46": 527985.0, - "47": 528131.0, - "48": 528209.0, - "49": 528359.0, - "50": 550860.0, - "51": 560328.0, - "52": 533340.0, - "53": 1055795.0, - "54": 555563.0, - "55": 554263.0, - "56": 1053101.0, - "57": 1052681.0, - "58": 1079130.0, - "59": 557347.0, - "60": 582286.0, - "61": 587119.0, - "62": 1578262.0, - "63": 557738.0, - "64": 537139.0, - "65": 1578266.0, - "66": 1058838.0, - "67": 1056227.0, - "68": 1053806.0, - "69": 561478.0, - "70": 531123.0, - "71": 1053985.0, - "72": 1053905.0, - "73": 1075978.0, - "74": 1075340.0, - "75": 557909.0, - "76": 1578167.0, - "77": 552293.0, - "78": 556281.0, - "79": 1077972.0, - "80": 578039.0, - "81": 1058970.0, - "82": 1080698.0, - "83": 1084664.0, - "84": 575523.0, - "85": 560438.0, - "86": 554843.0, - "87": 575549.0, - "88": 536306.0, - "89": 1053569.0, - "90": 1578469.0, - "91": 1053432.0, - "92": 1053657.0, - "93": 529105.0, - "94": 1053708.0, - "95": 558441.0, - "96": 1055928.0, - "97": 1054101.0, - "98": 1054213.0, - "99": 554460.0, - "100": 1578715.0 + "1": 2485.0, + "2": 2410.0, + "3": 2558.0, + "4": 2521.0, + "5": 2379.0, + "6": 2355.0, + "7": 2651.0, + "8": 2390.0, + "9": 2608.0, + "10": 2524.0, + "11": 2482.0, + "12": 2420.0, + "13": 2711.0, + "14": 2610.0, + "15": 2447.0, + "16": 2348.0, + "17": 2554.0, + "18": 2448.0, + "19": 2569.0, + "20": 2425.0, + "21": 2494.0, + "22": 2534.0, + "23": 2440.0, + "24": 2576.0, + "25": 2394.0, + "26": 2550.0, + "27": 2531.0, + "28": 2635.0, + "29": 2658.0, + "30": 2646.0, + "31": 2811.0, + "32": 2768.0, + "33": 32343.0, + "34": 2854.0, + "35": 527663.0, + "36": 527633.0, + "37": 25982.0, + "38": 527719.0, + "39": 527760.0, + "40": 527781.0, + "41": 31784.0, + "42": 527721.0, + "43": 528039.0, + "44": 527836.0, + "45": 30434.0, + "46": 528026.0, + "47": 528121.0, + "48": 528129.0, + "49": 528389.0, + "50": 550759.0, + "51": 531404.0, + "52": 533197.0, + "53": 1059399.0, + "54": 554922.0, + "55": 553732.0, + "56": 1053105.0, + "57": 1052632.0, + "58": 1078718.0, + "59": 557322.0, + "60": 582238.0, + "61": 1085352.0, + "62": 1578412.0, + "63": 557697.0, + "64": 554190.0, + "65": 1578355.0, + "66": 1059050.0, + "67": 1086225.0, + "68": 1053764.0, + "69": 561544.0, + "70": 531044.0, + "71": 1054678.0, + "72": 1053885.0, + "73": 1578126.0, + "74": 1055279.0, + "75": 556894.0, + "76": 1081506.0, + "77": 552287.0, + "78": 556331.0, + "79": 1077913.0, + "80": 578071.0, + "81": 1084863.0, + "82": 1080774.0, + "83": 1577846.0, + "84": 575407.0, + "85": 558901.0, + "86": 554895.0, + "87": 575490.0, + "88": 533778.0, + "89": 1053554.0, + "90": 1578611.0, + "91": 1053399.0, + "92": 1053583.0, + "93": 529118.0, + "94": 1053702.0, + "95": 558281.0, + "96": 1055792.0, + "97": 1054543.0, + "98": 1054324.0, + "99": 554749.0, + "100": 1578781.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 906272768.0, - "2": 906273792.0, - "3": 906273792.0, - "4": 906273792.0, - "5": 906273792.0, - "6": 906273792.0, - "7": 906273792.0, - "8": 906273792.0, - "9": 906273792.0, - "10": 906273792.0, - "11": 906273792.0, - "12": 906273792.0, - "13": 906273792.0, - "14": 906273792.0, - "15": 906273792.0, - "16": 906273792.0, - "17": 906273792.0, - "18": 906273792.0, - "19": 906273792.0, - "20": 906273792.0, - "21": 906273792.0, - "22": 906273792.0, - "23": 906273792.0, - "24": 906273792.0, - "25": 906273792.0, - "26": 906273792.0, - "27": 906273792.0, - "28": 906273792.0, - "29": 906273792.0, - "30": 906273792.0, - "31": 906273792.0, - "32": 906273792.0, - "33": 906273792.0, - "34": 906273792.0, - "35": 906273792.0, - "36": 906273792.0, - "37": 906273792.0, - "38": 906273792.0, - "39": 906273792.0, - "40": 906273792.0, - "41": 906273792.0, - "42": 906273792.0, - "43": 906273792.0, - "44": 906273792.0, - "45": 906273792.0, - "46": 906273792.0, - "47": 906273792.0, - "48": 906273792.0, - "49": 906273792.0, - "50": 906273792.0, - "51": 906273792.0, - "52": 906273792.0, - "53": 906273792.0, - "54": 906273792.0, - "55": 906273792.0, - "56": 906273792.0, - "57": 906273792.0, - "58": 906273792.0, - "59": 906273792.0, - "60": 906273792.0, - "61": 906273792.0, - "62": 906273792.0, - "63": 906273792.0, - "64": 906273792.0, - "65": 906273792.0, - "66": 906273792.0, - "67": 906273792.0, - "68": 906273792.0, - "69": 906273792.0, - "70": 906273792.0, - "71": 906273792.0, - "72": 906273792.0, - "73": 906273792.0, - "74": 906273792.0, - "75": 906273792.0, - "76": 906273792.0, - "77": 906273792.0, - "78": 906273792.0, - "79": 906273792.0, - "80": 906273792.0, - "81": 906273792.0, - "82": 906273792.0, - "83": 906273792.0, - "84": 906273792.0, - "85": 906273792.0, - "86": 906273792.0, - "87": 906273792.0, - "88": 906273792.0, - "89": 906273792.0, - "90": 906273792.0, - "91": 906273792.0, - "92": 906273792.0, - "93": 906273792.0, - "94": 906273792.0, - "95": 906273792.0, - "96": 906273792.0, - "97": 906273792.0, - "98": 906273792.0, - "99": 906273792.0, - "100": 906273792.0 + "1": 906272256.0, + "2": 906273280.0, + "3": 906273280.0, + "4": 906273280.0, + "5": 906273280.0, + "6": 906273280.0, + "7": 906273280.0, + "8": 906273280.0, + "9": 906273280.0, + "10": 906273280.0, + "11": 906273280.0, + "12": 906273280.0, + "13": 906273280.0, + "14": 906273280.0, + "15": 906273280.0, + "16": 906273280.0, + "17": 906273280.0, + "18": 906273280.0, + "19": 906273280.0, + "20": 906273280.0, + "21": 906273280.0, + "22": 906273280.0, + "23": 906273280.0, + "24": 906273280.0, + "25": 906273280.0, + "26": 906273280.0, + "27": 906273280.0, + "28": 906273280.0, + "29": 906273280.0, + "30": 906273280.0, + "31": 906273280.0, + "32": 906273280.0, + "33": 906273280.0, + "34": 906273280.0, + "35": 906273280.0, + "36": 906273280.0, + "37": 906273280.0, + "38": 906273280.0, + "39": 906273280.0, + "40": 906273280.0, + "41": 906273280.0, + "42": 906273280.0, + "43": 906273280.0, + "44": 906273280.0, + "45": 906273280.0, + "46": 906273280.0, + "47": 906273280.0, + "48": 906273280.0, + "49": 906273280.0, + "50": 906273280.0, + "51": 906273280.0, + "52": 906273280.0, + "53": 906273280.0, + "54": 906273280.0, + "55": 906273280.0, + "56": 906273280.0, + "57": 906273280.0, + "58": 906273280.0, + "59": 906273280.0, + "60": 906273280.0, + "61": 906273280.0, + "62": 906273280.0, + "63": 906273280.0, + "64": 906273280.0, + "65": 906273280.0, + "66": 906273280.0, + "67": 906273280.0, + "68": 906273280.0, + "69": 906273280.0, + "70": 906273280.0, + "71": 906273280.0, + "72": 906273280.0, + "73": 906273280.0, + "74": 906273280.0, + "75": 906273280.0, + "76": 906273280.0, + "77": 906273280.0, + "78": 906273280.0, + "79": 906273280.0, + "80": 906273280.0, + "81": 906273280.0, + "82": 906273280.0, + "83": 906273280.0, + "84": 906273280.0, + "85": 906273280.0, + "86": 906273280.0, + "87": 906273280.0, + "88": 906273280.0, + "89": 906273280.0, + "90": 906273280.0, + "91": 906273280.0, + "92": 906273280.0, + "93": 906273280.0, + "94": 906273280.0, + "95": 906273280.0, + "96": 906273280.0, + "97": 906273280.0, + "98": 906273280.0, + "99": 906273280.0, + "100": 906273280.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1323515904.0, - "2": 1613952512.0, - "3": 1617445376.0, - "4": 1619704832.0, - "5": 1619704832.0, - "6": 1619704832.0, - "7": 1619704832.0, - "8": 1619704832.0, - "9": 1619704832.0, - "10": 1619704832.0, - "11": 1619704832.0, - "12": 1619704832.0, - "13": 1619704832.0, - "14": 1619704832.0, - "15": 1619704832.0, - "16": 1619704832.0, - "17": 1619704832.0, - "18": 1619704832.0, - "19": 1619704832.0, - "20": 1619704832.0, - "21": 1619704832.0, - "22": 1619704832.0, - "23": 1619704832.0, - "24": 1619704832.0, - "25": 1619704832.0, - "26": 1619704832.0, - "27": 1619704832.0, - "28": 1619704832.0, - "29": 1619704832.0, - "30": 1619704832.0, - "31": 1619704832.0, - "32": 1619704832.0, - "33": 1619704832.0, - "34": 1619704832.0, - "35": 1619704832.0, - "36": 1619704832.0, - "37": 1619704832.0, - "38": 1619704832.0, - "39": 1619704832.0, - "40": 1619704832.0, - "41": 1619704832.0, - "42": 1619704832.0, - "43": 1619704832.0, - "44": 1619704832.0, - "45": 1619704832.0, - "46": 1619704832.0, - "47": 1619704832.0, - "48": 1619704832.0, - "49": 1619704832.0, - "50": 1619704832.0, - "51": 1619704832.0, - "52": 1619704832.0, - "53": 1619704832.0, - "54": 1619704832.0, - "55": 1619704832.0, - "56": 1619704832.0, - "57": 1619704832.0, - "58": 1619704832.0, - "59": 1619704832.0, - "60": 1619704832.0, - "61": 1619704832.0, - "62": 1619704832.0, - "63": 1619704832.0, - "64": 1619704832.0, - "65": 1619704832.0, - "66": 1619704832.0, - "67": 1619704832.0, - "68": 1619704832.0, - "69": 1619704832.0, - "70": 1619704832.0, - "71": 1619704832.0, - "72": 1619704832.0, - "73": 1619704832.0, - "74": 1619704832.0, - "75": 1619704832.0, - "76": 1619704832.0, - "77": 1619704832.0, - "78": 1619704832.0, - "79": 1619704832.0, - "80": 1619704832.0, - "81": 1619704832.0, - "82": 1619704832.0, - "83": 1619704832.0, - "84": 1619704832.0, - "85": 1619704832.0, - "86": 1619704832.0, - "87": 1619704832.0, - "88": 1619704832.0, - "89": 1619704832.0, - "90": 1619704832.0, - "91": 1619704832.0, - "92": 1619704832.0, - "93": 1619704832.0, - "94": 1619704832.0, - "95": 1619704832.0, - "96": 1619704832.0, - "97": 1619704832.0, - "98": 1619704832.0, - "99": 1619704832.0, - "100": 1619704832.0 + "1": 1322636288.0, + "2": 1613951488.0, + "3": 1617380864.0, + "4": 1620309504.0, + "5": 1620309504.0, + "6": 1620309504.0, + "7": 1620309504.0, + "8": 1620309504.0, + "9": 1620309504.0, + "10": 1620309504.0, + "11": 1620309504.0, + "12": 1620309504.0, + "13": 1620309504.0, + "14": 1620309504.0, + "15": 1620309504.0, + "16": 1620309504.0, + "17": 1620309504.0, + "18": 1620309504.0, + "19": 1620309504.0, + "20": 1620309504.0, + "21": 1620309504.0, + "22": 1620309504.0, + "23": 1620309504.0, + "24": 1620309504.0, + "25": 1620309504.0, + "26": 1620309504.0, + "27": 1620309504.0, + "28": 1620309504.0, + "29": 1620309504.0, + "30": 1620309504.0, + "31": 1620309504.0, + "32": 1620309504.0, + "33": 1620309504.0, + "34": 1620309504.0, + "35": 1620309504.0, + "36": 1620309504.0, + "37": 1620309504.0, + "38": 1620309504.0, + "39": 1620309504.0, + "40": 1620309504.0, + "41": 1620309504.0, + "42": 1620309504.0, + "43": 1620309504.0, + "44": 1620309504.0, + "45": 1620309504.0, + "46": 1620309504.0, + "47": 1620309504.0, + "48": 1620309504.0, + "49": 1620309504.0, + "50": 1620309504.0, + "51": 1620309504.0, + "52": 1620309504.0, + "53": 1620309504.0, + "54": 1620309504.0, + "55": 1620309504.0, + "56": 1620309504.0, + "57": 1620309504.0, + "58": 1620309504.0, + "59": 1620309504.0, + "60": 1620309504.0, + "61": 1620309504.0, + "62": 1620309504.0, + "63": 1620309504.0, + "64": 1620309504.0, + "65": 1620309504.0, + "66": 1620309504.0, + "67": 1620309504.0, + "68": 1620309504.0, + "69": 1620309504.0, + "70": 1620309504.0, + "71": 1620309504.0, + "72": 1620309504.0, + "73": 1620309504.0, + "74": 1620309504.0, + "75": 1620309504.0, + "76": 1620309504.0, + "77": 1620309504.0, + "78": 1620309504.0, + "79": 1620309504.0, + "80": 1620309504.0, + "81": 1620309504.0, + "82": 1620309504.0, + "83": 1620309504.0, + "84": 1620309504.0, + "85": 1620309504.0, + "86": 1620309504.0, + "87": 1620309504.0, + "88": 1620309504.0, + "89": 1620309504.0, + "90": 1620309504.0, + "91": 1620309504.0, + "92": 1620309504.0, + "93": 1620309504.0, + "94": 1620309504.0, + "95": 1620309504.0, + "96": 1620309504.0, + "97": 1620309504.0, + "98": 1620309504.0, + "99": 1620309504.0, + "100": 1620309504.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.37367, - "3": 2.54182, - "4": 2.09823, - "5": 1.74432, - "6": 2.41953, - "7": 1.6559, - "8": 2.65326, - "9": 2.63405, - "10": 2.42359, - "11": 2.59476, - "12": 2.405, - "13": 2.23734, - "14": 2.72146, - "15": 2.33172, - "16": 2.2102, - "17": 2.95714, - "18": 2.20431, - "19": 2.49222, - "20": 2.29905, - "21": 2.71979, - "22": 2.18441, - "23": 2.55797, - "24": 2.18328, - "25": 2.83896, - "26": 2.91067, - "27": 2.21319, - "28": 2.38566, - "29": 2.12367, - "30": 1.73958, - "31": 1.76394, - "32": 2.99398, - "33": 1.96459, - "34": 1.68131, - "35": 2.60891, - "36": 2.28892, - "37": 2.15773, - "38": 1.92535, - "39": 2.59033, - "40": 2.07581, - "41": 2.90767, - "42": 2.62746, - "43": 1.98056, - "44": 2.76516, - "45": 2.50896, - "46": 2.01713, - "47": 2.14119, - "48": 2.18834, - "49": 2.0346, - "50": 2.00386, - "51": 1.6825, - "52": 2.76421, - "53": 2.50338, - "54": 2.64345, - "55": 2.225, - "56": 3.26343, - "57": 1.48644, - "58": 2.71536, - "59": 2.00729, - "60": 2.16353, - "61": 2.44532, - "62": 3.04555, - "63": 1.69774, - "64": 2.24699, - "65": 2.03727, - "66": 2.14857, - "67": 2.48781, - "68": 1.71264, - "69": 2.41294, - "70": 2.17268, - "71": 2.73027, - "72": 2.13735, - "73": 2.2041, - "74": 2.20307, - "75": 2.20197, - "76": 2.52796, - "77": 2.27171, - "78": 2.21398, - "79": 2.90986, - "80": 2.33253, - "81": 2.7207, - "82": 2.52303, - "83": 2.27444, - "84": 1.94576, - "85": 2.07223, - "86": 2.12538, - "87": 2.79202, - "88": 2.4337, - "89": 2.17604, - "90": 2.44102, - "91": 2.46034, - "92": 2.84467, - "93": 1.98126, - "94": 2.94718, - "95": 2.3739, - "96": 2.44264, - "97": 2.2023, - "98": 2.19249, - "99": 3.07219, - "100": 2.08776 + "2": 8.44616, + "3": 4.35946, + "4": 3.58536, + "5": 3.43258, + "6": 4.01618, + "7": 3.41532, + "8": 4.69585, + "9": 4.42146, + "10": 3.2698, + "11": 4.19498, + "12": 3.71608, + "13": 4.078, + "14": 3.7934, + "15": 2.75437, + "16": 2.90972, + "17": 3.42375, + "18": 3.12947, + "19": 3.74111, + "20": 2.77586, + "21": 3.59019, + "22": 3.15397, + "23": 4.71787, + "24": 3.61079, + "25": 3.91574, + "26": 4.22139, + "27": 3.87761, + "28": 3.93585, + "29": 3.34081, + "30": 3.02171, + "31": 3.39522, + "32": 5.21075, + "33": 2.81038, + "34": 3.41835, + "35": 4.45162, + "36": 4.20517, + "37": 3.9982, + "38": 3.03186, + "39": 6.15837, + "40": 3.06211, + "41": 3.45761, + "42": 4.42809, + "43": 3.47773, + "44": 3.66168, + "45": 3.60833, + "46": 2.92395, + "47": 2.64427, + "48": 2.86604, + "49": 2.50808, + "50": 3.04166, + "51": 2.18664, + "52": 3.25348, + "53": 3.69136, + "54": 3.55702, + "55": 3.0236, + "56": 4.7145, + "57": 2.71755, + "58": 3.34597, + "59": 3.09264, + "60": 3.14001, + "61": 3.06867, + "62": 4.22075, + "63": 2.45818, + "64": 3.80461, + "65": 2.8975, + "66": 3.83227, + "67": 3.41683, + "68": 3.28576, + "69": 3.16606, + "70": 3.69457, + "71": 3.84527, + "72": 2.94589, + "73": 2.78343, + "74": 3.33954, + "75": 3.40883, + "76": 3.35809, + "77": 3.09918, + "78": 3.74403, + "79": 3.86757, + "80": 3.56784, + "81": 3.62657, + "82": 2.52733, + "83": 3.21777, + "84": 2.77215, + "85": 3.51637, + "86": 3.14139, + "87": 4.25648, + "88": 3.21899, + "89": 3.79927, + "90": 3.25394, + "91": 3.48754, + "92": 3.57614, + "93": 2.08265, + "94": 3.45887, + "95": 3.83381, + "96": 3.66511, + "97": 3.09335, + "98": 2.62447, + "99": 4.07289, + "100": 3.1867 } } } \ No newline at end of file From f690cee103ed8ea80a51988c579655bd7c560b39 Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Sat, 4 Jul 2026 17:35:33 -0700 Subject: [PATCH 3/7] Update golden values in functional tests for various models to enhance accuracy and consistency. Adjustments made to numerical values in JSON files across multiple test cases, ensuring improved results in model evaluations. Signed-off-by: Ajay Balasa --- .../golden_values_dev_dgx_gb200.json | 90 ++-- .../golden_values_dev_dgx_gb200.json | 398 +++++++++--------- .../golden_values_dev_dgx_gb200.json | 398 +++++++++--------- .../golden_values_dev_dgx_gb200.json | 190 ++++----- .../golden_values_dev_dgx_h100.json | 186 ++++---- .../golden_values_dev_dgx_h100.json | 196 ++++----- 6 files changed, 729 insertions(+), 729 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json index 84ecce2898f..c025959fae6 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json @@ -7,53 +7,53 @@ "1": 10.90136, "2": 10.93243, "3": 10.64755, - "4": 10.41185, - "5": 10.40023, - "6": 10.36738, - "7": 10.11272, - "8": 9.90223, - "9": 9.96421, - "10": 9.51353, - "11": 10.16205, - "12": 9.86207, - "13": 9.87005, - "14": 9.89979, - "15": 9.50628, - "16": 9.45068, - "17": 9.2599, - "18": 9.30257, - "19": 9.2097, - "20": 8.97236, - "21": 9.00551, - "22": 8.60549, - "23": 9.09502, - "24": 8.68553, + "4": 10.41183, + "5": 10.40045, + "6": 10.36588, + "7": 10.11237, + "8": 9.90152, + "9": 9.96409, + "10": 9.51308, + "11": 10.16314, + "12": 9.86212, + "13": 9.8691, + "14": 9.90016, + "15": 9.50636, + "16": 9.4505, + "17": 9.25987, + "18": 9.30236, + "19": 9.20973, + "20": 8.97225, + "21": 9.00508, + "22": 8.60641, + "23": 9.09405, + "24": 8.68494, "25": 8.50601, - "26": 8.73798, - "27": 8.82658, - "28": 8.95635, - "29": 8.94472, - "30": 8.49346, - "31": 7.95927, - "32": 8.67679, - "33": 8.74642, + "26": 8.73822, + "27": 8.82506, + "28": 8.95591, + "29": 8.94408, + "30": 8.49401, + "31": 7.96274, + "32": 8.67873, + "33": 8.74695, "34": 8.26598, - "35": 8.35285, - "36": 8.26951, - "37": 8.50672, - "38": 8.26594, - "39": 8.63279, - "40": 8.22554, - "41": 8.2712, - "42": 8.42397, - "43": 8.00502, - "44": 8.11408, - "45": 7.99641, - "46": 8.08273, - "47": 8.38354, - "48": 8.09419, - "49": 7.72682, - "50": 8.19229 + "35": 8.35396, + "36": 8.27, + "37": 8.507, + "38": 8.2667, + "39": 8.63224, + "40": 8.22542, + "41": 8.27217, + "42": 8.42381, + "43": 8.00559, + "44": 8.11452, + "45": 7.99742, + "46": 8.08286, + "47": 8.38391, + "48": 8.09449, + "49": 7.72776, + "50": 8.19243 } }, "num-zeros": { diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json index 7af7158c2f7..8ee325dec71 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.94223, "2": 10.91037, - "3": 10.92695, - "4": 10.91755, - "5": 10.92109, - "6": 10.92714, - "7": 10.92204, - "8": 10.91988, - "9": 10.93259, - "10": 10.92694, - "11": 10.91813, - "12": 10.93449, - "13": 10.91939, - "14": 10.90997, - "15": 10.91823, - "16": 10.92119, - "17": 10.93417, - "18": 10.91659, - "19": 10.92828, - "20": 10.90269, - "21": 10.92486, - "22": 10.92321, - "23": 10.92998, - "24": 10.9038, - "25": 10.91386, - "26": 10.915, - "27": 10.93016, - "28": 10.91934, - "29": 10.92036, - "30": 10.91169, - "31": 10.90623, - "32": 10.91809, - "33": 10.91676, - "34": 10.90345, - "35": 10.9267, - "36": 10.90712, - "37": 10.90571, - "38": 10.91714, - "39": 10.90958, - "40": 10.90392, - "41": 10.91141, - "42": 10.91504, - "43": 10.90745, - "44": 10.91699, - "45": 10.91067, - "46": 10.90596, - "47": 10.90033, - "48": 10.89363, - "49": 10.90514, - "50": 10.89275, - "51": 10.89862, - "52": 10.89719, - "53": 10.90118, - "54": 10.89132, - "55": 10.9069, - "56": 10.88484, - "57": 10.88827, - "58": 10.87969, - "59": 10.8783, - "60": 10.86783, - "61": 10.86994, - "62": 10.86303, - "63": 10.87019, - "64": 10.86276, - "65": 10.86493, - "66": 10.86769, - "67": 10.85571, - "68": 10.84807, - "69": 10.85373, - "70": 10.85849, - "71": 10.85049, - "72": 10.84976, - "73": 10.83918, - "74": 10.83438, - "75": 10.84061, - "76": 10.82898, - "77": 10.83019, - "78": 10.81284, - "79": 10.82943, - "80": 10.83362, - "81": 10.81529, - "82": 10.81451, - "83": 10.80445, - "84": 10.78089, - "85": 10.7817, - "86": 10.79887, - "87": 10.79855, - "88": 10.80194, - "89": 10.78447, - "90": 10.78272, - "91": 10.78271, - "92": 10.78221, - "93": 10.75837, - "94": 10.76705, - "95": 10.76515, - "96": 10.73834, - "97": 10.73022, - "98": 10.75364, - "99": 10.76033, - "100": 10.7384 + "3": 10.92694, + "4": 10.91781, + "5": 10.92105, + "6": 10.92688, + "7": 10.92235, + "8": 10.92031, + "9": 10.93219, + "10": 10.92611, + "11": 10.91797, + "12": 10.93443, + "13": 10.91919, + "14": 10.90942, + "15": 10.91807, + "16": 10.92132, + "17": 10.93511, + "18": 10.91657, + "19": 10.92834, + "20": 10.9022, + "21": 10.92588, + "22": 10.92401, + "23": 10.92945, + "24": 10.9046, + "25": 10.91421, + "26": 10.91452, + "27": 10.92959, + "28": 10.92001, + "29": 10.92055, + "30": 10.91213, + "31": 10.90591, + "32": 10.91832, + "33": 10.91666, + "34": 10.90319, + "35": 10.92638, + "36": 10.90801, + "37": 10.90602, + "38": 10.91644, + "39": 10.90971, + "40": 10.90394, + "41": 10.91095, + "42": 10.9147, + "43": 10.90761, + "44": 10.9166, + "45": 10.91017, + "46": 10.90603, + "47": 10.90082, + "48": 10.89309, + "49": 10.90441, + "50": 10.89335, + "51": 10.89814, + "52": 10.89698, + "53": 10.90087, + "54": 10.89136, + "55": 10.90606, + "56": 10.88392, + "57": 10.88847, + "58": 10.87993, + "59": 10.87881, + "60": 10.86754, + "61": 10.87005, + "62": 10.8627, + "63": 10.87072, + "64": 10.8622, + "65": 10.86482, + "66": 10.86786, + "67": 10.85568, + "68": 10.84834, + "69": 10.85356, + "70": 10.85766, + "71": 10.85007, + "72": 10.84966, + "73": 10.83916, + "74": 10.83516, + "75": 10.84065, + "76": 10.82979, + "77": 10.82946, + "78": 10.81214, + "79": 10.82951, + "80": 10.83397, + "81": 10.81606, + "82": 10.814, + "83": 10.8053, + "84": 10.78055, + "85": 10.78201, + "86": 10.79832, + "87": 10.79865, + "88": 10.80163, + "89": 10.78541, + "90": 10.78282, + "91": 10.78264, + "92": 10.78229, + "93": 10.75819, + "94": 10.76732, + "95": 10.76526, + "96": 10.73862, + "97": 10.73009, + "98": 10.75335, + "99": 10.76001, + "100": 10.73865 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1356.0, - "2": 1146.0, - "3": 1276.0, - "4": 1214.0, - "5": 1190.0, - "6": 1254.0, - "7": 1581.0, - "8": 1285.0, - "9": 1275.0, - "10": 1217.0, - "11": 1227.0, - "12": 1262.0, - "13": 1425.0, - "14": 1372.0, - "15": 1183.0, - "16": 1341.0, - "17": 1345.0, - "18": 1224.0, - "19": 1225.0, - "20": 1299.0, - "21": 1435.0, - "22": 1354.0, - "23": 1274.0, - "24": 1502.0, - "25": 1248.0, - "26": 1341.0, - "27": 1293.0, - "28": 1242.0, - "29": 1388.0, - "30": 1307.0, - "31": 1409.0, - "32": 1409.0, - "33": 1439.0, - "34": 1432.0, - "35": 1352.0, - "36": 1292.0, - "37": 1366.0, - "38": 1251.0, - "39": 1226.0, - "40": 1241.0, - "41": 1208.0, - "42": 1100.0, - "43": 1377.0, - "44": 1300.0, - "45": 1284.0, - "46": 1293.0, - "47": 1397.0, - "48": 1385.0, - "49": 1384.0, - "50": 1202.0, - "51": 1278.0, - "52": 1275.0, - "53": 1235.0, - "54": 1250.0, - "55": 1273.0, - "56": 1204.0, - "57": 1272.0, - "58": 1313.0, - "59": 1297.0, - "60": 1343.0, - "61": 1174.0, - "62": 1403.0, - "63": 1220.0, - "64": 1393.0, - "65": 1390.0, - "66": 1237.0, - "67": 1366.0, - "68": 1145.0, - "69": 1318.0, - "70": 1226.0, - "71": 1188.0, - "72": 1354.0, - "73": 1307.0, - "74": 1336.0, - "75": 1331.0, - "76": 1431.0, - "77": 1310.0, - "78": 1227.0, - "79": 1221.0, - "80": 1173.0, - "81": 1321.0, - "82": 1184.0, - "83": 1277.0, - "84": 1232.0, - "85": 1253.0, - "86": 1259.0, - "87": 1196.0, - "88": 1342.0, - "89": 1161.0, - "90": 1181.0, - "91": 1207.0, - "92": 1138.0, - "93": 1288.0, - "94": 1366.0, - "95": 1196.0, - "96": 1334.0, - "97": 1252.0, - "98": 1196.0, - "99": 1296.0, - "100": 1356.0 + "1": 1375.0, + "2": 1150.0, + "3": 1244.0, + "4": 1230.0, + "5": 1180.0, + "6": 1258.0, + "7": 1543.0, + "8": 1272.0, + "9": 1271.0, + "10": 1218.0, + "11": 1244.0, + "12": 1236.0, + "13": 1482.0, + "14": 1391.0, + "15": 1219.0, + "16": 1316.0, + "17": 1338.0, + "18": 1226.0, + "19": 1215.0, + "20": 1217.0, + "21": 1455.0, + "22": 1319.0, + "23": 1299.0, + "24": 1568.0, + "25": 1277.0, + "26": 1358.0, + "27": 1283.0, + "28": 1237.0, + "29": 1292.0, + "30": 1296.0, + "31": 1381.0, + "32": 1446.0, + "33": 1366.0, + "34": 1431.0, + "35": 1354.0, + "36": 1339.0, + "37": 1314.0, + "38": 1323.0, + "39": 1248.0, + "40": 1309.0, + "41": 1226.0, + "42": 1071.0, + "43": 1309.0, + "44": 1338.0, + "45": 1422.0, + "46": 1327.0, + "47": 1384.0, + "48": 1428.0, + "49": 1396.0, + "50": 1203.0, + "51": 1246.0, + "52": 1284.0, + "53": 1320.0, + "54": 1199.0, + "55": 1320.0, + "56": 1196.0, + "57": 1223.0, + "58": 1267.0, + "59": 1323.0, + "60": 1308.0, + "61": 1206.0, + "62": 1398.0, + "63": 1300.0, + "64": 1319.0, + "65": 1458.0, + "66": 1153.0, + "67": 1380.0, + "68": 1189.0, + "69": 1365.0, + "70": 1255.0, + "71": 1106.0, + "72": 1332.0, + "73": 1292.0, + "74": 1261.0, + "75": 1308.0, + "76": 1345.0, + "77": 1285.0, + "78": 1234.0, + "79": 1257.0, + "80": 1223.0, + "81": 1407.0, + "82": 1040.0, + "83": 1258.0, + "84": 1231.0, + "85": 1183.0, + "86": 1280.0, + "87": 1159.0, + "88": 1288.0, + "89": 1191.0, + "90": 1184.0, + "91": 1153.0, + "92": 1118.0, + "93": 1335.0, + "94": 1318.0, + "95": 1138.0, + "96": 1283.0, + "97": 1298.0, + "98": 1258.0, + "99": 1306.0, + "100": 1404.0 } }, "mem-allocated-bytes": { @@ -534,4 +534,4 @@ "100": 0.18358 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json index 08580990b15..a8b182d6793 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.94223, "2": 10.91037, - "3": 10.92689, - "4": 10.91732, - "5": 10.92104, - "6": 10.92713, - "7": 10.9221, - "8": 10.91999, - "9": 10.93273, - "10": 10.92667, - "11": 10.91821, - "12": 10.93435, - "13": 10.91957, - "14": 10.91029, - "15": 10.91823, - "16": 10.92142, - "17": 10.93495, - "18": 10.9164, - "19": 10.92839, - "20": 10.90219, - "21": 10.926, - "22": 10.92335, - "23": 10.92959, - "24": 10.90428, - "25": 10.91399, - "26": 10.91469, - "27": 10.92915, - "28": 10.91982, - "29": 10.92048, - "30": 10.9122, - "31": 10.90624, - "32": 10.91834, - "33": 10.91659, - "34": 10.90394, - "35": 10.9267, - "36": 10.90694, - "37": 10.90574, - "38": 10.91681, - "39": 10.90997, - "40": 10.90435, - "41": 10.91192, - "42": 10.9152, - "43": 10.90832, - "44": 10.91683, - "45": 10.91069, - "46": 10.90631, - "47": 10.90095, - "48": 10.89373, - "49": 10.90482, - "50": 10.89303, - "51": 10.8985, - "52": 10.89762, - "53": 10.90083, - "54": 10.89184, - "55": 10.9066, - "56": 10.88345, - "57": 10.88829, - "58": 10.87989, - "59": 10.87885, - "60": 10.86765, - "61": 10.8702, - "62": 10.86305, - "63": 10.87038, - "64": 10.86201, - "65": 10.86465, - "66": 10.86834, - "67": 10.85588, - "68": 10.84824, - "69": 10.85325, - "70": 10.85792, - "71": 10.85066, - "72": 10.84949, - "73": 10.83904, - "74": 10.83457, - "75": 10.84096, - "76": 10.82892, - "77": 10.82964, - "78": 10.81295, - "79": 10.82957, - "80": 10.83379, - "81": 10.81551, - "82": 10.81376, - "83": 10.80501, - "84": 10.78039, - "85": 10.7818, - "86": 10.79786, - "87": 10.79893, - "88": 10.80157, - "89": 10.78506, - "90": 10.78262, - "91": 10.78248, - "92": 10.78232, - "93": 10.75803, - "94": 10.76743, - "95": 10.76539, - "96": 10.73865, - "97": 10.73077, - "98": 10.75318, - "99": 10.7604, - "100": 10.7382 + "3": 10.92694, + "4": 10.91781, + "5": 10.92105, + "6": 10.92688, + "7": 10.92212, + "8": 10.92025, + "9": 10.9324, + "10": 10.92601, + "11": 10.91805, + "12": 10.93474, + "13": 10.91918, + "14": 10.90992, + "15": 10.91825, + "16": 10.9211, + "17": 10.93475, + "18": 10.91681, + "19": 10.92861, + "20": 10.90258, + "21": 10.92558, + "22": 10.92372, + "23": 10.92939, + "24": 10.90459, + "25": 10.91343, + "26": 10.91458, + "27": 10.92941, + "28": 10.91983, + "29": 10.92035, + "30": 10.91194, + "31": 10.90619, + "32": 10.91863, + "33": 10.91641, + "34": 10.90306, + "35": 10.92712, + "36": 10.90795, + "37": 10.90613, + "38": 10.9172, + "39": 10.90942, + "40": 10.90413, + "41": 10.91164, + "42": 10.91488, + "43": 10.90844, + "44": 10.91651, + "45": 10.91099, + "46": 10.9067, + "47": 10.90059, + "48": 10.89331, + "49": 10.90508, + "50": 10.89277, + "51": 10.89827, + "52": 10.89735, + "53": 10.90079, + "54": 10.89172, + "55": 10.90626, + "56": 10.88457, + "57": 10.88793, + "58": 10.87963, + "59": 10.87852, + "60": 10.86772, + "61": 10.87018, + "62": 10.86255, + "63": 10.87013, + "64": 10.86197, + "65": 10.86492, + "66": 10.8678, + "67": 10.85542, + "68": 10.84838, + "69": 10.8531, + "70": 10.85759, + "71": 10.85022, + "72": 10.84968, + "73": 10.83905, + "74": 10.83513, + "75": 10.84054, + "76": 10.82949, + "77": 10.82966, + "78": 10.81246, + "79": 10.82962, + "80": 10.8335, + "81": 10.81532, + "82": 10.81371, + "83": 10.80443, + "84": 10.78045, + "85": 10.78151, + "86": 10.79864, + "87": 10.79891, + "88": 10.80173, + "89": 10.7852, + "90": 10.78251, + "91": 10.78285, + "92": 10.78149, + "93": 10.75838, + "94": 10.76737, + "95": 10.76568, + "96": 10.73814, + "97": 10.7304, + "98": 10.75384, + "99": 10.75962, + "100": 10.73817 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1356.0, - "2": 1146.0, - "3": 1336.0, - "4": 1201.0, - "5": 1207.0, - "6": 1257.0, - "7": 1587.0, - "8": 1333.0, - "9": 1219.0, - "10": 1267.0, - "11": 1206.0, - "12": 1227.0, - "13": 1449.0, - "14": 1358.0, - "15": 1191.0, - "16": 1305.0, - "17": 1301.0, - "18": 1280.0, - "19": 1199.0, - "20": 1314.0, - "21": 1394.0, - "22": 1386.0, - "23": 1273.0, - "24": 1438.0, - "25": 1257.0, - "26": 1383.0, - "27": 1295.0, - "28": 1199.0, - "29": 1334.0, - "30": 1297.0, - "31": 1370.0, - "32": 1390.0, - "33": 1424.0, - "34": 1387.0, - "35": 1357.0, - "36": 1293.0, - "37": 1421.0, - "38": 1317.0, - "39": 1214.0, - "40": 1408.0, - "41": 1225.0, - "42": 1053.0, - "43": 1356.0, - "44": 1356.0, - "45": 1345.0, - "46": 1323.0, - "47": 1356.0, - "48": 1358.0, - "49": 1353.0, - "50": 1197.0, - "51": 1235.0, - "52": 1248.0, - "53": 1345.0, - "54": 1189.0, - "55": 1198.0, - "56": 1212.0, - "57": 1256.0, - "58": 1345.0, - "59": 1268.0, - "60": 1409.0, - "61": 1177.0, - "62": 1390.0, - "63": 1303.0, - "64": 1321.0, - "65": 1412.0, - "66": 1271.0, - "67": 1383.0, - "68": 1193.0, - "69": 1394.0, - "70": 1258.0, - "71": 1177.0, - "72": 1327.0, - "73": 1252.0, - "74": 1221.0, - "75": 1347.0, - "76": 1440.0, - "77": 1347.0, - "78": 1282.0, - "79": 1238.0, - "80": 1238.0, - "81": 1401.0, - "82": 1071.0, - "83": 1275.0, - "84": 1223.0, - "85": 1179.0, - "86": 1252.0, - "87": 1231.0, - "88": 1251.0, - "89": 1111.0, - "90": 1286.0, - "91": 1206.0, - "92": 1183.0, - "93": 1276.0, - "94": 1369.0, - "95": 1130.0, - "96": 1303.0, - "97": 1281.0, - "98": 1241.0, - "99": 1208.0, - "100": 1384.0 + "1": 1375.0, + "2": 1150.0, + "3": 1244.0, + "4": 1230.0, + "5": 1180.0, + "6": 1258.0, + "7": 1554.0, + "8": 1332.0, + "9": 1335.0, + "10": 1276.0, + "11": 1175.0, + "12": 1245.0, + "13": 1431.0, + "14": 1397.0, + "15": 1158.0, + "16": 1263.0, + "17": 1320.0, + "18": 1221.0, + "19": 1175.0, + "20": 1300.0, + "21": 1409.0, + "22": 1346.0, + "23": 1265.0, + "24": 1560.0, + "25": 1275.0, + "26": 1403.0, + "27": 1340.0, + "28": 1279.0, + "29": 1268.0, + "30": 1370.0, + "31": 1393.0, + "32": 1404.0, + "33": 1452.0, + "34": 1484.0, + "35": 1378.0, + "36": 1283.0, + "37": 1407.0, + "38": 1243.0, + "39": 1233.0, + "40": 1407.0, + "41": 1283.0, + "42": 1071.0, + "43": 1380.0, + "44": 1342.0, + "45": 1391.0, + "46": 1249.0, + "47": 1353.0, + "48": 1421.0, + "49": 1348.0, + "50": 1216.0, + "51": 1277.0, + "52": 1226.0, + "53": 1272.0, + "54": 1174.0, + "55": 1266.0, + "56": 1206.0, + "57": 1239.0, + "58": 1304.0, + "59": 1267.0, + "60": 1457.0, + "61": 1217.0, + "62": 1360.0, + "63": 1308.0, + "64": 1405.0, + "65": 1338.0, + "66": 1268.0, + "67": 1333.0, + "68": 1191.0, + "69": 1410.0, + "70": 1290.0, + "71": 1168.0, + "72": 1396.0, + "73": 1339.0, + "74": 1290.0, + "75": 1389.0, + "76": 1439.0, + "77": 1318.0, + "78": 1189.0, + "79": 1222.0, + "80": 1133.0, + "81": 1359.0, + "82": 1146.0, + "83": 1282.0, + "84": 1217.0, + "85": 1249.0, + "86": 1311.0, + "87": 1144.0, + "88": 1252.0, + "89": 1207.0, + "90": 1141.0, + "91": 1148.0, + "92": 1186.0, + "93": 1319.0, + "94": 1288.0, + "95": 1157.0, + "96": 1266.0, + "97": 1387.0, + "98": 1205.0, + "99": 1174.0, + "100": 1381.0 } }, "mem-allocated-bytes": { @@ -534,4 +534,4 @@ "100": 0.39309 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index 3890ac9b2a1..1039975b4f3 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.92137, "2": 10.91202, - "3": 10.91762, - "4": 10.91875, - "5": 10.90416, - "6": 10.90628, - "7": 10.91453, - "8": 10.91043, - "9": 10.91477, + "3": 10.91768, + "4": 10.91863, + "5": 10.90423, + "6": 10.9067, + "7": 10.91437, + "8": 10.91056, + "9": 10.91447, "10": 10.90704, - "11": 10.89631, - "12": 10.8962, - "13": 10.90115, - "14": 10.8933, - "15": 10.87249, - "16": 10.86129, - "17": 10.8741, - "18": 10.85931, - "19": 10.86604, - "20": 10.78205, - "21": 10.77976, - "22": 10.76536, - "23": 10.7581, - "24": 10.72011, - "25": 10.72271, - "26": 10.71647, - "27": 10.6839, - "28": 10.62326, - "29": 10.58091, - "30": 10.56525, - "31": 10.55932, - "32": 10.54826, - "33": 10.5132, - "34": 10.48724, - "35": 10.47882, - "36": 10.45973, - "37": 10.43072, - "38": 10.43542, - "39": 10.40203, - "40": 10.37962, - "41": 10.36342, - "42": 10.33785, - "43": 10.32307, - "44": 10.28579, - "45": 10.31085, - "46": 10.26832, - "47": 10.2563, - "48": 10.20763, - "49": 10.20839, - "50": 10.20889, - "51": 10.20926, - "52": 10.16623, - "53": 10.17263, - "54": 10.1328, - "55": 10.10861, - "56": 10.13282, - "57": 10.12153, - "58": 10.13393, - "59": 10.08186, - "60": 10.10037, - "61": 10.04693, - "62": 10.0199, - "63": 10.09433, - "64": 10.04939, - "65": 10.0234, - "66": 10.04781, - "67": 10.02641, - "68": 9.98517, - "69": 10.0078, - "70": 9.98473, - "71": 10.01266, - "72": 9.99391, - "73": 9.98666, - "74": 9.96476, - "75": 9.93796, - "76": 9.97191, - "77": 9.96324, - "78": 9.91576, + "11": 10.89659, + "12": 10.8959, + "13": 10.90064, + "14": 10.89352, + "15": 10.87253, + "16": 10.86144, + "17": 10.87413, + "18": 10.85919, + "19": 10.86552, + "20": 10.78225, + "21": 10.77925, + "22": 10.76543, + "23": 10.75797, + "24": 10.72021, + "25": 10.72236, + "26": 10.71659, + "27": 10.68375, + "28": 10.62333, + "29": 10.58113, + "30": 10.56502, + "31": 10.55937, + "32": 10.5483, + "33": 10.51334, + "34": 10.487, + "35": 10.47893, + "36": 10.45972, + "37": 10.43079, + "38": 10.4355, + "39": 10.40227, + "40": 10.37985, + "41": 10.36324, + "42": 10.33753, + "43": 10.32309, + "44": 10.28593, + "45": 10.31097, + "46": 10.26822, + "47": 10.25636, + "48": 10.2077, + "49": 10.2086, + "50": 10.20888, + "51": 10.20917, + "52": 10.16628, + "53": 10.17255, + "54": 10.13289, + "55": 10.10853, + "56": 10.13283, + "57": 10.12157, + "58": 10.13408, + "59": 10.08188, + "60": 10.10019, + "61": 10.04681, + "62": 10.01985, + "63": 10.09447, + "64": 10.04943, + "65": 10.02338, + "66": 10.04778, + "67": 10.02637, + "68": 9.98521, + "69": 10.00765, + "70": 9.98465, + "71": 10.01262, + "72": 9.99392, + "73": 9.9866, + "74": 9.96469, + "75": 9.93791, + "76": 9.97185, + "77": 9.96314, + "78": 9.91583, "79": 9.92237, - "80": 9.93276, + "80": 9.93257, "81": 9.96107, - "82": 9.89465, - "83": 9.86451, - "84": 9.79834, - "85": 9.78984, - "86": 9.89004, - "87": 9.9095, - "88": 9.88688, - "89": 9.82939, - "90": 9.8237, - "91": 9.83431, + "82": 9.8946, + "83": 9.86433, + "84": 9.79835, + "85": 9.78983, + "86": 9.89001, + "87": 9.90939, + "88": 9.88672, + "89": 9.82926, + "90": 9.82374, + "91": 9.83424, "92": 9.82048, - "93": 9.75663, - "94": 9.83607, - "95": 9.82846, - "96": 9.80856, - "97": 9.75437, - "98": 9.7862, - "99": 9.82588, - "100": 9.72102 + "93": 9.75668, + "94": 9.83611, + "95": 9.82852, + "96": 9.80839, + "97": 9.75439, + "98": 9.78617, + "99": 9.82596, + "100": 9.72104 } }, "num-zeros": { @@ -534,4 +534,4 @@ "100": 0.80202 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp1_pp1/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp1_pp1/golden_values_dev_dgx_h100.json index f73eb946b9e..9aa5a82da2b 100644 --- a/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp1_pp1/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp1_pp1/golden_values_dev_dgx_h100.json @@ -4,54 +4,54 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 9.14889, - "2": 9.1525, + "1": 9.14886, + "2": 9.15251, "3": 9.14729, - "4": 9.15371, - "5": 9.15097, - "6": 9.1479, - "7": 9.14456, - "8": 9.14434, - "9": 9.15048, - "10": 9.15301, + "4": 9.15372, + "5": 9.15094, + "6": 9.14793, + "7": 9.1446, + "8": 9.14425, + "9": 9.15047, + "10": 9.15302, "11": 9.14704, - "12": 9.14198, - "13": 9.14243, - "14": 9.14298, - "15": 9.13541, - "16": 9.12642, - "17": 9.12514, - "18": 9.1214, - "19": 9.11882, - "20": 9.09873, - "21": 9.07026, + "12": 9.14195, + "13": 9.14246, + "14": 9.14292, + "15": 9.1354, + "16": 9.12646, + "17": 9.1251, + "18": 9.12144, + "19": 9.11878, + "20": 9.09878, + "21": 9.07023, "22": 9.0709, - "23": 9.07173, - "24": 9.06194, - "25": 9.05599, - "26": 9.05798, - "27": 9.04141, + "23": 9.0717, + "24": 9.06191, + "25": 9.05605, + "26": 9.05795, + "27": 9.04146, "28": 9.0191, - "29": 9.00363, - "30": 8.99761, - "31": 8.99591, - "32": 8.98479, - "33": 8.97828, + "29": 9.0036, + "30": 8.99764, + "31": 8.99589, + "32": 8.98482, + "33": 8.97826, "34": 8.98705, - "35": 8.95076, - "36": 8.94654, + "35": 8.95072, + "36": 8.94651, "37": 8.92191, - "38": 8.94151, - "39": 8.92551, - "40": 8.87207, - "41": 8.89697, - "42": 8.87706, - "43": 8.87483, - "44": 8.84175, - "45": 8.81296, - "46": 8.79648, - "47": 8.84618, - "48": 8.77263, + "38": 8.94152, + "39": 8.92547, + "40": 8.8721, + "41": 8.897, + "42": 8.87703, + "43": 8.87481, + "44": 8.84172, + "45": 8.81293, + "46": 8.7965, + "47": 8.84615, + "48": 8.77262, "49": 8.78096, "50": 8.76244 } @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3477957.0, - "2": 3392110.0, - "3": 3629992.0, - "4": 3532325.0, - "5": 3783905.0, - "6": 3584695.0, - "7": 3478398.0, - "8": 3414315.0, - "9": 3511529.0, - "10": 3544384.0, - "11": 3475318.0, - "12": 3518920.0, - "13": 3591807.0, - "14": 3549684.0, - "15": 3421296.0, - "16": 3383259.0, - "17": 3424085.0, - "18": 3509306.0, - "19": 3426089.0, - "20": 3465855.0, - "21": 3700143.0, - "22": 3474221.0, - "23": 3693210.0, - "24": 3405660.0, - "25": 3457731.0, - "26": 3478932.0, - "27": 3555248.0, - "28": 3497383.0, - "29": 3562022.0, - "30": 3708115.0, - "31": 3397575.0, - "32": 3467927.0, - "33": 3515533.0, - "34": 3501303.0, - "35": 3432536.0, - "36": 3453820.0, - "37": 3959063.0, - "38": 3488434.0, - "39": 3410171.0, - "40": 3614582.0, - "41": 3425679.0, - "42": 3643722.0, - "43": 3472981.0, - "44": 3447803.0, - "45": 3452005.0, - "46": 3585525.0, - "47": 3467373.0, - "48": 3462973.0, - "49": 3529689.0, - "50": 3411773.0 + "1": 3477916.0, + "2": 3392189.0, + "3": 3629959.0, + "4": 3532399.0, + "5": 3783947.0, + "6": 3584706.0, + "7": 3478226.0, + "8": 3414243.0, + "9": 3511605.0, + "10": 3544388.0, + "11": 3475384.0, + "12": 3518947.0, + "13": 3591801.0, + "14": 3549533.0, + "15": 3421196.0, + "16": 3383330.0, + "17": 3424068.0, + "18": 3509371.0, + "19": 3426149.0, + "20": 3465928.0, + "21": 3700171.0, + "22": 3474428.0, + "23": 3693287.0, + "24": 3405771.0, + "25": 3457651.0, + "26": 3479148.0, + "27": 3555259.0, + "28": 3497157.0, + "29": 3561971.0, + "30": 3708195.0, + "31": 3397582.0, + "32": 3467937.0, + "33": 3515504.0, + "34": 3501429.0, + "35": 3432501.0, + "36": 3453950.0, + "37": 3958875.0, + "38": 3488166.0, + "39": 3410167.0, + "40": 3614402.0, + "41": 3425804.0, + "42": 3643574.0, + "43": 3473042.0, + "44": 3448095.0, + "45": 3452079.0, + "46": 3585487.0, + "47": 3467365.0, + "48": 3462763.0, + "49": 3529601.0, + "50": 3411713.0 } }, "mem-allocated-bytes": { @@ -284,4 +284,4 @@ "50": 0.17186 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp4_sp_cp2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp4_sp_cp2/golden_values_dev_dgx_h100.json index 6cde7a51e32..18a7eab2d52 100644 --- a/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp4_sp_cp2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/multimodal-llava/multimodal_llava_mcore_te_tp4_sp_cp2/golden_values_dev_dgx_h100.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 9.28559, - "2": 9.28324, - "3": 9.27826, - "4": 9.28841, - "5": 9.27504, - "6": 9.28568, - "7": 9.27731, - "8": 9.28114, + "1": 9.28561, + "2": 9.28322, + "3": 9.27821, + "4": 9.28847, + "5": 9.27509, + "6": 9.28576, + "7": 9.27727, + "8": 9.28111, "9": 9.28492, - "10": 9.28294, - "11": 9.28301, - "12": 9.27397, - "13": 9.27053, + "10": 9.28299, + "11": 9.28307, + "12": 9.27388, + "13": 9.27041, "14": 9.27114, - "15": 9.25315, - "16": 9.24478, - "17": 9.24797, - "18": 9.23003, - "19": 9.23035, - "20": 9.2076, + "15": 9.2532, + "16": 9.24492, + "17": 9.24791, + "18": 9.22992, + "19": 9.23034, + "20": 9.20767, "21": 9.17083, - "22": 9.14938, - "23": 9.16758, - "24": 9.15034, - "25": 9.14214, - "26": 9.14673, - "27": 9.1215, - "28": 9.09479, - "29": 9.09351, - "30": 9.07677, - "31": 8.97148, - "32": 9.03075, - "33": 9.02015, - "34": 8.98687, - "35": 8.95916, - "36": 8.97187, - "37": 8.91464, - "38": 8.88847, - "39": 8.88927, - "40": 8.90682, - "41": 8.81956, - "42": 8.87466, - "43": 8.85748, - "44": 8.81813, - "45": 8.81486, - "46": 8.84605, - "47": 8.73849, - "48": 8.6711, - "49": 8.70182, - "50": 8.73628 + "22": 9.14947, + "23": 9.1675, + "24": 9.15023, + "25": 9.14221, + "26": 9.1467, + "27": 9.12158, + "28": 9.09478, + "29": 9.09356, + "30": 9.0768, + "31": 8.97149, + "32": 9.03073, + "33": 9.02017, + "34": 8.98684, + "35": 8.95922, + "36": 8.97185, + "37": 8.91465, + "38": 8.88848, + "39": 8.88917, + "40": 8.90687, + "41": 8.81954, + "42": 8.87463, + "43": 8.85751, + "44": 8.81818, + "45": 8.81474, + "46": 8.84607, + "47": 8.73862, + "48": 8.67111, + "49": 8.70167, + "50": 8.73638 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5959405.0, - "2": 6553858.0, - "3": 7313621.0, - "4": 6377159.0, - "5": 6498204.0, - "6": 7151911.0, - "7": 6210199.0, - "8": 6334551.0, - "9": 6624834.0, - "10": 6529069.0, - "11": 7466600.0, - "12": 6471646.0, - "13": 6003542.0, - "14": 8072096.0, - "15": 6530028.0, - "16": 7526953.0, - "17": 6034944.0, - "18": 6289713.0, - "19": 6162181.0, - "20": 6527689.0, - "21": 6981905.0, - "22": 7132961.0, - "23": 5928410.0, - "24": 6210328.0, - "25": 6993334.0, - "26": 6471452.0, - "27": 6355265.0, - "28": 6876938.0, - "29": 6380179.0, - "30": 6468808.0, - "31": 8165266.0, - "32": 6765498.0, - "33": 6355533.0, - "34": 6662329.0, - "35": 7065327.0, - "36": 6076881.0, - "37": 7785788.0, - "38": 6727186.0, - "39": 7315785.0, - "40": 6555028.0, - "41": 7314448.0, - "42": 6592010.0, - "43": 6928116.0, - "44": 7274275.0, - "45": 6680226.0, - "46": 6232523.0, - "47": 6496796.0, - "48": 6809690.0, - "49": 6753543.0, - "50": 6238296.0 + "1": 5959428.0, + "2": 6553992.0, + "3": 7313522.0, + "4": 6377222.0, + "5": 6498148.0, + "6": 7151874.0, + "7": 6210232.0, + "8": 6334664.0, + "9": 6624720.0, + "10": 6529147.0, + "11": 7466719.0, + "12": 6471623.0, + "13": 6003501.0, + "14": 8072119.0, + "15": 6529887.0, + "16": 7526948.0, + "17": 6034963.0, + "18": 6289711.0, + "19": 6162402.0, + "20": 6527722.0, + "21": 6981867.0, + "22": 7132928.0, + "23": 5928526.0, + "24": 6210499.0, + "25": 6993156.0, + "26": 6471247.0, + "27": 6355330.0, + "28": 6877068.0, + "29": 6380144.0, + "30": 6468633.0, + "31": 8165065.0, + "32": 6765595.0, + "33": 6355562.0, + "34": 6662248.0, + "35": 7065305.0, + "36": 6076925.0, + "37": 7785598.0, + "38": 6727182.0, + "39": 7315768.0, + "40": 6554968.0, + "41": 7314632.0, + "42": 6592126.0, + "43": 6927988.0, + "44": 7274165.0, + "45": 6680054.0, + "46": 6232492.0, + "47": 6496568.0, + "48": 6809864.0, + "49": 6753589.0, + "50": 6238131.0 } }, "mem-allocated-bytes": { @@ -284,4 +284,4 @@ "50": 0.95087 } } -} \ No newline at end of file +} From 3961fc5150be2da93274fc7eca7a234adaad0b9a Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Mon, 6 Jul 2026 09:48:55 -0700 Subject: [PATCH 4/7] Update RO_API_TOKEN variable in functional tests configuration to use PROJECT_ACCESS_TOKEN_MCORE for improved access management. Signed-off-by: Ajay Balasa --- .gitlab/stages/04.functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index 8e0ac7812b4..0da0966f0c1 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -195,7 +195,7 @@ functional:configure: job: functional:configure strategy: depend variables: - RO_API_TOKEN: $PAT + RO_API_TOKEN: $PROJECT_ACCESS_TOKEN_MCORE CONTAINER_TAG: $CI_PIPELINE_ID CI_MCORE_LTS_IMAGE: $CI_MCORE_LTS_IMAGE GITLAB_ENDPOINT: $GITLAB_ENDPOINT From 1756122eb18590c5bc8716f3137846d266f26885 Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Mon, 6 Jul 2026 10:24:24 -0700 Subject: [PATCH 5/7] Update RO_API_TOKEN variable in functional tests configuration to use $PAT for enhanced security and access management. Signed-off-by: Ajay Balasa --- .gitlab/stages/04.functional-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index 0da0966f0c1..8e0ac7812b4 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -195,7 +195,7 @@ functional:configure: job: functional:configure strategy: depend variables: - RO_API_TOKEN: $PROJECT_ACCESS_TOKEN_MCORE + RO_API_TOKEN: $PAT CONTAINER_TAG: $CI_PIPELINE_ID CI_MCORE_LTS_IMAGE: $CI_MCORE_LTS_IMAGE GITLAB_ENDPOINT: $GITLAB_ENDPOINT From ed21e04fe01f046504103ec771a9f2b83e0d28fa Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Tue, 7 Jul 2026 06:36:55 +0000 Subject: [PATCH 6/7] update goldens Signed-off-by: Ajay Balasa --- .../golden_values_dev_dgx_h100.json | 410 +++--- .../golden_values_dev_dgx_h100.json | 446 +++---- .../golden_values_dev_dgx_gb200.json | 952 ++++++------- .../golden_values_dev_dgx_gb200.json | 954 ++++++------- .../golden_values_dev_dgx_gb200.json | 400 +++--- .../golden_values_dev_dgx_gb200.json | 960 +++++++------- .../golden_values_dev_dgx_gb200.json | 954 ++++++------- .../golden_values_dev_dgx_gb200.json | 480 +++---- .../golden_values_dev_dgx_gb200.json | 480 +++---- .../golden_values_dev_dgx_gb200.json | 494 +++---- .../golden_values_dev_dgx_gb200.json | 494 +++---- .../golden_values_dev_dgx_gb200.json | 494 +++---- .../golden_values_dev_dgx_gb200.json | 474 +++---- .../golden_values_dev_dgx_gb200.json | 474 +++---- .../golden_values_dev_dgx_gb200.json | 492 +++---- .../golden_values_dev_dgx_gb200.json | 990 +++++++------- .../golden_values_dev_dgx_gb200.json | 800 +++++------ .../golden_values_dev_dgx_gb200.json | 1176 ++++++++--------- 18 files changed, 5962 insertions(+), 5962 deletions(-) diff --git a/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp2/golden_values_dev_dgx_h100.json index be2f043d4b9..1d75976567a 100644 --- a/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp2/golden_values_dev_dgx_h100.json @@ -26,34 +26,34 @@ "20": 10.47714, "21": 10.45276, "22": 10.39141, - "23": 10.3972, - "24": 10.35475, - "25": 10.35246, - "26": 10.35041, - "27": 10.31147, - "28": 10.32877, - "29": 10.30861, - "30": 10.14449, - "31": 10.10687, - "32": 10.07856, - "33": 10.10424, - "34": 10.03458, - "35": 10.02764, - "36": 10.01018, - "37": 10.00276, - "38": 9.96156, - "39": 9.89009, - "40": 9.85306, - "41": 9.78426, - "42": 9.71982, - "43": 9.68658, - "44": 9.65517, - "45": 9.6502, - "46": 9.57537, - "47": 9.59269, - "48": 9.58288, - "49": 9.52859, - "50": 9.49558 + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "num-zeros": { @@ -83,34 +83,34 @@ "20": 2266.0, "21": 2428.0, "22": 2319.0, - "23": 2420.0, - "24": 2343.0, - "25": 2235.0, - "26": 2722.0, - "27": 2402.0, - "28": 2568.0, - "29": 1915.0, - "30": 2132.0, - "31": 2699.0, - "32": 2340.0, - "33": 2667.0, - "34": 2775.0, - "35": 2753.0, - "36": 1838.0, - "37": 2756.0, - "38": 2479.0, - "39": 2316.0, - "40": 3061.0, - "41": 3153.0, - "42": 3112.0, - "43": 2808.0, - "44": 3013.0, - "45": 3282.0, - "46": 3037.0, - "47": 3164.0, - "48": 3314.0, - "49": 2706.0, - "50": 2787.0 + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3433473536.0, - "2": 3434259968.0, - "3": 3434259968.0, - "4": 3433473536.0, - "5": 3434259968.0, - "6": 3433473536.0, - "7": 3433473536.0, - "8": 3433473536.0, - "9": 3433473536.0, - "10": 3433473536.0, - "11": 3433473536.0, - "12": 3433473536.0, - "13": 3433473536.0, - "14": 3433473536.0, - "15": 3433473536.0, - "16": 3433473536.0, - "17": 3433473536.0, - "18": 3433473536.0, - "19": 3433473536.0, - "20": 3433473536.0, - "21": 3433473536.0, - "22": 3433473536.0, - "23": 3433473536.0, - "24": 3433473536.0, - "25": 3433473536.0, - "26": 3433473536.0, - "27": 3433473536.0, - "28": 3434259968.0, - "29": 3433473536.0, - "30": 3433473536.0, - "31": 3433473536.0, - "32": 3433473536.0, - "33": 3434259968.0, - "34": 3434259968.0, - "35": 3434259968.0, - "36": 3434259968.0, - "37": 3434259968.0, - "38": 3434259968.0, - "39": 3433473536.0, - "40": 3433473536.0, - "41": 3434259968.0, - "42": 3434259968.0, - "43": 3434259968.0, - "44": 3434259968.0, - "45": 3433473536.0, - "46": 3433473536.0, - "47": 3433473536.0, - "48": 3433473536.0, - "49": 3433473536.0, - "50": 3433473536.0 + "1": 3433473024.0, + "2": 3433473024.0, + "3": 3433473024.0, + "4": 3433473024.0, + "5": 3433473024.0, + "6": 3433473024.0, + "7": 3433473024.0, + "8": 3433473024.0, + "9": 3433473024.0, + "10": 3433473024.0, + "11": 3433473024.0, + "12": 3433473024.0, + "13": 3433473024.0, + "14": 3433473024.0, + "15": 3433473024.0, + "16": 3433473024.0, + "17": 3433473024.0, + "18": 3433473024.0, + "19": 3433473024.0, + "20": 3433473024.0, + "21": 3433473024.0, + "22": 3433473024.0, + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4229407744.0, - "2": 5708179968.0, - "3": 5708179968.0, - "4": 5708179968.0, - "5": 5708179968.0, - "6": 5708180480.0, - "7": 5708180480.0, - "8": 5708180480.0, - "9": 5708180480.0, - "10": 5708180480.0, - "11": 5708180480.0, - "12": 5708180480.0, - "13": 5708180480.0, - "14": 5708180480.0, - "15": 5708180480.0, - "16": 5708180480.0, - "17": 5708180480.0, - "18": 5708180480.0, - "19": 5708180480.0, - "20": 5708180480.0, - "21": 5708180480.0, - "22": 5708180480.0, - "23": 5708180480.0, - "24": 5708180480.0, - "25": 5708180480.0, - "26": 5708180480.0, - "27": 5708180480.0, - "28": 5708180480.0, - "29": 5708180480.0, - "30": 5708180480.0, - "31": 5708180480.0, - "32": 5708180480.0, - "33": 5708180480.0, - "34": 5708180480.0, - "35": 5708180480.0, - "36": 5708180480.0, - "37": 5708180480.0, - "38": 5708180480.0, - "39": 5708180480.0, - "40": 5708180480.0, - "41": 5708180480.0, - "42": 5708180480.0, - "43": 5708180480.0, - "44": 5708180480.0, - "45": 5708180480.0, - "46": 5708180480.0, - "47": 5708180480.0, - "48": 5708180480.0, - "49": 5708180480.0, - "50": 5708180480.0 + "1": 4229407232.0, + "2": 5707393024.0, + "3": 5707393024.0, + "4": 5707393024.0, + "5": 5707393024.0, + "6": 5707393024.0, + "7": 5707393024.0, + "8": 5707393024.0, + "9": 5707393024.0, + "10": 5707393024.0, + "11": 5707393024.0, + "12": 5707393024.0, + "13": 5707393024.0, + "14": 5707393024.0, + "15": 5707393024.0, + "16": 5707393024.0, + "17": 5707393024.0, + "18": 5707393024.0, + "19": 5707393024.0, + "20": 5707393024.0, + "21": 5707393024.0, + "22": 5707393024.0, + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 6.41588, - "3": 0.51314, - "4": 0.49769, - "5": 0.4946, - "6": 0.88892, - "7": 0.57204, - "8": 0.81712, - "9": 0.81304, - "10": 0.52308, - "11": 1.03669, - "12": 0.46364, - "13": 0.46164, - "14": 0.47508, - "15": 0.48187, - "16": 0.48218, - "17": 0.47379, - "18": 0.4846, - "19": 0.47588, - "20": 0.48496, - "21": 0.75353, - "22": 0.47987, - "23": 0.47483, - "24": 0.75989, - "25": 0.51954, - "26": 0.46526, - "27": 0.466, - "28": 0.94359, - "29": 0.4704, - "30": 0.46604, - "31": 0.46713, - "32": 0.46632, - "33": 0.46589, - "34": 0.46699, - "35": 0.46297, - "36": 0.46316, - "37": 0.46511, - "38": 0.46467, - "39": 0.4622, - "40": 0.461, - "41": 0.46215, - "42": 0.46203, - "43": 0.4632, - "44": 0.46384, - "45": 0.48541, - "46": 0.47672, - "47": 0.47512, - "48": 0.47239, - "49": 0.48882, - "50": 0.49209 + "2": 5.74712, + "3": 0.49778, + "4": 0.45748, + "5": 0.45659, + "6": 0.45981, + "7": 0.46548, + "8": 0.46542, + "9": 0.46526, + "10": 0.46413, + "11": 0.45692, + "12": 0.46222, + "13": 0.46736, + "14": 0.46657, + "15": 0.46742, + "16": 0.46727, + "17": 0.4733, + "18": 0.469, + "19": 0.45727, + "20": 0.47259, + "21": 0.46632, + "22": 0.46891, + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp4_vp2/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp4_vp2/golden_values_dev_dgx_h100.json index 032b1902072..651f9de50bb 100644 --- a/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp4_vp2/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/bert/bert_mcore_tp1_pp4_vp2/golden_values_dev_dgx_h100.json @@ -17,43 +17,43 @@ "11": 10.49164, "12": 10.47821, "13": 10.47598, - "14": 10.48189, - "15": 10.48217, - "16": 10.46292, - "17": 10.45829, - "18": 10.45957, - "19": 10.43356, - "20": 10.45335, - "21": 10.42765, - "22": 10.37288, - "23": 10.3837, - "24": 10.34039, - "25": 10.30874, - "26": 10.32676, - "27": 10.3348, - "28": 10.31238, - "29": 10.20958, - "30": 10.10211, - "31": 10.07247, - "32": 10.04225, - "33": 10.04856, - "34": 9.96979, - "35": 9.96036, - "36": 9.94987, - "37": 9.93538, - "38": 9.91494, - "39": 9.81544, - "40": 9.7735, - "41": 9.73656, - "42": 9.68286, - "43": 9.66796, - "44": 9.64166, - "45": 9.64023, - "46": 9.56948, - "47": 9.60362, - "48": 9.59334, - "49": 9.54843, - "50": 9.50472 + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "num-zeros": { @@ -74,43 +74,43 @@ "11": 2246.0, "12": 1932.0, "13": 2162.0, - "14": 2390.0, - "15": 2034.0, - "16": 2039.0, - "17": 2152.0, - "18": 2153.0, - "19": 2104.0, - "20": 2360.0, - "21": 2119.0, - "22": 2155.0, - "23": 2343.0, - "24": 2293.0, - "25": 2243.0, - "26": 2619.0, - "27": 2396.0, - "28": 2465.0, - "29": 1927.0, - "30": 2561.0, - "31": 2128.0, - "32": 2812.0, - "33": 2801.0, - "34": 2333.0, - "35": 2276.0, - "36": 1626.0, - "37": 2759.0, - "38": 2349.0, - "39": 2650.0, - "40": 1821.0, - "41": 1530.0, - "42": 1737.0, - "43": 1415.0, - "44": 2609.0, - "45": 3604.0, - "46": 3307.0, - "47": 2903.0, - "48": 1855.0, - "49": 3165.0, - "50": 3023.0 + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2090120704.0, - "2": 2090120704.0, - "3": 2090120704.0, - "4": 2090120704.0, - "5": 2090120704.0, - "6": 2090120704.0, - "7": 2090120704.0, - "8": 2090120704.0, - "9": 2090120704.0, - "10": 2090120704.0, - "11": 2090120704.0, - "12": 2090120704.0, - "13": 2090120704.0, - "14": 2090120704.0, - "15": 2090120704.0, - "16": 2090120704.0, - "17": 2090120704.0, - "18": 2090120704.0, - "19": 2090120704.0, - "20": 2090120704.0, - "21": 2090120704.0, - "22": 2090120704.0, - "23": 2090120704.0, - "24": 2090120704.0, - "25": 2090120704.0, - "26": 2090120704.0, - "27": 2090120704.0, - "28": 2090120704.0, - "29": 2090120704.0, - "30": 2090120704.0, - "31": 2090120704.0, - "32": 2090120704.0, - "33": 2090120704.0, - "34": 2090120704.0, - "35": 2090120704.0, - "36": 2090120704.0, - "37": 2090120704.0, - "38": 2090120704.0, - "39": 2090120704.0, - "40": 2090120704.0, - "41": 2090120704.0, - "42": 2090120704.0, - "43": 2090120704.0, - "44": 2090120704.0, - "45": 2090120704.0, - "46": 2090120704.0, - "47": 2090120704.0, - "48": 2090120704.0, - "49": 2090120704.0, - "50": 2090120704.0 + "1": 2090120192.0, + "2": 2090120192.0, + "3": 2090120192.0, + "4": 2090120192.0, + "5": 2090120192.0, + "6": 2090120192.0, + "7": 2090120192.0, + "8": 2090120192.0, + "9": 2090120192.0, + "10": 2090120192.0, + "11": 2090120192.0, + "12": 2090120192.0, + "13": 2090120192.0, + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4420558848.0, - "2": 5293143552.0, - "3": 5293143552.0, - "4": 5293143552.0, - "5": 5293143552.0, - "6": 5293143552.0, - "7": 5293143552.0, - "8": 5293143552.0, - "9": 5293143552.0, - "10": 5293143552.0, - "11": 5293143552.0, - "12": 5293143552.0, - "13": 5293143552.0, - "14": 5293143552.0, - "15": 5293143552.0, - "16": 5293143552.0, - "17": 5293143552.0, - "18": 5293143552.0, - "19": 5293143552.0, - "20": 5293143552.0, - "21": 5293143552.0, - "22": 5293143552.0, - "23": 5293143552.0, - "24": 5293143552.0, - "25": 5293143552.0, - "26": 5293143552.0, - "27": 5293143552.0, - "28": 5293143552.0, - "29": 5293143552.0, - "30": 5293143552.0, - "31": 5293143552.0, - "32": 5293143552.0, - "33": 5293143552.0, - "34": 5293143552.0, - "35": 5293143552.0, - "36": 5293143552.0, - "37": 5293143552.0, - "38": 5293143552.0, - "39": 5293143552.0, - "40": 5293143552.0, - "41": 5293143552.0, - "42": 5293143552.0, - "43": 5293143552.0, - "44": 5293143552.0, - "45": 5293143552.0, - "46": 5293143552.0, - "47": 5293143552.0, - "48": 5293143552.0, - "49": 5293143552.0, - "50": 5293143552.0 + "1": 4420291584.0, + "2": 5293134848.0, + "3": 5293134848.0, + "4": 5293134848.0, + "5": 5293134848.0, + "6": 5293134848.0, + "7": 5293134848.0, + "8": 5293134848.0, + "9": 5293917696.0, + "10": 5293917696.0, + "11": 5293917696.0, + "12": 5293917696.0, + "13": 5293917696.0, + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 10.52401, - "3": 1.57552, - "4": 1.81828, - "5": 1.5526, - "6": 1.85419, - "7": 1.36624, - "8": 1.31137, - "9": 0.56701, - "10": 0.58911, - "11": 0.57613, - "12": 1.04751, - "13": 0.57846, - "14": 0.58988, - "15": 0.61613, - "16": 0.57419, - "17": 0.57512, - "18": 0.59, - "19": 0.56504, - "20": 0.55957, - "21": 0.59851, - "22": 0.58737, - "23": 0.59952, - "24": 0.58825, - "25": 0.57674, - "26": 0.58834, - "27": 0.57154, - "28": 0.55945, - "29": 0.55684, - "30": 0.58918, - "31": 0.59542, - "32": 0.56341, - "33": 0.58273, - "34": 0.57112, - "35": 0.58626, - "36": 0.58049, - "37": 0.60737, - "38": 0.62386, - "39": 0.58525, - "40": 0.58845, - "41": 0.58049, - "42": 0.61879, - "43": 0.5917, - "44": 0.6123, - "45": 0.85903, - "46": 0.58483, - "47": 0.98782, - "48": 0.87303, - "49": 1.5563, - "50": 1.29846 + "2": 8.61722, + "3": 0.57542, + "4": 0.56623, + "5": 0.5541, + "6": 0.56688, + "7": 0.54082, + "8": 0.54548, + "9": 0.55074, + "10": 0.55601, + "11": 0.55416, + "12": 0.5472, + "13": 1.6631, + "14": "nan", + "15": "nan", + "16": "nan", + "17": "nan", + "18": "nan", + "19": "nan", + "20": "nan", + "21": "nan", + "22": "nan", + "23": "nan", + "24": "nan", + "25": "nan", + "26": "nan", + "27": "nan", + "28": "nan", + "29": "nan", + "30": "nan", + "31": "nan", + "32": "nan", + "33": "nan", + "34": "nan", + "35": "nan", + "36": "nan", + "37": "nan", + "38": "nan", + "39": "nan", + "40": "nan", + "41": "nan", + "42": "nan", + "43": "nan", + "44": "nan", + "45": "nan", + "46": "nan", + "47": "nan", + "48": "nan", + "49": "nan", + "50": "nan" } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json index 37b40b075a8..6fda46fb04e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.92698, "2": 10.9227, "3": 10.9319, - "4": 10.93526, - "5": 10.93085, - "6": 10.92306, - "7": 10.93505, - "8": 10.92461, - "9": 10.92991, - "10": 10.92925, - "11": 10.91773, + "4": 10.93528, + "5": 10.93089, + "6": 10.92308, + "7": 10.93504, + "8": 10.92469, + "9": 10.92992, + "10": 10.92923, + "11": 10.91772, "12": 10.92574, - "13": 10.91873, - "14": 10.90866, - "15": 10.88358, - "16": 10.87866, - "17": 10.88886, - "18": 10.86471, - "19": 10.87233, - "20": 10.78255, - "21": 10.78375, - "22": 10.769, - "23": 10.76442, - "24": 10.72927, + "13": 10.91875, + "14": 10.90867, + "15": 10.88359, + "16": 10.87865, + "17": 10.88885, + "18": 10.86469, + "19": 10.87232, + "20": 10.78256, + "21": 10.7837, + "22": 10.76896, + "23": 10.76441, + "24": 10.72928, "25": 10.73567, - "26": 10.71336, - "27": 10.6818, - "28": 10.60193, + "26": 10.71337, + "27": 10.68181, + "28": 10.60186, "29": 10.58356, - "30": 10.55753, - "31": 10.5625, - "32": 10.5349, - "33": 10.50278, + "30": 10.5575, + "31": 10.56251, + "32": 10.53491, + "33": 10.50281, "34": 10.46248, - "35": 10.47715, - "36": 10.45697, + "35": 10.47714, + "36": 10.45696, "37": 10.4187, "38": 10.41664, - "39": 10.38143, - "40": 10.37437, - "41": 10.34276, + "39": 10.38141, + "40": 10.37439, + "41": 10.34271, "42": 10.33077, - "43": 10.30916, - "44": 10.2752, - "45": 10.30087, - "46": 10.25977, - "47": 10.23764, + "43": 10.30914, + "44": 10.27522, + "45": 10.30086, + "46": 10.2598, + "47": 10.23763, "48": 10.19356, - "49": 10.18482, - "50": 10.19738, - "51": 10.1947, + "49": 10.18478, + "50": 10.1974, + "51": 10.19469, "52": 10.14645, - "53": 10.15204, + "53": 10.15205, "54": 10.11846, - "55": 10.08345, + "55": 10.08346, "56": 10.12277, - "57": 10.1054, - "58": 10.12074, - "59": 10.06313, + "57": 10.10538, + "58": 10.12073, + "59": 10.06312, "60": 10.08108, - "61": 10.03883, - "62": 10.00458, - "63": 10.07813, - "64": 10.03224, + "61": 10.03882, + "62": 10.00457, + "63": 10.07811, + "64": 10.03223, "65": 10.00315, - "66": 10.03547, - "67": 10.00629, - "68": 9.96938, - "69": 9.99101, - "70": 9.97878, - "71": 9.99803, - "72": 9.97585, + "66": 10.03548, + "67": 10.00631, + "68": 9.96936, + "69": 9.99097, + "70": 9.97879, + "71": 9.99799, + "72": 9.97584, "73": 9.96598, - "74": 9.95985, - "75": 9.9258, - "76": 9.96563, - "77": 9.95588, - "78": 9.90231, - "79": 9.91042, - "80": 9.9262, - "81": 9.9445, - "82": 9.88904, - "83": 9.84931, - "84": 9.78866, - "85": 9.7764, - "86": 9.87986, + "74": 9.95989, + "75": 9.92578, + "76": 9.9656, + "77": 9.95592, + "78": 9.90232, + "79": 9.91043, + "80": 9.92619, + "81": 9.94449, + "82": 9.88905, + "83": 9.84929, + "84": 9.78865, + "85": 9.77636, + "86": 9.8799, "87": 9.91273, - "88": 9.88591, - "89": 9.8238, - "90": 9.81128, - "91": 9.82089, - "92": 9.81009, + "88": 9.88593, + "89": 9.82382, + "90": 9.8113, + "91": 9.82085, + "92": 9.8101, "93": 9.75217, - "94": 9.82529, - "95": 9.81925, - "96": 9.80588, + "94": 9.82528, + "95": 9.81926, + "96": 9.80585, "97": 9.7436, "98": 9.77346, "99": 9.81647, - "100": 9.71052 + "100": 9.71051 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1760.0, - "2": 1718.0, - "3": 1911.0, - "4": 1680.0, - "5": 1724.0, - "6": 1726.0, - "7": 1938.0, - "8": 1714.0, - "9": 1777.0, - "10": 1782.0, - "11": 1731.0, - "12": 1732.0, - "13": 1901.0, - "14": 1879.0, - "15": 1638.0, - "16": 1777.0, - "17": 1870.0, - "18": 1819.0, - "19": 1636.0, - "20": 1783.0, - "21": 1782.0, - "22": 1829.0, - "23": 1680.0, - "24": 1788.0, - "25": 1765.0, - "26": 1787.0, - "27": 1775.0, - "28": 1761.0, - "29": 1969.0, - "30": 1913.0, - "31": 2024.0, - "32": 1965.0, - "33": 1939.0, - "34": 2098.0, - "35": 2015.0, - "36": 1999.0, - "37": 2184.0, - "38": 2127.0, - "39": 2269.0, - "40": 2213.0, - "41": 2434.0, - "42": 2011.0, - "43": 2385.0, - "44": 2183.0, - "45": 2492.0, - "46": 2353.0, - "47": 2532.0, - "48": 2550.0, - "49": 2889.0, - "50": 2573.0, - "51": 2495.0, - "52": 2757.0, - "53": 2559.0, - "54": 2871.0, - "55": 2674.0, - "56": 2624.0, - "57": 2119.0, - "58": 3445.0, - "59": 2871.0, - "60": 3009.0, - "61": 2660.0, - "62": 3290.0, - "63": 3225.0, - "64": 3763.0, - "65": 2612.0, - "66": 3044.0, - "67": 3757.0, - "68": 3424.0, - "69": 2882.0, - "70": 3216.0, - "71": 3121.0, - "72": 3021.0, - "73": 3434.0, - "74": 3146.0, - "75": 3203.0, - "76": 3066.0, - "77": 3704.0, - "78": 3253.0, - "79": 3121.0, - "80": 3113.0, - "81": 3417.0, - "82": 3220.0, - "83": 3187.0, - "84": 2939.0, - "85": 2734.0, - "86": 3071.0, - "87": 3029.0, - "88": 3078.0, - "89": 3030.0, - "90": 3639.0, - "91": 3109.0, - "92": 2912.0, - "93": 3082.0, - "94": 3308.0, - "95": 3517.0, - "96": 3691.0, - "97": 3679.0, - "98": 3363.0, - "99": 3332.0, - "100": 3322.0 + "1": 1721.0, + "2": 1723.0, + "3": 1805.0, + "4": 1715.0, + "5": 1720.0, + "6": 1696.0, + "7": 2005.0, + "8": 1719.0, + "9": 1779.0, + "10": 1786.0, + "11": 1735.0, + "12": 1784.0, + "13": 1772.0, + "14": 2011.0, + "15": 1549.0, + "16": 1771.0, + "17": 1850.0, + "18": 1760.0, + "19": 1688.0, + "20": 1671.0, + "21": 1767.0, + "22": 1686.0, + "23": 1644.0, + "24": 1825.0, + "25": 1726.0, + "26": 1835.0, + "27": 1745.0, + "28": 1809.0, + "29": 1889.0, + "30": 1803.0, + "31": 2012.0, + "32": 1926.0, + "33": 1808.0, + "34": 2003.0, + "35": 2040.0, + "36": 2018.0, + "37": 2161.0, + "38": 2078.0, + "39": 2192.0, + "40": 2224.0, + "41": 2463.0, + "42": 1972.0, + "43": 2347.0, + "44": 2179.0, + "45": 2469.0, + "46": 2342.0, + "47": 2447.0, + "48": 2672.0, + "49": 2956.0, + "50": 2535.0, + "51": 2466.0, + "52": 2700.0, + "53": 2518.0, + "54": 2863.0, + "55": 2628.0, + "56": 2712.0, + "57": 2200.0, + "58": 3539.0, + "59": 2904.0, + "60": 3064.0, + "61": 2717.0, + "62": 3205.0, + "63": 3221.0, + "64": 3611.0, + "65": 2546.0, + "66": 3082.0, + "67": 3897.0, + "68": 3433.0, + "69": 2811.0, + "70": 3152.0, + "71": 3057.0, + "72": 2886.0, + "73": 3398.0, + "74": 3141.0, + "75": 3303.0, + "76": 3082.0, + "77": 3699.0, + "78": 3202.0, + "79": 3280.0, + "80": 2967.0, + "81": 3343.0, + "82": 3265.0, + "83": 3238.0, + "84": 2825.0, + "85": 2671.0, + "86": 3089.0, + "87": 2996.0, + "88": 3092.0, + "89": 2993.0, + "90": 3698.0, + "91": 2993.0, + "92": 2947.0, + "93": 2961.0, + "94": 3288.0, + "95": 3457.0, + "96": 3487.0, + "97": 3603.0, + "98": 3452.0, + "99": 3293.0, + "100": 3251.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 745146880.0, - "2": 745146880.0, - "3": 745146880.0, - "4": 745146880.0, - "5": 745146880.0, - "6": 745146880.0, - "7": 745146880.0, - "8": 745146880.0, - "9": 745146880.0, - "10": 745146880.0, - "11": 745146880.0, - "12": 745146880.0, - "13": 745146880.0, - "14": 745146880.0, - "15": 745146880.0, - "16": 745146880.0, - "17": 745146880.0, - "18": 745146880.0, - "19": 745146880.0, - "20": 745146880.0, - "21": 745146880.0, - "22": 745146880.0, - "23": 745146880.0, - "24": 745146880.0, - "25": 745146880.0, - "26": 745146880.0, - "27": 745146880.0, - "28": 745146880.0, - "29": 745146880.0, - "30": 745146880.0, - "31": 745146880.0, - "32": 745146880.0, - "33": 745146880.0, - "34": 745146880.0, - "35": 745146880.0, - "36": 745146880.0, - "37": 745146880.0, - "38": 745146880.0, - "39": 745146880.0, - "40": 745146880.0, - "41": 745146880.0, - "42": 745146880.0, - "43": 745146880.0, - "44": 745146880.0, - "45": 745146880.0, - "46": 745146880.0, - "47": 745146880.0, - "48": 745146880.0, - "49": 745146880.0, - "50": 745146880.0, - "51": 745146880.0, - "52": 745146880.0, - "53": 745146880.0, - "54": 745146880.0, - "55": 745146880.0, - "56": 745146880.0, - "57": 745146880.0, - "58": 745146880.0, - "59": 745146880.0, - "60": 745146880.0, - "61": 745146880.0, - "62": 745146880.0, - "63": 745146880.0, - "64": 745146880.0, - "65": 745146880.0, - "66": 745146880.0, - "67": 745146880.0, - "68": 745146880.0, - "69": 745146880.0, - "70": 745146880.0, - "71": 745146880.0, - "72": 745146880.0, - "73": 745146880.0, - "74": 745146880.0, - "75": 745146880.0, - "76": 745146880.0, - "77": 745146880.0, - "78": 745146880.0, - "79": 745146880.0, - "80": 745146880.0, - "81": 745146880.0, - "82": 745146880.0, - "83": 745146880.0, - "84": 745146880.0, - "85": 745146880.0, - "86": 745146880.0, - "87": 745146880.0, - "88": 745146880.0, - "89": 745146880.0, - "90": 745146880.0, - "91": 745146880.0, - "92": 745146880.0, - "93": 745146880.0, - "94": 745146880.0, - "95": 745146880.0, - "96": 745146880.0, - "97": 745146880.0, - "98": 745146880.0, - "99": 745146880.0, - "100": 745146880.0 + "1": 763168768.0, + "2": 763168768.0, + "3": 763168768.0, + "4": 763168768.0, + "5": 763168768.0, + "6": 763168768.0, + "7": 763168768.0, + "8": 763168768.0, + "9": 763168768.0, + "10": 763168768.0, + "11": 763168768.0, + "12": 763168768.0, + "13": 763168768.0, + "14": 763168768.0, + "15": 763168768.0, + "16": 763168768.0, + "17": 763168768.0, + "18": 763168768.0, + "19": 763168768.0, + "20": 763168768.0, + "21": 763168768.0, + "22": 763168768.0, + "23": 763168768.0, + "24": 763168768.0, + "25": 763168768.0, + "26": 763168768.0, + "27": 763168768.0, + "28": 763168768.0, + "29": 763168768.0, + "30": 763168768.0, + "31": 763168768.0, + "32": 763168768.0, + "33": 763168768.0, + "34": 763168768.0, + "35": 763168768.0, + "36": 763168768.0, + "37": 763168768.0, + "38": 763168768.0, + "39": 763168768.0, + "40": 763168768.0, + "41": 763168768.0, + "42": 763168768.0, + "43": 763168768.0, + "44": 763168768.0, + "45": 763168768.0, + "46": 763168768.0, + "47": 763168768.0, + "48": 763168768.0, + "49": 763168768.0, + "50": 763168768.0, + "51": 763168768.0, + "52": 763168768.0, + "53": 763168768.0, + "54": 763168768.0, + "55": 763168768.0, + "56": 763168768.0, + "57": 763168768.0, + "58": 763168768.0, + "59": 763168768.0, + "60": 763168768.0, + "61": 763168768.0, + "62": 763168768.0, + "63": 763168768.0, + "64": 763168768.0, + "65": 763168768.0, + "66": 763168768.0, + "67": 763168768.0, + "68": 763168768.0, + "69": 763168768.0, + "70": 763168768.0, + "71": 763168768.0, + "72": 763168768.0, + "73": 763168768.0, + "74": 763168768.0, + "75": 763168768.0, + "76": 763168768.0, + "77": 763168768.0, + "78": 763168768.0, + "79": 763168768.0, + "80": 763168768.0, + "81": 763168768.0, + "82": 763168768.0, + "83": 763168768.0, + "84": 763168768.0, + "85": 763168768.0, + "86": 763168768.0, + "87": 763168768.0, + "88": 763168768.0, + "89": 763168768.0, + "90": 763168768.0, + "91": 763168768.0, + "92": 763168768.0, + "93": 763168768.0, + "94": 763168768.0, + "95": 763168768.0, + "96": 763168768.0, + "97": 763168768.0, + "98": 763168768.0, + "99": 763168768.0, + "100": 763168768.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1938737152.0, - "2": 2222434304.0, - "3": 2222434304.0, - "4": 2222434304.0, - "5": 2222434304.0, - "6": 2222434304.0, - "7": 2222434304.0, - "8": 2222434304.0, - "9": 2222434304.0, - "10": 2222434304.0, - "11": 2222434304.0, - "12": 2222434304.0, - "13": 2222434304.0, - "14": 2222434304.0, - "15": 2222434304.0, - "16": 2222434304.0, - "17": 2222434304.0, - "18": 2222434304.0, - "19": 2222434304.0, - "20": 2222434304.0, - "21": 2222434304.0, - "22": 2222434304.0, - "23": 2222434304.0, - "24": 2222434304.0, - "25": 2222434304.0, - "26": 2222434304.0, - "27": 2222434304.0, - "28": 2222434304.0, - "29": 2222434304.0, - "30": 2222434304.0, - "31": 2222434304.0, - "32": 2222434304.0, - "33": 2222434304.0, - "34": 2222434304.0, - "35": 2222434304.0, - "36": 2222434304.0, - "37": 2222434304.0, - "38": 2222434304.0, - "39": 2222434304.0, - "40": 2222434304.0, - "41": 2222434304.0, - "42": 2222434304.0, - "43": 2222434304.0, - "44": 2222434304.0, - "45": 2222434304.0, - "46": 2222434304.0, - "47": 2222434304.0, - "48": 2222434304.0, - "49": 2222434304.0, - "50": 2222434304.0, - "51": 2222434304.0, - "52": 2222434304.0, - "53": 2222434304.0, - "54": 2222434304.0, - "55": 2222434304.0, - "56": 2222434304.0, - "57": 2222434304.0, - "58": 2222434304.0, - "59": 2222434304.0, - "60": 2222434304.0, - "61": 2222434304.0, - "62": 2222434304.0, - "63": 2222434304.0, - "64": 2222434304.0, - "65": 2222434304.0, - "66": 2222434304.0, - "67": 2222434304.0, - "68": 2222434304.0, - "69": 2222434304.0, - "70": 2222434304.0, - "71": 2222434304.0, - "72": 2222434304.0, - "73": 2222434304.0, - "74": 2222434304.0, - "75": 2222434304.0, - "76": 2222434304.0, - "77": 2222434304.0, - "78": 2222434304.0, - "79": 2222434304.0, - "80": 2222434304.0, - "81": 2222434304.0, - "82": 2222434304.0, - "83": 2222434304.0, - "84": 2222434304.0, - "85": 2222434304.0, - "86": 2222434304.0, - "87": 2222434304.0, - "88": 2222434304.0, - "89": 2222434304.0, - "90": 2222434304.0, - "91": 2222434304.0, - "92": 2222434304.0, - "93": 2222434304.0, - "94": 2222434304.0, - "95": 2222434304.0, - "96": 2222434304.0, - "97": 2222434304.0, - "98": 2222434304.0, - "99": 2222434304.0, - "100": 2222434304.0 + "1": 1951401472.0, + "2": 2235099648.0, + "3": 2235099648.0, + "4": 2235099648.0, + "5": 2235099648.0, + "6": 2235099648.0, + "7": 2235099648.0, + "8": 2235099648.0, + "9": 2235099648.0, + "10": 2235099648.0, + "11": 2235099648.0, + "12": 2235099648.0, + "13": 2235099648.0, + "14": 2235099648.0, + "15": 2235099648.0, + "16": 2235099648.0, + "17": 2235099648.0, + "18": 2235099648.0, + "19": 2235099648.0, + "20": 2235099648.0, + "21": 2235099648.0, + "22": 2235099648.0, + "23": 2235099648.0, + "24": 2235099648.0, + "25": 2235099648.0, + "26": 2235099648.0, + "27": 2235099648.0, + "28": 2235099648.0, + "29": 2235099648.0, + "30": 2235099648.0, + "31": 2235099648.0, + "32": 2235099648.0, + "33": 2235099648.0, + "34": 2235099648.0, + "35": 2235099648.0, + "36": 2235099648.0, + "37": 2235099648.0, + "38": 2235099648.0, + "39": 2235099648.0, + "40": 2235099648.0, + "41": 2235099648.0, + "42": 2235099648.0, + "43": 2235099648.0, + "44": 2235099648.0, + "45": 2235099648.0, + "46": 2235099648.0, + "47": 2235099648.0, + "48": 2235099648.0, + "49": 2235099648.0, + "50": 2235099648.0, + "51": 2235099648.0, + "52": 2235099648.0, + "53": 2235099648.0, + "54": 2235099648.0, + "55": 2235099648.0, + "56": 2235099648.0, + "57": 2235099648.0, + "58": 2235099648.0, + "59": 2235099648.0, + "60": 2235099648.0, + "61": 2235099648.0, + "62": 2235099648.0, + "63": 2235099648.0, + "64": 2235099648.0, + "65": 2235099648.0, + "66": 2235099648.0, + "67": 2235099648.0, + "68": 2235099648.0, + "69": 2235099648.0, + "70": 2235099648.0, + "71": 2235099648.0, + "72": 2235099648.0, + "73": 2235099648.0, + "74": 2235099648.0, + "75": 2235099648.0, + "76": 2235099648.0, + "77": 2235099648.0, + "78": 2235099648.0, + "79": 2235099648.0, + "80": 2235099648.0, + "81": 2235099648.0, + "82": 2235099648.0, + "83": 2235099648.0, + "84": 2235099648.0, + "85": 2235099648.0, + "86": 2235099648.0, + "87": 2235099648.0, + "88": 2235099648.0, + "89": 2235099648.0, + "90": 2235099648.0, + "91": 2235099648.0, + "92": 2235099648.0, + "93": 2235099648.0, + "94": 2235099648.0, + "95": 2235099648.0, + "96": 2235099648.0, + "97": 2235099648.0, + "98": 2235099648.0, + "99": 2235099648.0, + "100": 2235099648.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.56158, - "3": 0.15869, - "4": 0.10721, - "5": 0.14824, - "6": 0.1106, - "7": 0.10862, - "8": 0.1068, + "2": 4.78352, + "3": 0.14863, + "4": 0.13956, + "5": 0.10416, + "6": 0.10759, + "7": 0.10645, + "8": 0.20824, "9": 0.10527, - "10": 0.10817, - "11": 0.10884, - "12": 0.10913, - "13": 0.11023, - "14": 0.11167, - "15": 0.11229, - "16": 0.10881, - "17": 0.11078, - "18": 0.11025, - "19": 0.10791, - "20": 0.11031, - "21": 0.10748, - "22": 0.10902, - "23": 0.1093, - "24": 0.10897, - "25": 0.10851, - "26": 0.11185, - "27": 0.10834, - "28": 0.10977, - "29": 0.11163, - "30": 0.11039, - "31": 0.11522, - "32": 0.11993, - "33": 0.13234, - "34": 0.11833, - "35": 0.12647, - "36": 0.12247, - "37": 0.12178, - "38": 0.13371, - "39": 0.14995, - "40": 0.14872, - "41": 0.12884, - "42": 0.13805, - "43": 0.18641, - "44": 0.2213, - "45": 0.15476, - "46": 0.13438, - "47": 0.14588, - "48": 0.16245, - "49": 0.14501, - "50": 0.16698, - "51": 0.29692, - "52": 0.15313, - "53": 0.11607, - "54": 0.11231, - "55": 0.11258, - "56": 0.10959, - "57": 0.11041, - "58": 0.10862, - "59": 0.11219, - "60": 0.11002, - "61": 0.11193, - "62": 0.11166, - "63": 0.11218, - "64": 0.11323, - "65": 0.11166, - "66": 0.11224, - "67": 0.11118, - "68": 0.11162, - "69": 0.11266, - "70": 0.10998, - "71": 0.11022, - "72": 0.11131, - "73": 0.11191, - "74": 0.1131, - "75": 0.11379, - "76": 0.11029, - "77": 0.10806, - "78": 0.11148, - "79": 0.10984, - "80": 0.1145, - "81": 0.11238, - "82": 0.11156, - "83": 0.11285, - "84": 0.11078, - "85": 0.10979, - "86": 0.10896, - "87": 0.10794, - "88": 0.10787, - "89": 0.10902, - "90": 0.11007, - "91": 0.10875, - "92": 0.11026, - "93": 0.11078, - "94": 0.10982, - "95": 0.11005, - "96": 0.11147, - "97": 0.11317, - "98": 0.11103, - "99": 0.10962, - "100": 0.11047 + "10": 0.11087, + "11": 0.11051, + "12": 0.10823, + "13": 0.1052, + "14": 0.10573, + "15": 0.10448, + "16": 0.13987, + "17": 0.10488, + "18": 0.18815, + "19": 0.27888, + "20": 0.24425, + "21": 0.24961, + "22": 0.25227, + "23": 0.12849, + "24": 0.20611, + "25": 0.21781, + "26": 0.24771, + "27": 0.23387, + "28": 0.10539, + "29": 0.10767, + "30": 0.1054, + "31": 0.10343, + "32": 0.10624, + "33": 0.10543, + "34": 0.10526, + "35": 0.10457, + "36": 0.1068, + "37": 0.10794, + "38": 0.23907, + "39": 0.10751, + "40": 0.10452, + "41": 0.10482, + "42": 0.10361, + "43": 0.10773, + "44": 0.10487, + "45": 0.10412, + "46": 0.10566, + "47": 0.1042, + "48": 0.10699, + "49": 0.10759, + "50": 0.10496, + "51": 0.18594, + "52": 0.21891, + "53": 0.11196, + "54": 0.10722, + "55": 0.10848, + "56": 0.1095, + "57": 0.10661, + "58": 0.10744, + "59": 0.10882, + "60": 0.10835, + "61": 0.10875, + "62": 0.10777, + "63": 0.24068, + "64": 0.10969, + "65": 0.10962, + "66": 0.10783, + "67": 0.10717, + "68": 0.10768, + "69": 0.1069, + "70": 0.10745, + "71": 0.1082, + "72": 0.10788, + "73": 0.10945, + "74": 0.1063, + "75": 0.10905, + "76": 0.10666, + "77": 0.10815, + "78": 0.10897, + "79": 0.10838, + "80": 0.10731, + "81": 0.10509, + "82": 0.10767, + "83": 0.10827, + "84": 0.10804, + "85": 0.10805, + "86": 0.10674, + "87": 0.10614, + "88": 0.10633, + "89": 0.10892, + "90": 0.10888, + "91": 0.1087, + "92": 0.10608, + "93": 0.1078, + "94": 0.10679, + "95": 0.10872, + "96": 0.11635, + "97": 0.1167, + "98": 0.1162, + "99": 0.11702, + "100": 0.11716 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json index ddd60f5d746..ea445eabb39 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_resume_torch_dist_persistent_disable_bias_linear/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.92698, "2": 10.9227, "3": 10.9319, - "4": 10.93526, - "5": 10.93085, - "6": 10.92306, - "7": 10.93505, - "8": 10.92461, - "9": 10.92991, - "10": 10.92925, - "11": 10.91773, + "4": 10.93528, + "5": 10.93089, + "6": 10.92308, + "7": 10.93504, + "8": 10.92469, + "9": 10.92992, + "10": 10.92923, + "11": 10.91772, "12": 10.92574, - "13": 10.91873, - "14": 10.90866, - "15": 10.88358, - "16": 10.87866, - "17": 10.88886, - "18": 10.86471, - "19": 10.87233, - "20": 10.78255, - "21": 10.78375, - "22": 10.769, - "23": 10.76442, - "24": 10.72927, + "13": 10.91875, + "14": 10.90867, + "15": 10.88359, + "16": 10.87865, + "17": 10.88885, + "18": 10.86469, + "19": 10.87232, + "20": 10.78256, + "21": 10.7837, + "22": 10.76896, + "23": 10.76441, + "24": 10.72928, "25": 10.73567, - "26": 10.71336, - "27": 10.6818, - "28": 10.60193, + "26": 10.71337, + "27": 10.68181, + "28": 10.60186, "29": 10.58356, - "30": 10.55753, - "31": 10.5625, - "32": 10.5349, - "33": 10.50278, + "30": 10.5575, + "31": 10.56251, + "32": 10.53491, + "33": 10.50281, "34": 10.46248, - "35": 10.47715, - "36": 10.45697, + "35": 10.47714, + "36": 10.45696, "37": 10.4187, "38": 10.41664, - "39": 10.38143, - "40": 10.37437, - "41": 10.34276, + "39": 10.38141, + "40": 10.37439, + "41": 10.34271, "42": 10.33077, - "43": 10.30916, - "44": 10.2752, - "45": 10.30087, - "46": 10.25977, - "47": 10.23764, + "43": 10.30914, + "44": 10.27522, + "45": 10.30086, + "46": 10.2598, + "47": 10.23763, "48": 10.19356, - "49": 10.18482, - "50": 10.19738, - "51": 10.1947, + "49": 10.18478, + "50": 10.1974, + "51": 10.19469, "52": 10.14645, - "53": 10.15204, + "53": 10.15205, "54": 10.11846, - "55": 10.08345, + "55": 10.08346, "56": 10.12277, - "57": 10.1054, - "58": 10.12074, - "59": 10.06313, + "57": 10.10538, + "58": 10.12073, + "59": 10.06312, "60": 10.08108, - "61": 10.03883, - "62": 10.00458, - "63": 10.07813, - "64": 10.03224, + "61": 10.03882, + "62": 10.00457, + "63": 10.07811, + "64": 10.03223, "65": 10.00315, - "66": 10.03547, - "67": 10.00629, - "68": 9.96938, - "69": 9.99101, - "70": 9.97878, - "71": 9.99803, - "72": 9.97585, + "66": 10.03548, + "67": 10.00631, + "68": 9.96936, + "69": 9.99097, + "70": 9.97879, + "71": 9.99799, + "72": 9.97584, "73": 9.96598, - "74": 9.95985, - "75": 9.9258, - "76": 9.96563, - "77": 9.95588, - "78": 9.90231, - "79": 9.91042, - "80": 9.9262, - "81": 9.9445, - "82": 9.88904, - "83": 9.84931, - "84": 9.78866, - "85": 9.7764, - "86": 9.87986, + "74": 9.95989, + "75": 9.92578, + "76": 9.9656, + "77": 9.95592, + "78": 9.90232, + "79": 9.91043, + "80": 9.92619, + "81": 9.94449, + "82": 9.88905, + "83": 9.84929, + "84": 9.78865, + "85": 9.77636, + "86": 9.8799, "87": 9.91273, - "88": 9.88591, - "89": 9.8238, - "90": 9.81128, - "91": 9.82089, - "92": 9.81009, + "88": 9.88593, + "89": 9.82382, + "90": 9.8113, + "91": 9.82085, + "92": 9.8101, "93": 9.75217, - "94": 9.82529, - "95": 9.81925, - "96": 9.80588, + "94": 9.82528, + "95": 9.81926, + "96": 9.80585, "97": 9.7436, "98": 9.77346, "99": 9.81647, - "100": 9.71052 + "100": 9.71051 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1760.0, - "2": 1718.0, - "3": 1911.0, - "4": 1680.0, - "5": 1724.0, - "6": 1726.0, - "7": 1938.0, - "8": 1714.0, - "9": 1777.0, - "10": 1782.0, - "11": 1731.0, - "12": 1732.0, - "13": 1901.0, - "14": 1879.0, - "15": 1638.0, - "16": 1777.0, - "17": 1870.0, - "18": 1819.0, - "19": 1636.0, - "20": 1783.0, - "21": 1782.0, - "22": 1829.0, - "23": 1680.0, - "24": 1788.0, - "25": 1765.0, - "26": 1787.0, - "27": 1775.0, - "28": 1761.0, - "29": 1969.0, - "30": 1913.0, - "31": 2024.0, - "32": 1965.0, - "33": 1939.0, - "34": 2098.0, - "35": 2015.0, - "36": 1999.0, - "37": 2184.0, - "38": 2127.0, - "39": 2269.0, - "40": 2213.0, - "41": 2434.0, - "42": 2011.0, - "43": 2385.0, - "44": 2183.0, - "45": 2492.0, - "46": 2353.0, - "47": 2532.0, - "48": 2550.0, - "49": 2889.0, - "50": 2573.0, - "51": 2495.0, - "52": 2757.0, - "53": 2559.0, - "54": 2871.0, - "55": 2674.0, - "56": 2624.0, - "57": 2119.0, - "58": 3445.0, - "59": 2871.0, - "60": 3009.0, - "61": 2660.0, - "62": 3290.0, - "63": 3225.0, - "64": 3763.0, - "65": 2612.0, - "66": 3044.0, - "67": 3757.0, - "68": 3424.0, - "69": 2882.0, - "70": 3216.0, - "71": 3121.0, - "72": 3021.0, - "73": 3434.0, - "74": 3146.0, - "75": 3203.0, - "76": 3066.0, - "77": 3704.0, - "78": 3253.0, - "79": 3121.0, - "80": 3113.0, - "81": 3417.0, - "82": 3220.0, - "83": 3187.0, - "84": 2939.0, - "85": 2734.0, - "86": 3071.0, - "87": 3029.0, - "88": 3078.0, - "89": 3030.0, - "90": 3639.0, - "91": 3109.0, - "92": 2912.0, - "93": 3082.0, - "94": 3308.0, - "95": 3517.0, - "96": 3691.0, - "97": 3679.0, - "98": 3363.0, - "99": 3332.0, - "100": 3322.0 + "1": 1721.0, + "2": 1723.0, + "3": 1805.0, + "4": 1715.0, + "5": 1720.0, + "6": 1696.0, + "7": 2005.0, + "8": 1719.0, + "9": 1779.0, + "10": 1786.0, + "11": 1735.0, + "12": 1784.0, + "13": 1772.0, + "14": 2011.0, + "15": 1549.0, + "16": 1771.0, + "17": 1850.0, + "18": 1760.0, + "19": 1688.0, + "20": 1671.0, + "21": 1767.0, + "22": 1686.0, + "23": 1644.0, + "24": 1825.0, + "25": 1726.0, + "26": 1835.0, + "27": 1745.0, + "28": 1809.0, + "29": 1889.0, + "30": 1803.0, + "31": 2012.0, + "32": 1926.0, + "33": 1808.0, + "34": 2003.0, + "35": 2040.0, + "36": 2018.0, + "37": 2161.0, + "38": 2078.0, + "39": 2192.0, + "40": 2224.0, + "41": 2463.0, + "42": 1972.0, + "43": 2347.0, + "44": 2179.0, + "45": 2469.0, + "46": 2342.0, + "47": 2447.0, + "48": 2672.0, + "49": 2956.0, + "50": 2535.0, + "51": 2466.0, + "52": 2700.0, + "53": 2518.0, + "54": 2863.0, + "55": 2628.0, + "56": 2712.0, + "57": 2200.0, + "58": 3539.0, + "59": 2904.0, + "60": 3064.0, + "61": 2717.0, + "62": 3205.0, + "63": 3221.0, + "64": 3611.0, + "65": 2546.0, + "66": 3082.0, + "67": 3897.0, + "68": 3433.0, + "69": 2811.0, + "70": 3152.0, + "71": 3057.0, + "72": 2886.0, + "73": 3398.0, + "74": 3141.0, + "75": 3303.0, + "76": 3082.0, + "77": 3699.0, + "78": 3202.0, + "79": 3280.0, + "80": 2967.0, + "81": 3343.0, + "82": 3265.0, + "83": 3238.0, + "84": 2825.0, + "85": 2671.0, + "86": 3089.0, + "87": 2996.0, + "88": 3092.0, + "89": 2993.0, + "90": 3698.0, + "91": 2993.0, + "92": 2947.0, + "93": 2961.0, + "94": 3288.0, + "95": 3457.0, + "96": 3487.0, + "97": 3603.0, + "98": 3452.0, + "99": 3293.0, + "100": 3251.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 765266432.0, - "2": 765266432.0, - "3": 765266432.0, - "4": 765266432.0, - "5": 765266432.0, - "6": 765266432.0, - "7": 765266432.0, - "8": 765266432.0, - "9": 765266432.0, - "10": 765266432.0, - "11": 765266432.0, - "12": 765266432.0, - "13": 765266432.0, - "14": 765266432.0, - "15": 765266432.0, - "16": 765266432.0, - "17": 765266432.0, - "18": 765266432.0, - "19": 765266432.0, - "20": 765266432.0, - "21": 765266432.0, - "22": 765266432.0, - "23": 765266432.0, - "24": 765266432.0, - "25": 765266432.0, - "26": 765266432.0, - "27": 765266432.0, - "28": 765266432.0, - "29": 765266432.0, - "30": 765266432.0, - "31": 765266432.0, - "32": 765266432.0, - "33": 765266432.0, - "34": 765266432.0, - "35": 765266432.0, - "36": 765266432.0, - "37": 765266432.0, - "38": 765266432.0, - "39": 765266432.0, - "40": 765266432.0, - "41": 765266432.0, - "42": 765266432.0, - "43": 765266432.0, - "44": 765266432.0, - "45": 765266432.0, - "46": 765266432.0, - "47": 765266432.0, - "48": 765266432.0, - "49": 765266432.0, - "50": 765266432.0, - "51": 765266432.0, - "52": 765266432.0, - "53": 765266432.0, - "54": 765266432.0, - "55": 765266432.0, - "56": 765266432.0, - "57": 765266432.0, - "58": 765266432.0, - "59": 765266432.0, - "60": 765266432.0, - "61": 765266432.0, - "62": 765266432.0, - "63": 765266432.0, - "64": 765266432.0, - "65": 765266432.0, - "66": 765266432.0, - "67": 765266432.0, - "68": 765266432.0, - "69": 765266432.0, - "70": 765266432.0, - "71": 765266432.0, - "72": 765266432.0, - "73": 765266432.0, - "74": 765266432.0, - "75": 765266432.0, - "76": 765266432.0, - "77": 765266432.0, - "78": 765266432.0, - "79": 765266432.0, - "80": 765266432.0, - "81": 765266432.0, - "82": 765266432.0, - "83": 765266432.0, - "84": 765266432.0, - "85": 765266432.0, - "86": 765266432.0, - "87": 765266432.0, - "88": 765266432.0, - "89": 765266432.0, - "90": 765266432.0, - "91": 765266432.0, - "92": 765266432.0, - "93": 765266432.0, - "94": 765266432.0, - "95": 765266432.0, - "96": 765266432.0, - "97": 765266432.0, - "98": 765266432.0, - "99": 765266432.0, - "100": 765266432.0 + "1": 763168768.0, + "2": 763168768.0, + "3": 763168768.0, + "4": 763168768.0, + "5": 763168768.0, + "6": 763168768.0, + "7": 763168768.0, + "8": 763168768.0, + "9": 763168768.0, + "10": 763168768.0, + "11": 763168768.0, + "12": 763168768.0, + "13": 763168768.0, + "14": 763168768.0, + "15": 763168768.0, + "16": 763168768.0, + "17": 763168768.0, + "18": 763168768.0, + "19": 763168768.0, + "20": 763168768.0, + "21": 763168768.0, + "22": 763168768.0, + "23": 763168768.0, + "24": 763168768.0, + "25": 763168768.0, + "26": 763168768.0, + "27": 763168768.0, + "28": 763168768.0, + "29": 763168768.0, + "30": 763168768.0, + "31": 763168768.0, + "32": 763168768.0, + "33": 763168768.0, + "34": 763168768.0, + "35": 763168768.0, + "36": 763168768.0, + "37": 763168768.0, + "38": 763168768.0, + "39": 763168768.0, + "40": 763168768.0, + "41": 763168768.0, + "42": 763168768.0, + "43": 763168768.0, + "44": 763168768.0, + "45": 763168768.0, + "46": 763168768.0, + "47": 763168768.0, + "48": 763168768.0, + "49": 763168768.0, + "50": 763168768.0, + "51": 763168768.0, + "52": 763168768.0, + "53": 763168768.0, + "54": 763168768.0, + "55": 763168768.0, + "56": 763168768.0, + "57": 763168768.0, + "58": 763168768.0, + "59": 763168768.0, + "60": 763168768.0, + "61": 763168768.0, + "62": 763168768.0, + "63": 763168768.0, + "64": 763168768.0, + "65": 763168768.0, + "66": 763168768.0, + "67": 763168768.0, + "68": 763168768.0, + "69": 763168768.0, + "70": 763168768.0, + "71": 763168768.0, + "72": 763168768.0, + "73": 763168768.0, + "74": 763168768.0, + "75": 763168768.0, + "76": 763168768.0, + "77": 763168768.0, + "78": 763168768.0, + "79": 763168768.0, + "80": 763168768.0, + "81": 763168768.0, + "82": 763168768.0, + "83": 763168768.0, + "84": 763168768.0, + "85": 763168768.0, + "86": 763168768.0, + "87": 763168768.0, + "88": 763168768.0, + "89": 763168768.0, + "90": 763168768.0, + "91": 763168768.0, + "92": 763168768.0, + "93": 763168768.0, + "94": 763168768.0, + "95": 763168768.0, + "96": 763168768.0, + "97": 763168768.0, + "98": 763168768.0, + "99": 763168768.0, + "100": 763168768.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1951401984.0, - "2": 2237197312.0, - "3": 2237197312.0, - "4": 2237197312.0, - "5": 2237197312.0, - "6": 2237197312.0, - "7": 2237197312.0, - "8": 2237197312.0, - "9": 2237197312.0, - "10": 2237197312.0, - "11": 2237197312.0, - "12": 2237197312.0, - "13": 2237197312.0, - "14": 2237197312.0, - "15": 2237197312.0, - "16": 2237197312.0, - "17": 2237197312.0, - "18": 2237197312.0, - "19": 2237197312.0, - "20": 2237197312.0, - "21": 2237197312.0, - "22": 2237197312.0, - "23": 2237197312.0, - "24": 2237197312.0, - "25": 2237197312.0, - "26": 2237197312.0, - "27": 2237197312.0, - "28": 2237197312.0, - "29": 2237197312.0, - "30": 2237197312.0, - "31": 2237197312.0, - "32": 2237197312.0, - "33": 2237197312.0, - "34": 2237197312.0, - "35": 2237197312.0, - "36": 2237197312.0, - "37": 2237197312.0, - "38": 2237197312.0, - "39": 2237197312.0, - "40": 2237197312.0, - "41": 2237197312.0, - "42": 2237197312.0, - "43": 2237197312.0, - "44": 2237197312.0, - "45": 2237197312.0, - "46": 2237197312.0, - "47": 2237197312.0, - "48": 2237197312.0, - "49": 2237197312.0, - "50": 2237197312.0, - "51": 2237197312.0, - "52": 2237197312.0, - "53": 2237197312.0, - "54": 2237197312.0, - "55": 2237197312.0, - "56": 2237197312.0, - "57": 2237197312.0, - "58": 2237197312.0, - "59": 2237197312.0, - "60": 2237197312.0, - "61": 2237197312.0, - "62": 2237197312.0, - "63": 2237197312.0, - "64": 2237197312.0, - "65": 2237197312.0, - "66": 2237197312.0, - "67": 2237197312.0, - "68": 2237197312.0, - "69": 2237197312.0, - "70": 2237197312.0, - "71": 2237197312.0, - "72": 2237197312.0, - "73": 2237197312.0, - "74": 2237197312.0, - "75": 2237197312.0, - "76": 2237197312.0, - "77": 2237197312.0, - "78": 2237197312.0, - "79": 2237197312.0, - "80": 2237197312.0, - "81": 2237197312.0, - "82": 2237197312.0, - "83": 2237197312.0, - "84": 2237197312.0, - "85": 2237197312.0, - "86": 2237197312.0, - "87": 2237197312.0, - "88": 2237197312.0, - "89": 2237197312.0, - "90": 2237197312.0, - "91": 2237197312.0, - "92": 2237197312.0, - "93": 2237197312.0, - "94": 2237197312.0, - "95": 2237197312.0, - "96": 2237197312.0, - "97": 2237197312.0, - "98": 2237197312.0, - "99": 2237197312.0, - "100": 2237197312.0 + "1": 1951401472.0, + "2": 2235099648.0, + "3": 2235099648.0, + "4": 2235099648.0, + "5": 2235099648.0, + "6": 2235099648.0, + "7": 2235099648.0, + "8": 2235099648.0, + "9": 2235099648.0, + "10": 2235099648.0, + "11": 2235099648.0, + "12": 2235099648.0, + "13": 2235099648.0, + "14": 2235099648.0, + "15": 2235099648.0, + "16": 2235099648.0, + "17": 2235099648.0, + "18": 2235099648.0, + "19": 2235099648.0, + "20": 2235099648.0, + "21": 2235099648.0, + "22": 2235099648.0, + "23": 2235099648.0, + "24": 2235099648.0, + "25": 2235099648.0, + "26": 2235099648.0, + "27": 2235099648.0, + "28": 2235099648.0, + "29": 2235099648.0, + "30": 2235099648.0, + "31": 2235099648.0, + "32": 2235099648.0, + "33": 2235099648.0, + "34": 2235099648.0, + "35": 2235099648.0, + "36": 2235099648.0, + "37": 2235099648.0, + "38": 2235099648.0, + "39": 2235099648.0, + "40": 2235099648.0, + "41": 2235099648.0, + "42": 2235099648.0, + "43": 2235099648.0, + "44": 2235099648.0, + "45": 2235099648.0, + "46": 2235099648.0, + "47": 2235099648.0, + "48": 2235099648.0, + "49": 2235099648.0, + "50": 2235099648.0, + "51": 2235099648.0, + "52": 2235099648.0, + "53": 2235099648.0, + "54": 2235099648.0, + "55": 2235099648.0, + "56": 2235099648.0, + "57": 2235099648.0, + "58": 2235099648.0, + "59": 2235099648.0, + "60": 2235099648.0, + "61": 2235099648.0, + "62": 2235099648.0, + "63": 2235099648.0, + "64": 2235099648.0, + "65": 2235099648.0, + "66": 2235099648.0, + "67": 2235099648.0, + "68": 2235099648.0, + "69": 2235099648.0, + "70": 2235099648.0, + "71": 2235099648.0, + "72": 2235099648.0, + "73": 2235099648.0, + "74": 2235099648.0, + "75": 2235099648.0, + "76": 2235099648.0, + "77": 2235099648.0, + "78": 2235099648.0, + "79": 2235099648.0, + "80": 2235099648.0, + "81": 2235099648.0, + "82": 2235099648.0, + "83": 2235099648.0, + "84": 2235099648.0, + "85": 2235099648.0, + "86": 2235099648.0, + "87": 2235099648.0, + "88": 2235099648.0, + "89": 2235099648.0, + "90": 2235099648.0, + "91": 2235099648.0, + "92": 2235099648.0, + "93": 2235099648.0, + "94": 2235099648.0, + "95": 2235099648.0, + "96": 2235099648.0, + "97": 2235099648.0, + "98": 2235099648.0, + "99": 2235099648.0, + "100": 2235099648.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.43956, - "3": 0.15591, - "4": 0.11266, - "5": 0.11071, - "6": 0.11164, - "7": 0.11263, - "8": 0.11053, - "9": 0.1119, - "10": 0.11327, - "11": 0.11236, - "12": 0.11186, - "13": 0.11135, - "14": 0.11183, - "15": 0.11118, - "16": 0.11207, - "17": 0.11133, - "18": 0.1123, - "19": 0.11152, - "20": 0.11121, - "21": 0.11172, - "22": 0.11247, - "23": 0.11116, - "24": 0.11227, - "25": 0.11313, - "26": 0.11182, - "27": 0.11243, - "28": 0.11269, - "29": 0.11404, - "30": 0.11147, - "31": 0.11086, - "32": 0.11211, - "33": 0.11277, - "34": 0.11007, - "35": 0.11207, - "36": 0.11105, - "37": 0.1112, - "38": 0.11245, - "39": 0.112, - "40": 0.11066, - "41": 0.11274, - "42": 0.11112, - "43": 0.11204, - "44": 0.10911, - "45": 0.11112, - "46": 0.11171, - "47": 0.11241, - "48": 0.11141, - "49": 0.11332, - "50": 0.11241, - "51": 0.29996, - "52": 0.13199, - "53": 0.15643, - "54": 0.11642, - "55": 0.11353, - "56": 0.11454, - "57": 0.11525, - "58": 0.11381, - "59": 0.11329, - "60": 0.11422, - "61": 0.11409, - "62": 0.11194, - "63": 0.11257, - "64": 0.11326, - "65": 0.1138, - "66": 0.11399, - "67": 0.11526, - "68": 0.11439, - "69": 0.11328, - "70": 0.1133, - "71": 0.11543, - "72": 0.11457, - "73": 0.11312, - "74": 0.1134, - "75": 0.11278, - "76": 0.11378, - "77": 0.11398, - "78": 0.11654, - "79": 0.11604, - "80": 0.11511, - "81": 0.11532, - "82": 0.11535, - "83": 0.11554, - "84": 0.11277, - "85": 0.11342, - "86": 0.11073, - "87": 0.11279, - "88": 0.11241, - "89": 0.11427, - "90": 0.11169, - "91": 0.11291, - "92": 0.11217, - "93": 0.11356, - "94": 0.11284, - "95": 0.11405, - "96": 0.11437, - "97": 0.11406, - "98": 0.11274, - "99": 0.11268, - "100": 0.11335 + "2": 4.7245, + "3": 0.16483, + "4": 0.12552, + "5": 0.13055, + "6": 0.12956, + "7": 0.12865, + "8": 0.12901, + "9": 0.12281, + "10": 0.11612, + "11": 0.11663, + "12": 0.11767, + "13": 0.11793, + "14": 0.11413, + "15": 0.11713, + "16": 0.11773, + "17": 0.12378, + "18": 0.11913, + "19": 0.11572, + "20": 0.11423, + "21": 0.11481, + "22": 0.11565, + "23": 0.11567, + "24": 0.11202, + "25": 0.11089, + "26": 0.1106, + "27": 0.11097, + "28": 0.11083, + "29": 0.11228, + "30": 0.1119, + "31": 0.11204, + "32": 0.1118, + "33": 0.10876, + "34": 0.11162, + "35": 0.11078, + "36": 0.11215, + "37": 0.11223, + "38": 0.11117, + "39": 0.11118, + "40": 0.11208, + "41": 0.1108, + "42": 0.113, + "43": 0.12352, + "44": 0.12188, + "45": 0.12161, + "46": 0.12128, + "47": 0.11859, + "48": 0.12476, + "49": 0.12818, + "50": 0.12502, + "51": 0.19743, + "52": 0.16188, + "53": 0.1215, + "54": 0.11167, + "55": 0.10935, + "56": 0.11229, + "57": 0.1119, + "58": 0.11127, + "59": 0.112, + "60": 0.11188, + "61": 0.11222, + "62": 0.11041, + "63": 0.1101, + "64": 0.11237, + "65": 0.11352, + "66": 0.11518, + "67": 0.11072, + "68": 0.11374, + "69": 0.11088, + "70": 0.11202, + "71": 0.13077, + "72": 0.23073, + "73": 0.13575, + "74": 0.13364, + "75": 0.13338, + "76": 0.13041, + "77": 0.13312, + "78": 0.13183, + "79": 0.13538, + "80": 0.12854, + "81": 0.12646, + "82": 0.12526, + "83": 0.1271, + "84": 0.12702, + "85": 0.12604, + "86": 0.12725, + "87": 0.12614, + "88": 0.12518, + "89": 0.12824, + "90": 0.12616, + "91": 0.12779, + "92": 0.12593, + "93": 0.12815, + "94": 0.12779, + "95": 0.12397, + "96": 0.12464, + "97": 0.1233, + "98": 0.12148, + "99": 0.24012, + "100": 0.12575 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json index c025959fae6..12b4d73f9aa 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 30093520.0, - "2": 30296762.0, - "3": 29948276.0, - "4": 30610300.0, - "5": 30079840.0, - "6": 30406552.0, - "7": 30137690.0, - "8": 30305472.0, - "9": 30210696.0, - "10": 30295356.0, - "11": 29844032.0, - "12": 29800684.0, - "13": 30292756.0, - "14": 29728990.0, - "15": 30191228.0, - "16": 30201372.0, - "17": 30181048.0, - "18": 29934552.0, - "19": 29971900.0, - "20": 30057046.0, - "21": 30107032.0, - "22": 30162132.0, - "23": 29889884.0, - "24": 30139390.0, - "25": 30186684.0, - "26": 29894584.0, - "27": 29809380.0, - "28": 29798532.0, - "29": 29879004.0, - "30": 29980556.0, - "31": 30332928.0, - "32": 29938752.0, - "33": 29911076.0, - "34": 30200686.0, - "35": 30158358.0, - "36": 29940580.0, - "37": 29844900.0, - "38": 30271128.0, - "39": 30169744.0, - "40": 30020668.0, - "41": 30016156.0, - "42": 30027476.0, - "43": 30358288.0, - "44": 30110080.0, - "45": 30023836.0, - "46": 30256302.0, - "47": 29987144.0, - "48": 30298844.0, - "49": 30082964.0, - "50": 30275864.0 + "1": 30093584.0, + "2": 30296868.0, + "3": 29948216.0, + "4": 30610452.0, + "5": 30079954.0, + "6": 30406440.0, + "7": 30137332.0, + "8": 30305308.0, + "9": 30210788.0, + "10": 30295208.0, + "11": 29844004.0, + "12": 29800708.0, + "13": 30292576.0, + "14": 29729006.0, + "15": 30191246.0, + "16": 30201442.0, + "17": 30181084.0, + "18": 29934542.0, + "19": 29972104.0, + "20": 30057016.0, + "21": 30106920.0, + "22": 30162128.0, + "23": 29890266.0, + "24": 30139388.0, + "25": 30186512.0, + "26": 29894670.0, + "27": 29809340.0, + "28": 29798496.0, + "29": 29879152.0, + "30": 29980394.0, + "31": 30332976.0, + "32": 29938440.0, + "33": 29910984.0, + "34": 30200552.0, + "35": 30158306.0, + "36": 29940436.0, + "37": 29844712.0, + "38": 30271032.0, + "39": 30169892.0, + "40": 30020722.0, + "41": 30016120.0, + "42": 30027286.0, + "43": 30358212.0, + "44": 30109930.0, + "45": 30023796.0, + "46": 30256184.0, + "47": 29987288.0, + "48": 30298758.0, + "49": 30082702.0, + "50": 30275984.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 628049408.0, - "2": 628050944.0, - "3": 628050944.0, - "4": 628050944.0, - "5": 628050944.0, - "6": 628050944.0, - "7": 628050944.0, - "8": 628050944.0, - "9": 628050944.0, - "10": 628050944.0, - "11": 628050944.0, - "12": 628050944.0, - "13": 628050944.0, - "14": 628050944.0, - "15": 628050944.0, - "16": 628050944.0, - "17": 628050944.0, - "18": 628050944.0, - "19": 628050944.0, - "20": 628050944.0, - "21": 628050944.0, - "22": 628050944.0, - "23": 628050944.0, - "24": 628050944.0, - "25": 628050944.0, - "26": 628050944.0, - "27": 628050944.0, - "28": 628050944.0, - "29": 628050944.0, - "30": 628050944.0, - "31": 628050944.0, - "32": 628050944.0, - "33": 628050944.0, - "34": 628050944.0, - "35": 628050944.0, - "36": 628050944.0, - "37": 628050944.0, - "38": 628050944.0, - "39": 628050944.0, - "40": 628050944.0, - "41": 628050944.0, - "42": 628050944.0, - "43": 628050944.0, - "44": 628050944.0, - "45": 628050944.0, - "46": 628050944.0, - "47": 628050944.0, - "48": 628050944.0, - "49": 628050944.0, - "50": 628050944.0 + "1": 628048896.0, + "2": 628050432.0, + "3": 628050432.0, + "4": 628050432.0, + "5": 628050432.0, + "6": 628050432.0, + "7": 628050432.0, + "8": 628050432.0, + "9": 628050432.0, + "10": 628050432.0, + "11": 628050432.0, + "12": 628050432.0, + "13": 628050432.0, + "14": 628050432.0, + "15": 628050432.0, + "16": 628050432.0, + "17": 628050432.0, + "18": 628050432.0, + "19": 628050432.0, + "20": 628050432.0, + "21": 628050432.0, + "22": 628050432.0, + "23": 628050432.0, + "24": 628050432.0, + "25": 628050432.0, + "26": 628050432.0, + "27": 628050432.0, + "28": 628050432.0, + "29": 628050432.0, + "30": 628050432.0, + "31": 628050432.0, + "32": 628050432.0, + "33": 628050432.0, + "34": 628050432.0, + "35": 628050432.0, + "36": 628050432.0, + "37": 628050432.0, + "38": 628050432.0, + "39": 628050432.0, + "40": 628050432.0, + "41": 628050432.0, + "42": 628050432.0, + "43": 628050432.0, + "44": 628050432.0, + "45": 628050432.0, + "46": 628050432.0, + "47": 628050432.0, + "48": 628050432.0, + "49": 628050432.0, + "50": 628050432.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 2302538240.0, - "2": 2346255872.0, - "3": 2355607040.0, - "4": 2358100480.0, - "5": 2360282624.0, - "6": 2360282624.0, - "7": 2360282624.0, - "8": 2360282624.0, - "9": 2360282624.0, - "10": 2360282624.0, - "11": 2360282624.0, - "12": 2360282624.0, - "13": 2360282624.0, - "14": 2360905216.0, - "15": 2360905216.0, - "16": 2360905216.0, - "17": 2360905216.0, - "18": 2360905216.0, - "19": 2360905216.0, - "20": 2360905216.0, - "21": 2360905216.0, - "22": 2360905216.0, - "23": 2360905216.0, - "24": 2360905216.0, - "25": 2360905216.0, - "26": 2360905216.0, - "27": 2360905216.0, - "28": 2360905216.0, - "29": 2360905216.0, - "30": 2360905216.0, - "31": 2360905216.0, - "32": 2360905216.0, - "33": 2360905216.0, - "34": 2360905216.0, - "35": 2360905216.0, - "36": 2360905216.0, - "37": 2360905216.0, - "38": 2360905216.0, - "39": 2369320960.0, - "40": 2369320960.0, - "41": 2369320960.0, - "42": 2369320960.0, - "43": 2369320960.0, - "44": 2369320960.0, - "45": 2376177152.0, - "46": 2376177152.0, - "47": 2376177152.0, - "48": 2376177152.0, - "49": 2376177152.0, - "50": 2376177152.0 + "1": 2302537728.0, + "2": 2346255360.0, + "3": 2354982400.0, + "4": 2357787648.0, + "5": 2360281088.0, + "6": 2360281088.0, + "7": 2360281088.0, + "8": 2360281088.0, + "9": 2360281088.0, + "10": 2360281088.0, + "11": 2360281088.0, + "12": 2360281088.0, + "13": 2360281088.0, + "14": 2361527808.0, + "15": 2361527808.0, + "16": 2361527808.0, + "17": 2361527808.0, + "18": 2361527808.0, + "19": 2361527808.0, + "20": 2361527808.0, + "21": 2361527808.0, + "22": 2361527808.0, + "23": 2361527808.0, + "24": 2361527808.0, + "25": 2361527808.0, + "26": 2361527808.0, + "27": 2361527808.0, + "28": 2361527808.0, + "29": 2361527808.0, + "30": 2361527808.0, + "31": 2361527808.0, + "32": 2361527808.0, + "33": 2361527808.0, + "34": 2361527808.0, + "35": 2361527808.0, + "36": 2361527808.0, + "37": 2361527808.0, + "38": 2361527808.0, + "39": 2369631744.0, + "40": 2369631744.0, + "41": 2369631744.0, + "42": 2369631744.0, + "43": 2369631744.0, + "44": 2369631744.0, + "45": 2376800768.0, + "46": 2376800768.0, + "47": 2376800768.0, + "48": 2376800768.0, + "49": 2376800768.0, + "50": 2376800768.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 17.10942, - "3": 0.11674, - "4": 0.09185, - "5": 0.07633, - "6": 0.07651, - "7": 0.07484, - "8": 0.0706, - "9": 0.07357, - "10": 0.07127, - "11": 0.07069, - "12": 0.07632, - "13": 0.09386, - "14": 0.08494, - "15": 0.09136, - "16": 0.07536, - "17": 0.07425, - "18": 0.06628, - "19": 0.08648, - "20": 0.0728, - "21": 0.06799, - "22": 0.06666, - "23": 0.079, - "24": 0.0675, - "25": 0.06776, - "26": 0.07078, - "27": 0.06793, - "28": 0.07086, - "29": 0.06341, - "30": 0.06448, - "31": 0.0681, - "32": 0.07263, - "33": 0.06646, - "34": 0.06892, - "35": 0.06588, - "36": 0.06626, - "37": 0.06204, - "38": 0.06589, - "39": 0.07185, - "40": 0.07025, - "41": 0.06959, - "42": 0.06892, - "43": 0.06501, - "44": 0.06867, - "45": 0.06511, - "46": 0.06744, - "47": 0.06562, - "48": 0.06349, - "49": 0.06559, - "50": 0.06571 + "2": 9.87919, + "3": 0.1183, + "4": 0.07747, + "5": 0.07582, + "6": 0.07295, + "7": 0.07913, + "8": 0.08239, + "9": 0.08118, + "10": 0.09287, + "11": 0.08524, + "12": 0.07781, + "13": 0.08579, + "14": 0.08789, + "15": 0.0815, + "16": 0.07673, + "17": 0.08139, + "18": 0.07562, + "19": 0.08122, + "20": 0.08078, + "21": 0.07455, + "22": 0.07785, + "23": 0.07512, + "24": 0.0754, + "25": 0.07364, + "26": 0.07369, + "27": 0.07364, + "28": 0.07999, + "29": 0.07604, + "30": 0.07629, + "31": 0.07634, + "32": 0.07623, + "33": 0.07633, + "34": 0.07598, + "35": 0.07505, + "36": 0.07419, + "37": 0.07349, + "38": 0.07408, + "39": 0.07567, + "40": 0.07946, + "41": 0.07267, + "42": 0.07957, + "43": 0.07514, + "44": 0.07761, + "45": 0.07684, + "46": 0.07451, + "47": 0.07465, + "48": 0.07152, + "49": 0.07303, + "50": 0.07258 } } -} +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json index 8860e1b836a..8076a9a1481 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.91589, "2": 10.90332, - "3": 10.90347, - "4": 10.89774, - "5": 10.9187, - "6": 10.90932, - "7": 10.89761, - "8": 10.90061, - "9": 10.9039, - "10": 10.90834, - "11": 10.89574, - "12": 10.90275, - "13": 10.88282, - "14": 10.88306, - "15": 10.87697, - "16": 10.86788, - "17": 10.86436, - "18": 10.85948, - "19": 10.86322, - "20": 10.81598, - "21": 10.80064, - "22": 10.79033, - "23": 10.78083, - "24": 10.74886, - "25": 10.75938, - "26": 10.73731, - "27": 10.72307, - "28": 10.67446, - "29": 10.63463, - "30": 10.61775, - "31": 10.60463, - "32": 10.58936, - "33": 10.56185, - "34": 10.53587, - "35": 10.52959, - "36": 10.51926, - "37": 10.48229, + "3": 10.90407, + "4": 10.89821, + "5": 10.91875, + "6": 10.90975, + "7": 10.89778, + "8": 10.90082, + "9": 10.90382, + "10": 10.90689, + "11": 10.89562, + "12": 10.90277, + "13": 10.88364, + "14": 10.88193, + "15": 10.87673, + "16": 10.86723, + "17": 10.86388, + "18": 10.85869, + "19": 10.8634, + "20": 10.8156, + "21": 10.80014, + "22": 10.79085, + "23": 10.78152, + "24": 10.74885, + "25": 10.75953, + "26": 10.7368, + "27": 10.72342, + "28": 10.67507, + "29": 10.63529, + "30": 10.61804, + "31": 10.60399, + "32": 10.58931, + "33": 10.56203, + "34": 10.53574, + "35": 10.52913, + "36": 10.51959, + "37": 10.48264, "38": 10.48784, - "39": 10.45258, - "40": 10.43514, - "41": 10.41533, - "42": 10.40609, - "43": 10.38854, - "44": 10.34231, - "45": 10.36248, - "46": 10.32544, - "47": 10.30585, - "48": 10.27412, - "49": 10.26208, - "50": 10.26192, - "51": 10.26006, - "52": 10.21685, - "53": 10.22334, - "54": 10.17948, - "55": 10.16584, - "56": 10.17775, - "57": 10.16954, - "58": 10.17712, - "59": 10.13005, - "60": 10.14441, - "61": 10.09527, - "62": 10.0697, - "63": 10.12716, - "64": 10.09497, - "65": 10.07382, - "66": 10.08357, - "67": 10.06349, - "68": 10.02719, - "69": 10.05119, - "70": 10.02734, - "71": 10.04629, - "72": 10.03295, - "73": 10.02503, - "74": 10.00651, - "75": 9.98677, - "76": 10.00597, - "77": 10.00709, - "78": 9.95735, + "39": 10.45296, + "40": 10.43609, + "41": 10.41508, + "42": 10.40568, + "43": 10.38772, + "44": 10.34243, + "45": 10.36316, + "46": 10.32577, + "47": 10.30545, + "48": 10.27465, + "49": 10.2621, + "50": 10.26196, + "51": 10.25996, + "52": 10.21733, + "53": 10.22335, + "54": 10.17957, + "55": 10.16598, + "56": 10.17779, + "57": 10.16958, + "58": 10.17716, + "59": 10.13019, + "60": 10.14434, + "61": 10.09531, + "62": 10.06939, + "63": 10.12712, + "64": 10.09521, + "65": 10.07372, + "66": 10.08329, + "67": 10.06351, + "68": 10.0273, + "69": 10.05115, + "70": 10.02689, + "71": 10.04622, + "72": 10.03284, + "73": 10.02484, + "74": 10.00659, + "75": 9.98658, + "76": 10.00581, + "77": 10.00689, + "78": 9.95727, "79": 9.96947, - "80": 9.98336, - "81": 10.01128, - "82": 9.94198, - "83": 9.9208, - "84": 9.85035, - "85": 9.84797, - "86": 9.93302, - "87": 9.96425, - "88": 9.93278, + "80": 9.98335, + "81": 10.01126, + "82": 9.9418, + "83": 9.92067, + "84": 9.85009, + "85": 9.84809, + "86": 9.93341, + "87": 9.96458, + "88": 9.93316, "89": 9.87206, - "90": 9.87717, - "91": 9.89303, - "92": 9.87591, - "93": 9.81089, - "94": 9.89098, - "95": 9.87301, - "96": 9.854, - "97": 9.79985, - "98": 9.83386, - "99": 9.87162, - "100": 9.76501 + "90": 9.87737, + "91": 9.89316, + "92": 9.87589, + "93": 9.81148, + "94": 9.89101, + "95": 9.87284, + "96": 9.85381, + "97": 9.79974, + "98": 9.83405, + "99": 9.87114, + "100": 9.76527 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36227.0, - "2": 37201.0, - "3": 36774.0, - "4": 36570.0, - "5": 36726.0, - "6": 35453.0, - "7": 36952.0, - "8": 36230.0, - "9": 37164.0, - "10": 37327.0, - "11": 36787.0, - "12": 35527.0, - "13": 36236.0, - "14": 36776.0, - "15": 35521.0, - "16": 36024.0, - "17": 36158.0, - "18": 36460.0, - "19": 36280.0, - "20": 36245.0, - "21": 36531.0, - "22": 36435.0, - "23": 36450.0, - "24": 37114.0, - "25": 35514.0, - "26": 36289.0, - "27": 36220.0, - "28": 35094.0, - "29": 35685.0, - "30": 35235.0, - "31": 36128.0, - "32": 36397.0, - "33": 35851.0, - "34": 35612.0, - "35": 36610.0, - "36": 35768.0, - "37": 36402.0, - "38": 35569.0, - "39": 36310.0, - "40": 37593.0, - "41": 36111.0, - "42": 35320.0, - "43": 37666.0, - "44": 36410.0, - "45": 39411.0, - "46": 38337.0, - "47": 38141.0, - "48": 39163.0, - "49": 42791.0, - "50": 39350.0, - "51": 38770.0, - "52": 41454.0, - "53": 40064.0, - "54": 41412.0, - "55": 39457.0, - "56": 41555.0, - "57": 37183.0, - "58": 46390.0, - "59": 42779.0, - "60": 42192.0, - "61": 42281.0, - "62": 45440.0, - "63": 44644.0, - "64": 49137.0, - "65": 41769.0, - "66": 45654.0, - "67": 49704.0, - "68": 46553.0, - "69": 46016.0, - "70": 46937.0, - "71": 46350.0, - "72": 45770.0, - "73": 47491.0, - "74": 46137.0, - "75": 48941.0, - "76": 48261.0, - "77": 47890.0, - "78": 48949.0, - "79": 48015.0, - "80": 45941.0, - "81": 55431.0, - "82": 41811.0, - "83": 46772.0, - "84": 45180.0, - "85": 45564.0, - "86": 46154.0, - "87": 36767.0, - "88": 45300.0, - "89": 48760.0, - "90": 57596.0, - "91": 39387.0, - "92": 49929.0, - "93": 45177.0, - "94": 42214.0, - "95": 45091.0, - "96": 50439.0, - "97": 46128.0, - "98": 50578.0, - "99": 45725.0, - "100": 47131.0 + "1": 36420.0, + "2": 37155.0, + "3": 36644.0, + "4": 36934.0, + "5": 36624.0, + "6": 35528.0, + "7": 36602.0, + "8": 36137.0, + "9": 37058.0, + "10": 37638.0, + "11": 37147.0, + "12": 35256.0, + "13": 36589.0, + "14": 36757.0, + "15": 36060.0, + "16": 35696.0, + "17": 36643.0, + "18": 36333.0, + "19": 36297.0, + "20": 36321.0, + "21": 36213.0, + "22": 36327.0, + "23": 36718.0, + "24": 36750.0, + "25": 35567.0, + "26": 36216.0, + "27": 36465.0, + "28": 35402.0, + "29": 35596.0, + "30": 35576.0, + "31": 36306.0, + "32": 36472.0, + "33": 35767.0, + "34": 36040.0, + "35": 36172.0, + "36": 35581.0, + "37": 36269.0, + "38": 35631.0, + "39": 36503.0, + "40": 37789.0, + "41": 35801.0, + "42": 35628.0, + "43": 37094.0, + "44": 36427.0, + "45": 39416.0, + "46": 37619.0, + "47": 38315.0, + "48": 39306.0, + "49": 42105.0, + "50": 39301.0, + "51": 38683.0, + "52": 41456.0, + "53": 40117.0, + "54": 41408.0, + "55": 39529.0, + "56": 42106.0, + "57": 37267.0, + "58": 46199.0, + "59": 41914.0, + "60": 42984.0, + "61": 42296.0, + "62": 45372.0, + "63": 44098.0, + "64": 49051.0, + "65": 41645.0, + "66": 46171.0, + "67": 49569.0, + "68": 46939.0, + "69": 46269.0, + "70": 46721.0, + "71": 46207.0, + "72": 45459.0, + "73": 47588.0, + "74": 45737.0, + "75": 48621.0, + "76": 48260.0, + "77": 48175.0, + "78": 49398.0, + "79": 47843.0, + "80": 45523.0, + "81": 54755.0, + "82": 42367.0, + "83": 46969.0, + "84": 45147.0, + "85": 45749.0, + "86": 46128.0, + "87": 36474.0, + "88": 45383.0, + "89": 48647.0, + "90": 56883.0, + "91": 39433.0, + "92": 49245.0, + "93": 45029.0, + "94": 42576.0, + "95": 44773.0, + "96": 50005.0, + "97": 46352.0, + "98": 50263.0, + "99": 45829.0, + "100": 47142.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 892868608.0, - "2": 892867584.0, - "3": 892869120.0, - "4": 892869632.0, - "5": 892868096.0, - "6": 892866048.0, - "7": 892868608.0, - "8": 892870144.0, - "9": 892871680.0, - "10": 892871168.0, - "11": 892869632.0, - "12": 892870144.0, - "13": 892869632.0, - "14": 892871168.0, - "15": 892870656.0, + "1": 892868096.0, + "2": 892867072.0, + "3": 892869632.0, + "4": 892870144.0, + "5": 892867584.0, + "6": 892864512.0, + "7": 892867584.0, + "8": 892868608.0, + "9": 892871168.0, + "10": 892870656.0, + "11": 892869120.0, + "12": 892869632.0, + "13": 892868608.0, + "14": 892869632.0, + "15": 892869120.0, "16": 892870144.0, - "17": 892870656.0, - "18": 892870656.0, - "19": 892871680.0, - "20": 892870144.0, - "21": 892867584.0, - "22": 892866560.0, - "23": 892869632.0, - "24": 892869120.0, - "25": 892864512.0, - "26": 892870144.0, - "27": 892870144.0, - "28": 892865536.0, - "29": 892870144.0, - "30": 892870144.0, + "17": 892869632.0, + "18": 892870144.0, + "19": 892871168.0, + "20": 892870656.0, + "21": 892868096.0, + "22": 892865024.0, + "23": 892868608.0, + "24": 892869632.0, + "25": 892864000.0, + "26": 892871168.0, + "27": 892869632.0, + "28": 892866048.0, + "29": 892868096.0, + "30": 892869632.0, "31": 892871680.0, - "32": 892866048.0, - "33": 892870656.0, - "34": 892868608.0, - "35": 892869120.0, - "36": 892868608.0, - "37": 892866048.0, - "38": 892869120.0, - "39": 892866048.0, - "40": 892869120.0, - "41": 892869632.0, + "32": 892865536.0, + "33": 892870144.0, + "34": 892868096.0, + "35": 892867584.0, + "36": 892867584.0, + "37": 892866560.0, + "38": 892868096.0, + "39": 892867072.0, + "40": 892868608.0, + "41": 892869120.0, "42": 892866560.0, - "43": 892869120.0, - "44": 892867584.0, - "45": 892870144.0, - "46": 892869120.0, - "47": 892869632.0, - "48": 892869632.0, - "49": 892870656.0, - "50": 892868096.0, - "51": 892870144.0, - "52": 892870144.0, - "53": 892868608.0, - "54": 892867584.0, - "55": 892868096.0, - "56": 892867072.0, + "43": 892868608.0, + "44": 892867072.0, + "45": 892868608.0, + "46": 892868096.0, + "47": 892867584.0, + "48": 892868608.0, + "49": 892869632.0, + "50": 892867584.0, + "51": 892868608.0, + "52": 892869632.0, + "53": 892867072.0, + "54": 892866560.0, + "55": 892867072.0, + "56": 892864512.0, "57": 892869120.0, - "58": 892871168.0, - "59": 892868096.0, - "60": 892866560.0, - "61": 892865536.0, - "62": 892868096.0, + "58": 892870144.0, + "59": 892867072.0, + "60": 892866048.0, + "61": 892864512.0, + "62": 892867072.0, "63": 892868096.0, - "64": 892872192.0, - "65": 892866560.0, - "66": 892868096.0, - "67": 892869120.0, - "68": 892868096.0, - "69": 892868096.0, - "70": 892866048.0, - "71": 892868608.0, - "72": 892868096.0, - "73": 892868096.0, + "64": 892871680.0, + "65": 892867072.0, + "66": 892867584.0, + "67": 892867072.0, + "68": 892867072.0, + "69": 892867584.0, + "70": 892865536.0, + "71": 892869120.0, + "72": 892867072.0, + "73": 892867072.0, "74": 892866048.0, "75": 892866048.0, - "76": 892870144.0, - "77": 892868608.0, + "76": 892869632.0, + "77": 892866560.0, "78": 892865536.0, - "79": 892870144.0, + "79": 892868096.0, "80": 892868608.0, - "81": 892866560.0, - "82": 892868608.0, - "83": 892866560.0, - "84": 892870656.0, - "85": 892865536.0, + "81": 892865024.0, + "82": 892868096.0, + "83": 892866048.0, + "84": 892871680.0, + "85": 892866560.0, "86": 892868096.0, - "87": 892867584.0, + "87": 892867072.0, "88": 892866048.0, - "89": 892868608.0, - "90": 892871168.0, + "89": 892869632.0, + "90": 892871680.0, "91": 892866560.0, - "92": 892870656.0, + "92": 892869120.0, "93": 892866560.0, - "94": 892866560.0, - "95": 892876288.0, - "96": 892868096.0, - "97": 892870144.0, + "94": 892865536.0, + "95": 892875264.0, + "96": 892867584.0, + "97": 892869632.0, "98": 892872704.0, - "99": 892870656.0, - "100": 892876800.0 + "99": 892869120.0, + "100": 892874752.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1943461888.0, - "2": 2184493056.0, - "3": 2184493056.0, - "4": 2187152384.0, - "5": 2187152384.0, - "6": 2187152384.0, - "7": 2187152384.0, - "8": 2187152384.0, - "9": 2187152384.0, - "10": 2187152384.0, - "11": 2187152384.0, - "12": 2187152384.0, - "13": 2187152384.0, - "14": 2187152384.0, - "15": 2187152384.0, - "16": 2187152384.0, - "17": 2187152384.0, - "18": 2187152384.0, - "19": 2187152384.0, - "20": 2187152384.0, - "21": 2187152384.0, - "22": 2187152384.0, - "23": 2187152384.0, - "24": 2187152384.0, - "25": 2187152384.0, - "26": 2187152384.0, - "27": 2187152384.0, - "28": 2187152384.0, - "29": 2187152384.0, - "30": 2187152384.0, - "31": 2187152384.0, - "32": 2187152384.0, - "33": 2187152384.0, - "34": 2187152384.0, - "35": 2187152384.0, - "36": 2187152384.0, - "37": 2187152384.0, - "38": 2187152384.0, - "39": 2187152384.0, - "40": 2187152384.0, - "41": 2187152384.0, - "42": 2187152384.0, - "43": 2187152384.0, - "44": 2187152384.0, - "45": 2187152384.0, - "46": 2187152384.0, - "47": 2187152384.0, - "48": 2187152384.0, - "49": 2187152384.0, - "50": 2187152384.0, - "51": 2187152384.0, - "52": 2187152384.0, - "53": 2187152384.0, - "54": 2187152384.0, - "55": 2187152384.0, - "56": 2187152384.0, - "57": 2187152384.0, - "58": 2187152384.0, - "59": 2187152384.0, - "60": 2187152384.0, - "61": 2187152384.0, - "62": 2187152384.0, - "63": 2187152384.0, - "64": 2187152384.0, - "65": 2187152384.0, - "66": 2187152384.0, - "67": 2187152384.0, - "68": 2187152384.0, - "69": 2187152384.0, - "70": 2187152384.0, - "71": 2187152384.0, - "72": 2187152384.0, - "73": 2187152384.0, - "74": 2187152384.0, - "75": 2187152384.0, - "76": 2187152384.0, - "77": 2187152384.0, - "78": 2187152384.0, - "79": 2187152384.0, - "80": 2187152384.0, - "81": 2187152384.0, - "82": 2187152384.0, - "83": 2187152384.0, - "84": 2187152384.0, - "85": 2187152384.0, - "86": 2187152384.0, - "87": 2187152384.0, - "88": 2187152384.0, - "89": 2187152384.0, - "90": 2187152384.0, - "91": 2187152384.0, - "92": 2187152384.0, - "93": 2187152384.0, - "94": 2187152384.0, - "95": 2190771712.0, - "96": 2190771712.0, - "97": 2190771712.0, - "98": 2190771712.0, - "99": 2190771712.0, - "100": 2193000448.0 + "1": 1942770688.0, + "2": 2185258496.0, + "3": 2185258496.0, + "4": 2185258496.0, + "5": 2185258496.0, + "6": 2185258496.0, + "7": 2185258496.0, + "8": 2185258496.0, + "9": 2185258496.0, + "10": 2185258496.0, + "11": 2185258496.0, + "12": 2185258496.0, + "13": 2185258496.0, + "14": 2185258496.0, + "15": 2185258496.0, + "16": 2185258496.0, + "17": 2185258496.0, + "18": 2185258496.0, + "19": 2185258496.0, + "20": 2185258496.0, + "21": 2185258496.0, + "22": 2185258496.0, + "23": 2185258496.0, + "24": 2185258496.0, + "25": 2185258496.0, + "26": 2185258496.0, + "27": 2185258496.0, + "28": 2185258496.0, + "29": 2185258496.0, + "30": 2185258496.0, + "31": 2185258496.0, + "32": 2185258496.0, + "33": 2185258496.0, + "34": 2185258496.0, + "35": 2185258496.0, + "36": 2185258496.0, + "37": 2185258496.0, + "38": 2185258496.0, + "39": 2185258496.0, + "40": 2185258496.0, + "41": 2185258496.0, + "42": 2185258496.0, + "43": 2185258496.0, + "44": 2185258496.0, + "45": 2185258496.0, + "46": 2185258496.0, + "47": 2185258496.0, + "48": 2185258496.0, + "49": 2185258496.0, + "50": 2185258496.0, + "51": 2185258496.0, + "52": 2185258496.0, + "53": 2185258496.0, + "54": 2185258496.0, + "55": 2185258496.0, + "56": 2185258496.0, + "57": 2185258496.0, + "58": 2185258496.0, + "59": 2185258496.0, + "60": 2185258496.0, + "61": 2185258496.0, + "62": 2185258496.0, + "63": 2185258496.0, + "64": 2185258496.0, + "65": 2185258496.0, + "66": 2185258496.0, + "67": 2185258496.0, + "68": 2185258496.0, + "69": 2185258496.0, + "70": 2185258496.0, + "71": 2185258496.0, + "72": 2185258496.0, + "73": 2185258496.0, + "74": 2185258496.0, + "75": 2185258496.0, + "76": 2185258496.0, + "77": 2185258496.0, + "78": 2185258496.0, + "79": 2185258496.0, + "80": 2185258496.0, + "81": 2185258496.0, + "82": 2185258496.0, + "83": 2185258496.0, + "84": 2185258496.0, + "85": 2185258496.0, + "86": 2185258496.0, + "87": 2185258496.0, + "88": 2185258496.0, + "89": 2185258496.0, + "90": 2185258496.0, + "91": 2185258496.0, + "92": 2185498624.0, + "93": 2185498624.0, + "94": 2185498624.0, + "95": 2189356032.0, + "96": 2189356032.0, + "97": 2189356032.0, + "98": 2189356032.0, + "99": 2189356032.0, + "100": 2191795712.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.95433, - "3": 0.37445, - "4": 0.31336, - "5": 0.30408, - "6": 0.30658, - "7": 0.29824, - "8": 0.29602, - "9": 0.29625, - "10": 0.30327, - "11": 0.30372, - "12": 0.31271, - "13": 0.30177, - "14": 0.30035, - "15": 0.30274, - "16": 0.29771, - "17": 0.30582, - "18": 0.30105, - "19": 0.30687, - "20": 0.29956, - "21": 0.30307, - "22": 0.2999, - "23": 0.29862, - "24": 0.29981, - "25": 0.30031, - "26": 0.30337, - "27": 0.30206, - "28": 0.30359, - "29": 0.30048, - "30": 0.29481, - "31": 0.29481, - "32": 0.29786, - "33": 0.30394, - "34": 0.30565, - "35": 0.30321, - "36": 0.30416, - "37": 0.30756, - "38": 0.2977, - "39": 0.30042, - "40": 0.29684, - "41": 0.29738, - "42": 0.29797, - "43": 0.30171, - "44": 0.30328, - "45": 0.29965, - "46": 0.30131, - "47": 0.29627, - "48": 0.30145, - "49": 0.30216, - "50": 0.30155, - "51": 0.4538, - "52": 0.3082, - "53": 2.80815, - "54": 0.3134, - "55": 0.30732, - "56": 0.30211, - "57": 0.29715, - "58": 0.29216, - "59": 0.29861, - "60": 0.32094, - "61": 0.30018, - "62": 0.29725, - "63": 0.29693, - "64": 0.29859, - "65": 0.29834, - "66": 0.29586, - "67": 0.29746, - "68": 0.30081, - "69": 0.29873, - "70": 0.30002, - "71": 0.29214, - "72": 0.29315, - "73": 0.29683, - "74": 0.30289, - "75": 0.29809, - "76": 0.29966, - "77": 0.30004, - "78": 0.29577, - "79": 0.3007, - "80": 0.30789, - "81": 0.30664, - "82": 0.30394, - "83": 0.30658, - "84": 0.30198, - "85": 0.30765, - "86": 0.30323, - "87": 0.29594, - "88": 0.30431, - "89": 0.30423, - "90": 0.30189, - "91": 0.30993, - "92": 0.30641, - "93": 0.30609, - "94": 0.31019, - "95": 0.3179, - "96": 0.32326, - "97": 0.31841, - "98": 0.32224, - "99": 0.30746, - "100": 0.34198 + "2": 5.3415, + "3": 0.34618, + "4": 0.30829, + "5": 0.29867, + "6": 0.29963, + "7": 0.2967, + "8": 0.30158, + "9": 0.30005, + "10": 0.30268, + "11": 0.29425, + "12": 0.30337, + "13": 0.29877, + "14": 0.29977, + "15": 0.29619, + "16": 0.3187, + "17": 0.35504, + "18": 0.36138, + "19": 0.34546, + "20": 0.32879, + "21": 0.32188, + "22": 0.31587, + "23": 0.31338, + "24": 0.3113, + "25": 0.31355, + "26": 0.30794, + "27": 0.30415, + "28": 0.30302, + "29": 0.30024, + "30": 0.29985, + "31": 0.30354, + "32": 0.3052, + "33": 0.299, + "34": 0.29833, + "35": 0.29946, + "36": 0.3004, + "37": 0.29855, + "38": 0.29609, + "39": 0.29824, + "40": 0.29663, + "41": 0.29676, + "42": 0.29959, + "43": 0.30597, + "44": 0.29831, + "45": 0.29014, + "46": 0.28786, + "47": 0.29935, + "48": 0.29687, + "49": 0.29662, + "50": 0.29488, + "51": 0.38136, + "52": 0.3235, + "53": 2.77849, + "54": 0.29079, + "55": 0.28674, + "56": 0.2948, + "57": 0.28786, + "58": 0.33889, + "59": 0.34431, + "60": 0.33296, + "61": 0.32576, + "62": 0.31396, + "63": 0.31021, + "64": 0.31047, + "65": 0.31266, + "66": 0.29594, + "67": 0.30496, + "68": 0.30352, + "69": 0.30636, + "70": 0.29872, + "71": 0.29332, + "72": 0.30734, + "73": 0.29704, + "74": 0.29823, + "75": 0.29549, + "76": 0.29312, + "77": 0.30376, + "78": 0.29608, + "79": 0.30136, + "80": 0.29948, + "81": 0.3143, + "82": 0.35818, + "83": 0.34273, + "84": 0.33432, + "85": 0.33206, + "86": 0.31761, + "87": 0.30883, + "88": 0.31437, + "89": 0.31158, + "90": 0.32348, + "91": 0.31212, + "92": 0.3128, + "93": 0.30607, + "94": 0.31181, + "95": 0.32983, + "96": 0.30619, + "97": 0.32436, + "98": 0.31502, + "99": 0.31016, + "100": 0.3277 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json index 4439365e0f5..2e91c74d4f5 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.91589, "2": 10.90332, - "3": 10.90347, - "4": 10.89774, - "5": 10.91869, - "6": 10.90876, - "7": 10.89808, - "8": 10.90041, - "9": 10.90403, - "10": 10.90843, - "11": 10.8958, - "12": 10.90348, - "13": 10.88305, - "14": 10.88331, - "15": 10.87671, - "16": 10.86763, - "17": 10.86407, - "18": 10.85889, - "19": 10.86376, - "20": 10.81635, - "21": 10.80059, - "22": 10.79096, - "23": 10.78086, - "24": 10.74863, - "25": 10.75894, - "26": 10.73695, - "27": 10.72356, - "28": 10.67453, - "29": 10.6352, - "30": 10.61814, - "31": 10.60455, - "32": 10.58898, - "33": 10.56262, + "3": 10.90407, + "4": 10.89821, + "5": 10.91875, + "6": 10.90975, + "7": 10.89781, + "8": 10.90055, + "9": 10.90363, + "10": 10.9075, + "11": 10.8961, + "12": 10.90294, + "13": 10.88224, + "14": 10.88235, + "15": 10.87669, + "16": 10.86813, + "17": 10.86419, + "18": 10.85869, + "19": 10.86429, + "20": 10.81551, + "21": 10.80022, + "22": 10.79042, + "23": 10.78088, + "24": 10.74881, + "25": 10.75834, + "26": 10.73728, + "27": 10.72326, + "28": 10.67474, + "29": 10.63469, + "30": 10.61736, + "31": 10.60429, + "32": 10.58957, + "33": 10.56168, "34": 10.53561, - "35": 10.52922, - "36": 10.51896, - "37": 10.48236, - "38": 10.48771, - "39": 10.45244, - "40": 10.43529, - "41": 10.41515, - "42": 10.40591, - "43": 10.388, - "44": 10.34146, - "45": 10.36294, - "46": 10.32607, - "47": 10.30568, - "48": 10.27378, - "49": 10.26213, - "50": 10.26235, - "51": 10.25989, - "52": 10.2171, - "53": 10.22323, - "54": 10.17928, - "55": 10.16585, - "56": 10.17764, - "57": 10.16944, - "58": 10.17725, - "59": 10.13037, - "60": 10.14429, - "61": 10.09562, - "62": 10.0697, - "63": 10.12696, - "64": 10.09479, - "65": 10.07397, - "66": 10.08364, - "67": 10.06375, - "68": 10.02703, - "69": 10.05115, - "70": 10.02703, - "71": 10.04664, + "35": 10.52907, + "36": 10.51936, + "37": 10.48271, + "38": 10.48755, + "39": 10.45253, + "40": 10.43591, + "41": 10.41483, + "42": 10.40593, + "43": 10.38793, + "44": 10.34165, + "45": 10.36303, + "46": 10.32561, + "47": 10.30578, + "48": 10.27404, + "49": 10.2621, + "50": 10.26228, + "51": 10.25969, + "52": 10.21745, + "53": 10.22334, + "54": 10.1795, + "55": 10.16587, + "56": 10.17768, + "57": 10.16966, + "58": 10.17692, + "59": 10.12969, + "60": 10.14448, + "61": 10.09538, + "62": 10.06928, + "63": 10.12705, + "64": 10.09483, + "65": 10.07381, + "66": 10.0834, + "67": 10.06389, + "68": 10.02713, + "69": 10.05116, + "70": 10.02727, + "71": 10.04666, "72": 10.0328, - "73": 10.02496, - "74": 10.00629, - "75": 9.98643, - "76": 10.00588, - "77": 10.00712, - "78": 9.95731, - "79": 9.96935, - "80": 9.98348, - "81": 10.0112, - "82": 9.94191, - "83": 9.92046, - "84": 9.8503, - "85": 9.84797, - "86": 9.93316, - "87": 9.96433, - "88": 9.93294, - "89": 9.87213, - "90": 9.87714, - "91": 9.89324, - "92": 9.87604, - "93": 9.81103, - "94": 9.89137, - "95": 9.87307, - "96": 9.85376, - "97": 9.79965, - "98": 9.83401, - "99": 9.87123, - "100": 9.7653 + "73": 10.02494, + "74": 10.00636, + "75": 9.98651, + "76": 10.00586, + "77": 10.0069, + "78": 9.95735, + "79": 9.96897, + "80": 9.98358, + "81": 10.01113, + "82": 9.9421, + "83": 9.92053, + "84": 9.85013, + "85": 9.84771, + "86": 9.9333, + "87": 9.96422, + "88": 9.93283, + "89": 9.87201, + "90": 9.87755, + "91": 9.89335, + "92": 9.87566, + "93": 9.8112, + "94": 9.89073, + "95": 9.87272, + "96": 9.85381, + "97": 9.79964, + "98": 9.83421, + "99": 9.87142, + "100": 9.76525 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 36227.0, - "2": 37201.0, - "3": 36774.0, - "4": 36570.0, - "5": 36439.0, - "6": 35792.0, - "7": 37127.0, - "8": 36144.0, - "9": 36928.0, - "10": 37176.0, - "11": 37010.0, - "12": 35585.0, - "13": 37190.0, - "14": 36821.0, - "15": 35771.0, - "16": 36427.0, - "17": 36272.0, - "18": 36691.0, - "19": 36485.0, - "20": 36518.0, - "21": 36020.0, - "22": 36224.0, - "23": 36375.0, - "24": 36945.0, - "25": 35124.0, - "26": 36177.0, - "27": 36186.0, - "28": 35639.0, - "29": 35438.0, - "30": 35209.0, - "31": 36595.0, - "32": 36317.0, - "33": 35600.0, - "34": 35742.0, - "35": 36404.0, - "36": 35231.0, - "37": 36629.0, - "38": 35718.0, - "39": 36298.0, - "40": 37761.0, - "41": 36285.0, - "42": 35162.0, - "43": 37644.0, - "44": 36847.0, - "45": 39443.0, - "46": 38325.0, - "47": 38405.0, - "48": 39108.0, - "49": 42153.0, - "50": 39482.0, - "51": 38581.0, - "52": 41497.0, - "53": 40023.0, - "54": 41751.0, - "55": 39497.0, - "56": 41852.0, - "57": 37239.0, - "58": 46019.0, - "59": 42212.0, - "60": 42885.0, - "61": 42226.0, - "62": 45308.0, - "63": 44346.0, - "64": 49291.0, - "65": 41989.0, - "66": 46316.0, - "67": 50078.0, - "68": 46321.0, - "69": 46309.0, - "70": 46910.0, - "71": 46636.0, - "72": 45539.0, - "73": 47682.0, - "74": 45594.0, - "75": 49085.0, - "76": 48381.0, - "77": 48005.0, - "78": 48958.0, - "79": 48068.0, - "80": 45362.0, - "81": 54972.0, - "82": 41961.0, - "83": 46646.0, - "84": 45351.0, - "85": 45681.0, - "86": 45789.0, - "87": 36100.0, - "88": 45173.0, - "89": 48281.0, - "90": 56672.0, - "91": 38947.0, - "92": 48946.0, - "93": 45289.0, - "94": 42563.0, - "95": 45023.0, - "96": 49854.0, - "97": 46352.0, - "98": 50691.0, - "99": 45749.0, - "100": 46988.0 + "1": 36420.0, + "2": 37155.0, + "3": 36644.0, + "4": 36934.0, + "5": 36624.0, + "6": 35528.0, + "7": 36894.0, + "8": 36546.0, + "9": 36996.0, + "10": 37079.0, + "11": 36817.0, + "12": 35746.0, + "13": 36744.0, + "14": 36799.0, + "15": 36303.0, + "16": 36054.0, + "17": 36490.0, + "18": 36370.0, + "19": 36027.0, + "20": 36712.0, + "21": 36306.0, + "22": 36066.0, + "23": 36203.0, + "24": 36494.0, + "25": 35285.0, + "26": 36361.0, + "27": 36229.0, + "28": 35133.0, + "29": 35647.0, + "30": 35194.0, + "31": 36240.0, + "32": 36401.0, + "33": 35706.0, + "34": 36030.0, + "35": 36416.0, + "36": 35356.0, + "37": 36328.0, + "38": 35489.0, + "39": 36464.0, + "40": 37651.0, + "41": 36127.0, + "42": 35519.0, + "43": 37402.0, + "44": 36489.0, + "45": 39487.0, + "46": 38288.0, + "47": 38065.0, + "48": 39124.0, + "49": 42503.0, + "50": 40022.0, + "51": 38993.0, + "52": 41251.0, + "53": 40361.0, + "54": 41581.0, + "55": 39787.0, + "56": 41955.0, + "57": 37035.0, + "58": 46143.0, + "59": 42259.0, + "60": 42432.0, + "61": 42536.0, + "62": 45429.0, + "63": 44087.0, + "64": 49320.0, + "65": 42054.0, + "66": 46339.0, + "67": 49619.0, + "68": 47027.0, + "69": 46161.0, + "70": 47093.0, + "71": 46158.0, + "72": 45505.0, + "73": 47876.0, + "74": 45491.0, + "75": 48679.0, + "76": 47778.0, + "77": 48277.0, + "78": 48778.0, + "79": 48069.0, + "80": 45705.0, + "81": 55405.0, + "82": 41978.0, + "83": 47336.0, + "84": 45656.0, + "85": 45807.0, + "86": 45977.0, + "87": 36450.0, + "88": 45255.0, + "89": 48748.0, + "90": 57478.0, + "91": 39575.0, + "92": 49833.0, + "93": 44808.0, + "94": 42186.0, + "95": 44807.0, + "96": 50092.0, + "97": 46634.0, + "98": 50631.0, + "99": 45498.0, + "100": 46547.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1254505472.0, - "2": 1254504448.0, - "3": 1254505984.0, - "4": 1254506496.0, - "5": 1254504960.0, - "6": 1254502912.0, - "7": 1254505472.0, - "8": 1254507008.0, - "9": 1254508032.0, - "10": 1254507520.0, - "11": 1254506496.0, - "12": 1254507008.0, - "13": 1254506496.0, - "14": 1254506496.0, - "15": 1254507008.0, - "16": 1254507008.0, - "17": 1254505984.0, - "18": 1254508032.0, - "19": 1254508544.0, + "1": 1254504960.0, + "2": 1254503936.0, + "3": 1254506496.0, + "4": 1254507008.0, + "5": 1254504448.0, + "6": 1254501376.0, + "7": 1254504448.0, + "8": 1254507520.0, + "9": 1254508544.0, + "10": 1254507008.0, + "11": 1254505984.0, + "12": 1254506496.0, + "13": 1254505472.0, + "14": 1254504960.0, + "15": 1254507520.0, + "16": 1254506496.0, + "17": 1254507520.0, + "18": 1254507520.0, + "19": 1254508032.0, "20": 1254508544.0, "21": 1254505472.0, - "22": 1254501376.0, - "23": 1254505472.0, + "22": 1254501888.0, + "23": 1254504960.0, "24": 1254507008.0, - "25": 1254501376.0, - "26": 1254508544.0, - "27": 1254505984.0, - "28": 1254503424.0, - "29": 1254505984.0, - "30": 1254507008.0, - "31": 1254508544.0, - "32": 1254502912.0, - "33": 1254508544.0, - "34": 1254504448.0, - "35": 1254504960.0, - "36": 1254504448.0, - "37": 1254503936.0, + "25": 1254500864.0, + "26": 1254507520.0, + "27": 1254506496.0, + "28": 1254501376.0, + "29": 1254507520.0, + "30": 1254505984.0, + "31": 1254507520.0, + "32": 1254502400.0, + "33": 1254507008.0, + "34": 1254504960.0, + "35": 1254503424.0, + "36": 1254503936.0, + "37": 1254502912.0, "38": 1254505472.0, "39": 1254503936.0, - "40": 1254505472.0, - "41": 1254506496.0, - "42": 1254504960.0, - "43": 1254505472.0, - "44": 1254504960.0, - "45": 1254506496.0, - "46": 1254505472.0, - "47": 1254505984.0, - "48": 1254505984.0, - "49": 1254507520.0, - "50": 1254503936.0, - "51": 1254508032.0, + "40": 1254504448.0, + "41": 1254505472.0, + "42": 1254504448.0, + "43": 1254504960.0, + "44": 1254503936.0, + "45": 1254507520.0, + "46": 1254504960.0, + "47": 1254505472.0, + "48": 1254504960.0, + "49": 1254506496.0, + "50": 1254505472.0, + "51": 1254506496.0, "52": 1254506496.0, - "53": 1254504448.0, - "54": 1254503424.0, - "55": 1254505984.0, - "56": 1254502912.0, - "57": 1254506496.0, - "58": 1254507520.0, - "59": 1254504448.0, - "60": 1254503424.0, + "53": 1254503936.0, + "54": 1254503936.0, + "55": 1254503424.0, + "56": 1254503424.0, + "57": 1254504960.0, + "58": 1254507008.0, + "59": 1254503936.0, + "60": 1254502912.0, "61": 1254501888.0, - "62": 1254503936.0, - "63": 1254504448.0, - "64": 1254509568.0, + "62": 1254504448.0, + "63": 1254504960.0, + "64": 1254509056.0, "65": 1254503936.0, - "66": 1254504960.0, + "66": 1254503936.0, "67": 1254504448.0, - "68": 1254504448.0, - "69": 1254504960.0, - "70": 1254503424.0, + "68": 1254503936.0, + "69": 1254504448.0, + "70": 1254502400.0, "71": 1254505984.0, - "72": 1254504448.0, - "73": 1254504960.0, - "74": 1254502400.0, + "72": 1254504960.0, + "73": 1254504448.0, + "74": 1254502912.0, "75": 1254502400.0, - "76": 1254508032.0, - "77": 1254504960.0, - "78": 1254502912.0, - "79": 1254504448.0, - "80": 1254505984.0, - "81": 1254502912.0, - "82": 1254505984.0, - "83": 1254503424.0, - "84": 1254508032.0, + "76": 1254507520.0, + "77": 1254502912.0, + "78": 1254501376.0, + "79": 1254505984.0, + "80": 1254505472.0, + "81": 1254500864.0, + "82": 1254504448.0, + "83": 1254501888.0, + "84": 1254507008.0, "85": 1254502912.0, "86": 1254504960.0, "87": 1254502912.0, - "88": 1254502912.0, - "89": 1254507520.0, - "90": 1254508032.0, + "88": 1254503936.0, + "89": 1254507008.0, + "90": 1254507520.0, "91": 1254504448.0, - "92": 1254507008.0, - "93": 1254503424.0, + "92": 1254505984.0, + "93": 1254502912.0, "94": 1254503424.0, "95": 1254512640.0, - "96": 1254504960.0, - "97": 1254507520.0, - "98": 1254509056.0, + "96": 1254504448.0, + "97": 1254507008.0, + "98": 1254509568.0, "99": 1254507008.0, - "100": 1254513664.0 + "100": 1254513152.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2064315392.0, - "2": 2545652736.0, - "3": 2546057728.0, - "4": 2547006464.0, - "5": 2547006464.0, - "6": 2547006464.0, - "7": 2547006464.0, - "8": 2547006464.0, - "9": 2547006464.0, - "10": 2547006464.0, - "11": 2547006464.0, - "12": 2547006464.0, - "13": 2547006464.0, - "14": 2547006464.0, - "15": 2547006464.0, - "16": 2547006464.0, - "17": 2547006464.0, - "18": 2547006464.0, - "19": 2547006464.0, - "20": 2547006464.0, - "21": 2547006464.0, - "22": 2547006464.0, - "23": 2547006464.0, - "24": 2547006464.0, - "25": 2547006464.0, - "26": 2547006464.0, - "27": 2547006464.0, - "28": 2547006464.0, - "29": 2547006464.0, - "30": 2547006464.0, - "31": 2547006464.0, - "32": 2547006464.0, - "33": 2547006464.0, - "34": 2547006464.0, - "35": 2547006464.0, - "36": 2547006464.0, - "37": 2547006464.0, - "38": 2547006464.0, - "39": 2547006464.0, - "40": 2547006464.0, - "41": 2547006464.0, - "42": 2547006464.0, - "43": 2547006464.0, - "44": 2547006464.0, - "45": 2547006464.0, - "46": 2547006464.0, - "47": 2547006464.0, - "48": 2547006464.0, - "49": 2547006464.0, - "50": 2547006464.0, - "51": 2547006464.0, - "52": 2547006464.0, - "53": 2547006464.0, - "54": 2547006464.0, - "55": 2547006464.0, - "56": 2547006464.0, - "57": 2547006464.0, - "58": 2547006464.0, - "59": 2547006464.0, - "60": 2547006464.0, - "61": 2547006464.0, - "62": 2547006464.0, - "63": 2547006464.0, - "64": 2547006464.0, - "65": 2547006464.0, - "66": 2547006464.0, - "67": 2547006464.0, - "68": 2547006464.0, - "69": 2547006464.0, - "70": 2547006464.0, - "71": 2547006464.0, - "72": 2547006464.0, - "73": 2547006464.0, - "74": 2547006464.0, - "75": 2547006464.0, - "76": 2547006464.0, - "77": 2547006464.0, - "78": 2547006464.0, - "79": 2547006464.0, - "80": 2547006464.0, - "81": 2547006464.0, - "82": 2547006464.0, - "83": 2547006464.0, - "84": 2547006464.0, - "85": 2547006464.0, - "86": 2547006464.0, - "87": 2547006464.0, - "88": 2547006464.0, - "89": 2547006464.0, - "90": 2547006464.0, - "91": 2547474944.0, - "92": 2547474944.0, - "93": 2547474944.0, - "94": 2547474944.0, - "95": 2552599552.0, - "96": 2552599552.0, - "97": 2552599552.0, - "98": 2552599552.0, - "99": 2552599552.0, - "100": 2555065344.0 + "1": 2063404032.0, + "2": 2545785344.0, + "3": 2545785344.0, + "4": 2546376704.0, + "5": 2546376704.0, + "6": 2546376704.0, + "7": 2546376704.0, + "8": 2546376704.0, + "9": 2546376704.0, + "10": 2546376704.0, + "11": 2546376704.0, + "12": 2546788352.0, + "13": 2546788352.0, + "14": 2546788352.0, + "15": 2546788352.0, + "16": 2546788352.0, + "17": 2546788352.0, + "18": 2547357184.0, + "19": 2547357184.0, + "20": 2547357184.0, + "21": 2547357184.0, + "22": 2547357184.0, + "23": 2547357184.0, + "24": 2547357184.0, + "25": 2547357184.0, + "26": 2547357184.0, + "27": 2547357184.0, + "28": 2547357184.0, + "29": 2547357184.0, + "30": 2547357184.0, + "31": 2547357184.0, + "32": 2547357184.0, + "33": 2547357184.0, + "34": 2547357184.0, + "35": 2547357184.0, + "36": 2547357184.0, + "37": 2547357184.0, + "38": 2547357184.0, + "39": 2547357184.0, + "40": 2547357184.0, + "41": 2547357184.0, + "42": 2547357184.0, + "43": 2547357184.0, + "44": 2547357184.0, + "45": 2547357184.0, + "46": 2547357184.0, + "47": 2547357184.0, + "48": 2547357184.0, + "49": 2547357184.0, + "50": 2547357184.0, + "51": 2547357184.0, + "52": 2547357184.0, + "53": 2547357184.0, + "54": 2547357184.0, + "55": 2547357184.0, + "56": 2547357184.0, + "57": 2547357184.0, + "58": 2547357184.0, + "59": 2547357184.0, + "60": 2547357184.0, + "61": 2547357184.0, + "62": 2547357184.0, + "63": 2547357184.0, + "64": 2547357184.0, + "65": 2547357184.0, + "66": 2547357184.0, + "67": 2547357184.0, + "68": 2547357184.0, + "69": 2547357184.0, + "70": 2547357184.0, + "71": 2547357184.0, + "72": 2547357184.0, + "73": 2547357184.0, + "74": 2547357184.0, + "75": 2547357184.0, + "76": 2547357184.0, + "77": 2547357184.0, + "78": 2547357184.0, + "79": 2547357184.0, + "80": 2547357184.0, + "81": 2547357184.0, + "82": 2547357184.0, + "83": 2547357184.0, + "84": 2547357184.0, + "85": 2547357184.0, + "86": 2547357184.0, + "87": 2547357184.0, + "88": 2547357184.0, + "89": 2547357184.0, + "90": 2547357184.0, + "91": 2547357184.0, + "92": 2547357184.0, + "93": 2547357184.0, + "94": 2547357184.0, + "95": 2552076800.0, + "96": 2552076800.0, + "97": 2552076800.0, + "98": 2552076800.0, + "99": 2552076800.0, + "100": 2556050432.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 11.0819, - "3": 0.36833, - "4": 0.32683, - "5": 0.32942, - "6": 0.31935, - "7": 0.31498, - "8": 0.32015, - "9": 0.31753, - "10": 0.31685, - "11": 0.31806, - "12": 0.32793, - "13": 0.32197, - "14": 0.32077, - "15": 0.32115, - "16": 0.32273, - "17": 0.40718, - "18": 0.47062, - "19": 0.46423, - "20": 0.32225, - "21": 0.32195, - "22": 0.32051, - "23": 0.32322, - "24": 0.31898, - "25": 0.31927, - "26": 0.32161, - "27": 0.31547, - "28": 0.31646, - "29": 0.31934, - "30": 0.31854, - "31": 0.31731, - "32": 0.31914, - "33": 0.31985, - "34": 0.32103, - "35": 0.31933, - "36": 0.31631, - "37": 0.31677, - "38": 0.32117, - "39": 0.31703, - "40": 0.31662, - "41": 0.31589, - "42": 0.31588, - "43": 0.31899, - "44": 0.31738, - "45": 0.31932, - "46": 0.31426, - "47": 0.31636, - "48": 0.31679, - "49": 0.31809, - "50": 0.31714, - "51": 0.48772, - "52": 0.37919, - "53": 0.32359, - "54": 0.32004, - "55": 0.31905, - "56": 0.31773, - "57": 0.31814, - "58": 0.31827, - "59": 0.31803, - "60": 0.31498, - "61": 0.31895, - "62": 0.32768, - "63": 0.34422, - "64": 0.31828, - "65": 0.31622, - "66": 0.32066, - "67": 0.32222, - "68": 0.32008, - "69": 0.32226, - "70": 0.3186, - "71": 0.31724, - "72": 0.31783, - "73": 0.31766, - "74": 0.31789, - "75": 0.31795, - "76": 0.31727, - "77": 0.32985, - "78": 0.31459, - "79": 0.31766, - "80": 0.3231, - "81": 0.31855, - "82": 0.31971, - "83": 0.31933, - "84": 0.32224, - "85": 0.32846, - "86": 0.32265, - "87": 0.31531, - "88": 0.325, - "89": 0.32608, - "90": 0.32179, - "91": 0.32732, - "92": 0.33026, - "93": 0.32928, - "94": 0.32753, - "95": 0.34196, - "96": 0.33528, - "97": 0.34331, - "98": 0.33881, - "99": 0.3276, - "100": 0.34807 + "2": 6.25131, + "3": 0.34567, + "4": 0.30508, + "5": 0.30608, + "6": 0.30522, + "7": 0.30106, + "8": 0.30136, + "9": 0.29921, + "10": 0.30071, + "11": 0.29839, + "12": 0.30023, + "13": 0.29521, + "14": 0.29898, + "15": 0.29537, + "16": 0.28972, + "17": 0.29529, + "18": 0.30072, + "19": 0.2966, + "20": 0.29859, + "21": 0.2939, + "22": 0.29425, + "23": 0.29098, + "24": 0.29199, + "25": 0.29654, + "26": 0.35999, + "27": 0.35175, + "28": 0.34073, + "29": 0.3375, + "30": 0.34043, + "31": 0.32728, + "32": 0.32966, + "33": 0.31827, + "34": 0.30713, + "35": 0.30836, + "36": 0.30229, + "37": 0.30858, + "38": 0.30357, + "39": 0.303, + "40": 0.30087, + "41": 0.29878, + "42": 0.30079, + "43": 0.29728, + "44": 0.29391, + "45": 0.30111, + "46": 0.29438, + "47": 0.29749, + "48": 0.29609, + "49": 0.31803, + "50": 0.37291, + "51": 0.47072, + "52": 0.37629, + "53": 0.30739, + "54": 0.30635, + "55": 0.30558, + "56": 0.29893, + "57": 0.29704, + "58": 0.29635, + "59": 0.29713, + "60": 0.29835, + "61": 0.29494, + "62": 0.29707, + "63": 0.29443, + "64": 0.29314, + "65": 0.2904, + "66": 0.29605, + "67": 0.29645, + "68": 0.78223, + "69": 0.29613, + "70": 0.29023, + "71": 0.30038, + "72": 0.29657, + "73": 0.2888, + "74": 0.29985, + "75": 0.29443, + "76": 0.29419, + "77": 0.2945, + "78": 0.29658, + "79": 0.29584, + "80": 0.29949, + "81": 0.29594, + "82": 0.29555, + "83": 0.30807, + "84": 0.29609, + "85": 0.31218, + "86": 0.30588, + "87": 0.29675, + "88": 0.30058, + "89": 0.30782, + "90": 0.30071, + "91": 0.31007, + "92": 0.30781, + "93": 0.31571, + "94": 0.3125, + "95": 0.31963, + "96": 0.32198, + "97": 0.32168, + "98": 0.31241, + "99": 0.30797, + "100": 0.34606 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json index 6b4f0f44e0c..e17135639b3 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92072, "2": 10.91167, - "3": 10.92061, - "4": 10.9114, - "5": 10.92415, - "6": 10.91049, - "7": 10.9038, - "8": 10.91023, - "9": 10.91956, - "10": 10.91774, - "11": 10.90419, - "12": 10.90295, - "13": 10.89369, - "14": 10.8913, - "15": 10.88101, - "16": 10.87456, - "17": 10.87686, - "18": 10.8599, - "19": 10.86711, - "20": 10.82403, - "21": 10.80942, - "22": 10.79537, - "23": 10.79211, - "24": 10.75737, - "25": 10.76235, - "26": 10.74973, - "27": 10.74104, - "28": 10.67301, - "29": 10.64523, - "30": 10.62424, - "31": 10.61118, - "32": 10.59603, - "33": 10.57445, - "34": 10.54078, - "35": 10.53288, - "36": 10.52364, - "37": 10.48674, - "38": 10.49448, - "39": 10.45417, - "40": 10.44148, - "41": 10.42758, - "42": 10.41845, - "43": 10.38863, - "44": 10.35347, - "45": 10.37013, - "46": 10.33136, - "47": 10.31248, - "48": 10.27774, - "49": 10.27025, - "50": 10.27217 + "3": 10.92066, + "4": 10.91184, + "5": 10.92404, + "6": 10.91113, + "7": 10.90393, + "8": 10.90985, + "9": 10.91959, + "10": 10.91758, + "11": 10.90389, + "12": 10.90344, + "13": 10.89447, + "14": 10.89157, + "15": 10.88138, + "16": 10.87501, + "17": 10.87674, + "18": 10.8606, + "19": 10.868, + "20": 10.82424, + "21": 10.80928, + "22": 10.79632, + "23": 10.79123, + "24": 10.75749, + "25": 10.76202, + "26": 10.75071, + "27": 10.7414, + "28": 10.67372, + "29": 10.64549, + "30": 10.62348, + "31": 10.61147, + "32": 10.59676, + "33": 10.57395, + "34": 10.54145, + "35": 10.53203, + "36": 10.5234, + "37": 10.48714, + "38": 10.49435, + "39": 10.45409, + "40": 10.44144, + "41": 10.42824, + "42": 10.41826, + "43": 10.38934, + "44": 10.35343, + "45": 10.37024, + "46": 10.33167, + "47": 10.31245, + "48": 10.27749, + "49": 10.27081, + "50": 10.27222 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18671.0, - "2": 18984.0, - "3": 19142.0, - "4": 18835.0, - "5": 18819.0, - "6": 18559.0, - "7": 19133.0, - "8": 18877.0, - "9": 19162.0, - "10": 19030.0, - "11": 19199.0, - "12": 18543.0, - "13": 19032.0, - "14": 19223.0, - "15": 18484.0, - "16": 18564.0, - "17": 18982.0, - "18": 18794.0, - "19": 18825.0, - "20": 18819.0, - "21": 18855.0, - "22": 18718.0, - "23": 18533.0, - "24": 18777.0, - "25": 18363.0, - "26": 18675.0, - "27": 18723.0, - "28": 18192.0, - "29": 18449.0, - "30": 18151.0, - "31": 18678.0, - "32": 18899.0, - "33": 18450.0, - "34": 18645.0, - "35": 19043.0, - "36": 18511.0, - "37": 18899.0, - "38": 18362.0, - "39": 19027.0, - "40": 19215.0, - "41": 18812.0, - "42": 18311.0, - "43": 19125.0, - "44": 18694.0, - "45": 20457.0, - "46": 19515.0, - "47": 19834.0, - "48": 19869.0, - "49": 21552.0, - "50": 20234.0 + "1": 18555.0, + "2": 18999.0, + "3": 18950.0, + "4": 18669.0, + "5": 18777.0, + "6": 18519.0, + "7": 18807.0, + "8": 18670.0, + "9": 19307.0, + "10": 19430.0, + "11": 19062.0, + "12": 18644.0, + "13": 19076.0, + "14": 18936.0, + "15": 18541.0, + "16": 18430.0, + "17": 18751.0, + "18": 18698.0, + "19": 19066.0, + "20": 18677.0, + "21": 18789.0, + "22": 18783.0, + "23": 18643.0, + "24": 18816.0, + "25": 18303.0, + "26": 18784.0, + "27": 18692.0, + "28": 18297.0, + "29": 18282.0, + "30": 18205.0, + "31": 18931.0, + "32": 19019.0, + "33": 18449.0, + "34": 18535.0, + "35": 18869.0, + "36": 18390.0, + "37": 18979.0, + "38": 18529.0, + "39": 19178.0, + "40": 19142.0, + "41": 18772.0, + "42": 18360.0, + "43": 19127.0, + "44": 18528.0, + "45": 20133.0, + "46": 19760.0, + "47": 19924.0, + "48": 19649.0, + "49": 21808.0, + "50": 19795.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1027096064.0, - "2": 1027094016.0, - "3": 1027091968.0, - "4": 1027092992.0, - "5": 1027094528.0, - "6": 1027095040.0, - "7": 1027096064.0, - "8": 1027094528.0, - "9": 1027094528.0, + "1": 1027095552.0, + "2": 1027093504.0, + "3": 1027091456.0, + "4": 1027091968.0, + "5": 1027093504.0, + "6": 1027094528.0, + "7": 1027095040.0, + "8": 1027094016.0, + "9": 1027095040.0, "10": 1027092992.0, "11": 1027093504.0, - "12": 1027095552.0, + "12": 1027092992.0, "13": 1027091456.0, - "14": 1027094016.0, + "14": 1027092992.0, "15": 1027094528.0, - "16": 1027094528.0, - "17": 1027097088.0, + "16": 1027094016.0, + "17": 1027095552.0, "18": 1027096576.0, - "19": 1027093504.0, - "20": 1027094528.0, - "21": 1027095040.0, - "22": 1027092480.0, - "23": 1027094016.0, - "24": 1027098112.0, - "25": 1027093504.0, - "26": 1027095552.0, - "27": 1027092992.0, - "28": 1027096576.0, - "29": 1027094528.0, - "30": 1027098112.0, - "31": 1027094016.0, - "32": 1027091968.0, - "33": 1027097088.0, + "19": 1027095040.0, + "20": 1027092992.0, + "21": 1027095552.0, + "22": 1027091968.0, + "23": 1027091456.0, + "24": 1027097600.0, + "25": 1027092480.0, + "26": 1027094528.0, + "27": 1027092480.0, + "28": 1027094528.0, + "29": 1027091968.0, + "30": 1027097600.0, + "31": 1027093504.0, + "32": 1027090944.0, + "33": 1027098112.0, "34": 1027096064.0, - "35": 1027094016.0, - "36": 1027097088.0, - "37": 1027095040.0, - "38": 1027095040.0, - "39": 1027093504.0, - "40": 1027096576.0, - "41": 1027095552.0, - "42": 1027097600.0, - "43": 1027094016.0, + "35": 1027094528.0, + "36": 1027096064.0, + "37": 1027094528.0, + "38": 1027096064.0, + "39": 1027094016.0, + "40": 1027096064.0, + "41": 1027093504.0, + "42": 1027098112.0, + "43": 1027093504.0, "44": 1027093504.0, - "45": 1027096064.0, - "46": 1027094016.0, - "47": 1027096576.0, - "48": 1027094016.0, - "49": 1027096064.0, - "50": 1027095040.0 + "45": 1027095552.0, + "46": 1027093504.0, + "47": 1027096064.0, + "48": 1027093504.0, + "49": 1027096576.0, + "50": 1027096064.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3058638336.0, - "2": 3298349568.0, - "3": 3299699712.0, - "4": 3299699712.0, - "5": 3299985408.0, - "6": 3299985408.0, - "7": 3299985408.0, - "8": 3299985408.0, - "9": 3300500480.0, - "10": 3300500480.0, - "11": 3300500480.0, - "12": 3300500480.0, - "13": 3300500480.0, - "14": 3300500480.0, - "15": 3300500480.0, - "16": 3300500480.0, - "17": 3300500480.0, - "18": 3300500480.0, - "19": 3300500480.0, - "20": 3300500480.0, - "21": 3300914688.0, - "22": 3300914688.0, - "23": 3300914688.0, - "24": 3300914688.0, - "25": 3300914688.0, - "26": 3300914688.0, - "27": 3300914688.0, - "28": 3300914688.0, - "29": 3300914688.0, - "30": 3300914688.0, - "31": 3300914688.0, - "32": 3300914688.0, - "33": 3300914688.0, - "34": 3300914688.0, - "35": 3300914688.0, - "36": 3301115392.0, - "37": 3301115392.0, - "38": 3301115392.0, - "39": 3301115392.0, - "40": 3301262336.0, - "41": 3301262336.0, - "42": 3301687296.0, - "43": 3301687296.0, - "44": 3302436352.0, - "45": 3302436352.0, - "46": 3302436352.0, - "47": 3302436352.0, - "48": 3302436352.0, - "49": 3302436352.0, - "50": 3302436352.0 + "1": 3058330112.0, + "2": 3298373120.0, + "3": 3298764288.0, + "4": 3299075072.0, + "5": 3300025344.0, + "6": 3300025344.0, + "7": 3300025344.0, + "8": 3300025344.0, + "9": 3300025344.0, + "10": 3300025344.0, + "11": 3300025344.0, + "12": 3300025344.0, + "13": 3300025344.0, + "14": 3300025344.0, + "15": 3300025344.0, + "16": 3300025344.0, + "17": 3300025344.0, + "18": 3300135424.0, + "19": 3300135424.0, + "20": 3300135424.0, + "21": 3300135424.0, + "22": 3300135424.0, + "23": 3300135424.0, + "24": 3300135424.0, + "25": 3300135424.0, + "26": 3300135424.0, + "27": 3300135424.0, + "28": 3300135424.0, + "29": 3300135424.0, + "30": 3300135424.0, + "31": 3300135424.0, + "32": 3300135424.0, + "33": 3300135424.0, + "34": 3300452352.0, + "35": 3300452352.0, + "36": 3300452352.0, + "37": 3300452352.0, + "38": 3300452352.0, + "39": 3300452352.0, + "40": 3300525056.0, + "41": 3300525056.0, + "42": 3301999616.0, + "43": 3301999616.0, + "44": 3301999616.0, + "45": 3301999616.0, + "46": 3301999616.0, + "47": 3301999616.0, + "48": 3301999616.0, + "49": 3301999616.0, + "50": 3301999616.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.34877, - "3": 0.33751, - "4": 0.2905, - "5": 0.29669, - "6": 0.2873, - "7": 0.29357, - "8": 0.29108, - "9": 0.28732, - "10": 0.28631, - "11": 0.28348, - "12": 0.28397, - "13": 0.28835, - "14": 0.28581, - "15": 0.28383, - "16": 0.28331, - "17": 0.2832, - "18": 0.28606, - "19": 0.28319, - "20": 0.28581, - "21": 0.28796, - "22": 0.2897, - "23": 0.28617, - "24": 0.28274, - "25": 0.29014, - "26": 0.28915, - "27": 0.2854, - "28": 0.28204, - "29": 0.28533, - "30": 0.28908, - "31": 0.29087, - "32": 0.29191, - "33": 0.28836, - "34": 0.29404, - "35": 0.28626, - "36": 0.28436, - "37": 0.28666, - "38": 0.28224, - "39": 0.28463, - "40": 0.28174, - "41": 0.28126, - "42": 0.28093, - "43": 0.28304, - "44": 0.28165, - "45": 0.28418, - "46": 0.28392, - "47": 0.28118, - "48": 0.28435, - "49": 0.28707, - "50": 0.28431 + "2": 5.08499, + "3": 0.32344, + "4": 0.26578, + "5": 0.26136, + "6": 0.26073, + "7": 0.25791, + "8": 0.26042, + "9": 0.2894, + "10": 0.317, + "11": 0.3188, + "12": 0.32736, + "13": 0.31992, + "14": 0.31394, + "15": 0.30917, + "16": 0.31272, + "17": 0.3125, + "18": 0.31561, + "19": 0.31717, + "20": 0.30933, + "21": 0.31495, + "22": 0.31815, + "23": 0.31889, + "24": 0.32024, + "25": 0.3175, + "26": 0.31981, + "27": 0.31518, + "28": 0.32058, + "29": 0.31851, + "30": 0.32621, + "31": 0.31894, + "32": 0.31952, + "33": 0.31826, + "34": 0.3143, + "35": 0.31728, + "36": 0.31983, + "37": 0.31754, + "38": 0.31813, + "39": 0.32013, + "40": 0.31775, + "41": 0.3177, + "42": 0.31806, + "43": 0.31839, + "44": 0.3185, + "45": 0.31489, + "46": 0.31404, + "47": 0.3105, + "48": 0.31484, + "49": 0.31734, + "50": 0.31583 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json index 99e4e4ca6f5..d7084a96169 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92072, "2": 10.91167, - "3": 10.92061, - "4": 10.9114, - "5": 10.92415, - "6": 10.91049, - "7": 10.9038, - "8": 10.91023, - "9": 10.91956, - "10": 10.91774, - "11": 10.90419, - "12": 10.90295, - "13": 10.89369, - "14": 10.8913, - "15": 10.88101, - "16": 10.87456, - "17": 10.87686, - "18": 10.8599, - "19": 10.86711, - "20": 10.82403, - "21": 10.80942, - "22": 10.79537, - "23": 10.79211, - "24": 10.75737, - "25": 10.76235, - "26": 10.74973, - "27": 10.74104, - "28": 10.67301, - "29": 10.64523, - "30": 10.62424, - "31": 10.61118, - "32": 10.59603, - "33": 10.57445, - "34": 10.54078, - "35": 10.53288, - "36": 10.52364, - "37": 10.48674, - "38": 10.49448, - "39": 10.45417, - "40": 10.44148, - "41": 10.42758, - "42": 10.41845, - "43": 10.38863, - "44": 10.35347, - "45": 10.37013, - "46": 10.33136, - "47": 10.31248, - "48": 10.27774, - "49": 10.27025, - "50": 10.27217 + "3": 10.92066, + "4": 10.91184, + "5": 10.92404, + "6": 10.91113, + "7": 10.90393, + "8": 10.90985, + "9": 10.91959, + "10": 10.91758, + "11": 10.90389, + "12": 10.90344, + "13": 10.89447, + "14": 10.89157, + "15": 10.88138, + "16": 10.87501, + "17": 10.87674, + "18": 10.8606, + "19": 10.868, + "20": 10.82424, + "21": 10.80928, + "22": 10.79632, + "23": 10.79123, + "24": 10.75749, + "25": 10.76202, + "26": 10.75071, + "27": 10.7414, + "28": 10.67372, + "29": 10.64549, + "30": 10.62348, + "31": 10.61147, + "32": 10.59676, + "33": 10.57395, + "34": 10.54145, + "35": 10.53203, + "36": 10.5234, + "37": 10.48714, + "38": 10.49435, + "39": 10.45409, + "40": 10.44144, + "41": 10.42824, + "42": 10.41826, + "43": 10.38934, + "44": 10.35343, + "45": 10.37024, + "46": 10.33167, + "47": 10.31245, + "48": 10.27749, + "49": 10.27081, + "50": 10.27222 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 18671.0, - "2": 18984.0, - "3": 19142.0, - "4": 18835.0, - "5": 18819.0, - "6": 18559.0, - "7": 19133.0, - "8": 18877.0, - "9": 19162.0, - "10": 19030.0, - "11": 19199.0, - "12": 18543.0, - "13": 19032.0, - "14": 19223.0, - "15": 18484.0, - "16": 18564.0, - "17": 18982.0, - "18": 18794.0, - "19": 18825.0, - "20": 18819.0, - "21": 18855.0, - "22": 18718.0, - "23": 18533.0, - "24": 18777.0, - "25": 18363.0, - "26": 18675.0, - "27": 18723.0, - "28": 18192.0, - "29": 18449.0, - "30": 18151.0, - "31": 18678.0, - "32": 18899.0, - "33": 18450.0, - "34": 18645.0, - "35": 19043.0, - "36": 18511.0, - "37": 18899.0, - "38": 18362.0, - "39": 19027.0, - "40": 19215.0, - "41": 18812.0, - "42": 18311.0, - "43": 19125.0, - "44": 18694.0, - "45": 20457.0, - "46": 19515.0, - "47": 19834.0, - "48": 19869.0, - "49": 21552.0, - "50": 20234.0 + "1": 18555.0, + "2": 18999.0, + "3": 18950.0, + "4": 18669.0, + "5": 18777.0, + "6": 18519.0, + "7": 18807.0, + "8": 18670.0, + "9": 19307.0, + "10": 19430.0, + "11": 19062.0, + "12": 18644.0, + "13": 19076.0, + "14": 18936.0, + "15": 18541.0, + "16": 18430.0, + "17": 18751.0, + "18": 18698.0, + "19": 19066.0, + "20": 18677.0, + "21": 18789.0, + "22": 18783.0, + "23": 18643.0, + "24": 18816.0, + "25": 18303.0, + "26": 18784.0, + "27": 18692.0, + "28": 18297.0, + "29": 18282.0, + "30": 18205.0, + "31": 18931.0, + "32": 19019.0, + "33": 18449.0, + "34": 18535.0, + "35": 18869.0, + "36": 18390.0, + "37": 18979.0, + "38": 18529.0, + "39": 19178.0, + "40": 19142.0, + "41": 18772.0, + "42": 18360.0, + "43": 19127.0, + "44": 18528.0, + "45": 20133.0, + "46": 19760.0, + "47": 19924.0, + "48": 19649.0, + "49": 21808.0, + "50": 19795.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1027096064.0, - "2": 1027094016.0, - "3": 1027091968.0, - "4": 1027092992.0, - "5": 1027094528.0, - "6": 1027095040.0, - "7": 1027096064.0, - "8": 1027094528.0, - "9": 1027094528.0, + "1": 1027095552.0, + "2": 1027093504.0, + "3": 1027091456.0, + "4": 1027091968.0, + "5": 1027093504.0, + "6": 1027094528.0, + "7": 1027095040.0, + "8": 1027094016.0, + "9": 1027095040.0, "10": 1027092992.0, "11": 1027093504.0, - "12": 1027095552.0, + "12": 1027092992.0, "13": 1027091456.0, - "14": 1027094016.0, + "14": 1027092992.0, "15": 1027094528.0, - "16": 1027094528.0, - "17": 1027097088.0, + "16": 1027094016.0, + "17": 1027095552.0, "18": 1027096576.0, - "19": 1027093504.0, - "20": 1027094528.0, - "21": 1027095040.0, - "22": 1027092480.0, - "23": 1027094016.0, - "24": 1027098112.0, - "25": 1027093504.0, - "26": 1027095552.0, - "27": 1027092992.0, - "28": 1027096576.0, - "29": 1027094528.0, - "30": 1027098112.0, - "31": 1027094016.0, - "32": 1027091968.0, - "33": 1027097088.0, + "19": 1027095040.0, + "20": 1027092992.0, + "21": 1027095552.0, + "22": 1027091968.0, + "23": 1027091456.0, + "24": 1027097600.0, + "25": 1027092480.0, + "26": 1027094528.0, + "27": 1027092480.0, + "28": 1027094528.0, + "29": 1027091968.0, + "30": 1027097600.0, + "31": 1027093504.0, + "32": 1027090944.0, + "33": 1027098112.0, "34": 1027096064.0, - "35": 1027094016.0, - "36": 1027097088.0, - "37": 1027095040.0, - "38": 1027095040.0, - "39": 1027093504.0, - "40": 1027096576.0, - "41": 1027095552.0, - "42": 1027097600.0, - "43": 1027094016.0, + "35": 1027094528.0, + "36": 1027096064.0, + "37": 1027094528.0, + "38": 1027096064.0, + "39": 1027094016.0, + "40": 1027096064.0, + "41": 1027093504.0, + "42": 1027098112.0, + "43": 1027093504.0, "44": 1027093504.0, - "45": 1027096064.0, - "46": 1027094016.0, - "47": 1027096576.0, - "48": 1027094016.0, - "49": 1027096064.0, - "50": 1027095040.0 + "45": 1027095552.0, + "46": 1027093504.0, + "47": 1027096064.0, + "48": 1027093504.0, + "49": 1027096576.0, + "50": 1027096064.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3058638336.0, - "2": 3298349568.0, - "3": 3299699712.0, - "4": 3299699712.0, - "5": 3299985408.0, - "6": 3299985408.0, - "7": 3299985408.0, - "8": 3299985408.0, - "9": 3300500480.0, - "10": 3300500480.0, - "11": 3300500480.0, - "12": 3300500480.0, - "13": 3300500480.0, - "14": 3300500480.0, - "15": 3300500480.0, - "16": 3300500480.0, - "17": 3300500480.0, - "18": 3300500480.0, - "19": 3300500480.0, - "20": 3300500480.0, - "21": 3300914688.0, - "22": 3300914688.0, - "23": 3300914688.0, - "24": 3300914688.0, - "25": 3300914688.0, - "26": 3300914688.0, - "27": 3300914688.0, - "28": 3300914688.0, - "29": 3300914688.0, - "30": 3300914688.0, - "31": 3300914688.0, - "32": 3300914688.0, - "33": 3300914688.0, - "34": 3300914688.0, - "35": 3300914688.0, - "36": 3301120512.0, - "37": 3301120512.0, - "38": 3301120512.0, - "39": 3301120512.0, - "40": 3301178368.0, - "41": 3301178368.0, - "42": 3301648384.0, - "43": 3301648384.0, - "44": 3302436352.0, - "45": 3302436352.0, - "46": 3302436352.0, - "47": 3302436352.0, - "48": 3302436352.0, - "49": 3302436352.0, - "50": 3302436352.0 + "1": 3058330112.0, + "2": 3298373120.0, + "3": 3299813888.0, + "4": 3299813888.0, + "5": 3300025344.0, + "6": 3300025344.0, + "7": 3300025344.0, + "8": 3300025344.0, + "9": 3300025344.0, + "10": 3300025344.0, + "11": 3300025344.0, + "12": 3300025344.0, + "13": 3300025344.0, + "14": 3300025344.0, + "15": 3300025344.0, + "16": 3300025344.0, + "17": 3300025344.0, + "18": 3300135424.0, + "19": 3300135424.0, + "20": 3300135424.0, + "21": 3300973056.0, + "22": 3300973056.0, + "23": 3300973056.0, + "24": 3300973056.0, + "25": 3300973056.0, + "26": 3300973056.0, + "27": 3300973056.0, + "28": 3300973056.0, + "29": 3300973056.0, + "30": 3300973056.0, + "31": 3300973056.0, + "32": 3300973056.0, + "33": 3300973056.0, + "34": 3300973056.0, + "35": 3300973056.0, + "36": 3300973056.0, + "37": 3300973056.0, + "38": 3300973056.0, + "39": 3300973056.0, + "40": 3300973056.0, + "41": 3300973056.0, + "42": 3301952512.0, + "43": 3301952512.0, + "44": 3301952512.0, + "45": 3301952512.0, + "46": 3301952512.0, + "47": 3301952512.0, + "48": 3301952512.0, + "49": 3301952512.0, + "50": 3301952512.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.876, - "3": 0.39593, - "4": 0.36473, - "5": 0.31606, - "6": 0.30115, - "7": 0.30032, - "8": 0.37696, - "9": 0.39285, - "10": 0.44021, - "11": 0.28397, - "12": 0.28597, - "13": 0.28717, - "14": 0.28364, - "15": 0.28447, - "16": 0.28297, - "17": 0.28538, - "18": 0.28599, - "19": 0.29327, - "20": 0.28458, - "21": 0.28718, - "22": 0.28751, - "23": 0.28677, - "24": 0.2875, - "25": 0.28921, - "26": 0.28446, - "27": 0.28599, - "28": 0.28989, - "29": 0.28965, - "30": 0.28584, - "31": 0.28653, - "32": 0.28968, - "33": 0.28963, - "34": 0.28831, - "35": 0.28377, - "36": 0.28511, - "37": 0.28127, - "38": 0.28357, - "39": 0.28344, - "40": 0.28014, - "41": 0.28666, - "42": 0.40725, - "43": 0.38661, - "44": 0.37691, - "45": 0.28249, - "46": 0.28315, - "47": 0.28159, - "48": 0.28147, - "49": 0.28617, - "50": 0.28722 + "2": 5.09505, + "3": 0.37144, + "4": 0.33646, + "5": 0.32892, + "6": 0.31965, + "7": 0.32563, + "8": 0.32577, + "9": 0.32429, + "10": 0.32403, + "11": 0.31694, + "12": 0.31735, + "13": 0.35415, + "14": 0.47485, + "15": 0.51791, + "16": 0.31929, + "17": 0.34876, + "18": 0.45504, + "19": 0.36122, + "20": 0.33659, + "21": 0.59492, + "22": 0.31743, + "23": 0.32651, + "24": 0.30634, + "25": 0.28312, + "26": 0.26693, + "27": 0.26144, + "28": 0.25899, + "29": 0.25409, + "30": 0.25775, + "31": 0.25227, + "32": 0.26194, + "33": 0.27509, + "34": 0.25455, + "35": 0.25264, + "36": 0.25091, + "37": 0.25181, + "38": 0.25531, + "39": 0.25406, + "40": 0.25889, + "41": 0.25316, + "42": 0.25117, + "43": 0.25122, + "44": 0.25049, + "45": 0.24793, + "46": 0.24796, + "47": 0.25122, + "48": 0.25116, + "49": 0.24844, + "50": 0.2517 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json index dc4f0b9f903..e73c620b7ce 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.91963, "2": 10.9166, - "3": 10.91959, - "4": 10.92496, - "5": 10.92445, - "6": 10.91832, - "7": 10.9168, - "8": 10.915, - "9": 10.9164, - "10": 10.9155, - "11": 10.90429, - "12": 10.91186, - "13": 10.89945, - "14": 10.89612, - "15": 10.87857, - "16": 10.86639, - "17": 10.8663, - "18": 10.86381, - "19": 10.86403, - "20": 10.79992, - "21": 10.77911, - "22": 10.76248, - "23": 10.75542, - "24": 10.72136, - "25": 10.72037, - "26": 10.70567, - "27": 10.67974, - "28": 10.61399, - "29": 10.57394, - "30": 10.54745, - "31": 10.5479, - "32": 10.52103, - "33": 10.48777, - "34": 10.4617, - "35": 10.45895, - "36": 10.43997, - "37": 10.40416, - "38": 10.40807, - "39": 10.3671, - "40": 10.3552, - "41": 10.33084, - "42": 10.3281, - "43": 10.30275, - "44": 10.26511, - "45": 10.28204, - "46": 10.24736, - "47": 10.2271, - "48": 10.19687, - "49": 10.18995, - "50": 10.18947 + "3": 10.9199, + "4": 10.92456, + "5": 10.92484, + "6": 10.91858, + "7": 10.91748, + "8": 10.91501, + "9": 10.91647, + "10": 10.91536, + "11": 10.90479, + "12": 10.91195, + "13": 10.89944, + "14": 10.89559, + "15": 10.87847, + "16": 10.86591, + "17": 10.8659, + "18": 10.86335, + "19": 10.86334, + "20": 10.79958, + "21": 10.77882, + "22": 10.7628, + "23": 10.75541, + "24": 10.72183, + "25": 10.72015, + "26": 10.70582, + "27": 10.68006, + "28": 10.61348, + "29": 10.57442, + "30": 10.54775, + "31": 10.54743, + "32": 10.5213, + "33": 10.48753, + "34": 10.46188, + "35": 10.45971, + "36": 10.44025, + "37": 10.40376, + "38": 10.40842, + "39": 10.36701, + "40": 10.35543, + "41": 10.33079, + "42": 10.32796, + "43": 10.30255, + "44": 10.26445, + "45": 10.28208, + "46": 10.24743, + "47": 10.22724, + "48": 10.19664, + "49": 10.18999, + "50": 10.18976 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 16466.0, - "2": 16466.0, - "3": 16644.0, - "4": 16601.0, - "5": 16158.0, - "6": 16467.0, - "7": 16755.0, - "8": 16139.0, - "9": 16583.0, - "10": 16811.0, - "11": 16641.0, - "12": 16260.0, - "13": 16723.0, - "14": 16764.0, - "15": 16373.0, - "16": 16219.0, - "17": 16469.0, - "18": 16575.0, - "19": 16662.0, - "20": 16451.0, - "21": 16426.0, - "22": 16817.0, - "23": 16672.0, - "24": 16780.0, - "25": 16087.0, - "26": 16912.0, - "27": 16661.0, - "28": 16503.0, - "29": 16681.0, - "30": 16761.0, - "31": 17139.0, - "32": 16958.0, - "33": 17322.0, - "34": 16969.0, - "35": 17460.0, - "36": 17191.0, - "37": 18078.0, - "38": 17497.0, - "39": 18002.0, - "40": 18592.0, - "41": 17526.0, - "42": 17408.0, - "43": 18537.0, - "44": 17891.0, - "45": 20272.0, - "46": 19089.0, - "47": 19481.0, - "48": 19584.0, - "49": 22424.0, - "50": 20273.0 + "1": 16620.0, + "2": 16534.0, + "3": 17006.0, + "4": 16534.0, + "5": 16267.0, + "6": 16166.0, + "7": 16933.0, + "8": 16024.0, + "9": 16624.0, + "10": 16958.0, + "11": 16623.0, + "12": 16120.0, + "13": 17017.0, + "14": 16597.0, + "15": 16421.0, + "16": 16353.0, + "17": 16636.0, + "18": 16637.0, + "19": 16771.0, + "20": 16728.0, + "21": 16427.0, + "22": 16650.0, + "23": 16365.0, + "24": 16946.0, + "25": 16200.0, + "26": 16721.0, + "27": 16943.0, + "28": 16484.0, + "29": 16382.0, + "30": 16691.0, + "31": 17163.0, + "32": 17233.0, + "33": 17217.0, + "34": 17108.0, + "35": 17498.0, + "36": 17007.0, + "37": 17802.0, + "38": 17142.0, + "39": 17994.0, + "40": 18639.0, + "41": 17597.0, + "42": 17438.0, + "43": 18861.0, + "44": 17989.0, + "45": 20471.0, + "46": 19369.0, + "47": 19580.0, + "48": 19495.0, + "49": 22083.0, + "50": 20470.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1559857664.0, - "2": 1559555072.0, - "3": 1560364544.0, - "4": 1559656448.0, - "5": 1559605760.0, - "6": 1559820288.0, - "7": 1559605760.0, - "8": 1559681024.0, - "9": 1560861184.0, - "10": 1559795712.0, - "11": 1559907840.0, - "12": 1560763392.0, - "13": 1560167936.0, - "14": 1561400320.0, - "15": 1561538048.0, - "16": 1560900608.0, - "17": 1560706560.0, - "18": 1561003008.0, - "19": 1560310272.0, - "20": 1560138752.0, - "21": 1560665600.0, - "22": 1559991296.0, - "23": 1559808512.0, - "24": 1559757824.0, - "25": 1559808512.0, - "26": 1560541184.0, - "27": 1561412096.0, - "28": 1560797184.0, - "29": 1560370688.0, - "30": 1560295424.0, - "31": 1560733184.0, - "32": 1559859200.0, - "33": 1559808512.0, - "34": 1559859200.0, - "35": 1560512000.0, - "36": 1559859200.0, - "37": 1560176128.0, - "38": 1560649728.0, - "39": 1559909888.0, - "40": 1560648704.0, - "41": 1559922176.0, - "42": 1560095744.0, - "43": 1560192512.0, - "44": 1560950784.0, - "45": 1559909888.0, - "46": 1560251392.0, - "47": 1560357376.0, - "48": 1560075264.0, - "49": 1560888832.0, - "50": 1559960576.0 + "1": 1561525248.0, + "2": 1562172928.0, + "3": 1562373632.0, + "4": 1561779712.0, + "5": 1561811456.0, + "6": 1561242112.0, + "7": 1561976320.0, + "8": 1561222656.0, + "9": 1563140608.0, + "10": 1561320960.0, + "11": 1561682432.0, + "12": 1561222656.0, + "13": 1561222656.0, + "14": 1562254848.0, + "15": 1561312768.0, + "16": 1561992704.0, + "17": 1562004992.0, + "18": 1561914880.0, + "19": 1562181120.0, + "20": 1561401856.0, + "21": 1561222656.0, + "22": 1562348032.0, + "23": 1561468416.0, + "24": 1561222656.0, + "25": 1561222656.0, + "26": 1561222656.0, + "27": 1561222656.0, + "28": 1561951744.0, + "29": 1561513472.0, + "30": 1561222656.0, + "31": 1561559552.0, + "32": 1561222656.0, + "33": 1561524736.0, + "34": 1561222656.0, + "35": 1561222656.0, + "36": 1561222656.0, + "37": 1561669120.0, + "38": 1561282048.0, + "39": 1561837056.0, + "40": 1562086912.0, + "41": 1561222656.0, + "42": 1562263040.0, + "43": 1561263616.0, + "44": 1561871872.0, + "45": 1561865728.0, + "46": 1561222656.0, + "47": 1561570816.0, + "48": 1562082816.0, + "49": 1561452032.0, + "50": 1562226176.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 3475772416.0, - "2": 4054369792.0, - "3": 4054369792.0, - "4": 4054369792.0, - "5": 4058998784.0, - "6": 4058998784.0, - "7": 4058998784.0, - "8": 4063047168.0, - "9": 4063047168.0, - "10": 4063047168.0, - "11": 4063047168.0, - "12": 4063047168.0, - "13": 4063047168.0, - "14": 4063047168.0, - "15": 4063047168.0, - "16": 4063047168.0, - "17": 4063047168.0, - "18": 4063047168.0, - "19": 4063047168.0, - "20": 4063047168.0, - "21": 4063047168.0, - "22": 4063047168.0, - "23": 4063047168.0, - "24": 4063047168.0, - "25": 4063047168.0, - "26": 4063047168.0, - "27": 4063047168.0, - "28": 4063047168.0, - "29": 4063047168.0, - "30": 4063047168.0, - "31": 4063047168.0, - "32": 4063047168.0, - "33": 4063047168.0, - "34": 4063047168.0, - "35": 4063047168.0, - "36": 4063047168.0, - "37": 4063047168.0, - "38": 4063047168.0, - "39": 4063047168.0, - "40": 4063047168.0, - "41": 4063047168.0, - "42": 4063047168.0, - "43": 4063047168.0, - "44": 4063047168.0, - "45": 4063047168.0, - "46": 4063047168.0, - "47": 4063047168.0, - "48": 4063047168.0, - "49": 4063047168.0, - "50": 4063047168.0 + "1": 3478863360.0, + "2": 4057549824.0, + "3": 4057549824.0, + "4": 4057549824.0, + "5": 4064312320.0, + "6": 4064312320.0, + "7": 4064312320.0, + "8": 4064312320.0, + "9": 4064312320.0, + "10": 4064312320.0, + "11": 4064312320.0, + "12": 4064312320.0, + "13": 4064312320.0, + "14": 4064312320.0, + "15": 4064312320.0, + "16": 4064312320.0, + "17": 4064312320.0, + "18": 4064312320.0, + "19": 4064312320.0, + "20": 4064312320.0, + "21": 4064312320.0, + "22": 4064312320.0, + "23": 4064312320.0, + "24": 4064312320.0, + "25": 4064312320.0, + "26": 4064312320.0, + "27": 4064312320.0, + "28": 4064312320.0, + "29": 4064312320.0, + "30": 4064312320.0, + "31": 4064312320.0, + "32": 4064312320.0, + "33": 4064312320.0, + "34": 4064312320.0, + "35": 4064312320.0, + "36": 4064312320.0, + "37": 4064312320.0, + "38": 4064312320.0, + "39": 4064312320.0, + "40": 4064312320.0, + "41": 4064312320.0, + "42": 4064312320.0, + "43": 4064312320.0, + "44": 4064312320.0, + "45": 4064312320.0, + "46": 4064312320.0, + "47": 4064312320.0, + "48": 4064312320.0, + "49": 4064312320.0, + "50": 4064312320.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 12.87223, - "3": 0.45298, - "4": 0.37892, - "5": 0.45051, - "6": 0.45204, - "7": 0.47845, - "8": 0.51646, - "9": 0.53743, - "10": 0.36908, - "11": 0.36382, - "12": 0.35094, - "13": 0.36548, - "14": 0.36543, - "15": 0.35406, - "16": 0.35926, - "17": 0.35435, - "18": 0.35962, - "19": 0.35276, - "20": 0.34738, - "21": 0.35815, - "22": 0.34975, - "23": 0.34992, - "24": 0.34983, - "25": 0.35678, - "26": 0.35359, - "27": 0.35216, - "28": 0.36864, - "29": 0.38284, - "30": 0.3673, - "31": 0.36782, - "32": 0.37006, - "33": 0.37121, - "34": 0.36829, - "35": 0.37591, - "36": 0.36997, - "37": 0.36688, - "38": 0.35834, - "39": 0.36803, - "40": 0.36308, - "41": 0.36729, - "42": 0.3581, - "43": 0.36579, - "44": 0.35635, - "45": 0.35625, - "46": 0.36844, - "47": 0.36296, - "48": 0.37329, - "49": 0.35796, - "50": 0.36537 + "2": 7.28192, + "3": 0.48787, + "4": 0.45201, + "5": 0.43575, + "6": 0.44125, + "7": 0.45147, + "8": 0.43509, + "9": 0.43968, + "10": 0.43228, + "11": 0.44373, + "12": 0.44276, + "13": 0.44406, + "14": 0.43766, + "15": 0.43286, + "16": 0.44948, + "17": 0.44794, + "18": 0.45565, + "19": 0.45045, + "20": 0.4531, + "21": 0.46157, + "22": 0.45742, + "23": 0.45152, + "24": 0.45695, + "25": 0.46644, + "26": 0.4541, + "27": 0.4502, + "28": 0.4593, + "29": 0.45941, + "30": 0.45501, + "31": 0.46525, + "32": 0.48077, + "33": 0.43971, + "34": 0.44932, + "35": 0.44807, + "36": 0.43603, + "37": 0.4341, + "38": 0.43907, + "39": 0.43214, + "40": 0.44553, + "41": 0.43817, + "42": 0.42521, + "43": 0.42868, + "44": 0.42273, + "45": 0.4405, + "46": 0.46649, + "47": 0.44854, + "48": 0.44657, + "49": 0.42855, + "50": 0.44345 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index d14f30803a9..2ec8ec55b2d 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.89802, "2": 10.90021, - "3": 10.89823, - "4": 10.89669, - "5": 10.89951, - "6": 10.88562, - "7": 10.89028, - "8": 10.89225, - "9": 10.89399, - "10": 10.89786, - "11": 10.88695, - "12": 10.88994, - "13": 10.88377, - "14": 10.87719, - "15": 10.8672, - "16": 10.8546, - "17": 10.8549, - "18": 10.83796, - "19": 10.85411, - "20": 10.78881, - "21": 10.7772, - "22": 10.77399, - "23": 10.76223, - "24": 10.73066, - "25": 10.72292, - "26": 10.72271, - "27": 10.70096, - "28": 10.64012, - "29": 10.61024, - "30": 10.59081, - "31": 10.57683, - "32": 10.55893, - "33": 10.52562, - "34": 10.51045, - "35": 10.49804, - "36": 10.48157, - "37": 10.45275, - "38": 10.46063, - "39": 10.42883, - "40": 10.412, - "41": 10.39572, - "42": 10.3692, - "43": 10.34979, - "44": 10.32155, - "45": 10.33299, - "46": 10.29376, - "47": 10.2807, - "48": 10.23603, - "49": 10.24619, - "50": 10.23463 + "3": 10.89831, + "4": 10.8967, + "5": 10.8995, + "6": 10.88572, + "7": 10.88964, + "8": 10.8913, + "9": 10.89469, + "10": 10.89864, + "11": 10.88755, + "12": 10.89022, + "13": 10.88419, + "14": 10.87613, + "15": 10.86778, + "16": 10.85477, + "17": 10.85468, + "18": 10.83744, + "19": 10.85415, + "20": 10.78884, + "21": 10.77692, + "22": 10.77377, + "23": 10.76197, + "24": 10.73101, + "25": 10.7232, + "26": 10.723, + "27": 10.70065, + "28": 10.64004, + "29": 10.61056, + "30": 10.59133, + "31": 10.57684, + "32": 10.55855, + "33": 10.5257, + "34": 10.51116, + "35": 10.49829, + "36": 10.48199, + "37": 10.45262, + "38": 10.46114, + "39": 10.42884, + "40": 10.41235, + "41": 10.39539, + "42": 10.36979, + "43": 10.34936, + "44": 10.32171, + "45": 10.33284, + "46": 10.29371, + "47": 10.28075, + "48": 10.23593, + "49": 10.24626, + "50": 10.23483 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5949.0, - "2": 5859.0, - "3": 6058.0, - "4": 6009.0, - "5": 6077.0, - "6": 5629.0, - "7": 6046.0, - "8": 6056.0, - "9": 6076.0, - "10": 6057.0, - "11": 6040.0, - "12": 5768.0, - "13": 6176.0, - "14": 6151.0, - "15": 5912.0, - "16": 5989.0, - "17": 6088.0, - "18": 5925.0, - "19": 6083.0, - "20": 5820.0, - "21": 5694.0, - "22": 5956.0, - "23": 5936.0, - "24": 6145.0, - "25": 5845.0, - "26": 5951.0, - "27": 5898.0, - "28": 6086.0, - "29": 5955.0, - "30": 5874.0, - "31": 6090.0, - "32": 5974.0, - "33": 5926.0, - "34": 6205.0, - "35": 6237.0, - "36": 6061.0, - "37": 6564.0, - "38": 6296.0, - "39": 6436.0, - "40": 6544.0, - "41": 6819.0, - "42": 6065.0, - "43": 6763.0, - "44": 6546.0, - "45": 7297.0, - "46": 6821.0, - "47": 6931.0, - "48": 7259.0, - "49": 7526.0, - "50": 7287.0 + "1": 5858.0, + "2": 5844.0, + "3": 6012.0, + "4": 5997.0, + "5": 5898.0, + "6": 5796.0, + "7": 5882.0, + "8": 5932.0, + "9": 6015.0, + "10": 6087.0, + "11": 6036.0, + "12": 5855.0, + "13": 6186.0, + "14": 6013.0, + "15": 5954.0, + "16": 5946.0, + "17": 6240.0, + "18": 5776.0, + "19": 6004.0, + "20": 5739.0, + "21": 5744.0, + "22": 6042.0, + "23": 5734.0, + "24": 5934.0, + "25": 5747.0, + "26": 6006.0, + "27": 5918.0, + "28": 5966.0, + "29": 6001.0, + "30": 5972.0, + "31": 6042.0, + "32": 6109.0, + "33": 6111.0, + "34": 6240.0, + "35": 6197.0, + "36": 6019.0, + "37": 6607.0, + "38": 6320.0, + "39": 6542.0, + "40": 6467.0, + "41": 6730.0, + "42": 6170.0, + "43": 6724.0, + "44": 6532.0, + "45": 7109.0, + "46": 6853.0, + "47": 6848.0, + "48": 7283.0, + "49": 7503.0, + "50": 7260.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 627719168.0, - "2": 627717120.0, - "3": 627718144.0, - "4": 627719168.0, - "5": 627717632.0, - "6": 627717632.0, - "7": 627719168.0, - "8": 627717120.0, - "9": 627718144.0, - "10": 627717120.0, - "11": 627719680.0, - "12": 627718144.0, - "13": 627717120.0, - "14": 627720192.0, - "15": 627718144.0, - "16": 627719680.0, - "17": 627720192.0, - "18": 627719168.0, - "19": 627718656.0, - "20": 627718144.0, - "21": 627718144.0, - "22": 627718144.0, - "23": 627719168.0, - "24": 627716608.0, - "25": 627718144.0, - "26": 627720192.0, - "27": 627719168.0, - "28": 627719168.0, - "29": 627719168.0, - "30": 627719168.0, - "31": 627718144.0, - "32": 627720192.0, - "33": 627719168.0, - "34": 627719168.0, - "35": 627718144.0, - "36": 627718144.0, - "37": 627719168.0, - "38": 627719680.0, - "39": 627717632.0, - "40": 627718144.0, - "41": 627720192.0, - "42": 627718144.0, - "43": 627718656.0, - "44": 627719168.0, - "45": 627718656.0, - "46": 627720704.0, - "47": 627720704.0, - "48": 627718144.0, - "49": 627721216.0, - "50": 627720192.0 + "1": 627718656.0, + "2": 627716608.0, + "3": 627717632.0, + "4": 627718656.0, + "5": 627716608.0, + "6": 627718144.0, + "7": 627716608.0, + "8": 627716608.0, + "9": 627718656.0, + "10": 627716608.0, + "11": 627718144.0, + "12": 627717632.0, + "13": 627716608.0, + "14": 627719680.0, + "15": 627718656.0, + "16": 627719168.0, + "17": 627719680.0, + "18": 627718656.0, + "19": 627719168.0, + "20": 627719168.0, + "21": 627717632.0, + "22": 627717632.0, + "23": 627718144.0, + "24": 627716096.0, + "25": 627717632.0, + "26": 627718656.0, + "27": 627718656.0, + "28": 627718656.0, + "29": 627718144.0, + "30": 627718656.0, + "31": 627717632.0, + "32": 627719680.0, + "33": 627718656.0, + "34": 627718656.0, + "35": 627717120.0, + "36": 627717632.0, + "37": 627718656.0, + "38": 627719168.0, + "39": 627716096.0, + "40": 627717632.0, + "41": 627719680.0, + "42": 627717632.0, + "43": 627718144.0, + "44": 627718656.0, + "45": 627719168.0, + "46": 627720192.0, + "47": 627719680.0, + "48": 627716608.0, + "49": 627720704.0, + "50": 627719680.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 880417280.0, - "2": 1112464384.0, - "3": 1112464384.0, - "4": 1113576448.0, - "5": 1113804800.0, - "6": 1113804800.0, - "7": 1113804800.0, - "8": 1113804800.0, - "9": 1113804800.0, - "10": 1113804800.0, - "11": 1113804800.0, - "12": 1113804800.0, - "13": 1113804800.0, - "14": 1113804800.0, - "15": 1113804800.0, - "16": 1113804800.0, - "17": 1113804800.0, - "18": 1113804800.0, - "19": 1113804800.0, - "20": 1113804800.0, - "21": 1113804800.0, - "22": 1113804800.0, - "23": 1113804800.0, - "24": 1113804800.0, - "25": 1113804800.0, - "26": 1113804800.0, - "27": 1113804800.0, - "28": 1113804800.0, - "29": 1113804800.0, - "30": 1113804800.0, - "31": 1113804800.0, - "32": 1113804800.0, - "33": 1113804800.0, - "34": 1114211328.0, - "35": 1114211328.0, - "36": 1114211328.0, - "37": 1114211328.0, - "38": 1114211328.0, - "39": 1114211328.0, - "40": 1114211328.0, - "41": 1114211328.0, - "42": 1114211328.0, - "43": 1114211328.0, - "44": 1114211328.0, - "45": 1114211328.0, - "46": 1114211328.0, - "47": 1114211328.0, - "48": 1114211328.0, - "49": 1114211328.0, - "50": 1114211328.0 + "1": 880758272.0, + "2": 1113250304.0, + "3": 1113250304.0, + "4": 1114187264.0, + "5": 1114544640.0, + "6": 1114544640.0, + "7": 1114544640.0, + "8": 1114544640.0, + "9": 1114544640.0, + "10": 1114544640.0, + "11": 1114544640.0, + "12": 1114544640.0, + "13": 1114544640.0, + "14": 1114544640.0, + "15": 1114544640.0, + "16": 1114544640.0, + "17": 1114544640.0, + "18": 1114544640.0, + "19": 1114544640.0, + "20": 1114544640.0, + "21": 1114544640.0, + "22": 1114544640.0, + "23": 1114544640.0, + "24": 1114731008.0, + "25": 1114731008.0, + "26": 1114731008.0, + "27": 1114731008.0, + "28": 1114731008.0, + "29": 1114731008.0, + "30": 1114731008.0, + "31": 1114731008.0, + "32": 1114731008.0, + "33": 1114731008.0, + "34": 1114732032.0, + "35": 1114732032.0, + "36": 1114732032.0, + "37": 1114732032.0, + "38": 1114732032.0, + "39": 1114732032.0, + "40": 1114732032.0, + "41": 1114732032.0, + "42": 1114732032.0, + "43": 1114732032.0, + "44": 1114732032.0, + "45": 1114732032.0, + "46": 1114732032.0, + "47": 1114732032.0, + "48": 1114732032.0, + "49": 1114732032.0, + "50": 1114732032.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 13.483, - "3": 0.7312, - "4": 0.67116, - "5": 0.66974, - "6": 0.67336, - "7": 0.67737, - "8": 0.67005, - "9": 0.67248, - "10": 0.66633, - "11": 0.66801, - "12": 0.67109, - "13": 0.68147, - "14": 0.67377, - "15": 0.68024, - "16": 0.66845, - "17": 0.67015, - "18": 0.6805, - "19": 0.67134, - "20": 0.67231, - "21": 0.67494, - "22": 0.67105, - "23": 0.67228, - "24": 0.67125, - "25": 0.66877, - "26": 0.67124, - "27": 0.6716, - "28": 0.67345, - "29": 0.67161, - "30": 0.66868, - "31": 0.6693, - "32": 0.67028, - "33": 0.67091, - "34": 0.66779, - "35": 0.66824, - "36": 0.66646, - "37": 0.66883, - "38": 0.66822, - "39": 0.66707, - "40": 0.66596, - "41": 0.66832, - "42": 0.66834, - "43": 0.69874, - "44": 0.67435, - "45": 0.66356, - "46": 0.6675, - "47": 1.11895, - "48": 0.66977, - "49": 0.67647, - "50": 0.66556 + "2": 10.21234, + "3": 0.7015, + "4": 0.65111, + "5": 0.64723, + "6": 0.65244, + "7": 0.65229, + "8": 0.65142, + "9": 0.711, + "10": 0.77267, + "11": 0.75866, + "12": 0.75296, + "13": 0.73062, + "14": 0.71986, + "15": 0.72786, + "16": 0.78055, + "17": 0.74801, + "18": 0.79463, + "19": 0.76653, + "20": 0.73479, + "21": 1.26171, + "22": 0.64931, + "23": 0.64984, + "24": 0.65359, + "25": 0.65677, + "26": 0.64256, + "27": 0.65419, + "28": 0.67076, + "29": 0.82522, + "30": 1.26257, + "31": 0.8005, + "32": 0.79647, + "33": 0.79418, + "34": 0.80418, + "35": 0.78746, + "36": 0.80003, + "37": 0.79099, + "38": 0.80105, + "39": 0.81857, + "40": 0.80319, + "41": 0.79922, + "42": 0.79884, + "43": 0.80345, + "44": 0.81746, + "45": 0.79477, + "46": 0.79218, + "47": 0.79453, + "48": 0.79588, + "49": 0.80028, + "50": 0.78928 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json index 1a5c43059e3..4b07fa13dcf 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.89802, "2": 10.90021, - "3": 10.89823, - "4": 10.89669, - "5": 10.89951, - "6": 10.88562, - "7": 10.89028, - "8": 10.89225, - "9": 10.89399, - "10": 10.89786, - "11": 10.88695, - "12": 10.88994, - "13": 10.88377, - "14": 10.87719, - "15": 10.8672, - "16": 10.8546, - "17": 10.8549, - "18": 10.83796, - "19": 10.85411, - "20": 10.78881, - "21": 10.7772, - "22": 10.77399, - "23": 10.76223, - "24": 10.73066, - "25": 10.72292, - "26": 10.72271, - "27": 10.70096, - "28": 10.64012, - "29": 10.61024, - "30": 10.59081, - "31": 10.57683, - "32": 10.55893, - "33": 10.52562, - "34": 10.51045, - "35": 10.49804, - "36": 10.48157, - "37": 10.45275, - "38": 10.46063, - "39": 10.42883, - "40": 10.412, - "41": 10.39572, - "42": 10.3692, - "43": 10.34979, - "44": 10.32155, - "45": 10.33299, - "46": 10.29376, - "47": 10.2807, - "48": 10.23603, - "49": 10.24619, - "50": 10.23463 + "3": 10.89831, + "4": 10.8967, + "5": 10.8995, + "6": 10.88572, + "7": 10.88964, + "8": 10.8913, + "9": 10.89469, + "10": 10.89864, + "11": 10.88755, + "12": 10.89022, + "13": 10.88419, + "14": 10.87613, + "15": 10.86778, + "16": 10.85477, + "17": 10.85468, + "18": 10.83744, + "19": 10.85415, + "20": 10.78884, + "21": 10.77692, + "22": 10.77377, + "23": 10.76197, + "24": 10.73101, + "25": 10.7232, + "26": 10.723, + "27": 10.70065, + "28": 10.64004, + "29": 10.61056, + "30": 10.59133, + "31": 10.57684, + "32": 10.55855, + "33": 10.5257, + "34": 10.51116, + "35": 10.49829, + "36": 10.48199, + "37": 10.45262, + "38": 10.46114, + "39": 10.42884, + "40": 10.41235, + "41": 10.39539, + "42": 10.36979, + "43": 10.34936, + "44": 10.32171, + "45": 10.33284, + "46": 10.29371, + "47": 10.28075, + "48": 10.23593, + "49": 10.24626, + "50": 10.23483 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 5949.0, - "2": 5859.0, - "3": 6058.0, - "4": 6009.0, - "5": 6077.0, - "6": 5629.0, - "7": 6046.0, - "8": 6056.0, - "9": 6076.0, - "10": 6057.0, - "11": 6040.0, - "12": 5768.0, - "13": 6176.0, - "14": 6151.0, - "15": 5912.0, - "16": 5989.0, - "17": 6088.0, - "18": 5925.0, - "19": 6083.0, - "20": 5820.0, - "21": 5694.0, - "22": 5956.0, - "23": 5936.0, - "24": 6145.0, - "25": 5845.0, - "26": 5951.0, - "27": 5898.0, - "28": 6086.0, - "29": 5955.0, - "30": 5874.0, - "31": 6090.0, - "32": 5974.0, - "33": 5926.0, - "34": 6205.0, - "35": 6237.0, - "36": 6061.0, - "37": 6564.0, - "38": 6296.0, - "39": 6436.0, - "40": 6544.0, - "41": 6819.0, - "42": 6065.0, - "43": 6763.0, - "44": 6546.0, - "45": 7297.0, - "46": 6821.0, - "47": 6931.0, - "48": 7259.0, - "49": 7526.0, - "50": 7287.0 + "1": 5858.0, + "2": 5844.0, + "3": 6012.0, + "4": 5997.0, + "5": 5898.0, + "6": 5796.0, + "7": 5882.0, + "8": 5932.0, + "9": 6015.0, + "10": 6087.0, + "11": 6036.0, + "12": 5855.0, + "13": 6186.0, + "14": 6013.0, + "15": 5954.0, + "16": 5946.0, + "17": 6240.0, + "18": 5776.0, + "19": 6004.0, + "20": 5739.0, + "21": 5744.0, + "22": 6042.0, + "23": 5734.0, + "24": 5934.0, + "25": 5747.0, + "26": 6006.0, + "27": 5918.0, + "28": 5966.0, + "29": 6001.0, + "30": 5972.0, + "31": 6042.0, + "32": 6109.0, + "33": 6111.0, + "34": 6240.0, + "35": 6197.0, + "36": 6019.0, + "37": 6607.0, + "38": 6320.0, + "39": 6542.0, + "40": 6467.0, + "41": 6730.0, + "42": 6170.0, + "43": 6724.0, + "44": 6532.0, + "45": 7109.0, + "46": 6853.0, + "47": 6848.0, + "48": 7283.0, + "49": 7503.0, + "50": 7260.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 627719168.0, - "2": 627717120.0, - "3": 627718144.0, - "4": 627719168.0, - "5": 627717632.0, - "6": 627717632.0, - "7": 627719168.0, - "8": 627717120.0, - "9": 627718144.0, - "10": 627717120.0, - "11": 627719680.0, - "12": 627718144.0, - "13": 627717120.0, - "14": 627720192.0, - "15": 627718144.0, - "16": 627719680.0, - "17": 627720192.0, - "18": 627719168.0, - "19": 627718656.0, - "20": 627718144.0, - "21": 627718144.0, - "22": 627718144.0, - "23": 627719168.0, - "24": 627716608.0, - "25": 627718144.0, - "26": 627720192.0, - "27": 627719168.0, - "28": 627719168.0, - "29": 627719168.0, - "30": 627719168.0, - "31": 627718144.0, - "32": 627720192.0, - "33": 627719168.0, - "34": 627719168.0, - "35": 627718144.0, - "36": 627718144.0, - "37": 627719168.0, - "38": 627719680.0, - "39": 627717632.0, - "40": 627718144.0, - "41": 627720192.0, - "42": 627718144.0, - "43": 627718656.0, - "44": 627719168.0, - "45": 627718656.0, - "46": 627720704.0, - "47": 627720704.0, - "48": 627718144.0, - "49": 627721216.0, - "50": 627720192.0 + "1": 627718656.0, + "2": 627716608.0, + "3": 627717632.0, + "4": 627718656.0, + "5": 627716608.0, + "6": 627718144.0, + "7": 627716608.0, + "8": 627716608.0, + "9": 627718656.0, + "10": 627716608.0, + "11": 627718144.0, + "12": 627717632.0, + "13": 627716608.0, + "14": 627719680.0, + "15": 627718656.0, + "16": 627719168.0, + "17": 627719680.0, + "18": 627718656.0, + "19": 627719168.0, + "20": 627719168.0, + "21": 627717632.0, + "22": 627717632.0, + "23": 627718144.0, + "24": 627716096.0, + "25": 627717632.0, + "26": 627718656.0, + "27": 627718656.0, + "28": 627718656.0, + "29": 627718144.0, + "30": 627718656.0, + "31": 627717632.0, + "32": 627719680.0, + "33": 627718656.0, + "34": 627718656.0, + "35": 627717120.0, + "36": 627717632.0, + "37": 627718656.0, + "38": 627719168.0, + "39": 627716096.0, + "40": 627717632.0, + "41": 627719680.0, + "42": 627717632.0, + "43": 627718144.0, + "44": 627718656.0, + "45": 627719168.0, + "46": 627720192.0, + "47": 627719680.0, + "48": 627716608.0, + "49": 627720704.0, + "50": 627719680.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 880758784.0, - "2": 1113250816.0, - "3": 1113250816.0, - "4": 1114362880.0, - "5": 1114579968.0, - "6": 1114579968.0, - "7": 1114579968.0, - "8": 1114579968.0, - "9": 1114579968.0, - "10": 1114579968.0, - "11": 1114579968.0, - "12": 1114579968.0, - "13": 1114579968.0, - "14": 1114579968.0, - "15": 1114579968.0, - "16": 1114579968.0, - "17": 1114579968.0, - "18": 1114579968.0, - "19": 1114579968.0, - "20": 1114579968.0, - "21": 1114579968.0, - "22": 1114579968.0, - "23": 1114579968.0, - "24": 1114579968.0, - "25": 1114579968.0, - "26": 1114579968.0, - "27": 1114579968.0, - "28": 1114579968.0, - "29": 1114579968.0, - "30": 1114579968.0, - "31": 1114579968.0, - "32": 1114579968.0, - "33": 1114579968.0, - "34": 1114937344.0, - "35": 1114937344.0, - "36": 1114937344.0, - "37": 1114937344.0, - "38": 1114937344.0, - "39": 1114937344.0, - "40": 1114937344.0, - "41": 1114937344.0, - "42": 1114937344.0, - "43": 1114937344.0, - "44": 1114937344.0, - "45": 1114937344.0, - "46": 1114937344.0, - "47": 1114937344.0, - "48": 1114937344.0, - "49": 1114937344.0, - "50": 1114937344.0 + "1": 880758272.0, + "2": 1113250304.0, + "3": 1113250304.0, + "4": 1114187264.0, + "5": 1114544640.0, + "6": 1114544640.0, + "7": 1114544640.0, + "8": 1114544640.0, + "9": 1114544640.0, + "10": 1114544640.0, + "11": 1114544640.0, + "12": 1114544640.0, + "13": 1114544640.0, + "14": 1114544640.0, + "15": 1114544640.0, + "16": 1114544640.0, + "17": 1114544640.0, + "18": 1114544640.0, + "19": 1114544640.0, + "20": 1114544640.0, + "21": 1114544640.0, + "22": 1114544640.0, + "23": 1114544640.0, + "24": 1114731008.0, + "25": 1114731008.0, + "26": 1114731008.0, + "27": 1114731008.0, + "28": 1114731008.0, + "29": 1114731008.0, + "30": 1114731008.0, + "31": 1114731008.0, + "32": 1114731008.0, + "33": 1114731008.0, + "34": 1114732032.0, + "35": 1114732032.0, + "36": 1114732032.0, + "37": 1114732032.0, + "38": 1114732032.0, + "39": 1114732032.0, + "40": 1114732032.0, + "41": 1114732032.0, + "42": 1114732032.0, + "43": 1114732032.0, + "44": 1114732032.0, + "45": 1114732032.0, + "46": 1114732032.0, + "47": 1114732032.0, + "48": 1114732032.0, + "49": 1114732032.0, + "50": 1114732032.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 15.40919, - "3": 0.75426, - "4": 0.87534, - "5": 0.73868, - "6": 0.68354, - "7": 0.68563, - "8": 0.67934, - "9": 0.6814, - "10": 0.68143, - "11": 0.67986, - "12": 0.71601, - "13": 0.68559, - "14": 0.68837, - "15": 0.67963, - "16": 0.67492, - "17": 0.68242, - "18": 0.68731, - "19": 0.68382, - "20": 0.6857, - "21": 0.6885, - "22": 0.68262, - "23": 0.67761, - "24": 0.67638, - "25": 0.67197, - "26": 0.67301, - "27": 0.6741, - "28": 0.67606, - "29": 0.67143, - "30": 0.68474, - "31": 0.67407, - "32": 0.68845, - "33": 0.68723, - "34": 0.68059, - "35": 0.67852, - "36": 0.67773, - "37": 0.67297, - "38": 0.67585, - "39": 0.67629, - "40": 0.67402, - "41": 0.67594, - "42": 0.68121, - "43": 0.6947, - "44": 0.67403, - "45": 0.67901, - "46": 0.69073, - "47": 0.69234, - "48": 0.68458, - "49": 0.67565, - "50": 0.67788 + "2": 10.6396, + "3": 0.88808, + "4": 0.81903, + "5": 0.83842, + "6": 0.83047, + "7": 0.82722, + "8": 0.83388, + "9": 0.82641, + "10": 0.82108, + "11": 0.88756, + "12": 0.92781, + "13": 0.91005, + "14": 0.92756, + "15": 0.91831, + "16": 0.95742, + "17": 0.93774, + "18": 0.95694, + "19": 0.91706, + "20": 0.91244, + "21": 0.90981, + "22": 0.92203, + "23": 0.92101, + "24": 0.94481, + "25": 0.95514, + "26": 0.83433, + "27": 0.86786, + "28": 0.92449, + "29": 0.93244, + "30": 0.90295, + "31": 0.95313, + "32": 0.96658, + "33": 0.86027, + "34": 0.92597, + "35": 0.83618, + "36": 0.89351, + "37": 0.91155, + "38": 0.91197, + "39": 0.92484, + "40": 0.95373, + "41": 0.89966, + "42": 0.8958, + "43": 0.91397, + "44": 0.91131, + "45": 0.94879, + "46": 0.9542, + "47": 0.94332, + "48": 0.92787, + "49": 0.9775, + "50": 0.987 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index 566cf687183..fd99b14288d 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92142, "2": 10.92363, - "3": 10.92242, - "4": 10.92026, - "5": 10.91483, - "6": 10.90385, + "3": 10.92259, + "4": 10.92078, + "5": 10.91474, + "6": 10.90404, "7": 10.91109, - "8": 10.91595, - "9": 10.92047, - "10": 10.91896, - "11": 10.90617, - "12": 10.90579, - "13": 10.90562, - "14": 10.8989, - "15": 10.87723, - "16": 10.86957, - "17": 10.8846, - "18": 10.86603, - "19": 10.86663, - "20": 10.80107, - "21": 10.79745, - "22": 10.78608, - "23": 10.77958, - "24": 10.74282, - "25": 10.74782, - "26": 10.72794, - "27": 10.70948, - "28": 10.64378, - "29": 10.60676, - "30": 10.59805, - "31": 10.58664, - "32": 10.57936, - "33": 10.546, - "34": 10.50679, - "35": 10.50654, - "36": 10.49965, - "37": 10.45837, - "38": 10.46672, + "8": 10.91577, + "9": 10.92032, + "10": 10.91831, + "11": 10.90606, + "12": 10.90555, + "13": 10.90525, + "14": 10.89842, + "15": 10.87718, + "16": 10.87022, + "17": 10.88408, + "18": 10.86635, + "19": 10.8665, + "20": 10.80136, + "21": 10.79711, + "22": 10.78584, + "23": 10.77896, + "24": 10.74287, + "25": 10.74764, + "26": 10.72773, + "27": 10.70951, + "28": 10.64361, + "29": 10.60637, + "30": 10.59798, + "31": 10.58649, + "32": 10.5787, + "33": 10.54619, + "34": 10.50579, + "35": 10.50666, + "36": 10.49953, + "37": 10.45867, + "38": 10.46677, "39": 10.43117, - "40": 10.42314, - "41": 10.4007, - "42": 10.38462, - "43": 10.36964, - "44": 10.34028, - "45": 10.34887, - "46": 10.31013, - "47": 10.29593, - "48": 10.25007, - "49": 10.24404, - "50": 10.25153 + "40": 10.42323, + "41": 10.40064, + "42": 10.38491, + "43": 10.36918, + "44": 10.34051, + "45": 10.34921, + "46": 10.3103, + "47": 10.2959, + "48": 10.24992, + "49": 10.24412, + "50": 10.25152 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6494.0, - "2": 6740.0, - "3": 6956.0, - "4": 6747.0, - "5": 6648.0, - "6": 6407.0, - "7": 6764.0, - "8": 6471.0, - "9": 6733.0, - "10": 6750.0, - "11": 6865.0, - "12": 6288.0, - "13": 6782.0, - "14": 6749.0, - "15": 6437.0, - "16": 6530.0, - "17": 6644.0, - "18": 6336.0, - "19": 6734.0, - "20": 6356.0, - "21": 6588.0, - "22": 6652.0, - "23": 6530.0, - "24": 6665.0, - "25": 6702.0, - "26": 6687.0, - "27": 6681.0, - "28": 6510.0, - "29": 6670.0, - "30": 6556.0, - "31": 6782.0, - "32": 6863.0, - "33": 6884.0, - "34": 7126.0, - "35": 7032.0, - "36": 6895.0, - "37": 7186.0, - "38": 6999.0, - "39": 7246.0, - "40": 7364.0, - "41": 7488.0, - "42": 6879.0, - "43": 7620.0, - "44": 7231.0, - "45": 7956.0, - "46": 7554.0, - "47": 7513.0, - "48": 7742.0, - "49": 8157.0, - "50": 7799.0 + "1": 6607.0, + "2": 6632.0, + "3": 6855.0, + "4": 6848.0, + "5": 6647.0, + "6": 6417.0, + "7": 6691.0, + "8": 6372.0, + "9": 6894.0, + "10": 6664.0, + "11": 6802.0, + "12": 6322.0, + "13": 6741.0, + "14": 6743.0, + "15": 6389.0, + "16": 6586.0, + "17": 6799.0, + "18": 6379.0, + "19": 6676.0, + "20": 6418.0, + "21": 6669.0, + "22": 6660.0, + "23": 6599.0, + "24": 6655.0, + "25": 6486.0, + "26": 6717.0, + "27": 6750.0, + "28": 6524.0, + "29": 6685.0, + "30": 6492.0, + "31": 6976.0, + "32": 6727.0, + "33": 6930.0, + "34": 6992.0, + "35": 6730.0, + "36": 6634.0, + "37": 7187.0, + "38": 6996.0, + "39": 7190.0, + "40": 7312.0, + "41": 7491.0, + "42": 6775.0, + "43": 7635.0, + "44": 7281.0, + "45": 7816.0, + "46": 7550.0, + "47": 7588.0, + "48": 7762.0, + "49": 8253.0, + "50": 7895.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 491766272.0, - "2": 491769344.0, - "3": 491768320.0, - "4": 491769856.0, - "5": 491768320.0, - "6": 491767808.0, - "7": 491765248.0, - "8": 491767296.0, - "9": 491768832.0, - "10": 491769344.0, - "11": 491768832.0, - "12": 491769856.0, - "13": 491766784.0, - "14": 491769344.0, - "15": 491769344.0, - "16": 491767808.0, - "17": 491767296.0, + "1": 491765760.0, + "2": 491768832.0, + "3": 491767808.0, + "4": 491769344.0, + "5": 491768832.0, + "6": 491767296.0, + "7": 491765760.0, + "8": 491765760.0, + "9": 491769344.0, + "10": 491768832.0, + "11": 491768320.0, + "12": 491769344.0, + "13": 491766272.0, + "14": 491768832.0, + "15": 491768832.0, + "16": 491766784.0, + "17": 491765760.0, "18": 491769344.0, - "19": 491769344.0, - "20": 491768832.0, - "21": 491765760.0, - "22": 491770880.0, - "23": 491767808.0, + "19": 491768832.0, + "20": 491769856.0, + "21": 491766272.0, + "22": 491770368.0, + "23": 491767296.0, "24": 491768832.0, - "25": 491769344.0, + "25": 491768832.0, "26": 491766784.0, - "27": 491768320.0, - "28": 491767808.0, - "29": 491766272.0, + "27": 491767808.0, + "28": 491766272.0, + "29": 491766784.0, "30": 491768832.0, - "31": 491768832.0, - "32": 491769856.0, - "33": 491768320.0, - "34": 491768832.0, - "35": 491769344.0, + "31": 491768320.0, + "32": 491768320.0, + "33": 491767296.0, + "34": 491768320.0, + "35": 491767808.0, "36": 491770368.0, - "37": 491768320.0, - "38": 491769856.0, + "37": 491769344.0, + "38": 491769344.0, "39": 491767808.0, - "40": 491767296.0, - "41": 491769856.0, + "40": 491766272.0, + "41": 491769344.0, "42": 491770368.0, - "43": 491767808.0, - "44": 491768832.0, - "45": 491770368.0, - "46": 491771392.0, - "47": 491769344.0, - "48": 491770368.0, + "43": 491767296.0, + "44": 491768320.0, + "45": 491768832.0, + "46": 491770880.0, + "47": 491768320.0, + "48": 491769856.0, "49": 491770368.0, - "50": 491768832.0 + "50": 491769344.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1060007936.0, - "2": 1225918976.0, - "3": 1225918976.0, - "4": 1226195456.0, - "5": 1226195456.0, - "6": 1226195456.0, - "7": 1226195456.0, - "8": 1226195456.0, - "9": 1226195456.0, - "10": 1226195456.0, - "11": 1226195456.0, - "12": 1226195456.0, - "13": 1226195456.0, - "14": 1226288128.0, - "15": 1226288128.0, - "16": 1226288128.0, - "17": 1226288128.0, - "18": 1226288128.0, - "19": 1226288128.0, - "20": 1226370048.0, - "21": 1226370048.0, - "22": 1226370048.0, - "23": 1226370048.0, - "24": 1226370048.0, - "25": 1226370048.0, - "26": 1226370048.0, - "27": 1226370048.0, - "28": 1226370048.0, - "29": 1226370048.0, - "30": 1226370048.0, - "31": 1226370048.0, - "32": 1226370048.0, - "33": 1226413056.0, - "34": 1226413056.0, - "35": 1226587136.0, - "36": 1226587136.0, - "37": 1226587136.0, - "38": 1226587136.0, - "39": 1226587136.0, - "40": 1226587136.0, - "41": 1226592768.0, - "42": 1226592768.0, - "43": 1226592768.0, - "44": 1226592768.0, - "45": 1226592768.0, - "46": 1226592768.0, - "47": 1226592768.0, - "48": 1226592768.0, - "49": 1226592768.0, - "50": 1226592768.0 + "1": 1060007424.0, + "2": 1225918464.0, + "3": 1225918464.0, + "4": 1226219520.0, + "5": 1226219520.0, + "6": 1226219520.0, + "7": 1226223104.0, + "8": 1226223104.0, + "9": 1226223104.0, + "10": 1226223104.0, + "11": 1226223104.0, + "12": 1226223104.0, + "13": 1226223104.0, + "14": 1226223104.0, + "15": 1226223104.0, + "16": 1226223104.0, + "17": 1226267136.0, + "18": 1226267136.0, + "19": 1226267136.0, + "20": 1226267136.0, + "21": 1226267136.0, + "22": 1226267136.0, + "23": 1226267136.0, + "24": 1226267136.0, + "25": 1226267136.0, + "26": 1226267136.0, + "27": 1226267136.0, + "28": 1226267136.0, + "29": 1226267136.0, + "30": 1226267648.0, + "31": 1226267648.0, + "32": 1226267648.0, + "33": 1226351104.0, + "34": 1226351104.0, + "35": 1226351104.0, + "36": 1226351104.0, + "37": 1226351104.0, + "38": 1226351104.0, + "39": 1226351104.0, + "40": 1226423296.0, + "41": 1226491904.0, + "42": 1226491904.0, + "43": 1226491904.0, + "44": 1226491904.0, + "45": 1226491904.0, + "46": 1226491904.0, + "47": 1226491904.0, + "48": 1226491904.0, + "49": 1226491904.0, + "50": 1226491904.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 11.59297, - "3": 0.61117, - "4": 0.55518, - "5": 0.55352, - "6": 0.55419, - "7": 0.54854, - "8": 0.61911, - "9": 0.65501, - "10": 0.64508, - "11": 0.71416, - "12": 0.55167, - "13": 0.55492, - "14": 0.54884, - "15": 0.54681, - "16": 0.55185, - "17": 0.54866, - "18": 0.55156, - "19": 0.55251, - "20": 0.55242, - "21": 0.54509, - "22": 0.54782, - "23": 0.55094, - "24": 0.54959, - "25": 0.5476, - "26": 0.54683, - "27": 0.54882, - "28": 0.54789, - "29": 0.5489, - "30": 0.55627, - "31": 0.55243, - "32": 0.54838, - "33": 0.54675, - "34": 0.54772, - "35": 0.55016, - "36": 0.55247, - "37": 0.55075, - "38": 0.54782, - "39": 0.54863, - "40": 0.54751, - "41": 0.55018, - "42": 0.54783, - "43": 0.54888, - "44": 0.54735, - "45": 0.54997, - "46": 0.54875, - "47": 0.55091, - "48": 0.55445, - "49": 0.54824, - "50": 0.55019 + "2": 7.87373, + "3": 0.9906, + "4": 0.53943, + "5": 0.44181, + "6": 0.4461, + "7": 0.43769, + "8": 0.44438, + "9": 0.43907, + "10": 0.44439, + "11": 0.43668, + "12": 0.43528, + "13": 0.43461, + "14": 0.5253, + "15": 0.62362, + "16": 0.769, + "17": 0.52432, + "18": 0.59003, + "19": 0.53397, + "20": 0.43038, + "21": 0.42432, + "22": 0.42573, + "23": 0.42954, + "24": 0.42678, + "25": 0.42352, + "26": 0.42259, + "27": 0.42515, + "28": 0.42321, + "29": 0.42656, + "30": 0.43152, + "31": 0.43341, + "32": 0.43188, + "33": 0.43336, + "34": 0.42955, + "35": 0.43578, + "36": 0.42784, + "37": 0.43564, + "38": 0.43265, + "39": 0.42451, + "40": 0.42585, + "41": 0.42858, + "42": 0.43309, + "43": 0.43118, + "44": 0.43242, + "45": 0.43134, + "46": 0.43209, + "47": 0.43291, + "48": 0.44498, + "49": 0.44466, + "50": 0.45422 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json index 4761124bd0d..463f8705e51 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.92142, "2": 10.92363, - "3": 10.92242, - "4": 10.92026, - "5": 10.91483, - "6": 10.90385, + "3": 10.92259, + "4": 10.92078, + "5": 10.91474, + "6": 10.90404, "7": 10.91109, - "8": 10.91595, - "9": 10.92047, - "10": 10.91896, - "11": 10.90617, - "12": 10.90579, - "13": 10.90562, - "14": 10.8989, - "15": 10.87723, - "16": 10.86957, - "17": 10.8846, - "18": 10.86603, - "19": 10.86663, - "20": 10.80107, - "21": 10.79745, - "22": 10.78608, - "23": 10.77958, - "24": 10.74282, - "25": 10.74782, - "26": 10.72794, - "27": 10.70948, - "28": 10.64378, - "29": 10.60676, - "30": 10.59805, - "31": 10.58664, - "32": 10.57936, - "33": 10.546, - "34": 10.50679, - "35": 10.50654, - "36": 10.49965, - "37": 10.45837, - "38": 10.46672, + "8": 10.91577, + "9": 10.92032, + "10": 10.91831, + "11": 10.90606, + "12": 10.90555, + "13": 10.90525, + "14": 10.89842, + "15": 10.87718, + "16": 10.87022, + "17": 10.88408, + "18": 10.86635, + "19": 10.8665, + "20": 10.80136, + "21": 10.79711, + "22": 10.78584, + "23": 10.77896, + "24": 10.74287, + "25": 10.74764, + "26": 10.72773, + "27": 10.70951, + "28": 10.64361, + "29": 10.60637, + "30": 10.59798, + "31": 10.58649, + "32": 10.5787, + "33": 10.54619, + "34": 10.50579, + "35": 10.50666, + "36": 10.49953, + "37": 10.45867, + "38": 10.46677, "39": 10.43117, - "40": 10.42314, - "41": 10.4007, - "42": 10.38462, - "43": 10.36964, - "44": 10.34028, - "45": 10.34887, - "46": 10.31013, - "47": 10.29593, - "48": 10.25007, - "49": 10.24404, - "50": 10.25153 + "40": 10.42323, + "41": 10.40064, + "42": 10.38491, + "43": 10.36918, + "44": 10.34051, + "45": 10.34921, + "46": 10.3103, + "47": 10.2959, + "48": 10.24992, + "49": 10.24412, + "50": 10.25152 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6494.0, - "2": 6740.0, - "3": 6956.0, - "4": 6747.0, - "5": 6648.0, - "6": 6407.0, - "7": 6764.0, - "8": 6471.0, - "9": 6733.0, - "10": 6750.0, - "11": 6865.0, - "12": 6288.0, - "13": 6782.0, - "14": 6749.0, - "15": 6437.0, - "16": 6530.0, - "17": 6644.0, - "18": 6336.0, - "19": 6734.0, - "20": 6356.0, - "21": 6588.0, - "22": 6652.0, - "23": 6530.0, - "24": 6665.0, - "25": 6702.0, - "26": 6687.0, - "27": 6681.0, - "28": 6510.0, - "29": 6670.0, - "30": 6556.0, - "31": 6782.0, - "32": 6863.0, - "33": 6884.0, - "34": 7126.0, - "35": 7032.0, - "36": 6895.0, - "37": 7186.0, - "38": 6999.0, - "39": 7246.0, - "40": 7364.0, - "41": 7488.0, - "42": 6879.0, - "43": 7620.0, - "44": 7231.0, - "45": 7956.0, - "46": 7554.0, - "47": 7513.0, - "48": 7742.0, - "49": 8157.0, - "50": 7799.0 + "1": 6607.0, + "2": 6632.0, + "3": 6855.0, + "4": 6848.0, + "5": 6647.0, + "6": 6417.0, + "7": 6691.0, + "8": 6372.0, + "9": 6894.0, + "10": 6664.0, + "11": 6802.0, + "12": 6322.0, + "13": 6741.0, + "14": 6743.0, + "15": 6389.0, + "16": 6586.0, + "17": 6799.0, + "18": 6379.0, + "19": 6676.0, + "20": 6418.0, + "21": 6669.0, + "22": 6660.0, + "23": 6599.0, + "24": 6655.0, + "25": 6486.0, + "26": 6717.0, + "27": 6750.0, + "28": 6524.0, + "29": 6685.0, + "30": 6492.0, + "31": 6976.0, + "32": 6727.0, + "33": 6930.0, + "34": 6992.0, + "35": 6730.0, + "36": 6634.0, + "37": 7187.0, + "38": 6996.0, + "39": 7190.0, + "40": 7312.0, + "41": 7491.0, + "42": 6775.0, + "43": 7635.0, + "44": 7281.0, + "45": 7816.0, + "46": 7550.0, + "47": 7588.0, + "48": 7762.0, + "49": 8253.0, + "50": 7895.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 491766272.0, - "2": 491769344.0, - "3": 491768320.0, - "4": 491769856.0, - "5": 491768320.0, - "6": 491767808.0, - "7": 491765248.0, - "8": 491767296.0, - "9": 491768832.0, - "10": 491769344.0, - "11": 491768832.0, - "12": 491769856.0, - "13": 491766784.0, - "14": 491769344.0, - "15": 491769344.0, - "16": 491767808.0, - "17": 491767296.0, + "1": 491765760.0, + "2": 491768832.0, + "3": 491767808.0, + "4": 491769344.0, + "5": 491768832.0, + "6": 491767296.0, + "7": 491765760.0, + "8": 491765760.0, + "9": 491769344.0, + "10": 491768832.0, + "11": 491768320.0, + "12": 491769344.0, + "13": 491766272.0, + "14": 491768832.0, + "15": 491768832.0, + "16": 491766784.0, + "17": 491765760.0, "18": 491769344.0, - "19": 491769344.0, - "20": 491768832.0, - "21": 491765760.0, - "22": 491770880.0, - "23": 491767808.0, + "19": 491768832.0, + "20": 491769856.0, + "21": 491766272.0, + "22": 491770368.0, + "23": 491767296.0, "24": 491768832.0, - "25": 491769344.0, + "25": 491768832.0, "26": 491766784.0, - "27": 491768320.0, - "28": 491767808.0, - "29": 491766272.0, + "27": 491767808.0, + "28": 491766272.0, + "29": 491766784.0, "30": 491768832.0, - "31": 491768832.0, - "32": 491769856.0, - "33": 491768320.0, - "34": 491768832.0, - "35": 491769344.0, + "31": 491768320.0, + "32": 491768320.0, + "33": 491767296.0, + "34": 491768320.0, + "35": 491767808.0, "36": 491770368.0, - "37": 491768320.0, - "38": 491769856.0, + "37": 491769344.0, + "38": 491769344.0, "39": 491767808.0, - "40": 491767296.0, - "41": 491769856.0, + "40": 491766272.0, + "41": 491769344.0, "42": 491770368.0, - "43": 491767808.0, - "44": 491768832.0, - "45": 491770368.0, - "46": 491771392.0, - "47": 491769344.0, - "48": 491770368.0, + "43": 491767296.0, + "44": 491768320.0, + "45": 491768832.0, + "46": 491770880.0, + "47": 491768320.0, + "48": 491769856.0, "49": 491770368.0, - "50": 491768832.0 + "50": 491769344.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1059518464.0, - "2": 1224652288.0, - "3": 1224876544.0, - "4": 1225368064.0, - "5": 1225368064.0, - "6": 1226156544.0, - "7": 1226156544.0, - "8": 1226156544.0, - "9": 1226156544.0, - "10": 1226156544.0, - "11": 1226156544.0, - "12": 1226156544.0, - "13": 1226156544.0, - "14": 1226156544.0, - "15": 1226156544.0, - "16": 1226156544.0, - "17": 1226331136.0, - "18": 1226331136.0, - "19": 1226331136.0, - "20": 1226331136.0, - "21": 1226331136.0, - "22": 1226331136.0, - "23": 1226331136.0, - "24": 1226331136.0, - "25": 1226331136.0, - "26": 1226331136.0, - "27": 1226331136.0, - "28": 1226331136.0, - "29": 1226331136.0, - "30": 1226331136.0, - "31": 1226331136.0, - "32": 1226331136.0, - "33": 1226331136.0, - "34": 1226331136.0, - "35": 1226331136.0, - "36": 1226331136.0, - "37": 1226331136.0, - "38": 1226331136.0, - "39": 1226331136.0, - "40": 1226331136.0, - "41": 1226445312.0, - "42": 1226445312.0, - "43": 1226445312.0, - "44": 1226445312.0, - "45": 1226445312.0, - "46": 1226445312.0, - "47": 1226445312.0, - "48": 1226445312.0, - "49": 1226445312.0, - "50": 1226445312.0 + "1": 1060007424.0, + "2": 1225918464.0, + "3": 1225918464.0, + "4": 1226219520.0, + "5": 1226219520.0, + "6": 1226219520.0, + "7": 1226223104.0, + "8": 1226223104.0, + "9": 1226223104.0, + "10": 1226223104.0, + "11": 1226223104.0, + "12": 1226223104.0, + "13": 1226223104.0, + "14": 1226223104.0, + "15": 1226223104.0, + "16": 1226223104.0, + "17": 1226267136.0, + "18": 1226267136.0, + "19": 1226267136.0, + "20": 1226267136.0, + "21": 1226267136.0, + "22": 1226267136.0, + "23": 1226267136.0, + "24": 1226267136.0, + "25": 1226267136.0, + "26": 1226267136.0, + "27": 1226267136.0, + "28": 1226267136.0, + "29": 1226267136.0, + "30": 1226267648.0, + "31": 1226267648.0, + "32": 1226267648.0, + "33": 1226351104.0, + "34": 1226351104.0, + "35": 1226351104.0, + "36": 1226351104.0, + "37": 1226351104.0, + "38": 1226351104.0, + "39": 1226351104.0, + "40": 1226423296.0, + "41": 1226491904.0, + "42": 1226491904.0, + "43": 1226491904.0, + "44": 1226491904.0, + "45": 1226491904.0, + "46": 1226491904.0, + "47": 1226491904.0, + "48": 1226491904.0, + "49": 1226491904.0, + "50": 1226491904.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 10.18028, - "3": 0.60628, - "4": 0.55074, - "5": 0.53947, - "6": 0.54802, - "7": 0.58709, - "8": 0.59054, - "9": 0.69763, - "10": 0.60528, - "11": 0.54007, - "12": 0.54007, - "13": 0.54417, - "14": 0.54303, - "15": 0.54159, - "16": 0.54116, - "17": 0.53576, - "18": 0.54181, - "19": 0.54301, - "20": 0.54579, - "21": 0.53835, - "22": 0.54428, - "23": 0.54102, - "24": 0.53895, - "25": 0.53792, - "26": 0.54029, - "27": 0.53646, - "28": 0.54131, - "29": 0.54182, - "30": 0.54034, - "31": 0.54226, - "32": 0.54121, - "33": 0.5396, - "34": 0.53705, - "35": 0.53877, - "36": 0.53242, - "37": 0.53515, - "38": 0.53758, - "39": 0.53271, - "40": 0.53394, - "41": 0.54051, - "42": 0.53777, - "43": 0.536, - "44": 0.53692, - "45": 0.53218, - "46": 0.53218, - "47": 0.53654, - "48": 0.53637, - "49": 0.53063, - "50": 0.52999 + "2": 7.51559, + "3": 0.58186, + "4": 0.54901, + "5": 0.52875, + "6": 0.53374, + "7": 0.53171, + "8": 0.55101, + "9": 0.5594, + "10": 0.55775, + "11": 0.54701, + "12": 0.54695, + "13": 0.54823, + "14": 0.55026, + "15": 0.54701, + "16": 0.54635, + "17": 0.55049, + "18": 0.55236, + "19": 0.53939, + "20": 0.54788, + "21": 0.54015, + "22": 0.55043, + "23": 0.53578, + "24": 0.53661, + "25": 0.54689, + "26": 0.55716, + "27": 0.55122, + "28": 0.55205, + "29": 0.54336, + "30": 0.5549, + "31": 0.54891, + "32": 0.55515, + "33": 0.55464, + "34": 0.55497, + "35": 0.55348, + "36": 0.54338, + "37": 0.55207, + "38": 0.54166, + "39": 0.54909, + "40": 0.55943, + "41": 0.5653, + "42": 0.54663, + "43": 0.54781, + "44": 0.54912, + "45": 0.55054, + "46": 0.54292, + "47": 0.54964, + "48": 0.5447, + "49": 0.55051, + "50": 0.55167 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json index 5972aaba27f..bb99e290cd8 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel/golden_values_dev_dgx_gb200.json @@ -6,54 +6,54 @@ "values": { "1": 10.89842, "2": 10.89984, - "3": 10.89302, - "4": 10.89222, - "5": 10.89805, - "6": 10.88299, - "7": 10.88844, - "8": 10.89275, - "9": 10.8927, - "10": 10.89479, - "11": 10.88772, - "12": 10.88616, - "13": 10.88296, - "14": 10.87492, - "15": 10.86635, - "16": 10.85026, - "17": 10.85754, - "18": 10.84381, - "19": 10.85464, - "20": 10.78787, - "21": 10.77781, - "22": 10.77446, - "23": 10.7613, - "24": 10.73152, - "25": 10.72157, - "26": 10.72003, - "27": 10.70155, - "28": 10.64166, - "29": 10.61095, - "30": 10.59234, - "31": 10.57783, - "32": 10.55928, - "33": 10.52447, - "34": 10.51226, - "35": 10.49693, - "36": 10.48657, - "37": 10.45357, - "38": 10.46163, - "39": 10.42974, - "40": 10.41128, - "41": 10.3959, - "42": 10.36906, - "43": 10.35092, - "44": 10.32343, - "45": 10.33546, - "46": 10.29381, - "47": 10.28133, - "48": 10.23785, - "49": 10.24676, - "50": 10.23623 + "3": 10.89315, + "4": 10.89205, + "5": 10.89791, + "6": 10.88314, + "7": 10.88865, + "8": 10.892, + "9": 10.89278, + "10": 10.89466, + "11": 10.88863, + "12": 10.88572, + "13": 10.88349, + "14": 10.87478, + "15": 10.8671, + "16": 10.85042, + "17": 10.85782, + "18": 10.84365, + "19": 10.85392, + "20": 10.78747, + "21": 10.77765, + "22": 10.7745, + "23": 10.76114, + "24": 10.73188, + "25": 10.72197, + "26": 10.7201, + "27": 10.70167, + "28": 10.64187, + "29": 10.61116, + "30": 10.59268, + "31": 10.57812, + "32": 10.5589, + "33": 10.52408, + "34": 10.51189, + "35": 10.49655, + "36": 10.48676, + "37": 10.45356, + "38": 10.4618, + "39": 10.43031, + "40": 10.41152, + "41": 10.39616, + "42": 10.36855, + "43": 10.35081, + "44": 10.32344, + "45": 10.33551, + "46": 10.29368, + "47": 10.28138, + "48": 10.2377, + "49": 10.24679, + "50": 10.23614 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 15620.0, - "2": 15986.0, - "3": 15761.0, - "4": 15774.0, - "5": 15335.0, - "6": 15244.0, - "7": 15809.0, - "8": 15219.0, - "9": 15710.0, - "10": 16019.0, - "11": 15723.0, - "12": 15095.0, - "13": 15669.0, - "14": 15761.0, - "15": 15201.0, - "16": 15269.0, - "17": 15843.0, - "18": 15928.0, - "19": 15871.0, - "20": 15667.0, - "21": 15501.0, - "22": 15619.0, - "23": 15820.0, - "24": 15841.0, - "25": 15040.0, - "26": 15714.0, - "27": 15416.0, - "28": 15566.0, - "29": 15586.0, - "30": 15394.0, - "31": 15755.0, - "32": 16063.0, - "33": 15930.0, - "34": 15945.0, - "35": 16443.0, - "36": 15888.0, - "37": 16158.0, - "38": 15774.0, - "39": 16738.0, - "40": 16971.0, - "41": 16352.0, - "42": 16318.0, - "43": 17671.0, - "44": 16604.0, - "45": 18323.0, - "46": 17918.0, - "47": 17975.0, - "48": 18914.0, - "49": 20266.0, - "50": 18720.0 + "1": 15774.0, + "2": 15623.0, + "3": 15734.0, + "4": 15487.0, + "5": 15394.0, + "6": 15279.0, + "7": 15822.0, + "8": 15087.0, + "9": 15675.0, + "10": 15956.0, + "11": 15587.0, + "12": 15142.0, + "13": 15699.0, + "14": 15935.0, + "15": 15325.0, + "16": 15262.0, + "17": 15818.0, + "18": 15619.0, + "19": 15676.0, + "20": 15730.0, + "21": 15268.0, + "22": 15714.0, + "23": 15626.0, + "24": 16060.0, + "25": 14966.0, + "26": 15899.0, + "27": 15556.0, + "28": 15323.0, + "29": 15652.0, + "30": 15671.0, + "31": 15886.0, + "32": 16014.0, + "33": 16093.0, + "34": 15576.0, + "35": 16392.0, + "36": 15732.0, + "37": 16419.0, + "38": 15869.0, + "39": 16764.0, + "40": 17078.0, + "41": 16421.0, + "42": 16250.0, + "43": 17480.0, + "44": 16603.0, + "45": 18553.0, + "46": 17722.0, + "47": 17951.0, + "48": 18749.0, + "49": 20262.0, + "50": 18842.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 659439104.0, - "2": 659481088.0, - "3": 659502080.0, - "4": 659428352.0, - "5": 659430400.0, - "6": 659415040.0, - "7": 659451392.0, - "8": 659438592.0, - "9": 659425280.0, - "10": 659442176.0, - "11": 659412992.0, - "12": 659431936.0, - "13": 659432448.0, - "14": 659432960.0, - "15": 659524096.0, - "16": 659438592.0, - "17": 659414016.0, - "18": 660136448.0, - "19": 659442688.0, - "20": 659418624.0, - "21": 659413504.0, - "22": 659919872.0, - "23": 659412992.0, - "24": 659448320.0, + "1": 659438592.0, + "2": 659480576.0, + "3": 659502592.0, + "4": 659420672.0, + "5": 660441600.0, + "6": 659411456.0, + "7": 659427840.0, + "8": 660042240.0, + "9": 659415552.0, + "10": 659430912.0, + "11": 659907584.0, + "12": 659438592.0, + "13": 659524096.0, + "14": 659417088.0, + "15": 659412992.0, + "16": 659446784.0, + "17": 659423744.0, + "18": 659431424.0, + "19": 659497472.0, + "20": 659421696.0, + "21": 659434496.0, + "22": 659441664.0, + "23": 659501568.0, + "24": 659449344.0, "25": 659407872.0, - "26": 659604992.0, - "27": 659447296.0, - "28": 660154368.0, - "29": 659434496.0, - "30": 659445760.0, - "31": 659923968.0, - "32": 659480064.0, - "33": 659409920.0, - "34": 659439616.0, - "35": 659409408.0, - "36": 659428864.0, - "37": 659410432.0, - "38": 659426304.0, - "39": 659411456.0, - "40": 659436544.0, - "41": 659472896.0, - "42": 660295168.0, - "43": 659449856.0, - "44": 659676672.0, - "45": 659410432.0, - "46": 659873280.0, - "47": 659778560.0, - "48": 659733504.0, - "49": 659573248.0, - "50": 659980800.0 + "26": 659444736.0, + "27": 659439104.0, + "28": 659413504.0, + "29": 659448320.0, + "30": 659885568.0, + "31": 659410432.0, + "32": 659411968.0, + "33": 659421696.0, + "34": 659433472.0, + "35": 659408384.0, + "36": 659414528.0, + "37": 659418112.0, + "38": 659435008.0, + "39": 659876864.0, + "40": 659411456.0, + "41": 659463168.0, + "42": 659411456.0, + "43": 659411456.0, + "44": 659550208.0, + "45": 659916800.0, + "46": 659543040.0, + "47": 659501568.0, + "48": 659476992.0, + "49": 659962368.0, + "50": 659985408.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 1878177792.0, - "2": 2110117376.0, - "3": 2110117376.0, - "4": 2110137344.0, - "5": 2110137344.0, - "6": 2110137344.0, - "7": 2110137344.0, - "8": 2110137344.0, - "9": 2111263744.0, - "10": 2111263744.0, - "11": 2111263744.0, - "12": 2111263744.0, - "13": 2111263744.0, - "14": 2111263744.0, - "15": 2111263744.0, - "16": 2111263744.0, - "17": 2111263744.0, - "18": 2111337984.0, - "19": 2111337984.0, - "20": 2111337984.0, - "21": 2111337984.0, - "22": 2111337984.0, - "23": 2111919104.0, - "24": 2111919104.0, - "25": 2111919104.0, - "26": 2111919104.0, - "27": 2111919104.0, - "28": 2111919104.0, - "29": 2111919104.0, - "30": 2111919104.0, - "31": 2111919104.0, - "32": 2111919104.0, - "33": 2111919104.0, - "34": 2111919104.0, - "35": 2111919104.0, - "36": 2111919104.0, - "37": 2111919104.0, - "38": 2111919104.0, - "39": 2111919104.0, - "40": 2111919104.0, - "41": 2111919104.0, - "42": 2111919104.0, - "43": 2111919104.0, - "44": 2111919104.0, - "45": 2111919104.0, - "46": 2111919104.0, - "47": 2111919104.0, - "48": 2111919104.0, - "49": 2111919104.0, - "50": 2111959040.0 + "1": 1878177280.0, + "2": 2110116864.0, + "3": 2110116864.0, + "4": 2110116864.0, + "5": 2110116864.0, + "6": 2110116864.0, + "7": 2110116864.0, + "8": 2110116864.0, + "9": 2110466560.0, + "10": 2110466560.0, + "11": 2111707136.0, + "12": 2111707136.0, + "13": 2111707136.0, + "14": 2111707136.0, + "15": 2111707136.0, + "16": 2111707136.0, + "17": 2111707136.0, + "18": 2111912960.0, + "19": 2111912960.0, + "20": 2111912960.0, + "21": 2111912960.0, + "22": 2111912960.0, + "23": 2111912960.0, + "24": 2111912960.0, + "25": 2111912960.0, + "26": 2111912960.0, + "27": 2111912960.0, + "28": 2111912960.0, + "29": 2111912960.0, + "30": 2111912960.0, + "31": 2111912960.0, + "32": 2111912960.0, + "33": 2111912960.0, + "34": 2111912960.0, + "35": 2111912960.0, + "36": 2111912960.0, + "37": 2111912960.0, + "38": 2111912960.0, + "39": 2111912960.0, + "40": 2111912960.0, + "41": 2111912960.0, + "42": 2111912960.0, + "43": 2111912960.0, + "44": 2111912960.0, + "45": 2111912960.0, + "46": 2111912960.0, + "47": 2111912960.0, + "48": 2111912960.0, + "49": 2111912960.0, + "50": 2111965696.0 } }, "iteration-time": { @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 11.42407, - "3": 0.38415, - "4": 0.3299, - "5": 0.32996, - "6": 0.32375, - "7": 0.32537, - "8": 0.32327, - "9": 0.32106, - "10": 0.32401, - "11": 0.32197, - "12": 0.32176, - "13": 0.32124, - "14": 0.31811, - "15": 0.3222, - "16": 0.32268, - "17": 0.31987, - "18": 0.32003, - "19": 0.32111, - "20": 0.31732, - "21": 0.31955, - "22": 0.31955, - "23": 0.31871, - "24": 0.3179, - "25": 0.31902, - "26": 0.31956, - "27": 0.31879, - "28": 0.31788, - "29": 0.31741, - "30": 0.31969, - "31": 0.31983, - "32": 0.32008, - "33": 0.31666, - "34": 0.31972, - "35": 0.32091, - "36": 0.32017, - "37": 0.32136, - "38": 0.31891, - "39": 0.31546, - "40": 0.31598, - "41": 0.31808, - "42": 0.31567, - "43": 0.31653, - "44": 0.31621, - "45": 0.31706, - "46": 0.32732, - "47": 0.33584, - "48": 0.38861, - "49": 0.34149, - "50": 0.31583 + "2": 7.13771, + "3": 0.34013, + "4": 0.27708, + "5": 0.2731, + "6": 0.26745, + "7": 0.26802, + "8": 0.26061, + "9": 0.2608, + "10": 0.25555, + "11": 0.25974, + "12": 0.262, + "13": 0.25761, + "14": 0.25504, + "15": 0.25701, + "16": 0.26624, + "17": 0.26036, + "18": 0.26357, + "19": 0.26084, + "20": 0.25702, + "21": 0.25849, + "22": 0.25961, + "23": 0.26392, + "24": 0.26286, + "25": 0.26165, + "26": 0.26259, + "27": 0.25964, + "28": 0.26522, + "29": 0.26099, + "30": 0.26107, + "31": 0.26109, + "32": 0.26311, + "33": 0.26427, + "34": 0.26323, + "35": 0.26145, + "36": 0.25473, + "37": 0.2583, + "38": 0.26088, + "39": 0.2588, + "40": 0.25986, + "41": 0.25782, + "42": 0.26173, + "43": 0.26282, + "44": 0.26485, + "45": 0.2635, + "46": 0.26039, + "47": 0.25988, + "48": 0.26279, + "49": 0.26413, + "50": 0.26165 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index 39af4dd28cc..a06ed9b3457 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.94223, "2": 10.91037, - "3": 10.92695, - "4": 10.91767, - "5": 10.92123, - "6": 10.92664, - "7": 10.9217, - "8": 10.91836, - "9": 10.92965, - "10": 10.92315, - "11": 10.90829, - "12": 10.92307, - "13": 10.90609, - "14": 10.89523, - "15": 10.88222, - "16": 10.87896, - "17": 10.89049, - "18": 10.86024, - "19": 10.8748, - "20": 10.80806, - "21": 10.79625, - "22": 10.77972, - "23": 10.7812, - "24": 10.73749, - "25": 10.73943, - "26": 10.72335, - "27": 10.71554, - "28": 10.65056, - "29": 10.62001, - "30": 10.58815, - "31": 10.5838, - "32": 10.57437, - "33": 10.54146, - "34": 10.5048, - "35": 10.52203, - "36": 10.48035, - "37": 10.4436, - "38": 10.45979, - "39": 10.42469, - "40": 10.41474, - "41": 10.39207, - "42": 10.38861, - "43": 10.34961, - "44": 10.33972, - "45": 10.34145, - "46": 10.30945, - "47": 10.28903, + "3": 10.9268, + "4": 10.91762, + "5": 10.92118, + "6": 10.92668, + "7": 10.92166, + "8": 10.91845, + "9": 10.9291, + "10": 10.92309, + "11": 10.90873, + "12": 10.92257, + "13": 10.90544, + "14": 10.89519, + "15": 10.8824, + "16": 10.87904, + "17": 10.89076, + "18": 10.85966, + "19": 10.87445, + "20": 10.80814, + "21": 10.79611, + "22": 10.77988, + "23": 10.78063, + "24": 10.7376, + "25": 10.73909, + "26": 10.724, + "27": 10.71573, + "28": 10.65057, + "29": 10.62057, + "30": 10.58798, + "31": 10.58345, + "32": 10.57445, + "33": 10.54151, + "34": 10.50476, + "35": 10.52172, + "36": 10.48005, + "37": 10.44351, + "38": 10.45961, + "39": 10.42496, + "40": 10.4146, + "41": 10.39181, + "42": 10.38818, + "43": 10.34918, + "44": 10.34001, + "45": 10.34176, + "46": 10.30957, + "47": 10.28885, "48": 10.25379, - "49": 10.24217, - "50": 10.24935, - "51": 10.24386, - "52": 10.19468, - "53": 10.19068, - "54": 10.1649, - "55": 10.13346, - "56": 10.16662, - "57": 10.1368, - "58": 10.15637, - "59": 10.10414, - "60": 10.10781, - "61": 10.07602, - "62": 10.03947, - "63": 10.11164, - "64": 10.05938, - "65": 10.03151, - "66": 10.05805, - "67": 10.03102, - "68": 9.98965, - "69": 10.00291, - "70": 9.99282, - "71": 10.02377, - "72": 9.98607, - "73": 9.97326, - "74": 9.96181, - "75": 9.92948, - "76": 9.97549, + "49": 10.24252, + "50": 10.24934, + "51": 10.24409, + "52": 10.19443, + "53": 10.19073, + "54": 10.16499, + "55": 10.13335, + "56": 10.16659, + "57": 10.13691, + "58": 10.15642, + "59": 10.10438, + "60": 10.10778, + "61": 10.07593, + "62": 10.03941, + "63": 10.11173, + "64": 10.05927, + "65": 10.03133, + "66": 10.05809, + "67": 10.03116, + "68": 9.98939, + "69": 10.00278, + "70": 9.99296, + "71": 10.02393, + "72": 9.98614, + "73": 9.97319, + "74": 9.9619, + "75": 9.92942, + "76": 9.97547, "77": 9.96656, - "78": 9.90586, - "79": 9.91757, - "80": 9.92926, - "81": 9.95031, - "82": 9.90425, - "83": 9.86098, - "84": 9.79635, - "85": 9.78073, - "86": 9.89398, - "87": 9.92005, - "88": 9.89597, - "89": 9.83309, - "90": 9.8208, - "91": 9.83504, - "92": 9.8266, - "93": 9.76812, - "94": 9.84073, - "95": 9.83814, - "96": 9.82054, - "97": 9.75795, - "98": 9.79373, - "99": 9.83563, - "100": 9.73571 + "78": 9.90579, + "79": 9.91769, + "80": 9.92928, + "81": 9.95016, + "82": 9.90429, + "83": 9.86097, + "84": 9.79636, + "85": 9.78064, + "86": 9.89399, + "87": 9.92021, + "88": 9.89604, + "89": 9.83311, + "90": 9.82077, + "91": 9.83514, + "92": 9.82665, + "93": 9.76804, + "94": 9.84074, + "95": 9.83826, + "96": 9.82057, + "97": 9.7579, + "98": 9.79362, + "99": 9.83572, + "100": 9.73556 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 1356.0, - "2": 1146.0, - "3": 1308.0, - "4": 1201.0, - "5": 1181.0, - "6": 1356.0, - "7": 1661.0, - "8": 1309.0, - "9": 1246.0, - "10": 1257.0, - "11": 1204.0, - "12": 1346.0, - "13": 1385.0, - "14": 1393.0, - "15": 1241.0, - "16": 1189.0, - "17": 1180.0, - "18": 1293.0, - "19": 1321.0, - "20": 1383.0, - "21": 1333.0, - "22": 1235.0, - "23": 1212.0, - "24": 1255.0, - "25": 1268.0, - "26": 1121.0, - "27": 1302.0, - "28": 1254.0, - "29": 1139.0, - "30": 1069.0, - "31": 1202.0, - "32": 1194.0, - "33": 1138.0, - "34": 1179.0, - "35": 1157.0, - "36": 1262.0, - "37": 1041.0, - "38": 1010.0, - "39": 1126.0, - "40": 1282.0, - "41": 1192.0, - "42": 1131.0, - "43": 1163.0, - "44": 47171.0, - "45": 1307.0, - "46": 1144.0, - "47": 2329.0, - "48": 1185.0, - "49": 2275.0, - "50": 1311.0, - "51": 50408.0, - "52": 1050367.0, - "53": 1115953.0, - "54": 13597.0, - "55": 1104617.0, - "56": 52522.0, - "57": 1050274.0, - "58": 120266.0, - "59": 1055549.0, - "60": 1064790.0, - "61": 98631.0, - "62": 1050443.0, - "63": 140751.0, - "64": 2103804.0, - "65": 2164940.0, - "66": 45473.0, - "67": 1117211.0, - "68": 1163244.0, - "69": 2145629.0, - "70": 2142654.0, - "71": 2197856.0, - "72": 2113996.0, - "73": 2099710.0, - "74": 1111040.0, - "75": 2162030.0, - "76": 2180613.0, - "77": 1148043.0, - "78": 2103914.0, - "79": 2145703.0, - "80": 1109006.0, - "81": 1124413.0, - "82": 2209422.0, - "83": 57887.0, - "84": 1099708.0, - "85": 1143598.0, - "86": 2100695.0, - "87": 3190841.0, - "88": 106080.0, - "89": 3148860.0, - "90": 3206232.0, - "91": 3151748.0, - "92": 3148863.0, - "93": 1103759.0, - "94": 3203989.0, - "95": 3148998.0, - "96": 2207348.0, - "97": 3154093.0, - "98": 3204328.0, - "99": 2171573.0, - "100": 3198980.0 + "1": 1375.0, + "2": 1150.0, + "3": 1296.0, + "4": 1308.0, + "5": 1185.0, + "6": 1287.0, + "7": 1578.0, + "8": 1288.0, + "9": 1256.0, + "10": 1323.0, + "11": 1325.0, + "12": 1374.0, + "13": 1372.0, + "14": 1403.0, + "15": 1256.0, + "16": 1277.0, + "17": 1205.0, + "18": 1276.0, + "19": 1320.0, + "20": 1315.0, + "21": 1287.0, + "22": 1252.0, + "23": 1150.0, + "24": 1269.0, + "25": 1177.0, + "26": 1073.0, + "27": 1258.0, + "28": 1220.0, + "29": 1078.0, + "30": 1153.0, + "31": 1147.0, + "32": 1222.0, + "33": 1195.0, + "34": 1200.0, + "35": 1286.0, + "36": 1253.0, + "37": 1122.0, + "38": 1062.0, + "39": 1038.0, + "40": 1247.0, + "41": 1201.0, + "42": 1095.0, + "43": 1159.0, + "44": 48217.0, + "45": 1231.0, + "46": 1177.0, + "47": 2334.0, + "48": 1152.0, + "49": 2245.0, + "50": 1302.0, + "51": 50382.0, + "52": 1050435.0, + "53": 1060661.0, + "54": 65824.0, + "55": 1104639.0, + "56": 51480.0, + "57": 1050285.0, + "58": 15825.0, + "59": 1050406.0, + "60": 1052423.0, + "61": 98579.0, + "62": 1050472.0, + "63": 97735.0, + "64": 2104778.0, + "65": 2164974.0, + "66": 48525.0, + "67": 1117202.0, + "68": 1160164.0, + "69": 2144604.0, + "70": 1139656.0, + "71": 2149748.0, + "72": 2114027.0, + "73": 2099701.0, + "74": 1104859.0, + "75": 2156875.0, + "76": 2179634.0, + "77": 1147985.0, + "78": 2103878.0, + "79": 1144745.0, + "80": 1109066.0, + "81": 2105989.0, + "82": 2209447.0, + "83": 57872.0, + "84": 1099748.0, + "85": 1092367.0, + "86": 2100729.0, + "87": 3190800.0, + "88": 1101874.0, + "89": 3148784.0, + "90": 3205216.0, + "91": 3151744.0, + "92": 3148782.0, + "93": 1102795.0, + "94": 3197855.0, + "95": 2145942.0, + "96": 2207356.0, + "97": 3155079.0, + "98": 3204409.0, + "99": 2171467.0, + "100": 3200006.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 788639744.0, - "2": 788651008.0, - "3": 788643328.0, - "4": 788628480.0, - "5": 788636672.0, - "6": 788622336.0, - "7": 788641280.0, - "8": 788612608.0, - "9": 788637184.0, - "10": 788629504.0, - "11": 788607488.0, - "12": 788597248.0, - "13": 788684288.0, - "14": 788691456.0, - "15": 788621824.0, - "16": 788622336.0, - "17": 788631552.0, - "18": 788590592.0, - "19": 788645888.0, - "20": 788592640.0, - "21": 788622336.0, - "22": 788581888.0, - "23": 788545024.0, - "24": 788574208.0, - "25": 788550144.0, - "26": 788628992.0, - "27": 788562432.0, - "28": 788528640.0, - "29": 788578816.0, - "30": 788515840.0, - "31": 788516352.0, - "32": 788528128.0, - "33": 788499968.0, - "34": 788546560.0, - "35": 788528640.0, - "36": 788487680.0, - "37": 788518400.0, - "38": 788532224.0, - "39": 788476928.0, - "40": 788488192.0, - "41": 788506624.0, - "42": 788504064.0, - "43": 788497920.0, - "44": 788470784.0, - "45": 788451328.0, - "46": 788493824.0, - "47": 788476416.0, - "48": 788452864.0, - "49": 788434432.0, - "50": 788404224.0, - "51": 788392448.0, - "52": 788389376.0, - "53": 788388864.0, - "54": 788375552.0, - "55": 788362240.0, - "56": 788380672.0, - "57": 788392448.0, - "58": 788394496.0, - "59": 788360704.0, - "60": 788393984.0, - "61": 788348928.0, - "62": 788369920.0, - "63": 788376576.0, - "64": 788382720.0, - "65": 788378112.0, - "66": 788334592.0, - "67": 788378624.0, - "68": 788372480.0, - "69": 788373504.0, - "70": 788344320.0, - "71": 788338688.0, - "72": 788396032.0, - "73": 788360704.0, - "74": 788320256.0, - "75": 788337152.0, - "76": 788312064.0, - "77": 788334592.0, - "78": 788309504.0, - "79": 788289024.0, - "80": 788264448.0, - "81": 788267520.0, - "82": 788224000.0, - "83": 788265984.0, - "84": 788276224.0, - "85": 788275712.0, - "86": 788277248.0, - "87": 788162048.0, - "88": 788196352.0, - "89": 788263424.0, - "90": 788261376.0, - "91": 788287488.0, - "92": 788261376.0, - "93": 788253184.0, - "94": 788343808.0, - "95": 788255232.0, - "96": 788274688.0, - "97": 788295680.0, - "98": 788281856.0, - "99": 788251136.0, - "100": 788296192.0 + "1": 792501760.0, + "2": 792513024.0, + "3": 792505344.0, + "4": 792491008.0, + "5": 792498176.0, + "6": 792484352.0, + "7": 792503296.0, + "8": 792475136.0, + "9": 792500224.0, + "10": 792493568.0, + "11": 792468992.0, + "12": 792460288.0, + "13": 792544256.0, + "14": 792552960.0, + "15": 792484352.0, + "16": 792482304.0, + "17": 792494592.0, + "18": 792454656.0, + "19": 792507904.0, + "20": 792454144.0, + "21": 792485376.0, + "22": 792443392.0, + "23": 792409600.0, + "24": 792435200.0, + "25": 792412672.0, + "26": 792489472.0, + "27": 792425472.0, + "28": 792393216.0, + "29": 792442368.0, + "30": 792377344.0, + "31": 792377856.0, + "32": 792391680.0, + "33": 792360960.0, + "34": 792407552.0, + "35": 792390656.0, + "36": 792349696.0, + "37": 792379904.0, + "38": 792396800.0, + "39": 792339456.0, + "40": 792350720.0, + "41": 792368128.0, + "42": 792364544.0, + "43": 792359936.0, + "44": 792333824.0, + "45": 792313344.0, + "46": 792354816.0, + "47": 792339456.0, + "48": 792316416.0, + "49": 792296448.0, + "50": 792264704.0, + "51": 792253440.0, + "52": 792249856.0, + "53": 792249856.0, + "54": 792237056.0, + "55": 792222208.0, + "56": 792242176.0, + "57": 792254464.0, + "58": 792259584.0, + "59": 792221696.0, + "60": 792256000.0, + "61": 792208384.0, + "62": 792233472.0, + "63": 792239616.0, + "64": 792245248.0, + "65": 792241664.0, + "66": 792196608.0, + "67": 792240640.0, + "68": 792234496.0, + "69": 792233984.0, + "70": 792206848.0, + "71": 792200192.0, + "72": 792258048.0, + "73": 792222208.0, + "74": 792182272.0, + "75": 792199680.0, + "76": 792172032.0, + "77": 792197120.0, + "78": 792171520.0, + "79": 792152576.0, + "80": 792125440.0, + "81": 792129024.0, + "82": 792086528.0, + "83": 792126976.0, + "84": 792137216.0, + "85": 792136704.0, + "86": 792138240.0, + "87": 792024576.0, + "88": 792057856.0, + "89": 792125440.0, + "90": 792122880.0, + "91": 792150528.0, + "92": 792123392.0, + "93": 792115200.0, + "94": 792206848.0, + "95": 792116736.0, + "96": 792136704.0, + "97": 792155136.0, + "98": 792143360.0, + "99": 792111104.0, + "100": 792157184.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2942716416.0, - "2": 3126817792.0, - "3": 3126817792.0, - "4": 3126817792.0, - "5": 3126817792.0, - "6": 3126817792.0, - "7": 3126817792.0, - "8": 3126817792.0, - "9": 3126817792.0, - "10": 3126817792.0, - "11": 3126817792.0, - "12": 3126817792.0, - "13": 3146392576.0, - "14": 3152364544.0, - "15": 3152364544.0, - "16": 3152364544.0, - "17": 3152364544.0, - "18": 3152364544.0, - "19": 3152364544.0, - "20": 3152364544.0, - "21": 3152364544.0, - "22": 3152364544.0, - "23": 3152364544.0, - "24": 3152364544.0, - "25": 3152364544.0, - "26": 3152364544.0, - "27": 3152364544.0, - "28": 3152364544.0, - "29": 3152364544.0, - "30": 3152364544.0, - "31": 3152364544.0, - "32": 3152364544.0, - "33": 3152364544.0, - "34": 3152364544.0, - "35": 3152364544.0, - "36": 3152364544.0, - "37": 3152364544.0, - "38": 3152364544.0, - "39": 3152364544.0, - "40": 3152364544.0, - "41": 3152364544.0, - "42": 3152364544.0, - "43": 3152364544.0, - "44": 3152364544.0, - "45": 3152364544.0, - "46": 3152364544.0, - "47": 3152364544.0, - "48": 3152364544.0, - "49": 3152364544.0, - "50": 3152364544.0, - "51": 3152364544.0, - "52": 3152364544.0, - "53": 3152364544.0, - "54": 3152364544.0, - "55": 3152364544.0, - "56": 3152364544.0, - "57": 3152364544.0, - "58": 3152364544.0, - "59": 3152364544.0, - "60": 3152364544.0, - "61": 3152364544.0, - "62": 3152364544.0, - "63": 3152364544.0, - "64": 3152364544.0, - "65": 3152364544.0, - "66": 3152364544.0, - "67": 3152364544.0, - "68": 3152364544.0, - "69": 3152364544.0, - "70": 3152364544.0, - "71": 3152364544.0, - "72": 3152364544.0, - "73": 3152364544.0, - "74": 3152364544.0, - "75": 3152364544.0, - "76": 3152364544.0, - "77": 3152364544.0, - "78": 3152364544.0, - "79": 3152364544.0, - "80": 3152364544.0, - "81": 3152364544.0, - "82": 3152364544.0, - "83": 3152364544.0, - "84": 3152364544.0, - "85": 3152364544.0, - "86": 3152364544.0, - "87": 3152364544.0, - "88": 3152364544.0, - "89": 3152364544.0, - "90": 3152364544.0, - "91": 3152364544.0, - "92": 3152364544.0, - "93": 3152364544.0, - "94": 3152364544.0, - "95": 3152364544.0, - "96": 3152364544.0, - "97": 3152364544.0, - "98": 3152364544.0, - "99": 3152364544.0, - "100": 3152364544.0 + "1": 2942715904.0, + "2": 3126370816.0, + "3": 3126370816.0, + "4": 3126370816.0, + "5": 3126370816.0, + "6": 3126370816.0, + "7": 3126370816.0, + "8": 3126370816.0, + "9": 3126370816.0, + "10": 3126370816.0, + "11": 3126370816.0, + "12": 3126370816.0, + "13": 3146655744.0, + "14": 3153188864.0, + "15": 3153188864.0, + "16": 3153188864.0, + "17": 3153188864.0, + "18": 3153188864.0, + "19": 3153188864.0, + "20": 3153188864.0, + "21": 3153188864.0, + "22": 3153188864.0, + "23": 3153188864.0, + "24": 3153188864.0, + "25": 3153188864.0, + "26": 3153188864.0, + "27": 3153188864.0, + "28": 3153188864.0, + "29": 3153188864.0, + "30": 3153188864.0, + "31": 3153188864.0, + "32": 3153188864.0, + "33": 3153188864.0, + "34": 3153188864.0, + "35": 3153188864.0, + "36": 3153188864.0, + "37": 3153188864.0, + "38": 3153188864.0, + "39": 3153188864.0, + "40": 3153188864.0, + "41": 3153188864.0, + "42": 3153188864.0, + "43": 3153188864.0, + "44": 3153188864.0, + "45": 3153188864.0, + "46": 3153188864.0, + "47": 3153188864.0, + "48": 3153188864.0, + "49": 3153188864.0, + "50": 3153188864.0, + "51": 3153188864.0, + "52": 3153188864.0, + "53": 3153188864.0, + "54": 3153188864.0, + "55": 3153188864.0, + "56": 3153188864.0, + "57": 3153188864.0, + "58": 3153188864.0, + "59": 3153188864.0, + "60": 3153188864.0, + "61": 3153188864.0, + "62": 3153188864.0, + "63": 3153188864.0, + "64": 3153188864.0, + "65": 3153188864.0, + "66": 3153188864.0, + "67": 3153188864.0, + "68": 3153188864.0, + "69": 3153188864.0, + "70": 3153188864.0, + "71": 3153188864.0, + "72": 3153188864.0, + "73": 3153188864.0, + "74": 3153188864.0, + "75": 3153188864.0, + "76": 3153188864.0, + "77": 3153188864.0, + "78": 3153188864.0, + "79": 3153188864.0, + "80": 3153188864.0, + "81": 3153188864.0, + "82": 3153188864.0, + "83": 3153188864.0, + "84": 3153188864.0, + "85": 3153188864.0, + "86": 3153188864.0, + "87": 3153188864.0, + "88": 3153188864.0, + "89": 3153188864.0, + "90": 3153188864.0, + "91": 3153188864.0, + "92": 3153188864.0, + "93": 3153188864.0, + "94": 3153188864.0, + "95": 3153188864.0, + "96": 3153188864.0, + "97": 3153188864.0, + "98": 3153188864.0, + "99": 3153188864.0, + "100": 3153188864.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.21557, - "3": 0.19982, - "4": 0.17659, - "5": 0.17978, - "6": 0.15377, - "7": 0.1543, - "8": 0.14889, - "9": 0.15506, - "10": 0.14482, - "11": 0.14246, - "12": 0.13468, - "13": 0.14271, - "14": 0.14414, - "15": 0.14587, - "16": 0.134, - "17": 0.13546, - "18": 0.13992, - "19": 0.13955, - "20": 0.13591, - "21": 0.13964, - "22": 0.1393, - "23": 0.13393, - "24": 0.13614, - "25": 0.14201, - "26": 0.14551, - "27": 0.14312, - "28": 0.13986, - "29": 0.14225, - "30": 0.14286, - "31": 0.14352, - "32": 0.14113, - "33": 0.13912, - "34": 0.13929, - "35": 0.1384, - "36": 0.1412, - "37": 0.1409, - "38": 0.13278, - "39": 0.13855, - "40": 0.14225, - "41": 0.13667, - "42": 0.13704, - "43": 0.14144, - "44": 0.13481, - "45": 0.1342, - "46": 0.1345, - "47": 0.14111, - "48": 0.13987, - "49": 0.13877, - "50": 0.13934, - "51": 0.36379, - "52": 0.21206, - "53": 0.147, - "54": 0.14716, - "55": 0.13694, - "56": 0.13963, - "57": 0.14193, - "58": 0.13675, - "59": 0.13806, - "60": 0.13886, - "61": 0.13888, - "62": 0.13655, - "63": 0.14146, - "64": 0.14389, - "65": 0.13439, - "66": 0.14384, - "67": 0.15144, - "68": 0.15384, - "69": 0.15657, - "70": 0.16466, - "71": 0.16654, - "72": 0.17206, - "73": 0.22115, - "74": 0.24397, - "75": 0.19099, - "76": 0.141, - "77": 0.13565, - "78": 0.13834, - "79": 0.13745, - "80": 0.13454, - "81": 0.13549, - "82": 0.13414, - "83": 0.13789, - "84": 0.13434, - "85": 0.13784, - "86": 0.13638, - "87": 0.14387, - "88": 0.13082, - "89": 0.13031, - "90": 0.13152, - "91": 0.13377, - "92": 0.1349, - "93": 0.13374, - "94": 0.13109, - "95": 0.12918, - "96": 0.13031, - "97": 0.13041, - "98": 0.13841, - "99": 0.13048, - "100": 0.13379 + "2": 7.34903, + "3": 0.20596, + "4": 0.1627, + "5": 0.17191, + "6": 0.15415, + "7": 0.16956, + "8": 0.15447, + "9": 0.15218, + "10": 0.1487, + "11": 0.14813, + "12": 0.13935, + "13": 0.13984, + "14": 0.15185, + "15": 0.15873, + "16": 0.13285, + "17": 0.14613, + "18": 0.1535, + "19": 0.15274, + "20": 0.1436, + "21": 0.14376, + "22": 0.14439, + "23": 0.13695, + "24": 0.14087, + "25": 0.14388, + "26": 0.13736, + "27": 0.14187, + "28": 0.14751, + "29": 0.14677, + "30": 0.14289, + "31": 0.13836, + "32": 0.14005, + "33": 0.14214, + "34": 0.14702, + "35": 0.14824, + "36": 0.14799, + "37": 0.15295, + "38": 0.14523, + "39": 0.14684, + "40": 0.15296, + "41": 0.14676, + "42": 0.13996, + "43": 0.14028, + "44": 0.13914, + "45": 0.14165, + "46": 0.13919, + "47": 0.14461, + "48": 0.14574, + "49": 0.14199, + "50": 0.14698, + "51": 0.28837, + "52": 0.14672, + "53": 0.16328, + "54": 0.15808, + "55": 0.19843, + "56": 0.1442, + "57": 0.14188, + "58": 0.13997, + "59": 0.14018, + "60": 0.14439, + "61": 0.13569, + "62": 0.13741, + "63": 0.14959, + "64": 0.1404, + "65": 0.13567, + "66": 0.13664, + "67": 0.13424, + "68": 0.14699, + "69": 0.15206, + "70": 0.15705, + "71": 0.16014, + "72": 0.14857, + "73": 0.20289, + "74": 0.20362, + "75": 0.14205, + "76": 0.1433, + "77": 0.14477, + "78": 0.15303, + "79": 0.15187, + "80": 0.14892, + "81": 0.14242, + "82": 0.15278, + "83": 0.14311, + "84": 0.14226, + "85": 0.1345, + "86": 0.13802, + "87": 0.13778, + "88": 0.13612, + "89": 0.13446, + "90": 0.13786, + "91": 0.13891, + "92": 0.13722, + "93": 0.13317, + "94": 0.13917, + "95": 0.13437, + "96": 0.13672, + "97": 0.14551, + "98": 0.14696, + "99": 0.145, + "100": 0.14208 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index 1039975b4f3..9c55b67895a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2600.0, - "2": 2575.0, - "3": 2663.0, - "4": 2540.0, - "5": 2495.0, - "6": 2456.0, - "7": 2665.0, - "8": 2456.0, - "9": 2542.0, - "10": 2587.0, - "11": 2615.0, - "12": 2562.0, - "13": 2416.0, - "14": 2579.0, - "15": 2483.0, - "16": 2437.0, - "17": 2606.0, - "18": 2652.0, - "19": 2592.0, - "20": 2430.0, - "21": 2403.0, - "22": 2519.0, - "23": 2542.0, - "24": 2699.0, - "25": 2599.0, - "26": 2623.0, - "27": 2642.0, - "28": 2627.0, - "29": 2828.0, - "30": 2746.0, - "31": 2652.0, - "32": 2866.0, - "33": 2775.0, - "34": 2753.0, - "35": 2900.0, - "36": 2819.0, - "37": 2968.0, - "38": 3036.0, - "39": 3043.0, - "40": 3106.0, - "41": 3141.0, - "42": 2962.0, - "43": 3120.0, - "44": 2927.0, - "45": 3372.0, - "46": 3106.0, - "47": 3358.0, - "48": 3449.0, - "49": 3622.0, - "50": 3388.0, - "51": 3437.0, - "52": 3538.0, - "53": 3463.0, - "54": 7425.0, - "55": 3951.0, - "56": 4056.0, - "57": 5345.0, - "58": 529055.0, - "59": 6405.0, - "60": 3758.0, - "61": 4234.0, - "62": 4198.0, - "63": 5243.0, - "64": 5070.0, - "65": 6092.0, - "66": 5869.0, - "67": 4429.0, - "68": 37149.0, - "69": 7328.0, - "70": 26606.0, - "71": 4062.0, - "72": 26940.0, - "73": 7034.0, - "74": 36839.0, - "75": 28085.0, - "76": 1053896.0, - "77": 557118.0, - "78": 36431.0, - "79": 529144.0, - "80": 554052.0, - "81": 529453.0, - "82": 529602.0, - "83": 528928.0, - "84": 528260.0, - "85": 528286.0, - "86": 529261.0, - "87": 582853.0, - "88": 529146.0, - "89": 528960.0, - "90": 529211.0, - "91": 528846.0, - "92": 528547.0, - "93": 529017.0, - "94": 528697.0, - "95": 528634.0, - "96": 531185.0, - "97": 529299.0, - "98": 529348.0, - "99": 38149.0, - "100": 530991.0 + "1": 2666.0, + "2": 2644.0, + "3": 2751.0, + "4": 2603.0, + "5": 2552.0, + "6": 2425.0, + "7": 2606.0, + "8": 2505.0, + "9": 2490.0, + "10": 2506.0, + "11": 2668.0, + "12": 2586.0, + "13": 2519.0, + "14": 2483.0, + "15": 2434.0, + "16": 2528.0, + "17": 2682.0, + "18": 2604.0, + "19": 2510.0, + "20": 2434.0, + "21": 2425.0, + "22": 2551.0, + "23": 2511.0, + "24": 2495.0, + "25": 2503.0, + "26": 2631.0, + "27": 2652.0, + "28": 2669.0, + "29": 2733.0, + "30": 2613.0, + "31": 2775.0, + "32": 2808.0, + "33": 2750.0, + "34": 2788.0, + "35": 2882.0, + "36": 2782.0, + "37": 3001.0, + "38": 2950.0, + "39": 2915.0, + "40": 3165.0, + "41": 3101.0, + "42": 2987.0, + "43": 3038.0, + "44": 3017.0, + "45": 3336.0, + "46": 3207.0, + "47": 3368.0, + "48": 3472.0, + "49": 3487.0, + "50": 3453.0, + "51": 3438.0, + "52": 3540.0, + "53": 3438.0, + "54": 7447.0, + "55": 3892.0, + "56": 4010.0, + "57": 5332.0, + "58": 529109.0, + "59": 6516.0, + "60": 3780.0, + "61": 4406.0, + "62": 4225.0, + "63": 5252.0, + "64": 4844.0, + "65": 9249.0, + "66": 5773.0, + "67": 4636.0, + "68": 37715.0, + "69": 7172.0, + "70": 528692.0, + "71": 4227.0, + "72": 26929.0, + "73": 7106.0, + "74": 36832.0, + "75": 28664.0, + "76": 556371.0, + "77": 557196.0, + "78": 36991.0, + "79": 529101.0, + "80": 554041.0, + "81": 529463.0, + "82": 529423.0, + "83": 529040.0, + "84": 528140.0, + "85": 528770.0, + "86": 529266.0, + "87": 583382.0, + "88": 529195.0, + "89": 529009.0, + "90": 529255.0, + "91": 528899.0, + "92": 528696.0, + "93": 529174.0, + "94": 528741.0, + "95": 528662.0, + "96": 531201.0, + "97": 529233.0, + "98": 529714.0, + "99": 40294.0, + "100": 560372.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 628064256.0, - "2": 628065280.0, - "3": 628065280.0, - "4": 628065280.0, - "5": 628065280.0, - "6": 628065280.0, - "7": 628065280.0, - "8": 628065280.0, - "9": 628065280.0, - "10": 628065280.0, - "11": 628065280.0, - "12": 628065280.0, - "13": 628065280.0, - "14": 628065280.0, - "15": 628065280.0, - "16": 628065280.0, - "17": 628065280.0, - "18": 628065280.0, - "19": 628065280.0, - "20": 628065280.0, - "21": 628065280.0, - "22": 628065280.0, - "23": 628065280.0, - "24": 628065280.0, - "25": 628065280.0, - "26": 628065280.0, - "27": 628065280.0, - "28": 628065280.0, - "29": 628065280.0, - "30": 628065280.0, - "31": 628065280.0, - "32": 628065280.0, - "33": 628065280.0, - "34": 628065280.0, - "35": 628065280.0, - "36": 628065280.0, - "37": 628065280.0, - "38": 628065280.0, - "39": 628065280.0, - "40": 628065280.0, - "41": 628065280.0, - "42": 628065280.0, - "43": 628065280.0, - "44": 628065280.0, - "45": 628065280.0, - "46": 628065280.0, - "47": 628065280.0, - "48": 628065280.0, - "49": 628065280.0, - "50": 628065280.0, - "51": 628065280.0, - "52": 628065280.0, - "53": 628065280.0, - "54": 628065280.0, - "55": 628065280.0, - "56": 628065280.0, - "57": 628065280.0, - "58": 628065280.0, - "59": 628065280.0, - "60": 628065280.0, - "61": 628065280.0, - "62": 628065280.0, - "63": 628065280.0, - "64": 628065280.0, - "65": 628065280.0, - "66": 628065280.0, - "67": 628065280.0, - "68": 628065280.0, - "69": 628065280.0, - "70": 628065280.0, - "71": 628065280.0, - "72": 628065280.0, - "73": 628065280.0, - "74": 628065280.0, - "75": 628065280.0, - "76": 628065280.0, - "77": 628065280.0, - "78": 628065280.0, - "79": 628065280.0, - "80": 628065280.0, - "81": 628065280.0, - "82": 628065280.0, - "83": 628065280.0, - "84": 628065280.0, - "85": 628065280.0, - "86": 628065280.0, - "87": 628065280.0, - "88": 628065280.0, - "89": 628065280.0, - "90": 628065280.0, - "91": 628065280.0, - "92": 628065280.0, - "93": 628065280.0, - "94": 628065280.0, - "95": 628065280.0, - "96": 628065280.0, - "97": 628065280.0, - "98": 628065280.0, - "99": 628065280.0, - "100": 628065280.0 + "1": 628063744.0, + "2": 628064768.0, + "3": 628064768.0, + "4": 628064768.0, + "5": 628064768.0, + "6": 628064768.0, + "7": 628064768.0, + "8": 628064768.0, + "9": 628064768.0, + "10": 628064768.0, + "11": 628064768.0, + "12": 628064768.0, + "13": 628064768.0, + "14": 628064768.0, + "15": 628064768.0, + "16": 628064768.0, + "17": 628064768.0, + "18": 628064768.0, + "19": 628064768.0, + "20": 628064768.0, + "21": 628064768.0, + "22": 628064768.0, + "23": 628064768.0, + "24": 628064768.0, + "25": 628064768.0, + "26": 628064768.0, + "27": 628064768.0, + "28": 628064768.0, + "29": 628064768.0, + "30": 628064768.0, + "31": 628064768.0, + "32": 628064768.0, + "33": 628064768.0, + "34": 628064768.0, + "35": 628064768.0, + "36": 628064768.0, + "37": 628064768.0, + "38": 628064768.0, + "39": 628064768.0, + "40": 628064768.0, + "41": 628064768.0, + "42": 628064768.0, + "43": 628064768.0, + "44": 628064768.0, + "45": 628064768.0, + "46": 628064768.0, + "47": 628064768.0, + "48": 628064768.0, + "49": 628064768.0, + "50": 628064768.0, + "51": 628064768.0, + "52": 628064768.0, + "53": 628064768.0, + "54": 628064768.0, + "55": 628064768.0, + "56": 628064768.0, + "57": 628064768.0, + "58": 628064768.0, + "59": 628064768.0, + "60": 628064768.0, + "61": 628064768.0, + "62": 628064768.0, + "63": 628064768.0, + "64": 628064768.0, + "65": 628064768.0, + "66": 628064768.0, + "67": 628064768.0, + "68": 628064768.0, + "69": 628064768.0, + "70": 628064768.0, + "71": 628064768.0, + "72": 628064768.0, + "73": 628064768.0, + "74": 628064768.0, + "75": 628064768.0, + "76": 628064768.0, + "77": 628064768.0, + "78": 628064768.0, + "79": 628064768.0, + "80": 628064768.0, + "81": 628064768.0, + "82": 628064768.0, + "83": 628064768.0, + "84": 628064768.0, + "85": 628064768.0, + "86": 628064768.0, + "87": 628064768.0, + "88": 628064768.0, + "89": 628064768.0, + "90": 628064768.0, + "91": 628064768.0, + "92": 628064768.0, + "93": 628064768.0, + "94": 628064768.0, + "95": 628064768.0, + "96": 628064768.0, + "97": 628064768.0, + "98": 628064768.0, + "99": 628064768.0, + "100": 628064768.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 981955072.0, - "2": 1152957952.0, - "3": 1152957952.0, - "4": 1152957952.0, - "5": 1152957952.0, - "6": 1153114112.0, - "7": 1153114112.0, - "8": 1153114112.0, - "9": 1153114112.0, - "10": 1153114112.0, - "11": 1153114112.0, - "12": 1153114112.0, - "13": 1153114112.0, - "14": 1153114112.0, - "15": 1153114112.0, - "16": 1153114112.0, - "17": 1153114112.0, - "18": 1153242112.0, - "19": 1153242112.0, - "20": 1153242112.0, - "21": 1153242112.0, - "22": 1153242112.0, - "23": 1153551360.0, - "24": 1153551360.0, - "25": 1153551360.0, - "26": 1153551360.0, - "27": 1154565120.0, - "28": 1154565120.0, - "29": 1154565120.0, - "30": 1154565120.0, - "31": 1154565120.0, - "32": 1154565120.0, - "33": 1154565120.0, - "34": 1154565120.0, - "35": 1154565120.0, - "36": 1154565120.0, - "37": 1155892224.0, - "38": 1155892224.0, - "39": 1155892224.0, - "40": 1155892224.0, - "41": 1155892224.0, - "42": 1155892224.0, - "43": 1155892224.0, - "44": 1155892224.0, - "45": 1155892224.0, - "46": 1155892224.0, - "47": 1155892224.0, - "48": 1155892224.0, - "49": 1155892224.0, - "50": 1155892224.0, - "51": 1155892224.0, - "52": 1155892224.0, - "53": 1155892224.0, - "54": 1155892224.0, - "55": 1155892224.0, - "56": 1155892224.0, - "57": 1155892224.0, - "58": 1155892224.0, - "59": 1155892224.0, - "60": 1155892224.0, - "61": 1155892224.0, - "62": 1155892224.0, - "63": 1155892224.0, - "64": 1155892224.0, - "65": 1155892224.0, - "66": 1155892224.0, - "67": 1155892224.0, - "68": 1155892224.0, - "69": 1155892224.0, - "70": 1155892224.0, - "71": 1155892224.0, - "72": 1155892224.0, - "73": 1155892224.0, - "74": 1155892224.0, - "75": 1155892224.0, - "76": 1155892224.0, - "77": 1155892224.0, - "78": 1155892224.0, - "79": 1155892224.0, - "80": 1155892224.0, - "81": 1155892224.0, - "82": 1155892224.0, - "83": 1155892224.0, - "84": 1155892224.0, - "85": 1155892224.0, - "86": 1155892224.0, - "87": 1155892224.0, - "88": 1155892224.0, - "89": 1155892224.0, - "90": 1155892224.0, - "91": 1155892224.0, - "92": 1155892224.0, - "93": 1155892224.0, - "94": 1155892224.0, - "95": 1155892224.0, - "96": 1155892224.0, - "97": 1155892224.0, - "98": 1155892224.0, - "99": 1155892224.0, - "100": 1155892224.0 + "1": 981954560.0, + "2": 1152957440.0, + "3": 1152957440.0, + "4": 1152957440.0, + "5": 1152957440.0, + "6": 1153038848.0, + "7": 1153038848.0, + "8": 1153038848.0, + "9": 1153038848.0, + "10": 1153038848.0, + "11": 1153038848.0, + "12": 1153038848.0, + "13": 1153038848.0, + "14": 1153038848.0, + "15": 1153038848.0, + "16": 1153038848.0, + "17": 1153038848.0, + "18": 1154730496.0, + "19": 1154730496.0, + "20": 1154730496.0, + "21": 1154730496.0, + "22": 1154730496.0, + "23": 1154730496.0, + "24": 1154730496.0, + "25": 1154730496.0, + "26": 1154730496.0, + "27": 1154730496.0, + "28": 1154730496.0, + "29": 1154730496.0, + "30": 1154730496.0, + "31": 1154730496.0, + "32": 1154730496.0, + "33": 1154730496.0, + "34": 1154730496.0, + "35": 1154730496.0, + "36": 1154730496.0, + "37": 1155892736.0, + "38": 1155892736.0, + "39": 1155892736.0, + "40": 1155892736.0, + "41": 1155892736.0, + "42": 1155892736.0, + "43": 1155892736.0, + "44": 1155892736.0, + "45": 1155892736.0, + "46": 1155892736.0, + "47": 1155892736.0, + "48": 1155892736.0, + "49": 1155892736.0, + "50": 1155892736.0, + "51": 1155892736.0, + "52": 1155892736.0, + "53": 1155892736.0, + "54": 1155892736.0, + "55": 1155892736.0, + "56": 1155892736.0, + "57": 1155892736.0, + "58": 1155892736.0, + "59": 1155892736.0, + "60": 1155892736.0, + "61": 1155892736.0, + "62": 1155892736.0, + "63": 1155892736.0, + "64": 1155892736.0, + "65": 1155892736.0, + "66": 1155892736.0, + "67": 1155892736.0, + "68": 1155892736.0, + "69": 1155892736.0, + "70": 1155892736.0, + "71": 1155892736.0, + "72": 1155892736.0, + "73": 1155892736.0, + "74": 1155892736.0, + "75": 1155892736.0, + "76": 1155892736.0, + "77": 1155892736.0, + "78": 1155892736.0, + "79": 1155892736.0, + "80": 1155892736.0, + "81": 1155892736.0, + "82": 1155892736.0, + "83": 1155892736.0, + "84": 1155892736.0, + "85": 1155892736.0, + "86": 1155892736.0, + "87": 1155892736.0, + "88": 1155892736.0, + "89": 1155892736.0, + "90": 1155892736.0, + "91": 1155892736.0, + "92": 1155892736.0, + "93": 1155892736.0, + "94": 1155892736.0, + "95": 1155892736.0, + "96": 1155892736.0, + "97": 1155892736.0, + "98": 1155892736.0, + "99": 1155892736.0, + "100": 1155892736.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 16.46485, - "3": 0.87334, - "4": 0.8227, - "5": 0.80522, - "6": 0.81568, - "7": 0.80259, - "8": 0.80542, - "9": 0.80548, - "10": 0.80462, - "11": 0.80607, - "12": 0.80034, - "13": 0.80923, - "14": 0.8404, - "15": 0.86601, - "16": 0.78878, - "17": 0.78928, - "18": 0.79067, - "19": 0.79768, - "20": 0.80393, - "21": 0.79691, - "22": 0.78928, - "23": 0.79412, - "24": 0.79516, - "25": 0.79051, - "26": 0.79054, - "27": 0.80099, - "28": 0.79419, - "29": 0.80041, - "30": 0.79309, - "31": 0.79303, - "32": 0.79027, - "33": 0.79387, - "34": 0.79327, - "35": 0.79149, - "36": 0.80003, - "37": 0.79309, - "38": 0.78975, - "39": 0.79328, - "40": 0.7967, - "41": 0.78522, - "42": 0.80037, - "43": 0.78843, - "44": 0.78953, - "45": 0.79678, - "46": 0.7942, - "47": 0.79576, - "48": 0.78958, - "49": 0.79823, - "50": 0.79019, - "51": 1.15823, - "52": 0.86866, - "53": 0.80141, - "54": 0.79914, - "55": 0.79267, - "56": 0.78606, - "57": 0.79044, - "58": 0.79505, - "59": 0.79648, - "60": 0.78671, - "61": 0.79532, - "62": 0.78982, - "63": 0.78299, - "64": 0.79512, - "65": 0.79022, - "66": 0.8071, - "67": 0.78832, - "68": 0.78567, - "69": 0.79323, - "70": 0.78591, - "71": 0.79336, - "72": 0.78788, - "73": 0.78546, - "74": 0.79288, - "75": 0.79825, - "76": 0.78914, - "77": 0.78843, - "78": 0.78655, - "79": 0.79502, - "80": 0.79628, - "81": 0.79458, - "82": 0.78779, - "83": 0.78904, - "84": 0.78403, - "85": 0.78448, - "86": 0.79348, - "87": 0.79533, - "88": 0.78494, - "89": 0.78828, - "90": 0.78964, - "91": 0.78366, - "92": 0.78162, - "93": 0.79491, - "94": 0.80666, - "95": 0.78192, - "96": 0.79074, - "97": 0.80245, - "98": 0.80015, - "99": 0.79781, - "100": 0.80202 + "2": 11.10133, + "3": 1.00423, + "4": 0.87576, + "5": 0.85174, + "6": 0.89147, + "7": 0.85652, + "8": 0.81429, + "9": 0.78782, + "10": 0.77925, + "11": 0.77442, + "12": 0.76661, + "13": 0.75464, + "14": 0.73604, + "15": 0.75554, + "16": 0.73924, + "17": 0.72982, + "18": 0.73443, + "19": 0.80729, + "20": 0.86288, + "21": 0.8837, + "22": 0.87191, + "23": 0.86056, + "24": 0.83907, + "25": 0.83018, + "26": 0.82874, + "27": 0.81482, + "28": 0.80051, + "29": 0.80103, + "30": 0.79464, + "31": 0.78839, + "32": 0.78643, + "33": 0.7887, + "34": 0.7918, + "35": 0.78506, + "36": 0.77353, + "37": 0.77466, + "38": 0.84635, + "39": 0.86024, + "40": 0.8233, + "41": 0.78652, + "42": 0.76746, + "43": 0.75677, + "44": 0.76688, + "45": 0.75709, + "46": 0.7512, + "47": 0.76635, + "48": 0.74707, + "49": 0.75704, + "50": 0.74817, + "51": 1.53778, + "52": 1.40548, + "53": 0.8928, + "54": 0.89804, + "55": 0.89234, + "56": 0.88819, + "57": 0.87448, + "58": 0.84218, + "59": 0.81327, + "60": 0.80856, + "61": 0.78612, + "62": 0.77474, + "63": 0.75246, + "64": 0.74889, + "65": 0.76004, + "66": 1.23226, + "67": 0.76409, + "68": 0.76577, + "69": 1.18807, + "70": 0.74565, + "71": 0.73788, + "72": 0.74486, + "73": 0.83307, + "74": 0.85133, + "75": 0.83092, + "76": 0.78755, + "77": 0.76405, + "78": 0.75659, + "79": 0.74733, + "80": 0.74014, + "81": 0.75757, + "82": 0.73935, + "83": 0.74015, + "84": 0.74067, + "85": 0.73626, + "86": 0.74291, + "87": 0.7382, + "88": 0.73417, + "89": 0.73046, + "90": 0.73499, + "91": 0.73824, + "92": 0.73677, + "93": 0.74129, + "94": 0.74578, + "95": 0.737, + "96": 1.17233, + "97": 0.74753, + "98": 0.74302, + "99": 0.73302, + "100": 0.734 } } -} +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json index b71266ec7bf..93c18afda0e 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.98117, "2": 10.9645, - "3": 10.96971, - "4": 10.96034, - "5": 10.9694, - "6": 10.96819, - "7": 10.96521, - "8": 10.97102, - "9": 10.9613, - "10": 10.96449, - "11": 10.95681, - "12": 10.94738, - "13": 10.94814, - "14": 10.95045, - "15": 10.91678, - "16": 10.90065, - "17": 10.89914, - "18": 10.90066, - "19": 10.8855, - "20": 10.81847, - "21": 10.77563, - "22": 10.7745, - "23": 10.76288, - "24": 10.74152, - "25": 10.71518, - "26": 10.70616, - "27": 10.67036, - "28": 10.60676, - "29": 10.57569, - "30": 10.54329, - "31": 10.53963, - "32": 10.48644, - "33": 10.45246, - "34": 10.44946, - "35": 10.4173, - "36": 10.40476, - "37": 10.38047, - "38": 10.37279, - "39": 10.33077, - "40": 10.33851, - "41": 10.29289, - "42": 10.24356, - "43": 10.23805, - "44": 10.19154, - "45": 10.22779, - "46": 10.16374, - "47": 10.15035, - "48": 10.11933, - "49": 10.11166, - "50": 10.09761, - "51": 10.11122, - "52": 10.05443, - "53": 10.03191, + "3": 10.96986, + "4": 10.96038, + "5": 10.9701, + "6": 10.96815, + "7": 10.96507, + "8": 10.97054, + "9": 10.96141, + "10": 10.96424, + "11": 10.95664, + "12": 10.94737, + "13": 10.94819, + "14": 10.95025, + "15": 10.91641, + "16": 10.90025, + "17": 10.89894, + "18": 10.90045, + "19": 10.88544, + "20": 10.8187, + "21": 10.77645, + "22": 10.77489, + "23": 10.76234, + "24": 10.74094, + "25": 10.71574, + "26": 10.70618, + "27": 10.66985, + "28": 10.60725, + "29": 10.57586, + "30": 10.54319, + "31": 10.53935, + "32": 10.48648, + "33": 10.45188, + "34": 10.44968, + "35": 10.41723, + "36": 10.40441, + "37": 10.38008, + "38": 10.3728, + "39": 10.33083, + "40": 10.3386, + "41": 10.29305, + "42": 10.24315, + "43": 10.23829, + "44": 10.19158, + "45": 10.22758, + "46": 10.16368, + "47": 10.1506, + "48": 10.11899, + "49": 10.11135, + "50": 10.09763, + "51": 10.11138, + "52": 10.05432, + "53": 10.03179, "54": 10.01418, - "55": 9.96812, - "56": 10.0153, - "57": 9.99385, - "58": 9.99648, - "59": 9.93409, - "60": 9.9616, - "61": 9.91387, - "62": 9.85864, - "63": 9.96286, - "64": 9.90737, - "65": 9.87389, - "66": 9.89542, - "67": 9.873, - "68": 9.79964, - "69": 9.84556, - "70": 9.82573, - "71": 9.84951, - "72": 9.83178, - "73": 9.79607, - "74": 9.78221, - "75": 9.73012, - "76": 9.79552, + "55": 9.96809, + "56": 10.01525, + "57": 9.99335, + "58": 9.99662, + "59": 9.93395, + "60": 9.96146, + "61": 9.91376, + "62": 9.85848, + "63": 9.96287, + "64": 9.90765, + "65": 9.87405, + "66": 9.89551, + "67": 9.87298, + "68": 9.79949, + "69": 9.84522, + "70": 9.82568, + "71": 9.84955, + "72": 9.83187, + "73": 9.79639, + "74": 9.7821, + "75": 9.72976, + "76": 9.79558, "77": 9.79604, - "78": 9.75154, - "79": 9.73366, - "80": 9.75373, - "81": 9.78969, - "82": 9.71894, - "83": 9.65361, - "84": 9.61879, - "85": 9.57948, - "86": 9.72637, - "87": 9.73078, - "88": 9.73265, - "89": 9.63477, - "90": 9.62616, - "91": 9.66684, - "92": 9.63608, - "93": 9.53913, - "94": 9.65, - "95": 9.63655, - "96": 9.6297, - "97": 9.54241, - "98": 9.58521, - "99": 9.63026, - "100": 9.52521 + "78": 9.75162, + "79": 9.73342, + "80": 9.75354, + "81": 9.78965, + "82": 9.71909, + "83": 9.65341, + "84": 9.61892, + "85": 9.57955, + "86": 9.72668, + "87": 9.73079, + "88": 9.73212, + "89": 9.6346, + "90": 9.62591, + "91": 9.66704, + "92": 9.63591, + "93": 9.53928, + "94": 9.64991, + "95": 9.6367, + "96": 9.62983, + "97": 9.54251, + "98": 9.58549, + "99": 9.63024, + "100": 9.52511 } }, "num-zeros": { @@ -111,105 +111,105 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 22985560.0, - "2": 22866748.0, - "3": 22718818.0, - "4": 22792960.0, - "5": 22800260.0, - "6": 22758768.0, - "7": 22889378.0, - "8": 22616924.0, - "9": 22770568.0, - "10": 22482246.0, - "11": 22767856.0, - "12": 22646592.0, - "13": 23376208.0, - "14": 23020854.0, - "15": 22728548.0, - "16": 22844226.0, - "17": 22956420.0, - "18": 23025498.0, + "1": 22985550.0, + "2": 22866696.0, + "3": 22718816.0, + "4": 22793004.0, + "5": 22800288.0, + "6": 22758748.0, + "7": 22889372.0, + "8": 22616932.0, + "9": 22770608.0, + "10": 22482312.0, + "11": 22767860.0, + "12": 22646624.0, + "13": 23376188.0, + "14": 23020892.0, + "15": 22728500.0, + "16": 22844224.0, + "17": 22956374.0, + "18": 23025448.0, "19": 23121808.0, - "20": 22737768.0, - "21": 22939064.0, - "22": 22975254.0, - "23": 22636424.0, - "24": 22885696.0, - "25": 22646658.0, - "26": 23036304.0, - "27": 22820206.0, + "20": 22737748.0, + "21": 22939010.0, + "22": 22975312.0, + "23": 22636400.0, + "24": 22885738.0, + "25": 22646644.0, + "26": 23036360.0, + "27": 22820198.0, "28": 23031728.0, - "29": 23007778.0, - "30": 22978112.0, - "31": 22931456.0, - "32": 22671864.0, - "33": 22753828.0, - "34": 23115308.0, - "35": 22764132.0, - "36": 22708340.0, + "29": 23007826.0, + "30": 22977996.0, + "31": 22931432.0, + "32": 22671932.0, + "33": 22753836.0, + "34": 23115324.0, + "35": 22764116.0, + "36": 22708324.0, "37": 23140486.0, - "38": 22990924.0, - "39": 23017588.0, - "40": 22766870.0, - "41": 23101558.0, - "42": 22699616.0, - "43": 23018640.0, - "44": 22716672.0, - "45": 22868262.0, - "46": 22742782.0, - "47": 22872448.0, - "48": 22852388.0, - "49": 22908228.0, - "50": 22654392.0, - "51": 22713780.0, - "52": 22833194.0, - "53": 22987584.0, - "54": 22807296.0, - "55": 22951004.0, - "56": 22670110.0, - "57": 23234448.0, - "58": 22699600.0, - "59": 22862066.0, - "60": 23046696.0, - "61": 22688716.0, - "62": 22743728.0, - "63": 22644392.0, - "64": 23031340.0, - "65": 23243132.0, - "66": 22705426.0, - "67": 22986340.0, - "68": 22948972.0, - "69": 23192944.0, - "70": 22837866.0, - "71": 22749410.0, - "72": 23154740.0, - "73": 23167860.0, - "74": 22969416.0, - "75": 22902876.0, - "76": 22713622.0, - "77": 23010920.0, - "78": 23009832.0, - "79": 22844970.0, - "80": 22957820.0, - "81": 22849744.0, - "82": 22745680.0, + "38": 22991004.0, + "39": 23017608.0, + "40": 22766856.0, + "41": 23101476.0, + "42": 22699692.0, + "43": 23019168.0, + "44": 22716684.0, + "45": 22868822.0, + "46": 22742870.0, + "47": 22872520.0, + "48": 22852580.0, + "49": 22908216.0, + "50": 22654296.0, + "51": 22713688.0, + "52": 22833222.0, + "53": 22987634.0, + "54": 22807092.0, + "55": 22950776.0, + "56": 22669972.0, + "57": 23234460.0, + "58": 22699660.0, + "59": 22861762.0, + "60": 23046726.0, + "61": 22688404.0, + "62": 22743736.0, + "63": 22644196.0, + "64": 23031364.0, + "65": 23243098.0, + "66": 22705374.0, + "67": 22986432.0, + "68": 22949096.0, + "69": 23193052.0, + "70": 22837850.0, + "71": 22749418.0, + "72": 23154708.0, + "73": 23168010.0, + "74": 22969342.0, + "75": 22902846.0, + "76": 22713620.0, + "77": 23010912.0, + "78": 23009874.0, + "79": 22845022.0, + "80": 22957728.0, + "81": 22849644.0, + "82": 22745748.0, "83": 22741216.0, - "84": 23135188.0, - "85": 22945246.0, - "86": 23107702.0, - "87": 22369120.0, - "88": 22564712.0, - "89": 22737614.0, - "90": 22782128.0, - "91": 22940596.0, - "92": 22680148.0, - "93": 22647868.0, - "94": 23168378.0, - "95": 22701712.0, - "96": 22866930.0, - "97": 22852060.0, - "98": 22897236.0, - "99": 22645292.0, + "84": 23135212.0, + "85": 22945266.0, + "86": 23107608.0, + "87": 22369124.0, + "88": 22564568.0, + "89": 22737612.0, + "90": 22782104.0, + "91": 22940644.0, + "92": 22680084.0, + "93": 22647824.0, + "94": 23168470.0, + "95": 22701628.0, + "96": 22866924.0, + "97": 22852120.0, + "98": 22897152.0, + "99": 22645312.0, "100": 23029176.0 } }, @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 793281024.0, - "2": 821341184.0, - "3": 816778240.0, - "4": 803063808.0, - "5": 803063808.0, - "6": 803063808.0, - "7": 807159808.0, - "8": 803063808.0, - "9": 803063808.0, - "10": 805963776.0, - "11": 808945664.0, - "12": 803063808.0, - "13": 803063808.0, - "14": 809617408.0, - "15": 809576448.0, - "16": 803063808.0, - "17": 803063808.0, - "18": 803063808.0, - "19": 803063808.0, - "20": 803063808.0, - "21": 806301696.0, - "22": 809705472.0, - "23": 809128960.0, - "24": 803063808.0, - "25": 803063808.0, - "26": 803063808.0, - "27": 805701632.0, - "28": 806543360.0, - "29": 803063808.0, - "30": 803063808.0, - "31": 803063808.0, - "32": 803063808.0, - "33": 805914624.0, - "34": 808945664.0, - "35": 803063808.0, - "36": 803063808.0, - "37": 803063808.0, - "38": 803063808.0, - "39": 805909504.0, - "40": 805701632.0, - "41": 809805824.0, - "42": 803063808.0, - "43": 803063808.0, - "44": 803063808.0, - "45": 803063808.0, - "46": 809083904.0, - "47": 803063808.0, - "48": 803063808.0, - "49": 803063808.0, - "50": 803063808.0, - "51": 803063808.0, - "52": 803063808.0, - "53": 803063808.0, - "54": 806274048.0, - "55": 805701632.0, - "56": 809486336.0, - "57": 808945664.0, - "58": 805718016.0, - "59": 803063808.0, - "60": 803063808.0, - "61": 803063808.0, - "62": 803063808.0, - "63": 805701632.0, - "64": 808945664.0, - "65": 809230336.0, - "66": 803063808.0, - "67": 805701632.0, - "68": 803063808.0, - "69": 805701632.0, - "70": 809149440.0, - "71": 803063808.0, - "72": 809519104.0, - "73": 803063808.0, - "74": 805701632.0, - "75": 803063808.0, - "76": 805701632.0, - "77": 803063808.0, - "78": 803063808.0, - "79": 808945664.0, - "80": 803063808.0, - "81": 805701632.0, - "82": 809391104.0, - "83": 803063808.0, - "84": 805954560.0, - "85": 803063808.0, - "86": 803063808.0, - "87": 806307840.0, - "88": 803063808.0, - "89": 805701632.0, - "90": 808945664.0, - "91": 803063808.0, - "92": 809469952.0, - "93": 803063808.0, - "94": 806692864.0, - "95": 803063808.0, - "96": 803063808.0, - "97": 809764864.0, - "98": 803063808.0, - "99": 805701632.0, - "100": 803063808.0 + "1": 776184832.0, + "2": 792302080.0, + "3": 776188416.0, + "4": 803877376.0, + "5": 807465472.0, + "6": 801239552.0, + "7": 801239552.0, + "8": 801239552.0, + "9": 801239552.0, + "10": 803877376.0, + "11": 806515200.0, + "12": 801239552.0, + "13": 803877376.0, + "14": 807175680.0, + "15": 807137792.0, + "16": 808035840.0, + "17": 801239552.0, + "18": 801239552.0, + "19": 801239552.0, + "20": 801239552.0, + "21": 808100864.0, + "22": 807793152.0, + "23": 801239552.0, + "24": 804506112.0, + "25": 801239552.0, + "26": 807669248.0, + "27": 801239552.0, + "28": 801239552.0, + "29": 803877376.0, + "30": 808099328.0, + "31": 801239552.0, + "32": 804414976.0, + "33": 801239552.0, + "34": 801239552.0, + "35": 806515200.0, + "36": 801239552.0, + "37": 801239552.0, + "38": 803877376.0, + "39": 807889408.0, + "40": 801239552.0, + "41": 803877376.0, + "42": 806515200.0, + "43": 801239552.0, + "44": 801239552.0, + "45": 806515200.0, + "46": 806515200.0, + "47": 801239552.0, + "48": 806515200.0, + "49": 806515200.0, + "50": 806515200.0, + "51": 803877376.0, + "52": 807052800.0, + "53": 801239552.0, + "54": 801239552.0, + "55": 807037440.0, + "56": 803877376.0, + "57": 801239552.0, + "58": 806515200.0, + "59": 804414976.0, + "60": 803877376.0, + "61": 801239552.0, + "62": 801239552.0, + "63": 801239552.0, + "64": 804414976.0, + "65": 803877376.0, + "66": 801239552.0, + "67": 801239552.0, + "68": 801239552.0, + "69": 803877376.0, + "70": 804414976.0, + "71": 801239552.0, + "72": 801239552.0, + "73": 801239552.0, + "74": 806515200.0, + "75": 803877376.0, + "76": 801239552.0, + "77": 801239552.0, + "78": 803877376.0, + "79": 803877376.0, + "80": 803877376.0, + "81": 801239552.0, + "82": 807052800.0, + "83": 803877376.0, + "84": 801239552.0, + "85": 801239552.0, + "86": 801239552.0, + "87": 807052800.0, + "88": 806515200.0, + "89": 804082176.0, + "90": 801239552.0, + "91": 801239552.0, + "92": 807152128.0, + "93": 803877376.0, + "94": 801239552.0, + "95": 804414976.0, + "96": 801239552.0, + "97": 801239552.0, + "98": 807473664.0, + "99": 804164096.0, + "100": 801239552.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 953233408.0, - "2": 1168079360.0, - "3": 1171789824.0, - "4": 1171789824.0, - "5": 1171789824.0, - "6": 1171789824.0, - "7": 1171789824.0, - "8": 1171789824.0, - "9": 1171789824.0, - "10": 1171789824.0, - "11": 1171789824.0, - "12": 1171789824.0, - "13": 1171789824.0, - "14": 1171789824.0, - "15": 1171789824.0, - "16": 1171789824.0, - "17": 1171789824.0, - "18": 1171789824.0, - "19": 1171789824.0, - "20": 1171789824.0, - "21": 1171789824.0, - "22": 1171789824.0, - "23": 1171789824.0, - "24": 1171789824.0, - "25": 1171789824.0, - "26": 1171789824.0, - "27": 1171789824.0, - "28": 1171789824.0, - "29": 1171789824.0, - "30": 1171789824.0, - "31": 1171789824.0, - "32": 1171789824.0, - "33": 1171789824.0, - "34": 1171789824.0, - "35": 1171789824.0, - "36": 1171789824.0, - "37": 1171789824.0, - "38": 1171789824.0, - "39": 1171789824.0, - "40": 1171789824.0, - "41": 1171789824.0, - "42": 1171789824.0, - "43": 1171789824.0, - "44": 1171789824.0, - "45": 1171789824.0, - "46": 1171789824.0, - "47": 1171789824.0, - "48": 1171789824.0, - "49": 1171789824.0, - "50": 1171789824.0, - "51": 1171789824.0, - "52": 1171789824.0, - "53": 1171789824.0, - "54": 1171789824.0, - "55": 1171789824.0, - "56": 1171789824.0, - "57": 1171789824.0, - "58": 1171789824.0, - "59": 1171789824.0, - "60": 1171789824.0, - "61": 1171789824.0, - "62": 1171789824.0, - "63": 1171789824.0, - "64": 1171789824.0, - "65": 1171789824.0, - "66": 1171789824.0, - "67": 1171789824.0, - "68": 1171789824.0, - "69": 1171789824.0, - "70": 1171789824.0, - "71": 1171789824.0, - "72": 1171789824.0, - "73": 1171789824.0, - "74": 1171789824.0, - "75": 1171789824.0, - "76": 1171789824.0, - "77": 1171789824.0, - "78": 1171789824.0, - "79": 1171789824.0, - "80": 1171789824.0, - "81": 1171789824.0, - "82": 1171789824.0, - "83": 1171789824.0, - "84": 1171789824.0, - "85": 1171789824.0, - "86": 1171789824.0, - "87": 1171789824.0, - "88": 1171789824.0, - "89": 1171789824.0, - "90": 1171789824.0, - "91": 1171789824.0, - "92": 1171789824.0, - "93": 1171789824.0, - "94": 1171789824.0, - "95": 1171789824.0, - "96": 1171789824.0, - "97": 1171789824.0, - "98": 1171789824.0, - "99": 1171789824.0, - "100": 1171789824.0 + "1": 934987264.0, + "2": 1151992320.0, + "3": 1151992320.0, + "4": 1151992320.0, + "5": 1151992320.0, + "6": 1151992320.0, + "7": 1151992320.0, + "8": 1151992320.0, + "9": 1151992320.0, + "10": 1151992320.0, + "11": 1151992320.0, + "12": 1151992320.0, + "13": 1151992320.0, + "14": 1151992320.0, + "15": 1151992320.0, + "16": 1151992320.0, + "17": 1151992320.0, + "18": 1151992320.0, + "19": 1151992320.0, + "20": 1151992320.0, + "21": 1151992320.0, + "22": 1151992320.0, + "23": 1151992320.0, + "24": 1151992320.0, + "25": 1151992320.0, + "26": 1151992320.0, + "27": 1151992320.0, + "28": 1151992320.0, + "29": 1151992320.0, + "30": 1151992320.0, + "31": 1151992320.0, + "32": 1151992320.0, + "33": 1151992320.0, + "34": 1151992320.0, + "35": 1151992320.0, + "36": 1151992320.0, + "37": 1151992320.0, + "38": 1151992320.0, + "39": 1151992320.0, + "40": 1151992320.0, + "41": 1151992320.0, + "42": 1151992320.0, + "43": 1151992320.0, + "44": 1151992320.0, + "45": 1151992320.0, + "46": 1151992320.0, + "47": 1151992320.0, + "48": 1151992320.0, + "49": 1151992320.0, + "50": 1151992320.0, + "51": 1151992320.0, + "52": 1151992320.0, + "53": 1151992320.0, + "54": 1151992320.0, + "55": 1151992320.0, + "56": 1151992320.0, + "57": 1151992320.0, + "58": 1151992320.0, + "59": 1151992320.0, + "60": 1151992320.0, + "61": 1151992320.0, + "62": 1151992320.0, + "63": 1151992320.0, + "64": 1151992320.0, + "65": 1151992320.0, + "66": 1151992320.0, + "67": 1151992320.0, + "68": 1151992320.0, + "69": 1151992320.0, + "70": 1151992320.0, + "71": 1151992320.0, + "72": 1151992320.0, + "73": 1151992320.0, + "74": 1151992320.0, + "75": 1151992320.0, + "76": 1151992320.0, + "77": 1151992320.0, + "78": 1151992320.0, + "79": 1151992320.0, + "80": 1151992320.0, + "81": 1151992320.0, + "82": 1151992320.0, + "83": 1151992320.0, + "84": 1151992320.0, + "85": 1151992320.0, + "86": 1151992320.0, + "87": 1151992320.0, + "88": 1151992320.0, + "89": 1151992320.0, + "90": 1151992320.0, + "91": 1151992320.0, + "92": 1151992320.0, + "93": 1151992320.0, + "94": 1151992320.0, + "95": 1151992320.0, + "96": 1151992320.0, + "97": 1151992320.0, + "98": 1151992320.0, + "99": 1151992320.0, + "100": 1151992320.0 } }, "mtp_1 loss": { @@ -434,104 +434,104 @@ "values": { "1": 10.91979, "2": 10.92264, - "3": 10.92219, - "4": 10.91957, - "5": 10.90846, - "6": 10.9229, - "7": 10.93085, - "8": 10.92446, - "9": 10.91998, - "10": 10.92341, - "11": 10.91588, - "12": 10.90792, - "13": 10.92213, - "14": 10.91569, - "15": 10.91529, - "16": 10.91901, - "17": 10.90055, - "18": 10.91177, - "19": 10.917, - "20": 10.90018, - "21": 10.90603, - "22": 10.90322, - "23": 10.90169, - "24": 10.8909, - "25": 10.8825, - "26": 10.89359, - "27": 10.8887, - "28": 10.87619, - "29": 10.86751, - "30": 10.87702, - "31": 10.8625, - "32": 10.86729, - "33": 10.86131, - "34": 10.85898, - "35": 10.85734, - "36": 10.86421, - "37": 10.84409, - "38": 10.85709, - "39": 10.84395, - "40": 10.85568, - "41": 10.84958, - "42": 10.84639, - "43": 10.84328, - "44": 10.82898, - "45": 10.84347, - "46": 10.83298, - "47": 10.83911, - "48": 10.82542, - "49": 10.83132, - "50": 10.82117, - "51": 10.82453, - "52": 10.81985, - "53": 10.80688, - "54": 10.80117, - "55": 10.79733, - "56": 10.79545, - "57": 10.80261, - "58": 10.80053, - "59": 10.77927, - "60": 10.77895, - "61": 10.77556, - "62": 10.76277, - "63": 10.77594, - "64": 10.76136, - "65": 10.7585, - "66": 10.75798, - "67": 10.74787, - "68": 10.73059, - "69": 10.73669, - "70": 10.73703, - "71": 10.72519, - "72": 10.73051, - "73": 10.71804, - "74": 10.7032, - "75": 10.69024, - "76": 10.70557, - "77": 10.68506, - "78": 10.67502, - "79": 10.66887, - "80": 10.67858, - "81": 10.67147, - "82": 10.65165, - "83": 10.63057, - "84": 10.61714, - "85": 10.60392, - "86": 10.63183, - "87": 10.62791, - "88": 10.62833, - "89": 10.59789, - "90": 10.59506, - "91": 10.60606, - "92": 10.58205, - "93": 10.55314, - "94": 10.58516, - "95": 10.57313, - "96": 10.56963, - "97": 10.55522, - "98": 10.55779, - "99": 10.54891, - "100": 10.52231 + "3": 10.92209, + "4": 10.91955, + "5": 10.90835, + "6": 10.92272, + "7": 10.93087, + "8": 10.92437, + "9": 10.91985, + "10": 10.92389, + "11": 10.91598, + "12": 10.90802, + "13": 10.92185, + "14": 10.91605, + "15": 10.91495, + "16": 10.91889, + "17": 10.90071, + "18": 10.91147, + "19": 10.91668, + "20": 10.8999, + "21": 10.90635, + "22": 10.90341, + "23": 10.90177, + "24": 10.89122, + "25": 10.8824, + "26": 10.89437, + "27": 10.88841, + "28": 10.87614, + "29": 10.86752, + "30": 10.87704, + "31": 10.86234, + "32": 10.86731, + "33": 10.86105, + "34": 10.85919, + "35": 10.85696, + "36": 10.86409, + "37": 10.84378, + "38": 10.85715, + "39": 10.84402, + "40": 10.85552, + "41": 10.84961, + "42": 10.84625, + "43": 10.84286, + "44": 10.82915, + "45": 10.84334, + "46": 10.83246, + "47": 10.83934, + "48": 10.8254, + "49": 10.83159, + "50": 10.8209, + "51": 10.82446, + "52": 10.81983, + "53": 10.80681, + "54": 10.80125, + "55": 10.79716, + "56": 10.79536, + "57": 10.80256, + "58": 10.80015, + "59": 10.77957, + "60": 10.77884, + "61": 10.77508, + "62": 10.76253, + "63": 10.77603, + "64": 10.761, + "65": 10.75846, + "66": 10.75776, + "67": 10.74768, + "68": 10.7305, + "69": 10.73663, + "70": 10.73712, + "71": 10.72532, + "72": 10.73045, + "73": 10.71784, + "74": 10.70335, + "75": 10.69014, + "76": 10.70558, + "77": 10.68498, + "78": 10.6753, + "79": 10.66888, + "80": 10.67872, + "81": 10.67143, + "82": 10.65162, + "83": 10.63075, + "84": 10.61692, + "85": 10.60406, + "86": 10.63193, + "87": 10.62773, + "88": 10.62836, + "89": 10.5979, + "90": 10.59492, + "91": 10.60603, + "92": 10.58196, + "93": 10.55322, + "94": 10.58501, + "95": 10.57284, + "96": 10.56957, + "97": 10.55499, + "98": 10.55773, + "99": 10.54915, + "100": 10.52235 } }, "iteration-time": { @@ -540,105 +540,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 22.16572, - "3": 2.30504, - "4": 5.77447, - "5": 1.26048, - "6": 1.23877, - "7": 1.23731, - "8": 1.23013, - "9": 1.23957, - "10": 1.22497, - "11": 1.23106, - "12": 1.23134, - "13": 1.23358, - "14": 1.2428, - "15": 1.20904, - "16": 1.2076, - "17": 1.21721, - "18": 1.20791, - "19": 1.21512, - "20": 1.21794, - "21": 1.22641, - "22": 1.2177, - "23": 1.20978, - "24": 1.21639, - "25": 1.21895, - "26": 1.22003, - "27": 1.22059, - "28": 1.22036, - "29": 1.25052, - "30": 1.20676, - "31": 1.20496, - "32": 1.21515, - "33": 1.21874, - "34": 1.23076, - "35": 1.21931, - "36": 1.21764, - "37": 1.21674, - "38": 1.21767, - "39": 1.21586, - "40": 1.21111, - "41": 1.21523, - "42": 1.22066, - "43": 1.21904, - "44": 1.22086, - "45": 1.22269, - "46": 1.2322, - "47": 1.22039, - "48": 1.21113, - "49": 1.21706, - "50": 1.21703, - "51": 1.30383, - "52": 1.29128, - "53": 1.23455, - "54": 1.2112, - "55": 1.21663, - "56": 1.21638, - "57": 1.21461, - "58": 1.26721, - "59": 1.24337, - "60": 1.21314, - "61": 1.21899, - "62": 1.22185, - "63": 1.21986, - "64": 1.21819, - "65": 1.21905, - "66": 1.22392, - "67": 1.24026, - "68": 1.2538, - "69": 1.25153, - "70": 1.24541, - "71": 1.24874, - "72": 1.24253, - "73": 1.24483, - "74": 1.24404, - "75": 1.25519, - "76": 1.24466, - "77": 1.24356, - "78": 1.24303, - "79": 1.24055, - "80": 1.23989, - "81": 1.23949, - "82": 1.23834, - "83": 1.23369, - "84": 1.24091, - "85": 1.24654, - "86": 1.23966, - "87": 1.23859, - "88": 1.24545, - "89": 1.24179, - "90": 1.24094, - "91": 1.2434, - "92": 1.24862, - "93": 1.24606, - "94": 1.24163, - "95": 1.24411, - "96": 1.24133, - "97": 1.24141, - "98": 1.2405, - "99": 1.24618, - "100": 1.2418 + "2": 35.74123, + "3": 2.18315, + "4": 5.01562, + "5": 1.03725, + "6": 1.02347, + "7": 1.02133, + "8": 1.02003, + "9": 1.01236, + "10": 1.02816, + "11": 1.02785, + "12": 1.0306, + "13": 3.07844, + "14": 1.08648, + "15": 1.90378, + "16": 1.42103, + "17": 1.03321, + "18": 1.03208, + "19": 1.08999, + "20": 1.12218, + "21": 1.13644, + "22": 1.12541, + "23": 1.11741, + "24": 1.14956, + "25": 1.16864, + "26": 1.15669, + "27": 1.13971, + "28": 1.12588, + "29": 1.13278, + "30": 1.13947, + "31": 1.12834, + "32": 1.14288, + "33": 1.1523, + "34": 1.14619, + "35": 1.14037, + "36": 1.13045, + "37": 1.13097, + "38": 1.12346, + "39": 1.12979, + "40": 1.12916, + "41": 1.14953, + "42": 1.14055, + "43": 1.12561, + "44": 1.13109, + "45": 1.14685, + "46": 1.12948, + "47": 1.15187, + "48": 1.13531, + "49": 1.09506, + "50": 1.32392, + "51": 1.3855, + "52": 1.32904, + "53": 1.33371, + "54": 1.33298, + "55": 1.36204, + "56": 1.36401, + "57": 1.38403, + "58": 1.36486, + "59": 1.38035, + "60": 1.38823, + "61": 1.38756, + "62": 1.39908, + "63": 1.38566, + "64": 1.36931, + "65": 1.39594, + "66": 1.37573, + "67": 1.38916, + "68": 1.39546, + "69": 1.36946, + "70": 1.39074, + "71": 1.39027, + "72": 1.39378, + "73": 1.3835, + "74": 1.37947, + "75": 1.38571, + "76": 1.36867, + "77": 1.38108, + "78": 1.38407, + "79": 1.39463, + "80": 1.396, + "81": 1.36373, + "82": 1.3735, + "83": 1.37647, + "84": 1.39929, + "85": 1.38807, + "86": 1.38388, + "87": 1.3898, + "88": 1.39568, + "89": 1.3878, + "90": 1.37603, + "91": 1.38357, + "92": 1.38586, + "93": 1.3676, + "94": 1.36346, + "95": 1.36686, + "96": 1.37126, + "97": 1.37851, + "98": 1.38658, + "99": 1.38347, + "100": 1.39137 } } } \ No newline at end of file From 07f13ece993db484d095ddeffff372288c6e249e Mon Sep 17 00:00:00 2001 From: Ajay Balasa Date: Tue, 7 Jul 2026 17:31:56 +0000 Subject: [PATCH 7/7] update goldens and mark tests as allow failure as the issues need to be resolved Signed-off-by: Ajay Balasa --- .../golden_values_dev_dgx_gb200.json | 98 +++++++++---------- .../recipes/gb200/gpt-perf-dp4.yaml | 1 + .../test_utils/recipes/gb200/hybrid-perf.yaml | 1 + 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json index 12b4d73f9aa..857dfc6f69e 100644 --- a/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/deepseek_proxy_fsdp_ep2_fsdp2/golden_values_dev_dgx_gb200.json @@ -233,55 +233,55 @@ "step_interval": 1, "values": { "1": "nan", - "2": 9.87919, - "3": 0.1183, - "4": 0.07747, - "5": 0.07582, - "6": 0.07295, - "7": 0.07913, - "8": 0.08239, - "9": 0.08118, - "10": 0.09287, - "11": 0.08524, - "12": 0.07781, - "13": 0.08579, - "14": 0.08789, - "15": 0.0815, - "16": 0.07673, - "17": 0.08139, - "18": 0.07562, - "19": 0.08122, - "20": 0.08078, - "21": 0.07455, - "22": 0.07785, - "23": 0.07512, - "24": 0.0754, - "25": 0.07364, - "26": 0.07369, - "27": 0.07364, - "28": 0.07999, - "29": 0.07604, - "30": 0.07629, - "31": 0.07634, - "32": 0.07623, - "33": 0.07633, - "34": 0.07598, - "35": 0.07505, - "36": 0.07419, - "37": 0.07349, - "38": 0.07408, - "39": 0.07567, - "40": 0.07946, - "41": 0.07267, - "42": 0.07957, - "43": 0.07514, - "44": 0.07761, - "45": 0.07684, - "46": 0.07451, - "47": 0.07465, - "48": 0.07152, - "49": 0.07303, - "50": 0.07258 + "2": 12.8321, + "3": 0.12373, + "4": 0.08346, + "5": 0.08416, + "6": 0.07239, + "7": 0.08224, + "8": 0.07207, + "9": 0.08251, + "10": 0.08562, + "11": 0.08731, + "12": 0.08106, + "13": 0.07755, + "14": 0.07683, + "15": 0.08155, + "16": 0.07425, + "17": 0.07476, + "18": 0.07543, + "19": 0.07529, + "20": 0.07404, + "21": 0.07799, + "22": 0.07111, + "23": 0.07327, + "24": 0.07359, + "25": 0.07084, + "26": 0.07257, + "27": 0.07256, + "28": 0.07299, + "29": 0.07575, + "30": 0.07277, + "31": 0.0723, + "32": 0.0756, + "33": 0.07168, + "34": 0.0718, + "35": 0.07182, + "36": 0.07978, + "37": 0.07074, + "38": 0.07522, + "39": 0.07461, + "40": 0.07262, + "41": 0.07271, + "42": 0.07214, + "43": 0.07172, + "44": 0.07212, + "45": 0.07086, + "46": 0.07628, + "47": 0.07302, + "48": 0.07252, + "49": 0.07203, + "50": 0.07695 } } } \ No newline at end of file diff --git a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml index fb02a6f167d..9f4839d28fe 100644 --- a/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml +++ b/tests/test_utils/recipes/gb200/gpt-perf-dp4.yaml @@ -45,3 +45,4 @@ products: - environment: [dev] scope: [mr] platforms: [dgx_gb200] + allow_failure: [true] # TODO: remove after https://github.com/NVIDIA/Megatron-LM/issues/5692 diff --git a/tests/test_utils/recipes/gb200/hybrid-perf.yaml b/tests/test_utils/recipes/gb200/hybrid-perf.yaml index 78585557c42..be09301b004 100644 --- a/tests/test_utils/recipes/gb200/hybrid-perf.yaml +++ b/tests/test_utils/recipes/gb200/hybrid-perf.yaml @@ -46,3 +46,4 @@ products: - environment: [dev] scope: [mr] platforms: [dgx_gb200] + allow_failure: [true] # TODO: remove after https://github.com/NVIDIA/Megatron-LM/issues/5693