From f3e86e371b1e016e4b225979d7e5c78fc864bbb4 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Mon, 3 Aug 2026 19:07:22 -0500 Subject: [PATCH 1/2] tests: update golden values for inference_optimized_swiglu and moe_resume_gb200 Fixes MCORE-610: logprob[5] for gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch exceeded abs_tol=0.001 by ~0.00279 due to numerical drift after PR #5700 (inference optimized qwen moe) and subsequent commits (bs-invar MoE #4871). Also refreshes lm_loss golden values for gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer on GB200 which drifted in the same window. Downloaded from GitLab pipeline 60908036 (--only-failing). Co-Authored-By: Claude Sonnet 4.6 (1M context) Signed-off-by: Kajal Jain --- .../golden_values_dev_dgx_gb200.json | 782 +++++++++--------- .../golden_values_dev_dgx_h100.json | 470 +++++------ 2 files changed, 626 insertions(+), 626 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index 9c55b67895a..08dbbe539d8 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -7,103 +7,103 @@ "1": 10.92137, "2": 10.91202, "3": 10.91768, - "4": 10.91863, - "5": 10.90423, - "6": 10.9067, - "7": 10.91437, - "8": 10.91056, - "9": 10.91447, - "10": 10.90704, - "11": 10.89659, - "12": 10.8959, - "13": 10.90064, - "14": 10.89352, - "15": 10.87253, - "16": 10.86144, - "17": 10.87413, - "18": 10.85919, - "19": 10.86552, - "20": 10.78225, - "21": 10.77925, - "22": 10.76543, - "23": 10.75797, - "24": 10.72021, - "25": 10.72236, - "26": 10.71659, - "27": 10.68375, - "28": 10.62333, - "29": 10.58113, - "30": 10.56502, - "31": 10.55937, - "32": 10.5483, - "33": 10.51334, - "34": 10.487, - "35": 10.47893, - "36": 10.45972, - "37": 10.43079, - "38": 10.4355, - "39": 10.40227, - "40": 10.37985, - "41": 10.36324, - "42": 10.33753, - "43": 10.32309, - "44": 10.28593, - "45": 10.31097, - "46": 10.26822, - "47": 10.25636, - "48": 10.2077, - "49": 10.2086, - "50": 10.20888, - "51": 10.20917, - "52": 10.16628, - "53": 10.17255, - "54": 10.13289, - "55": 10.10853, - "56": 10.13283, - "57": 10.12157, - "58": 10.13408, - "59": 10.08188, - "60": 10.10019, - "61": 10.04681, - "62": 10.01985, - "63": 10.09447, - "64": 10.04943, - "65": 10.02338, - "66": 10.04778, - "67": 10.02637, - "68": 9.98521, - "69": 10.00765, - "70": 9.98465, - "71": 10.01262, - "72": 9.99392, - "73": 9.9866, - "74": 9.96469, - "75": 9.93791, - "76": 9.97185, - "77": 9.96314, - "78": 9.91583, - "79": 9.92237, - "80": 9.93257, - "81": 9.96107, - "82": 9.8946, - "83": 9.86433, - "84": 9.79835, - "85": 9.78983, - "86": 9.89001, - "87": 9.90939, - "88": 9.88672, - "89": 9.82926, - "90": 9.82374, - "91": 9.83424, - "92": 9.82048, - "93": 9.75668, - "94": 9.83611, - "95": 9.82852, - "96": 9.80839, - "97": 9.75439, - "98": 9.78617, - "99": 9.82596, - "100": 9.72104 + "4": 10.91867, + "5": 10.90429, + "6": 10.9062, + "7": 10.914, + "8": 10.91006, + "9": 10.91461, + "10": 10.90711, + "11": 10.89655, + "12": 10.89605, + "13": 10.90084, + "14": 10.89353, + "15": 10.87232, + "16": 10.86136, + "17": 10.87416, + "18": 10.85925, + "19": 10.86561, + "20": 10.78245, + "21": 10.7792, + "22": 10.7655, + "23": 10.75784, + "24": 10.72029, + "25": 10.72272, + "26": 10.71634, + "27": 10.68397, + "28": 10.62345, + "29": 10.58126, + "30": 10.56533, + "31": 10.5593, + "32": 10.54822, + "33": 10.51337, + "34": 10.48685, + "35": 10.47871, + "36": 10.45967, + "37": 10.43094, + "38": 10.43582, + "39": 10.40263, + "40": 10.37971, + "41": 10.36341, + "42": 10.33811, + "43": 10.3231, + "44": 10.28605, + "45": 10.31118, + "46": 10.26833, + "47": 10.25651, + "48": 10.20784, + "49": 10.20864, + "50": 10.20894, + "51": 10.20907, + "52": 10.1664, + "53": 10.17258, + "54": 10.1329, + "55": 10.10858, + "56": 10.13295, + "57": 10.12151, + "58": 10.13389, + "59": 10.08174, + "60": 10.10018, + "61": 10.04694, + "62": 10.01976, + "63": 10.09419, + "64": 10.04926, + "65": 10.02317, + "66": 10.04748, + "67": 10.02609, + "68": 9.98481, + "69": 10.00719, + "70": 9.98435, + "71": 10.01225, + "72": 9.99337, + "73": 9.98595, + "74": 9.96408, + "75": 9.93716, + "76": 9.97108, + "77": 9.96223, + "78": 9.91482, + "79": 9.92103, + "80": 9.93141, + "81": 9.95995, + "82": 9.89348, + "83": 9.86307, + "84": 9.79718, + "85": 9.78856, + "86": 9.88919, + "87": 9.90846, + "88": 9.88568, + "89": 9.82832, + "90": 9.82266, + "91": 9.83293, + "92": 9.81925, + "93": 9.75539, + "94": 9.83457, + "95": 9.82715, + "96": 9.80685, + "97": 9.75264, + "98": 9.78463, + "99": 9.82436, + "100": 9.71945 } }, "num-zeros": { @@ -111,106 +111,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 2666.0, - "2": 2644.0, - "3": 2751.0, - "4": 2603.0, - "5": 2552.0, - "6": 2425.0, - "7": 2606.0, - "8": 2505.0, - "9": 2490.0, - "10": 2506.0, - "11": 2668.0, - "12": 2586.0, - "13": 2519.0, - "14": 2483.0, - "15": 2434.0, - "16": 2528.0, - "17": 2682.0, - "18": 2604.0, - "19": 2510.0, - "20": 2434.0, - "21": 2425.0, - "22": 2551.0, - "23": 2511.0, - "24": 2495.0, - "25": 2503.0, - "26": 2631.0, - "27": 2652.0, - "28": 2669.0, - "29": 2733.0, - "30": 2613.0, - "31": 2775.0, - "32": 2808.0, - "33": 2750.0, - "34": 2788.0, - "35": 2882.0, - "36": 2782.0, - "37": 3001.0, - "38": 2950.0, - "39": 2915.0, - "40": 3165.0, - "41": 3101.0, - "42": 2987.0, - "43": 3038.0, - "44": 3017.0, - "45": 3336.0, - "46": 3207.0, - "47": 3368.0, - "48": 3472.0, - "49": 3487.0, - "50": 3453.0, - "51": 3438.0, - "52": 3540.0, - "53": 3438.0, - "54": 7447.0, - "55": 3892.0, - "56": 4010.0, - "57": 5332.0, - "58": 529109.0, - "59": 6516.0, - "60": 3780.0, - "61": 4406.0, - "62": 4225.0, - "63": 5252.0, - "64": 4844.0, - "65": 9249.0, - "66": 5773.0, - "67": 4636.0, - "68": 37715.0, - "69": 7172.0, - "70": 528692.0, - "71": 4227.0, - "72": 26929.0, - "73": 7106.0, - "74": 36832.0, - "75": 28664.0, - "76": 556371.0, - "77": 557196.0, - "78": 36991.0, - "79": 529101.0, - "80": 554041.0, - "81": 529463.0, - "82": 529423.0, - "83": 529040.0, - "84": 528140.0, - "85": 528770.0, - "86": 529266.0, - "87": 583382.0, - "88": 529195.0, - "89": 529009.0, - "90": 529255.0, - "91": 528899.0, - "92": 528696.0, - "93": 529174.0, - "94": 528741.0, - "95": 528662.0, - "96": 531201.0, - "97": 529233.0, - "98": 529714.0, - "99": 40294.0, - "100": 560372.0 + "1": 6670.0, + "2": 6580.0, + "3": 6653.0, + "4": 6526.0, + "5": 6462.0, + "6": 6209.0, + "7": 6831.0, + "8": 6551.0, + "9": 6643.0, + "10": 6568.0, + "11": 6679.0, + "12": 6265.0, + "13": 6608.0, + "14": 6484.0, + "15": 6336.0, + "16": 6452.0, + "17": 6592.0, + "18": 6571.0, + "19": 6523.0, + "20": 6191.0, + "21": 6672.0, + "22": 6529.0, + "23": 6372.0, + "24": 6646.0, + "25": 6532.0, + "26": 6596.0, + "27": 6675.0, + "28": 6831.0, + "29": 7222.0, + "30": 6842.0, + "31": 7049.0, + "32": 7290.0, + "33": 7229.0, + "34": 7344.0, + "35": 7383.0, + "36": 7250.0, + "37": 7828.0, + "38": 7533.0, + "39": 7790.0, + "40": 7960.0, + "41": 8186.0, + "42": 7584.0, + "43": 8105.0, + "44": 7859.0, + "45": 8684.0, + "46": 8605.0, + "47": 8515.0, + "48": 8762.0, + "49": 9131.0, + "50": 8830.0, + "51": 10301.0, + "52": 9384.0, + "53": 8941.0, + "54": 17166.0, + "55": 9783.0, + "56": 10246.0, + "57": 12994.0, + "58": 1060368.0, + "59": 15622.0, + "60": 59852.0, + "61": 11305.0, + "62": 18030.0, + "63": 1060940.0, + "64": 1059913.0, + "65": 1063020.0, + "66": 17265.0, + "67": 58014.0, + "68": 137665.0, + "69": 1064689.0, + "70": 1108015.0, + "71": 19901.0, + "72": 1103930.0, + "73": 1063927.0, + "74": 1170080.0, + "75": 1105577.0, + "76": 3156838.0, + "77": 2163555.0, + "78": 1122743.0, + "79": 2107511.0, + "80": 2157105.0, + "81": 2108230.0, + "82": 2108359.0, + "83": 2106919.0, + "84": 1104784.0, + "85": 2106376.0, + "86": 2107704.0, + "87": 2215262.0, + "88": 2107733.0, + "89": 2107132.0, + "90": 1109659.0, + "91": 2106948.0, + "92": 1109424.0, + "93": 2106782.0, + "94": 2106957.0, + "95": 1111714.0, + "96": 1122793.0, + "97": 2107258.0, + "98": 2107901.0, + "99": 1125984.0, + "100": 2113039.0 } }, "mem-allocated-bytes": { @@ -330,101 +330,101 @@ "3": 1152957440.0, "4": 1152957440.0, "5": 1152957440.0, - "6": 1153038848.0, - "7": 1153038848.0, - "8": 1153038848.0, - "9": 1153038848.0, - "10": 1153038848.0, - "11": 1153038848.0, - "12": 1153038848.0, - "13": 1153038848.0, - "14": 1153038848.0, - "15": 1153038848.0, - "16": 1153038848.0, - "17": 1153038848.0, - "18": 1154730496.0, - "19": 1154730496.0, - "20": 1154730496.0, - "21": 1154730496.0, - "22": 1154730496.0, - "23": 1154730496.0, - "24": 1154730496.0, - "25": 1154730496.0, - "26": 1154730496.0, - "27": 1154730496.0, - "28": 1154730496.0, - "29": 1154730496.0, - "30": 1154730496.0, - "31": 1154730496.0, - "32": 1154730496.0, - "33": 1154730496.0, - "34": 1154730496.0, - "35": 1154730496.0, - "36": 1154730496.0, - "37": 1155892736.0, - "38": 1155892736.0, - "39": 1155892736.0, - "40": 1155892736.0, - "41": 1155892736.0, - "42": 1155892736.0, - "43": 1155892736.0, - "44": 1155892736.0, - "45": 1155892736.0, - "46": 1155892736.0, - "47": 1155892736.0, - "48": 1155892736.0, - "49": 1155892736.0, - "50": 1155892736.0, - "51": 1155892736.0, - "52": 1155892736.0, - "53": 1155892736.0, - "54": 1155892736.0, - "55": 1155892736.0, - "56": 1155892736.0, - "57": 1155892736.0, - "58": 1155892736.0, - "59": 1155892736.0, - "60": 1155892736.0, - "61": 1155892736.0, - "62": 1155892736.0, - "63": 1155892736.0, - "64": 1155892736.0, - "65": 1155892736.0, - "66": 1155892736.0, - "67": 1155892736.0, - "68": 1155892736.0, - "69": 1155892736.0, - "70": 1155892736.0, - "71": 1155892736.0, - "72": 1155892736.0, - "73": 1155892736.0, - "74": 1155892736.0, - "75": 1155892736.0, - "76": 1155892736.0, - "77": 1155892736.0, - "78": 1155892736.0, - "79": 1155892736.0, - "80": 1155892736.0, - "81": 1155892736.0, - "82": 1155892736.0, - "83": 1155892736.0, - "84": 1155892736.0, - "85": 1155892736.0, - "86": 1155892736.0, - "87": 1155892736.0, - "88": 1155892736.0, - "89": 1155892736.0, - "90": 1155892736.0, - "91": 1155892736.0, - "92": 1155892736.0, - "93": 1155892736.0, - "94": 1155892736.0, - "95": 1155892736.0, - "96": 1155892736.0, - "97": 1155892736.0, - "98": 1155892736.0, - "99": 1155892736.0, - "100": 1155892736.0 + "6": 1153049088.0, + "7": 1153049088.0, + "8": 1153049088.0, + "9": 1153049088.0, + "10": 1153049088.0, + "11": 1153049088.0, + "12": 1153049088.0, + "13": 1153049088.0, + "14": 1153049088.0, + "15": 1153049088.0, + "16": 1153049088.0, + "17": 1153049088.0, + "18": 1153221120.0, + "19": 1153221120.0, + "20": 1153221120.0, + "21": 1153221120.0, + "22": 1153221120.0, + "23": 1153649664.0, + "24": 1153649664.0, + "25": 1153649664.0, + "26": 1153649664.0, + "27": 1153682432.0, + "28": 1153682432.0, + "29": 1153682432.0, + "30": 1153682432.0, + "31": 1153682432.0, + "32": 1153682432.0, + "33": 1153682432.0, + "34": 1154268672.0, + "35": 1154268672.0, + "36": 1154268672.0, + "37": 1155870208.0, + "38": 1155870208.0, + "39": 1155870208.0, + "40": 1155870208.0, + "41": 1155870208.0, + "42": 1155870208.0, + "43": 1155870208.0, + "44": 1155870208.0, + "45": 1155870208.0, + "46": 1155870208.0, + "47": 1155870208.0, + "48": 1155870208.0, + "49": 1155870208.0, + "50": 1155870208.0, + "51": 1155870208.0, + "52": 1155870208.0, + "53": 1155870208.0, + "54": 1155870208.0, + "55": 1155870208.0, + "56": 1155870208.0, + "57": 1155870208.0, + "58": 1155870208.0, + "59": 1155870208.0, + "60": 1155870208.0, + "61": 1155870208.0, + "62": 1155870208.0, + "63": 1155870208.0, + "64": 1155870208.0, + "65": 1155870208.0, + "66": 1155870208.0, + "67": 1155870208.0, + "68": 1155870208.0, + "69": 1155870208.0, + "70": 1155870208.0, + "71": 1155870208.0, + "72": 1155870208.0, + "73": 1155870208.0, + "74": 1155870208.0, + "75": 1155870208.0, + "76": 1155870208.0, + "77": 1155870208.0, + "78": 1155870208.0, + "79": 1155870208.0, + "80": 1155870208.0, + "81": 1155870208.0, + "82": 1155870208.0, + "83": 1155870208.0, + "84": 1155870208.0, + "85": 1155870208.0, + "86": 1155870208.0, + "87": 1155870208.0, + "88": 1155870208.0, + "89": 1155870208.0, + "90": 1155870208.0, + "91": 1155870208.0, + "92": 1155870208.0, + "93": 1155870208.0, + "94": 1155870208.0, + "95": 1155870208.0, + "96": 1155870208.0, + "97": 1155870208.0, + "98": 1155870208.0, + "99": 1155870208.0, + "100": 1155870208.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 11.10133, - "3": 1.00423, - "4": 0.87576, - "5": 0.85174, - "6": 0.89147, - "7": 0.85652, - "8": 0.81429, - "9": 0.78782, - "10": 0.77925, - "11": 0.77442, - "12": 0.76661, - "13": 0.75464, - "14": 0.73604, - "15": 0.75554, - "16": 0.73924, - "17": 0.72982, - "18": 0.73443, - "19": 0.80729, - "20": 0.86288, - "21": 0.8837, - "22": 0.87191, - "23": 0.86056, - "24": 0.83907, - "25": 0.83018, - "26": 0.82874, - "27": 0.81482, - "28": 0.80051, - "29": 0.80103, - "30": 0.79464, - "31": 0.78839, - "32": 0.78643, - "33": 0.7887, - "34": 0.7918, - "35": 0.78506, - "36": 0.77353, - "37": 0.77466, - "38": 0.84635, - "39": 0.86024, - "40": 0.8233, - "41": 0.78652, - "42": 0.76746, - "43": 0.75677, - "44": 0.76688, - "45": 0.75709, - "46": 0.7512, - "47": 0.76635, - "48": 0.74707, - "49": 0.75704, - "50": 0.74817, - "51": 1.53778, - "52": 1.40548, - "53": 0.8928, - "54": 0.89804, - "55": 0.89234, - "56": 0.88819, - "57": 0.87448, - "58": 0.84218, - "59": 0.81327, - "60": 0.80856, - "61": 0.78612, - "62": 0.77474, - "63": 0.75246, - "64": 0.74889, - "65": 0.76004, - "66": 1.23226, - "67": 0.76409, - "68": 0.76577, - "69": 1.18807, - "70": 0.74565, - "71": 0.73788, - "72": 0.74486, - "73": 0.83307, - "74": 0.85133, - "75": 0.83092, - "76": 0.78755, - "77": 0.76405, - "78": 0.75659, - "79": 0.74733, - "80": 0.74014, - "81": 0.75757, - "82": 0.73935, - "83": 0.74015, - "84": 0.74067, - "85": 0.73626, - "86": 0.74291, - "87": 0.7382, - "88": 0.73417, - "89": 0.73046, - "90": 0.73499, - "91": 0.73824, - "92": 0.73677, - "93": 0.74129, - "94": 0.74578, - "95": 0.737, - "96": 1.17233, - "97": 0.74753, - "98": 0.74302, - "99": 0.73302, - "100": 0.734 + "2": 11.83058, + "3": 0.82508, + "4": 0.74972, + "5": 0.74042, + "6": 1.00259, + "7": 0.89449, + "8": 0.8759, + "9": 0.87184, + "10": 0.86509, + "11": 0.85748, + "12": 1.32158, + "13": 0.86046, + "14": 1.03395, + "15": 0.77839, + "16": 0.74425, + "17": 0.74869, + "18": 0.75046, + "19": 0.74142, + "20": 1.21365, + "21": 0.73819, + "22": 0.77621, + "23": 1.20813, + "24": 0.73173, + "25": 0.71672, + "26": 0.71557, + "27": 0.70966, + "28": 0.71834, + "29": 0.85056, + "30": 0.75759, + "31": 0.85622, + "32": 0.92568, + "33": 0.87335, + "34": 0.81007, + "35": 0.80328, + "36": 0.77227, + "37": 0.84838, + "38": 0.92855, + "39": 0.8909, + "40": 0.8787, + "41": 0.84717, + "42": 0.8157, + "43": 0.77531, + "44": 0.76867, + "45": 0.75715, + "46": 0.75285, + "47": 0.76756, + "48": 0.78106, + "49": 0.75731, + "50": 0.78164, + "51": 1.37059, + "52": 0.86322, + "53": 0.79098, + "54": 0.78768, + "55": 0.75874, + "56": 0.75701, + "57": 0.76475, + "58": 0.76646, + "59": 0.76405, + "60": 0.79722, + "61": 0.83895, + "62": 0.75428, + "63": 0.74588, + "64": 0.80382, + "65": 0.7593, + "66": 0.76944, + "67": 0.74185, + "68": 0.74406, + "69": 0.75134, + "70": 0.74691, + "71": 0.74555, + "72": 0.74547, + "73": 0.74761, + "74": 0.80938, + "75": 0.89836, + "76": 0.90394, + "77": 0.86227, + "78": 0.80477, + "79": 0.78357, + "80": 0.78429, + "81": 0.76924, + "82": 0.76603, + "83": 0.78278, + "84": 0.79874, + "85": 0.85956, + "86": 0.77729, + "87": 0.78864, + "88": 0.77975, + "89": 0.79747, + "90": 0.77097, + "91": 0.77472, + "92": 0.76719, + "93": 0.76669, + "94": 0.85451, + "95": 0.91211, + "96": 0.8871, + "97": 0.85004, + "98": 0.80822, + "99": 0.80291, + "100": 0.81481 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch/golden_values_dev_dgx_h100.json index bf510060fa9..c10845b6439 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_pp1_ep4_16B_inference_optimized_swiglu_logitsmatch/golden_values_dev_dgx_h100.json @@ -34,8 +34,8 @@ 1394, 1636 ], - "latency": 4.658271074295044, - "ttft": 0.15854811668395996, + "latency": 3.448763370513916, + "ttft": 0.11750078201293945, "cuda_graph_request_count_map": null, "step_count": 30, "top_n_logprobs": null, @@ -46,120 +46,120 @@ -2.8317055702209473, -1.2464544773101807, -0.2591174840927124, - -1.791695475578308, - -2.3808443546295166, - -2.00931453704834, - -2.1425540447235107, - -6.077279090881348, - -0.858441948890686, - -2.413856267929077, - -3.5147483348846436, - -4.1333465576171875, - -1.9693496227264404, - -1.8686254024505615, - -2.318774700164795, - -7.2168731689453125, - -0.03999466449022293, - -1.8926615715026855, - -5.031061172485352, - -8.748196601867676, - -9.905678749084473, - -0.7831851840019226, - -4.823017597198486, - -0.8517996072769165, - -2.2906551361083984, - -0.02102702669799328, - -0.03639509528875351, - -3.4814400672912598, - -8.717294692993164, - -1.2549344301223755, - -6.63994026184082, - -3.754265785217285, - -3.775294303894043, - -4.187918663024902, - -2.2120625972747803, - -1.0773918628692627, - -0.22668257355690002, - -0.8210052847862244, - -4.717593193054199, - -9.101178169250488, - -0.013496698811650276, - -3.161458730697632, - -1.2833906412124634, - -3.9933042526245117, - -0.7499399185180664, - -0.002040805760771036, - -2.96846866607666, - -10.53866195678711, - -3.2272119522094727, - -1.1545158624649048, - -4.900886058807373, - -0.21137981116771698, - -0.06306853890419006, - -1.3629783391952515, - -2.2003962993621826, - -4.379840850830078, - -0.36900594830513, - -4.048164367675781, - -0.39393556118011475, - -0.14044521749019623, - -2.711949348449707, - -10.646138191223145, - -0.04656114801764488, - -3.380455493927002, - -0.8181529641151428, - -4.758600234985352, - -0.2629872262477875, - -2.5767107009887695, - -0.843501091003418, - -1.6030611991882324, - -5.800514221191406, - -16.95305061340332, - -2.9422059059143066, - -0.13231781125068665, - -7.411016464233398, - -1.090275764465332, - -2.1541879177093506, - -1.556337833404541, - -0.28904998302459717, - -5.916282653808594, - -0.007025183644145727, - -7.742945194244385, - -2.650775909423828, - -2.9270660877227783, - -3.0080981254577637 + -1.7944855690002441, + -2.380171537399292, + -1.9556585550308228, + -2.132157564163208, + -6.080101013183594, + -0.8113913536071777, + -2.4728193283081055, + -3.5239360332489014, + -4.15925407409668, + -1.9680509567260742, + -1.8262128829956055, + -2.2131776809692383, + -7.124510288238525, + -0.044891878962516785, + -1.991797924041748, + -4.9725847244262695, + -8.745280265808105, + -9.882224082946777, + -0.9239863753318787, + -4.809059143066406, + -0.8529053330421448, + -2.338634490966797, + -0.019403060898184776, + -0.038808632642030716, + -3.3657867908477783, + -8.731738090515137, + -1.372592806816101, + -6.612602233886719, + -3.6236889362335205, + -3.745239734649658, + -4.204811096191406, + -2.2149853706359863, + -1.070363163948059, + -0.23609621822834015, + -0.799513578414917, + -4.759500503540039, + -9.26357650756836, + -0.013302621431648731, + -3.1332390308380127, + -1.3096261024475098, + -3.944545269012451, + -0.7864221334457397, + -0.002206393051892519, + -2.9709091186523438, + -10.505013465881348, + -2.955021381378174, + -1.1547832489013672, + -4.878579616546631, + -0.2092953622341156, + -0.06599172204732895, + -1.2958307266235352, + -2.195488214492798, + -4.321039199829102, + -0.35520139336586, + -4.061367988586426, + -0.36226996779441833, + -0.140281543135643, + -2.8274178504943848, + -10.6322660446167, + -0.05513199791312218, + -3.277632713317871, + -0.9312729835510254, + -4.775629997253418, + -0.26216623187065125, + -2.691115379333496, + -0.8495481610298157, + -1.637433409690857, + -5.811770439147949, + -17.00593376159668, + -2.6000914573669434, + -0.1386266052722931, + -7.422258377075195, + -1.0928596258163452, + -2.132650852203369, + -1.5687007904052734, + -0.25539320707321167, + -5.677168846130371, + -0.0061931367963552475, + -7.793409824371338, + -2.706460952758789, + -2.9823734760284424, + -2.97084903717041 ], "generated_logprobs": [ - -2.342200517654419, - -0.439301460981369, - -1.5168120861053467, - -2.386837959289551, - -0.5973772406578064, - -1.307364821434021, - -1.9625905752182007, - -1.704308271408081, - -0.7901337742805481, - -0.46505871415138245, - -1.2885141372680664, - -1.5861717462539673, - -0.960959792137146, - -0.40262749791145325, - -0.4481161832809448, - -0.0424172468483448, - -1.275164008140564, - -2.1298739910125732, - -2.6976256370544434, - -0.8189972639083862, - -0.4352183938026428, - -2.8580269813537598, - -1.5504716634750366, - -1.5744516849517822, - -0.04782837629318237, - -1.3560791015625, - -1.3296492099761963, - -1.230303406715393, - -1.301802396774292, - -0.4764270484447479 + -2.3568782806396484, + -0.3986137807369232, + -1.4268791675567627, + -2.368211507797241, + -0.56053227186203, + -1.2728139162063599, + -1.972313404083252, + -1.7016775608062744, + -0.7880533933639526, + -0.46236151456832886, + -1.3692306280136108, + -1.5098048448562622, + -0.9540380239486694, + -0.43997034430503845, + -0.4529658555984497, + -0.04222561791539192, + -1.3501358032226562, + -2.237499237060547, + -2.704948663711548, + -0.8287879228591919, + -0.41727471351623535, + -2.8360228538513184, + -1.4974873065948486, + -1.6276496648788452, + -0.05315329506993294, + -1.3325340747833252, + -1.3301773071289062, + -1.3194408416748047, + -1.3469845056533813, + -0.5280233025550842 ], "logprobs": [ -10.293252944946289, @@ -167,131 +167,131 @@ -2.8317055702209473, -1.2464544773101807, -0.2591174840927124, - -1.791695475578308, - -2.3808443546295166, - -2.00931453704834, - -2.1425540447235107, - -6.077279090881348, - -0.858441948890686, - -2.413856267929077, - -3.5147483348846436, - -4.1333465576171875, - -1.9693496227264404, - -1.8686254024505615, - -2.318774700164795, - -7.2168731689453125, - -0.03999466449022293, - -1.8926615715026855, - -5.031061172485352, - -8.748196601867676, - -9.905678749084473, - -0.7831851840019226, - -4.823017597198486, - -0.8517996072769165, - -2.2906551361083984, - -0.02102702669799328, - -0.03639509528875351, - -3.4814400672912598, - -8.717294692993164, - -1.2549344301223755, - -6.63994026184082, - -3.754265785217285, - -3.775294303894043, - -4.187918663024902, - -2.2120625972747803, - -1.0773918628692627, - -0.22668257355690002, - -0.8210052847862244, - -4.717593193054199, - -9.101178169250488, - -0.013496698811650276, - -3.161458730697632, - -1.2833906412124634, - -3.9933042526245117, - -0.7499399185180664, - -0.002040805760771036, - -2.96846866607666, - -10.53866195678711, - -3.2272119522094727, - -1.1545158624649048, - -4.900886058807373, - -0.21137981116771698, - -0.06306853890419006, - -1.3629783391952515, - -2.2003962993621826, - -4.379840850830078, - -0.36900594830513, - -4.048164367675781, - -0.39393556118011475, - -0.14044521749019623, - -2.711949348449707, - -10.646138191223145, - -0.04656114801764488, - -3.380455493927002, - -0.8181529641151428, - -4.758600234985352, - -0.2629872262477875, - -2.5767107009887695, - -0.843501091003418, - -1.6030611991882324, - -5.800514221191406, - -16.95305061340332, - -2.9422059059143066, - -0.13231781125068665, - -7.411016464233398, - -1.090275764465332, - -2.1541879177093506, - -1.556337833404541, - -0.28904998302459717, - -5.916282653808594, - -0.007025183644145727, - -7.742945194244385, - -2.650775909423828, - -2.9270660877227783, - -3.0080981254577637, - -2.342200517654419, - -0.439301460981369, - -1.5168120861053467, - -2.386837959289551, - -0.5973772406578064, - -1.307364821434021, - -1.9625905752182007, - -1.704308271408081, - -0.7901337742805481, - -0.46505871415138245, - -1.2885141372680664, - -1.5861717462539673, - -0.960959792137146, - -0.40262749791145325, - -0.4481161832809448, - -0.0424172468483448, - -1.275164008140564, - -2.1298739910125732, - -2.6976256370544434, - -0.8189972639083862, - -0.4352183938026428, - -2.8580269813537598, - -1.5504716634750366, - -1.5744516849517822, - -0.04782837629318237, - -1.3560791015625, - -1.3296492099761963, - -1.230303406715393, - -1.301802396774292, - -0.4764270484447479 + -1.7944855690002441, + -2.380171537399292, + -1.9556585550308228, + -2.132157564163208, + -6.080101013183594, + -0.8113913536071777, + -2.4728193283081055, + -3.5239360332489014, + -4.15925407409668, + -1.9680509567260742, + -1.8262128829956055, + -2.2131776809692383, + -7.124510288238525, + -0.044891878962516785, + -1.991797924041748, + -4.9725847244262695, + -8.745280265808105, + -9.882224082946777, + -0.9239863753318787, + -4.809059143066406, + -0.8529053330421448, + -2.338634490966797, + -0.019403060898184776, + -0.038808632642030716, + -3.3657867908477783, + -8.731738090515137, + -1.372592806816101, + -6.612602233886719, + -3.6236889362335205, + -3.745239734649658, + -4.204811096191406, + -2.2149853706359863, + -1.070363163948059, + -0.23609621822834015, + -0.799513578414917, + -4.759500503540039, + -9.26357650756836, + -0.013302621431648731, + -3.1332390308380127, + -1.3096261024475098, + -3.944545269012451, + -0.7864221334457397, + -0.002206393051892519, + -2.9709091186523438, + -10.505013465881348, + -2.955021381378174, + -1.1547832489013672, + -4.878579616546631, + -0.2092953622341156, + -0.06599172204732895, + -1.2958307266235352, + -2.195488214492798, + -4.321039199829102, + -0.35520139336586, + -4.061367988586426, + -0.36226996779441833, + -0.140281543135643, + -2.8274178504943848, + -10.6322660446167, + -0.05513199791312218, + -3.277632713317871, + -0.9312729835510254, + -4.775629997253418, + -0.26216623187065125, + -2.691115379333496, + -0.8495481610298157, + -1.637433409690857, + -5.811770439147949, + -17.00593376159668, + -2.6000914573669434, + -0.1386266052722931, + -7.422258377075195, + -1.0928596258163452, + -2.132650852203369, + -1.5687007904052734, + -0.25539320707321167, + -5.677168846130371, + -0.0061931367963552475, + -7.793409824371338, + -2.706460952758789, + -2.9823734760284424, + -2.97084903717041, + -2.3568782806396484, + -0.3986137807369232, + -1.4268791675567627, + -2.368211507797241, + -0.56053227186203, + -1.2728139162063599, + -1.972313404083252, + -1.7016775608062744, + -0.7880533933639526, + -0.46236151456832886, + -1.3692306280136108, + -1.5098048448562622, + -0.9540380239486694, + -0.43997034430503845, + -0.4529658555984497, + -0.04222561791539192, + -1.3501358032226562, + -2.237499237060547, + -2.704948663711548, + -0.8287879228591919, + -0.41727471351623535, + -2.8360228538513184, + -1.4974873065948486, + -1.6276496648788452, + -0.05315329506993294, + -1.3325340747833252, + -1.3301773071289062, + -1.3194408416748047, + -1.3469845056533813, + -0.5280233025550842 ] }, "throughput": [ - 2.3304007637648834, - 6.32366590848818, - 6.396973111935408, - 6.371799291627996, - 6.431814924368525, - 6.440822539177039, - 6.433822662809243, - 6.438126830194765 + 2.7643179788012695, + 8.583974100134604, + 8.625953999617474, + 8.614690206850545, + 8.645243790984122, + 8.649400693317933, + 8.642985467277763, + 8.687274169335302 ], - "mem-max-allocated-bytes": 30659137536, + "mem-max-allocated-bytes": 30659134464, "lifetime_prefill_token_count": 88, "async_sched_step_count": 0, "async_sched_compaction_step_count": 0 From 17b98147ba70255e16aa56ba08d3a48acb069307 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Wed, 5 Aug 2026 18:14:26 -0500 Subject: [PATCH 2/2] tests: remove nan from iteration-time step 1 in moe_resume gb200 golden values The download script pulled a string "nan" at iteration-time step 1 (warmup noise), which fails the check_golden_values.py non-finite validator in CI. Co-Authored-By: Claude Sonnet 4.6 (1M context) Signed-off-by: Kajal Jain --- .../golden_values_dev_dgx_gb200.json | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json index 08dbbe539d8..1a71b23dc39 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_resume_torch_dist_dist_optimizer/golden_values_dev_dgx_gb200.json @@ -432,7 +432,6 @@ "end_step": 100, "step_interval": 1, "values": { - "1": "nan", "2": 11.83058, "3": 0.82508, "4": 0.74972,