diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4f715adc7f8..a84f259defc 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -31,30 +31,10 @@ flowchart LR #### (Step 1): Add PR label `Expert Review` -#### (Step 2): Assign expert reviewers - -Assign the expert reviewers that correspond to the areas the PR touches (if no match, assign someone from `DEFAULT`): - -- Model: - - [ ] Bert: `aot@nvidia.com` - - [ ] GPT: `ksanthanam@nvidia.com` - - [ ] MoE: `akhattar@nvidia.com`, `vkorthikanti@nvidia.com`, `ykarnati@nvidia.com`, `shiqingf@nvidia.com`, `xiny@nvidia.com`, `zijiey@nvidia.com` - - [ ] Multimodal: `trintamaki@nvidia.com` - - [ ] Retro: `lmcafee@nvidia.com` - - [ ] T5: `huvu@nvidia.com` -- Distributed: - - [ ] Distributed Checkpointing: `skierat@nvidia.com`, `mblaz@nvidia.com`, `dpykhtar@nvidia.com` - - [ ] Distributed optimizer: `slym@nvidia.com` -- [ ] Inference: `vkorthikanti@nvidia.com` -- [ ] Quantization and Inference (QAT): `chenhany@nvidia.com` -- Parallelism: - - [ ] Context Parallelism: `boxiangw@nvidia.com`, `xren@nvidia.com` - - [ ] Pipeline Parallelism: `xren@nvidia.com` -- Transformer: `ksanthanam@nvidia.com` -- Post-training: `chenhany@nvidia.com`, `aanoosheh@nvidia.com`, `kmorabia@nvidia.com` -- [ ] Datasets: `jkamalu@nvidia.com` -- [ ] Tests: okoenig, donghyukc, pagaray, chtruong -- [ ] __DEFAULT__: shanmugamr jcasper eharper okoenig +#### (Step 2): Collect the expert reviewers reviews + +1. Attach the `Expert Review` label when your PR is ready for review. +2. GitHub auto-assigns expert reviewers based on your changes. They will get notified and pick up your PR soon. :warning: Only proceed to the next step once all reviewers have approved, merge-conflict are resolved and the CI is passing. Final Review might get declined if these requirements are not fulfilled. @@ -62,10 +42,7 @@ Final Review might get declined if these requirements are not fulfilled. #### (Step 3): Final Review 1. Add `Final Review` label -2. Assign - -- ADLR: `jcasper@nvidia.com` || `dnarayanan@nvidia.com` || `vkorthikanti@nvidia.com` -- NeMo: `eharper@nvidia.com` || `shanmugamr@nvidia.com` || `ansubramania@nvidia.com` || `yuya@nvidia.com` +2. GitHub auto-assigns final reviewers based on your changes. They will get notified and pick up your PR soon. #### (Optional Step 4): Cherry-pick into release branch diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgxh100_eos.json index 5e0ca24c497..3f213856697 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgxh100_eos.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_dev_dgxh100_eos.json @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 759681536.0, - "2": 759681536.0, - "3": 759681536.0, - "4": 759681536.0, - "5": 759681536.0, - "6": 759681536.0, - "7": 759681536.0, - "8": 759681536.0, - "9": 759681536.0, - "10": 759681536.0, - "11": 759681536.0, - "12": 759681536.0, - "13": 759681536.0, - "14": 759681536.0, - "15": 759681536.0, - "16": 759681536.0, - "17": 759681536.0, - "18": 759681536.0, - "19": 759681536.0, - "20": 759681536.0, - "21": 759681536.0, - "22": 759681536.0, - "23": 759681536.0, - "24": 759681536.0, - "25": 759681536.0, - "26": 759681536.0, - "27": 759681536.0, - "28": 759681536.0, - "29": 759681536.0, - "30": 759681536.0, - "31": 759681536.0, - "32": 759681536.0, - "33": 759681536.0, - "34": 759681536.0, - "35": 759681536.0, - "36": 759681536.0, - "37": 759681536.0, - "38": 759681536.0, - "39": 759681536.0, - "40": 759681536.0, - "41": 759681536.0, - "42": 759681536.0, - "43": 759681536.0, - "44": 759681536.0, - "45": 759681536.0, - "46": 759681536.0, - "47": 759681536.0, - "48": 759681536.0, - "49": 759681536.0, - "50": 759681536.0 + "1": 759682560.0, + "2": 759682560.0, + "3": 759682560.0, + "4": 759682560.0, + "5": 759682560.0, + "6": 759682560.0, + "7": 759682560.0, + "8": 759682560.0, + "9": 759682560.0, + "10": 759682560.0, + "11": 759682560.0, + "12": 759682560.0, + "13": 759682560.0, + "14": 759682560.0, + "15": 759682560.0, + "16": 759682560.0, + "17": 759682560.0, + "18": 759682560.0, + "19": 759682560.0, + "20": 759682560.0, + "21": 759682560.0, + "22": 759682560.0, + "23": 759682560.0, + "24": 759682560.0, + "25": 759682560.0, + "26": 759682560.0, + "27": 759682560.0, + "28": 759682560.0, + "29": 759682560.0, + "30": 759682560.0, + "31": 759682560.0, + "32": 759682560.0, + "33": 759682560.0, + "34": 759682560.0, + "35": 759682560.0, + "36": 759682560.0, + "37": 759682560.0, + "38": 759682560.0, + "39": 759682560.0, + "40": 759682560.0, + "41": 759682560.0, + "42": 759682560.0, + "43": 759682560.0, + "44": 759682560.0, + "45": 759682560.0, + "46": 759682560.0, + "47": 759682560.0, + "48": 759682560.0, + "49": 759682560.0, + "50": 759682560.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4340902912.0, - "2": 4622614528.0, - "3": 4622614528.0, - "4": 4622614528.0, - "5": 4622614528.0, - "6": 4622614528.0, - "7": 4622614528.0, - "8": 4622614528.0, - "9": 4622614528.0, - "10": 4622614528.0, - "11": 4622614528.0, - "12": 4622614528.0, - "13": 4622614528.0, - "14": 4622614528.0, - "15": 4622614528.0, - "16": 4622614528.0, - "17": 4622614528.0, - "18": 4622614528.0, - "19": 4622614528.0, - "20": 4622614528.0, - "21": 4622614528.0, - "22": 4622614528.0, - "23": 4622614528.0, - "24": 4622614528.0, - "25": 4622614528.0, - "26": 4622614528.0, - "27": 4622614528.0, - "28": 4622614528.0, - "29": 4622614528.0, - "30": 4622614528.0, - "31": 4622614528.0, - "32": 4622614528.0, - "33": 4622614528.0, - "34": 4622614528.0, - "35": 4622614528.0, - "36": 4622614528.0, - "37": 4622614528.0, - "38": 4622614528.0, - "39": 4622614528.0, - "40": 4622614528.0, - "41": 4622614528.0, - "42": 4622614528.0, - "43": 4622614528.0, - "44": 4622614528.0, - "45": 4622614528.0, - "46": 4622614528.0, - "47": 4622614528.0, - "48": 4622614528.0, - "49": 4622614528.0, - "50": 4622614528.0 + "1": 4340903936.0, + "2": 4622615552.0, + "3": 4622615552.0, + "4": 4622615552.0, + "5": 4622615552.0, + "6": 4622615552.0, + "7": 4622615552.0, + "8": 4622615552.0, + "9": 4622615552.0, + "10": 4622615552.0, + "11": 4622615552.0, + "12": 4622615552.0, + "13": 4622615552.0, + "14": 4622615552.0, + "15": 4622615552.0, + "16": 4622615552.0, + "17": 4622615552.0, + "18": 4622615552.0, + "19": 4622615552.0, + "20": 4622615552.0, + "21": 4622615552.0, + "22": 4622615552.0, + "23": 4622615552.0, + "24": 4622615552.0, + "25": 4622615552.0, + "26": 4622615552.0, + "27": 4622615552.0, + "28": 4622615552.0, + "29": 4622615552.0, + "30": 4622615552.0, + "31": 4622615552.0, + "32": 4622615552.0, + "33": 4622615552.0, + "34": 4622615552.0, + "35": 4622615552.0, + "36": 4622615552.0, + "37": 4622615552.0, + "38": 4622615552.0, + "39": 4622615552.0, + "40": 4622615552.0, + "41": 4622615552.0, + "42": 4622615552.0, + "43": 4622615552.0, + "44": 4622615552.0, + "45": 4622615552.0, + "46": 4622615552.0, + "47": 4622615552.0, + "48": 4622615552.0, + "49": 4622615552.0, + "50": 4622615552.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 12.785, - "2": 0.28429, - "3": 0.25654, - "4": 0.25675, - "5": 0.25763, - "6": 0.25556, - "7": 0.25403, - "8": 0.25276, - "9": 0.25351, - "10": 0.25546, - "11": 0.25488, - "12": 0.25607, - "13": 0.25404, - "14": 0.25256, - "15": 0.25733, - "16": 0.25987, - "17": 0.25778, - "18": 0.25053, - "19": 0.25288, - "20": 0.258, - "21": 0.25606, - "22": 0.25231, - "23": 0.25223, - "24": 0.26464, - "25": 0.26469, - "26": 0.25015, - "27": 0.25378, - "28": 0.25459, - "29": 0.26134, - "30": 0.26129, - "31": 0.2595, - "32": 0.26444, - "33": 0.25568, - "34": 0.25514, - "35": 0.25087, - "36": 0.25275, - "37": 0.25383, - "38": 0.24953, - "39": 0.24996, - "40": 0.25393, - "41": 0.25556, - "42": 0.25158, - "43": 0.25124, - "44": 0.25, - "45": 0.25586, - "46": 0.26057, - "47": 0.25868, - "48": 0.26304, - "49": 0.2615, - "50": 0.26261 + "1": 13.74796, + "2": 0.16361, + "3": 0.12487, + "4": 0.11772, + "5": 0.11849, + "6": 0.11989, + "7": 0.11765, + "8": 0.11845, + "9": 0.11909, + "10": 0.11808, + "11": 0.11972, + "12": 0.12, + "13": 0.11843, + "14": 0.11918, + "15": 0.11921, + "16": 0.11744, + "17": 0.11954, + "18": 0.11987, + "19": 0.12032, + "20": 0.11887, + "21": 0.16664, + "22": 0.14091, + "23": 0.11946, + "24": 0.11878, + "25": 0.12175, + "26": 0.16637, + "27": 0.12057, + "28": 0.11963, + "29": 0.11766, + "30": 0.11771, + "31": 0.11891, + "32": 0.11873, + "33": 0.12109, + "34": 0.12022, + "35": 0.11979, + "36": 0.12012, + "37": 0.11942, + "38": 0.12115, + "39": 0.1194, + "40": 0.12047, + "41": 0.12028, + "42": 0.12169, + "43": 0.12404, + "44": 0.12402, + "45": 0.12356, + "46": 0.12029, + "47": 0.11637, + "48": 0.11959, + "49": 0.11817, + "50": 0.12162 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..024ac59e078 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_fp8_logitsmatch/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,178 @@ +{ + "0": { + "input_prompt": "Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.", + "generated_text": " And then you get to the end of the movie, and you realize that this is not New York at all. This is New York at the end", + "generated_tokens": [ + 3060, + 2430, + 1636, + 2012, + 1317, + 1278, + 2362, + 1307, + 1278, + 16070, + 1044, + 1321, + 1636, + 23067, + 1455, + 1593, + 1395, + 1605, + 3140, + 5152, + 1513, + 1747, + 1046, + 2409, + 1395, + 3140, + 5152, + 1513, + 1278, + 2362 + ], + "latency": 0.29413437843322754, + "cuda_graph_request_count_map": { + "372": 0, + "360": 0, + "336": 0, + "312": 0, + "288": 0, + "264": 0, + "240": 0, + "216": 0, + "192": 0, + "168": 0, + "144": 0, + "120": 0, + "96": 0, + "72": 0, + "48": 0, + "24": 29 + }, + "step_count": 240, + "logprobs": [ + -9.362494468688965, + -2.827894449234009, + -4.557381629943848, + -1.4968647956848145, + -0.717312216758728, + -1.7262351512908936, + -2.522736072540283, + -2.1782360076904297, + -2.3603432178497314, + -6.136383533477783, + -1.4676916599273682, + -3.468963384628296, + -4.424870491027832, + -3.7345848083496094, + -2.012619972229004, + -1.8833301067352295, + -3.5708768367767334, + -6.8197832107543945, + -0.3122292757034302, + -0.9820290207862854, + -6.532033443450928, + -7.498172760009766, + -12.615165710449219, + -2.409003496170044, + -3.8550546169281006, + -0.5105050802230835, + -4.2802581787109375, + -0.06971167027950287, + -0.054025799036026, + -3.319596767425537, + -9.703240394592285, + -1.0997297763824463, + -6.224854469299316, + -5.234503269195557, + -3.934987783432007, + -2.5263679027557373, + -3.1843955516815186, + -5.880871295928955, + -1.8436813354492188, + -5.906496047973633, + -12.15787410736084, + -12.5841064453125, + -0.0819428563117981, + -2.6212656497955322, + -1.4329369068145752, + -2.885145425796509, + -1.2901865243911743, + -0.006647023372352123, + -3.5115818977355957, + -12.945953369140625, + -3.793078899383545, + -3.0094375610351562, + -5.966838836669922, + -0.8998424410820007, + -0.040962252765893936, + -1.5467679500579834, + -1.0785343647003174, + -5.73494815826416, + -0.38491737842559814, + -5.017007827758789, + -0.5568072199821472, + -0.5968841910362244, + -2.3609962463378906, + -13.582086563110352, + -0.09050048142671585, + -3.7264108657836914, + -1.1208789348602295, + -6.052675247192383, + -0.5848909616470337, + -3.5906238555908203, + -0.9494907855987549, + -1.5676641464233398, + -5.127577781677246, + -17.19189453125, + -6.698403835296631, + -1.0449178218841553, + -4.365664958953857, + -1.1243419647216797, + -2.2092156410217285, + -1.8081634044647217, + -0.23330983519554138, + -9.439546585083008, + -0.2947109341621399, + -7.253565788269043, + -2.3855936527252197, + -4.629369258880615, + -3.4186267852783203, + -1.9727531671524048, + -2.331681251525879, + -1.5606917142868042, + -2.454296588897705, + -1.5334703922271729, + -1.2631131410598755, + -2.657367706298828, + -0.6480202078819275, + -0.4550393521785736, + -1.3625166416168213, + -0.8142069578170776, + -0.4496593475341797, + -0.9312890768051147, + -1.732723355293274, + -0.44613128900527954, + -1.6895122528076172, + -0.6082233190536499, + -1.0978344678878784, + -1.1122435331344604, + -0.002520838286727667, + -1.4072327613830566, + -0.007462364621460438, + -0.7548662424087524, + -0.9937503337860107, + -0.0675487294793129, + -0.9595617055892944, + -0.029961343854665756, + -2.205785036087036, + -1.2615025043487549, + -0.7878209352493286 + ] + }, + "throughput": 104.98559493782837 +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..18a5978a127 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_cuda_graphs_logitsmatch_decode_graphs_only/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,178 @@ +{ + "0": { + "input_prompt": "Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.", + "generated_text": " And then you get to the end of the movie, and you realize that this is not New York at all. This is New York at the end", + "generated_tokens": [ + 3060, + 2430, + 1636, + 2012, + 1317, + 1278, + 2362, + 1307, + 1278, + 16070, + 1044, + 1321, + 1636, + 23067, + 1455, + 1593, + 1395, + 1605, + 3140, + 5152, + 1513, + 1747, + 1046, + 2409, + 1395, + 3140, + 5152, + 1513, + 1278, + 2362 + ], + "latency": 0.3712351322174072, + "cuda_graph_request_count_map": { + "372": 0, + "360": 0, + "336": 0, + "312": 0, + "288": 0, + "264": 0, + "240": 0, + "216": 0, + "192": 0, + "168": 0, + "144": 0, + "120": 0, + "96": 0, + "72": 0, + "48": 0, + "24": 29 + }, + "step_count": 240, + "logprobs": [ + -9.362494468688965, + -2.827894449234009, + -4.557381629943848, + -1.4968647956848145, + -0.717312216758728, + -1.7262351512908936, + -2.522736072540283, + -2.1782360076904297, + -2.3603432178497314, + -6.136383533477783, + -1.4676916599273682, + -3.468963384628296, + -4.424870491027832, + -3.7345848083496094, + -2.012619972229004, + -1.8833301067352295, + -3.5708768367767334, + -6.8197832107543945, + -0.3122292757034302, + -0.9820290207862854, + -6.532033443450928, + -7.498172760009766, + -12.615165710449219, + -2.409003496170044, + -3.8550546169281006, + -0.5105050802230835, + -4.2802581787109375, + -0.06971167027950287, + -0.054025799036026, + -3.319596767425537, + -9.703240394592285, + -1.0997297763824463, + -6.224854469299316, + -5.234503269195557, + -3.934987783432007, + -2.5263679027557373, + -3.1843955516815186, + -5.880871295928955, + -1.8436813354492188, + -5.906496047973633, + -12.15787410736084, + -12.5841064453125, + -0.0819428563117981, + -2.6212656497955322, + -1.4329369068145752, + -2.885145425796509, + -1.2901865243911743, + -0.006647023372352123, + -3.5115818977355957, + -12.945953369140625, + -3.793078899383545, + -3.0094375610351562, + -5.966838836669922, + -0.8998424410820007, + -0.040962252765893936, + -1.5467679500579834, + -1.0785343647003174, + -5.73494815826416, + -0.38491737842559814, + -5.017007827758789, + -0.5568072199821472, + -0.5968841910362244, + -2.3609962463378906, + -13.582086563110352, + -0.09050048142671585, + -3.7264108657836914, + -1.1208789348602295, + -6.052675247192383, + -0.5848909616470337, + -3.5906238555908203, + -0.9494907855987549, + -1.5676641464233398, + -5.127577781677246, + -17.19189453125, + -6.698403835296631, + -1.0449178218841553, + -4.365664958953857, + -1.1243419647216797, + -2.2092156410217285, + -1.8081634044647217, + -0.23330983519554138, + -9.439546585083008, + -0.2947109341621399, + -7.253565788269043, + -2.3855936527252197, + -4.629369258880615, + -3.4186267852783203, + -1.9727531671524048, + -2.331681251525879, + -1.5606917142868042, + -2.454296588897705, + -1.5334703922271729, + -1.2631131410598755, + -2.657367706298828, + -0.6480202078819275, + -0.4550393521785736, + -1.3625166416168213, + -0.8142069578170776, + -0.4496593475341797, + -0.9312890768051147, + -1.732723355293274, + -0.44613128900527954, + -1.6895122528076172, + -0.6082233190536499, + -1.0978344678878784, + -1.1122435331344604, + -0.002520838286727667, + -1.4072327613830566, + -0.007462364621460438, + -0.7548662424087524, + -0.9937503337860107, + -0.0675487294793129, + -0.9595617055892944, + -0.029961343854665756, + -2.205785036087036, + -1.2615025043487549, + -0.7878209352493286 + ] + }, + "throughput": 79.88988160240554 +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..3ba402cbcae --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp1_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,2699 @@ +{ + "0": { + "input_prompt": "The $500 Cup of coffee?\nConsider this, most Americans spend an average of $1,500-2,000 a year on this bean water.\nI have a few question for you: \nHow has business been the past few months?\nDo you ever feel like your business is stuck?\nDon't feel like you're able to improve performance and make changes required to achieve success ?\nAre your customers spneding less and less and wanting more?\nHave the gas prices affected your business?\nDo you have employees and do they hate you or wish they could quit?\n\nNow, before you and I can decide wheter or not I will be a good fit for your business we should talk this over with coffee.\nAnd, just to warn you this isn't some casual thing. This is not a date or time to be personal or social (but by all means share what you will coz I'll gladly listen).\nTher eare two major talking points and stratagies we will focios on in our lil coffee social\nFor one, we will find your unique selling Proposition (USP).\nDo have the best price? Are you the cheapest in town? Are your customers jerks? Do you haVE REGULARS? Why do people come back?\nwe'll also look for the holes in your business bucket. I'm willing to bet there's a hole or two in your business we'll find together that'll make this 500 dollar cup of Joe pay for itse;f immedietly.\nMany find themselves to be more profitable by just finding out where the dollars are escaping in their business and I like to think of myself as a guy that comes along with some spakel or putty and patch those holes up for you.\nBeleive me, just fixing one hole can mean a lot...just think about a sinking boat that has a hole in it that's about 3\u201d in diameter... it doesn't take long to sink.\nI have no agenda, besides f=getting to know your business and seeing wher I can patch the holes and find what makes you do darn unique (I know this won't take long.)\nMany folks, I bet, will find what they need to get off their chest with a quick phone call and they just paypal me the money and make a coffee at home. Look, that's fine too.\nI just to get you ot of your comfort zone, because this is where it all starts my frind.\nSome smart GOAT entrepreneur will probably get everything they need just out of our lil mini consulatant for the more extensive business consukting I offer, and look, that's fine, too.\nMaybe this cup of coffee will be all you need to gtet back on your feet, but not only surive, but thrive!\nI'm not trying to make frineds, or make a bunch of money, or look down your shirt\nBut this is only going to be a 45 minute (max) coffee chat\nAnd, I know you (and me) both have a lot on our plates. So no messing around\nAfter our consultation I will follow up with you in a few days and see how things are going, then I will be emailing you about once or twice every two weeks, just to make sure you're staying on task and implementing what we went over.\nTghere is no obligation to go any further and will gladly give you back your money if this pre-consultation doesn't put you on the right path or you don't get any value out of it...", + "generated_text": " $ is a$ is a $ is a $ is a $ is a $ is a $$1, you\n$ $$ $\n$ $- $\n$\n$\n$ $$$\n$\n$\n$\n$\n$\n$\n$\n$???????, $\n-1\n$\n1.5.\n$\n$, you\n$.\n$\n1,1.1\nI\n$.\nI\n\n\nHow\n$,,,0,1,0,0.0\nIn\nThe\nThe\nThe\n", + "generated_tokens": [ + 1659, + 1395, + 1261, + 1036, + 1395, + 1261, + 1659, + 1395, + 1261, + 1659, + 1395, + 1261, + 1659, + 1395, + 1261, + 1659, + 1395, + 1261, + 1659, + 1036, + 1049, + 1044, + 1636, + 1010, + 1036, + 1659, + 1036, + 1659, + 1010, + 1036, + 1659, + 1045, + 1659, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1659, + 1036, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1010, + 1036, + 1063, + 1063, + 1063, + 1063, + 1063, + 1063, + 1063, + 1044, + 1659, + 1010, + 1045, + 1049, + 1010, + 1036, + 1010, + 1049, + 1046, + 1053, + 1046, + 1010, + 1036, + 1010, + 1036, + 1044, + 1636, + 1010, + 1036, + 1046, + 1010, + 1036, + 1010, + 1049, + 1044, + 1049, + 1046, + 1049, + 1010, + 1073, + 1010, + 1036, + 1046, + 1010, + 1073, + 1010, + 1010, + 1010, + 7801, + 1010, + 1036, + 1044, + 1044, + 1044, + 1048, + 1044, + 1049, + 1044, + 1048, + 1044, + 1048, + 1046, + 1048, + 1010, + 1785, + 1010, + 1784, + 1010, + 1784, + 1010, + 1784, + 1010 + ], + "latency": 9.61214280128479, + "cuda_graph_request_count_map": null, + "step_count": 6144, + "logprobs": [ + -7.7319135665893555, + -2.188307285308838, + -0.7547445297241211, + -0.7294313311576843, + -10.238386154174805, + -3.3775341510772705, + -6.394498825073242, + -7.354557037353516, + -9.018157958984375, + -3.012073040008545, + -3.2584073543548584, + -5.220732688903809, + -4.620487213134766, + -2.5078930854797363, + -3.752683162689209, + -0.13360372185707092, + -0.05705544352531433, + -0.41462242603302, + -1.585279941558838, + -1.6438164710998535, + -1.9557222127914429, + -0.3989897072315216, + -0.0365302674472332, + -6.368816375732422, + -0.8731719255447388, + -0.022585075348615646, + -0.2775891423225403, + -0.0027362785767763853, + -0.0006812873762100935, + -1.581446647644043, + -0.008688976056873798, + -0.3532317280769348, + -6.071163177490234, + -9.162371635437012, + -9.965556144714355, + -2.400461196899414, + -2.9898362159729004, + -2.9803032875061035, + -2.12601900100708, + -3.500912666320801, + -7.015069007873535, + -2.278961420059204, + -0.46380555629730225, + -4.078739166259766, + -1.9430254697799683, + -3.5642244815826416, + -3.689701795578003, + -6.201474189758301, + -6.580833911895752, + -2.3081111907958984, + -5.42717170715332, + -1.1886008977890015, + -1.172760248184204, + -1.3571951389312744, + -1.3551844358444214, + -3.376784324645996, + -0.05118789151310921, + -4.064360618591309, + -2.575554847717285, + -0.6994737386703491, + -2.56724214553833, + -2.1888976097106934, + -0.4816131591796875, + -4.070178985595703, + -2.0060782432556152, + -6.858033180236816, + -0.059200502932071686, + -3.214278221130371, + -0.9671833515167236, + -0.823198676109314, + -1.0130078792572021, + -4.595561981201172, + -0.012724989093840122, + -5.214311599731445, + -8.246870040893555, + -3.1476030349731445, + -3.299684524536133, + -4.218191146850586, + -7.318399429321289, + -0.8580498695373535, + -3.0894036293029785, + -1.886361002922058, + -7.217658996582031, + -3.271679639816284, + -3.9717154502868652, + -1.8835484981536865, + -10.034332275390625, + -11.382490158081055, + -5.417011260986328, + -7.505967140197754, + -2.33837890625, + -0.07904055714607239, + -3.294971227645874, + -7.813640594482422, + -1.7646901607513428, + -4.025320053100586, + -3.5977325439453125, + -4.390352249145508, + -9.147806167602539, + -0.5303041934967041, + -7.721246242523193, + -0.6311959028244019, + -0.8119025230407715, + -0.7227814197540283, + -1.8369406461715698, + -0.20933297276496887, + -1.5395950078964233, + -4.424448490142822, + -4.084965705871582, + -3.355497360229492, + -1.0475609302520752, + -6.479413986206055, + -0.7810530662536621, + -2.132437229156494, + -6.648703098297119, + -2.9522438049316406, + -1.2485712766647339, + -4.040503025054932, + -2.3415768146514893, + -5.358206748962402, + -1.6258506774902344, + -3.956300973892212, + -0.732298731803894, + -7.441117286682129, + -1.5242161750793457, + -2.4555861949920654, + -4.295163154602051, + -9.687600135803223, + -0.8213484883308411, + -1.2446978092193604, + -0.01942702941596508, + -4.619411468505859, + -3.3297007083892822, + -2.2139487266540527, + -3.691431999206543, + -2.6574106216430664, + -6.075929641723633, + -0.6123450994491577, + -1.2942559719085693, + -0.6262839436531067, + -7.398006439208984, + -4.4869890213012695, + -4.202048301696777, + -4.982994079589844, + -0.637227475643158, + -3.061023235321045, + -10.117584228515625, + -3.8567495346069336, + -4.0480828285217285, + -2.472019672393799, + -4.246374607086182, + -1.3939155340194702, + -7.132441520690918, + -0.20108745992183685, + -4.986658573150635, + -4.387957572937012, + -0.01108358334749937, + -4.209756851196289, + -7.271108627319336, + -4.047314643859863, + -2.6497321128845215, + -1.4763175249099731, + -0.28365400433540344, + -3.5247769355773926, + -1.4226995706558228, + -4.327237129211426, + -2.0407187938690186, + -6.1437907218933105, + -1.5190880298614502, + -2.5511486530303955, + -7.504094123840332, + -2.152172565460205, + -6.708334922790527, + -6.913146495819092, + -3.6959621906280518, + -6.752341270446777, + -0.63083815574646, + -0.12433214485645294, + -5.0525641441345215, + -4.435934066772461, + -0.45601028203964233, + -6.3459577560424805, + -9.882917404174805, + -3.1422882080078125, + -2.550520658493042, + -3.2099051475524902, + -6.278127193450928, + -0.07764133810997009, + -3.155696153640747, + -1.933587670326233, + -9.61027717590332, + -6.211391925811768, + -4.664543151855469, + -6.783782005310059, + -5.676271438598633, + -8.605900764465332, + -0.0824289619922638, + -3.5463995933532715, + -13.374168395996094, + -1.2401021718978882, + -1.8734056949615479, + -3.4154422283172607, + -1.6733763217926025, + -17.633970260620117, + -9.345113754272461, + -0.6277351975440979, + -2.9617538452148438, + -2.5565333366394043, + -10.10580825805664, + -7.130337715148926, + -7.36820125579834, + -4.098911285400391, + -5.747079372406006, + -2.945054769515991, + -0.7887389063835144, + -1.6583149433135986, + -1.0165244340896606, + -6.581666946411133, + -5.926386833190918, + -5.845194339752197, + -0.9657630920410156, + -7.868755340576172, + -1.3244551420211792, + -0.2657390236854553, + -0.06403665244579315, + -2.983020782470703, + -5.943899631500244, + -7.877285957336426, + -3.593116283416748, + -3.819509506225586, + -7.226177215576172, + -2.5206997394561768, + -3.385587215423584, + -0.37499159574508667, + -1.4698283672332764, + -3.1460342407226562, + -0.0077166082337498665, + -4.350916862487793, + -3.2183218002319336, + -0.6242184638977051, + -1.4782464504241943, + -2.8054311275482178, + -3.0831401348114014, + -12.17662525177002, + -2.113419532775879, + -1.6448111534118652, + -2.1834323406219482, + -0.7630388140678406, + -10.1896390914917, + -6.234405517578125, + -11.46288776397705, + -1.003785490989685, + -4.211658477783203, + -1.5010679960250854, + -5.859302043914795, + -2.0465080738067627, + -3.7468819618225098, + -4.684195518493652, + -4.318704128265381, + -2.7234389781951904, + -9.00437068939209, + -3.043811321258545, + -3.1384406089782715, + -2.713779926300049, + -2.095993995666504, + -2.1484954357147217, + -10.274479866027832, + -0.682350754737854, + -0.25973302125930786, + -3.6964316368103027, + -13.434456825256348, + -2.3368239402770996, + -5.382724761962891, + -1.9073458909988403, + -5.905669212341309, + -0.032165709882974625, + -1.6530004739761353, + -2.728893280029297, + -1.640552043914795, + -1.1391171216964722, + -1.4353511333465576, + -4.003787994384766, + -0.3450564742088318, + -0.7168521285057068, + -0.34650325775146484, + -0.3616408705711365, + -7.062709331512451, + -1.2851682901382446, + -2.299129009246826, + -8.800156593322754, + -5.208735466003418, + -4.780910491943359, + -2.78342342376709, + -4.469717979431152, + -6.909726619720459, + -2.5114197731018066, + -0.659822404384613, + -0.6915416121482849, + -3.2363741397857666, + -0.5283617377281189, + -0.10473938286304474, + -6.215325832366943, + -7.283237934112549, + -1.6797031164169312, + -11.50100040435791, + -7.5822978019714355, + -3.387317657470703, + -11.407575607299805, + -5.441976547241211, + -3.3264851570129395, + -0.7265786528587341, + -1.382750153541565, + -7.841699600219727, + -8.105277061462402, + -3.9569506645202637, + -4.963083267211914, + -0.5492897629737854, + -4.6081390380859375, + -5.870400905609131, + -3.957930088043213, + -5.275494575500488, + -4.105091094970703, + -2.15435528755188, + -2.8472700119018555, + -1.1278448104858398, + -8.226571083068848, + -0.40629008412361145, + -9.916461944580078, + -4.616743087768555, + -1.691868543624878, + -0.6639478802680969, + -2.5716753005981445, + -6.676954746246338, + -6.535329818725586, + -0.4170510768890381, + -1.443942904472351, + -3.145481824874878, + -1.440589427947998, + -0.26935356855392456, + -0.9647155404090881, + -4.335958957672119, + -1.5647850036621094, + -5.890466690063477, + -3.01654052734375, + -1.9168468713760376, + -3.7365682125091553, + -8.001864433288574, + -10.680083274841309, + -4.489352226257324, + -4.6058149337768555, + -7.69011116027832, + -3.6247005462646484, + -1.5600426197052002, + -10.2160062789917, + -5.004643440246582, + -0.19602319598197937, + -3.375545024871826, + -2.669325590133667, + -1.3932737112045288, + -1.6410658359527588, + -6.847603797912598, + -6.744344711303711, + -0.5215591192245483, + -0.25840020179748535, + -1.1448237895965576, + -5.57253885269165, + -7.251138687133789, + -4.221924781799316, + -0.7688062787055969, + -2.504502534866333, + -3.146519660949707, + -2.206653356552124, + -1.4295082092285156, + -7.96943998336792, + -4.332189083099365, + -2.5750505924224854, + -1.7102608680725098, + -5.311381816864014, + -8.897522926330566, + -2.994919538497925, + -3.3397974967956543, + -2.1794328689575195, + -2.437566041946411, + -0.3181810975074768, + -0.27412793040275574, + -0.7914466857910156, + -2.3470635414123535, + -2.4099245071411133, + -2.491870880126953, + -3.024170160293579, + -1.9719040393829346, + -11.373910903930664, + -1.4279751777648926, + -0.14573107659816742, + -2.055763006210327, + -6.366893291473389, + -4.24091911315918, + -0.00709194503724575, + -2.0199716091156006, + -2.524750232696533, + -1.4272525310516357, + -0.5185190439224243, + -2.927150011062622, + -2.7070627212524414, + -3.365638017654419, + -4.318085193634033, + -7.773144721984863, + -1.7947180271148682, + -7.657534599304199, + -8.767786026000977, + -14.74280071258545, + -1.8042558431625366, + -3.2712037563323975, + -1.4002125263214111, + -4.887944221496582, + -1.4821010828018188, + -1.5255622863769531, + -5.879070281982422, + -4.463839530944824, + -5.1955976486206055, + -5.665647506713867, + -0.3775045573711395, + -5.9350481033325195, + -2.800539255142212, + -0.13162286579608917, + -3.034379720687866, + -4.729524612426758, + -4.6252641677856445, + -3.850942611694336, + -2.4760568141937256, + -6.059760093688965, + -10.12075138092041, + -0.9469369649887085, + -11.595907211303711, + -6.875324726104736, + -4.268826007843018, + -2.835529088973999, + -3.8626279830932617, + -4.876199245452881, + -0.013071090914309025, + -4.964417934417725, + -0.7445687055587769, + -5.707155227661133, + -6.10660457611084, + -4.317755699157715, + -4.440443992614746, + -2.9202542304992676, + -4.743522644042969, + -1.2569392919540405, + -2.8675737380981445, + -2.3151841163635254, + -4.318130970001221, + -1.9054772853851318, + -1.1808521747589111, + -0.765956461429596, + -2.768916606903076, + -6.237791061401367, + -1.7224305868148804, + -7.137521743774414, + -4.512486457824707, + -1.9069950580596924, + -4.145983695983887, + -5.365190505981445, + -0.059828490018844604, + -2.273892879486084, + -3.4013004302978516, + -5.035730361938477, + -6.501443386077881, + -9.903446197509766, + -1.6332892179489136, + -2.1572084426879883, + -1.6149548292160034, + -1.4698481559753418, + -6.01010799407959, + -2.2243528366088867, + -6.900836944580078, + -6.0930986404418945, + -2.974020481109619, + -3.225423574447632, + -8.423272132873535, + -1.3423724174499512, + -3.626147508621216, + -0.4862469434738159, + -6.860866546630859, + -3.8910953998565674, + -2.33319354057312, + -1.7229185104370117, + -2.215972423553467, + -8.99046516418457, + -4.099084854125977, + -2.4191012382507324, + -8.288970947265625, + -2.9641928672790527, + -1.5036451816558838, + -3.0544614791870117, + -0.0715634673833847, + -2.444031238555908, + -4.520998954772949, + -3.972568988800049, + -0.4985870122909546, + -2.1651363372802734, + -3.4427435398101807, + -1.730639100074768, + -0.9458961486816406, + -7.740211009979248, + -9.39163875579834, + -3.895984172821045, + -1.7523534297943115, + -5.41331672668457, + -8.910720825195312, + -12.971094131469727, + -3.0455880165100098, + -10.501265525817871, + -3.3864927291870117, + -4.842309951782227, + -3.9964733123779297, + -7.3046793937683105, + -2.6607093811035156, + -1.3541781902313232, + -5.003270626068115, + -3.944551944732666, + -0.11356143653392792, + -5.174440383911133, + -9.628616333007812, + -8.654989242553711, + -8.980416297912598, + -6.670101642608643, + -5.488286018371582, + -5.943419933319092, + -2.126483201980591, + -8.054739952087402, + -7.458671569824219, + -2.5267202854156494, + -6.455472946166992, + -8.655346870422363, + -7.903901100158691, + -6.221062660217285, + -7.129237174987793, + -4.2345380783081055, + -2.5375306606292725, + -7.697700500488281, + -1.567080020904541, + -2.084331750869751, + -0.25020831823349, + -1.5145041942596436, + -4.619244575500488, + -0.2970108985900879, + -0.4977554678916931, + -6.197869300842285, + -4.030620098114014, + -7.232107639312744, + -0.21076253056526184, + -1.563366174697876, + -1.133756160736084, + -2.708237648010254, + -4.080535888671875, + -0.6818401217460632, + -0.1864331066608429, + -0.49012088775634766, + -8.732468605041504, + -11.945040702819824, + -5.243098735809326, + -1.5294703245162964, + -0.8935543298721313, + -0.6174070835113525, + -1.5068217515945435, + -3.5766501426696777, + -5.393096923828125, + -4.202867031097412, + -14.765748023986816, + -5.2513813972473145, + -0.7597705721855164, + -0.2502063810825348, + -1.7403976917266846, + -2.8000779151916504, + -1.9808133840560913, + -2.1654744148254395, + -1.8629226684570312, + -3.222038745880127, + -0.040942225605249405, + -2.3384013175964355, + -10.210381507873535, + -4.5859761238098145, + -0.5805734395980835, + -3.7019288539886475, + -2.001936674118042, + -2.7876083850860596, + -2.9799084663391113, + -4.349887371063232, + -0.0792960673570633, + -1.4366114139556885, + -1.0813264846801758, + -1.3510822057724, + -6.7060699462890625, + -5.436615943908691, + -3.978389263153076, + -6.785447597503662, + -6.147171497344971, + -3.97414231300354, + -4.332991600036621, + -0.9269428253173828, + -5.1237101554870605, + -4.486598968505859, + -0.04678357392549515, + -1.0307552814483643, + -1.4249452352523804, + -4.517682075500488, + -3.561821699142456, + -2.0815205574035645, + -0.6041194200515747, + -5.992964744567871, + -7.092092514038086, + -0.48916709423065186, + -2.6405677795410156, + -4.3345723152160645, + -3.533582925796509, + -3.1233346462249756, + -3.107872486114502, + -1.9901115894317627, + -3.1052846908569336, + -1.8440347909927368, + -6.21368408203125, + -1.8796799182891846, + -2.705214738845825, + -0.2987763583660126, + -4.070865154266357, + -1.6675832271575928, + -1.3896636962890625, + -1.5731089115142822, + -3.526170015335083, + -2.5088443756103516, + -1.208929419517517, + -3.673125743865967, + -2.501532554626465, + -6.875064373016357, + -8.512459754943848, + -1.042314052581787, + -3.657850980758667, + -7.0950798988342285, + -4.974049091339111, + -8.14085578918457, + -3.529888153076172, + -1.9389504194259644, + -7.0902204513549805, + -2.409292459487915, + -2.9428021907806396, + -1.688283085823059, + -3.622368335723877, + -2.0903351306915283, + -4.160663604736328, + -3.1683764457702637, + -1.2135626077651978, + -7.566033363342285, + -3.1186251640319824, + -5.899919509887695, + -0.9518840312957764, + -2.656729221343994, + -2.2994377613067627, + -6.806836128234863, + -1.280236840248108, + -2.838846206665039, + -1.3598848581314087, + -11.707776069641113, + -3.134333372116089, + -0.6230669617652893, + -8.219222068786621, + -7.562507152557373, + -7.489459037780762, + -1.5368008613586426, + -7.149652481079102, + -5.749268054962158, + -3.162869691848755, + -2.7235195636749268, + -6.128931999206543, + -1.1934199333190918, + -3.986410617828369, + -3.76609468460083, + -1.712721586227417, + -3.195504903793335, + -8.397743225097656, + -3.1260581016540527, + -9.792022705078125, + -4.217884540557861, + -11.583260536193848, + -5.987588882446289, + -5.178754806518555, + -6.994749069213867, + -5.167606353759766, + -7.124668121337891, + -6.201416015625, + -10.203682899475098, + -6.858526229858398, + -2.733592987060547, + -5.078882217407227, + -9.003358840942383, + -4.704894542694092, + -3.9085562229156494, + -7.247268199920654, + -7.091092109680176, + -4.4150166511535645, + -7.56699275970459, + -9.485116004943848, + -1.9977033138275146, + -6.65272331237793, + -2.236643075942993, + -7.518955707550049, + -5.525973320007324, + -4.67877721786499, + -6.608670234680176, + -5.536133766174316, + -10.772479057312012, + -10.8853178024292, + -3.6156129837036133, + -6.751470565795898, + -6.4537434577941895, + -3.4220399856567383, + -8.251005172729492, + -3.2146153450012207, + -6.330069541931152, + -1.5551663637161255, + -6.520583629608154, + -10.450878143310547, + -5.8788957595825195, + -3.7398200035095215, + -3.9084208011627197, + -0.3640081584453583, + -6.961522102355957, + -6.066243648529053, + -7.270624160766602, + -5.098455429077148, + -2.7642822265625, + -5.460171699523926, + -7.362828731536865, + -2.558631658554077, + -2.186410427093506, + -2.5309929847717285, + -2.46756649017334, + -2.0306026935577393, + -1.8713470697402954, + -2.108008623123169, + -1.2698389291763306, + -2.1712756156921387, + -2.4432802200317383, + -1.1477653980255127, + -1.8417484760284424, + -2.5971946716308594, + -1.8250831365585327, + -2.103092670440674, + -2.5183165073394775, + -2.9367291927337646, + -1.9412965774536133, + -1.7692793607711792, + -2.864521026611328, + -3.1332175731658936, + -1.098311185836792, + -2.946441173553467, + -2.2800471782684326, + -3.1929852962493896, + -2.754260778427124, + -3.485616445541382, + -3.3010287284851074, + -2.5537776947021484, + -2.6752865314483643, + -3.1617612838745117, + -2.4571690559387207, + -2.060081958770752, + -2.425969362258911, + -2.212725877761841, + -2.4232254028320312, + -3.0587053298950195, + -2.4074010848999023, + -2.457937479019165, + -2.319617986679077, + -2.6340954303741455, + -2.599524736404419, + -2.5302212238311768, + -1.6849274635314941, + -2.2609786987304688, + -2.039928674697876, + -1.9474098682403564, + -2.3550753593444824, + -1.718749761581421, + -2.413884162902832, + -1.6247628927230835, + -2.4784040451049805, + -1.828325629234314, + -1.3880831003189087, + -1.4448199272155762, + -1.1477117538452148, + -1.1669728755950928, + -1.8787822723388672, + -1.5565840005874634, + -1.6666553020477295, + -1.747725248336792, + -1.959598422050476, + -2.0376486778259277, + -2.345367431640625, + -2.055098533630371, + -1.3940613269805908, + -3.4385242462158203, + -2.7489635944366455, + -3.2590157985687256, + -3.1128957271575928, + -1.7070379257202148, + -3.9010369777679443, + -3.21574068069458, + -3.3850393295288086, + -1.8778185844421387, + -2.698211908340454, + -1.8060741424560547, + -2.0845324993133545, + -3.4797585010528564, + -2.263254404067993, + -3.083108901977539, + -1.6589758396148682, + -2.687279224395752, + -1.77505624294281, + -2.6142921447753906, + -1.934045672416687, + -1.8834377527236938, + -2.8038980960845947, + -1.550542950630188, + -3.4054152965545654, + -1.724036693572998, + -2.3146564960479736, + -1.5134503841400146, + -2.9289023876190186, + -1.5285141468048096, + -3.421035051345825, + -1.3757282495498657, + -3.441431760787964, + -1.5286564826965332, + -3.4372904300689697, + -3.173043966293335, + -1.1313854455947876, + -1.486415147781372, + -2.506413221359253, + -3.5165903568267822, + -1.4112176895141602, + -3.7175354957580566, + -2.2170844078063965, + -0.704839289188385, + -0.6626103520393372, + -2.5483946800231934, + -0.949668288230896, + -3.1339564323425293, + -1.3326977491378784, + -3.5493476390838623, + -0.6785370111465454, + -3.236161470413208, + -2.1347084045410156, + -3.802447557449341, + -2.585503339767456, + -3.9361765384674072, + -2.290905714035034, + -3.714280605316162, + -1.793616533279419, + -3.7252492904663086, + -1.450188159942627, + -3.11938738822937, + -1.25174880027771 + ] + }, + "32": { + "input_prompt": "create a conversational article", + "generated_text": " about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article. The article should be about the topic of the article.", + "generated_tokens": [ + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046, + 1531, + 9369, + 2715, + 1402, + 2314, + 1278, + 17915, + 1307, + 1278, + 9369, + 1046 + ], + "latency": 47.09983468055725, + "cuda_graph_request_count_map": null, + "step_count": 6144, + "logprobs": [ + -4.4165568351745605, + -11.358176231384277, + -0.0701780766248703, + -7.797665119171143, + -2.6805992126464844, + -1.4707680940628052, + -3.0390255451202393, + -1.6902849674224854, + -1.270594835281372, + -1.1936196088790894, + -1.2523558139801025, + -2.7270259857177734, + -1.2371309995651245, + -0.9618493318557739, + -0.4379909038543701, + -1.3917063474655151, + -1.1055524349212646, + -0.9122569561004639, + -0.9911308288574219, + -0.08436793833971024, + -0.5424078106880188, + -0.9181017279624939, + -0.5873759388923645, + -0.19014373421669006, + -0.06655456870794296, + -0.15252672135829926, + -0.09415211528539658, + -0.009787309914827347, + -0.013910251669585705, + -0.005296128336340189, + -0.005677408073097467, + -0.02013739012181759, + -0.21594694256782532, + -0.07153760641813278, + -0.0066444179974496365, + -0.010198505595326424, + -0.011980246752500534, + -0.003686776151880622, + -0.0037619550712406635, + -0.0022467151284217834, + -0.004088377580046654, + -0.021828632801771164, + -0.0012669878778979182, + -0.09768074005842209, + -0.02652405947446823, + -0.0019286142196506262, + -0.002283824374899268, + -0.0032225127797573805, + -0.0009741804678924382, + -0.0009415484382770956, + -0.001211624126881361, + -0.001135300612077117, + -0.002340436913073063, + -0.0010846928926184773, + -0.0509282611310482, + -0.03832047060132027, + -0.00257422705180943, + -0.0022806129418313503, + -0.00262785074301064, + -0.0008195855189114809, + -0.0010239601833745837, + -0.0013777059502899647, + -0.0009899006690829992, + -0.0018756669014692307, + -0.0015304292319342494, + -0.08506463468074799, + -0.01893703266978264, + -0.0013797297142446041, + -0.0014461545506492257, + -0.0013971101725474, + -0.0005869334563612938, + -0.0005212855176068842, + -0.000876757490914315, + -0.0005256939912214875, + -0.0012863941956311464, + -0.0015691122971475124, + -0.051276568323373795, + -0.00973513163626194, + -0.0010469438275322318, + -0.0011531615164130926, + -0.0009969270322471857, + -0.00038342276820912957, + -0.0004032037395518273, + -0.000730247818864882, + -0.0003275334893260151, + -0.0008700875914655626, + -0.0017572689102962613, + -0.044393111020326614, + -0.013102858327329159, + -0.0011463745031505823, + -0.0012070996453985572, + -0.0012325793504714966, + -0.0005048430757597089, + -0.0004876854654867202, + -0.0007901645149104297, + -0.00041500062798149884, + -0.0009869233472272754, + -0.0018687656847760081, + -0.03943866863846779, + -0.014425630681216717, + -0.0014756753807887435, + -0.001423775334842503, + -0.001209719106554985, + -0.0005046047735959291, + -0.00042167355422861874, + -0.0007688426994718611, + -0.0002699726028367877, + -0.0006598440813831985, + -0.0017849955474957824, + -0.038999658077955246, + -0.012665312737226486, + -0.0014427024871110916, + -0.0014492495683953166, + -0.001016576774418354, + -0.00042083943844772875, + -0.00033241944038309157, + -0.0006403064471669495, + -0.00022373080719262362, + -0.0007053509471006691, + -0.0018597226589918137, + -0.030997740104794502, + -0.011259939521551132, + -0.0012655591126531363, + -0.00134151556994766, + -0.0008106521563604474, + -0.0003513672563713044, + -0.0002964295563288033, + -0.0006368515896610916, + -0.00020180096908006817, + -0.0005779979983344674, + -0.0016014858847483993, + -0.0271126888692379 + ] + }, + "64": { + "input_prompt": "App Concept: \"Eggy's Interactive Adventure World\"\n\nEggy's Interactive Adventure World is a cutting-edge app designed for phones and tablets that combines immersive storytelling with engaging, interactive experiences for both parents and children. This innovative app allows users to explore the world of Eggy and participate in a series of adventures that teach essential life skills, foster personal growth, and strengthen parent-child bonds.\n\nKey Features:\n\n1. Interactive Stories: The app brings the 20 Eggy stories to life through interactive, animated storytelling. Users can read, listen, and actively participate in the stories, making choices that guide Eggy through various adventures, each with unique lessons and experiences.\n2. Augmented Reality (AR) Integration: The app utilizes AR technology, allowing users to interact with Eggy and the story characters in their real-world environment. This immersive experience creates a deeper connection with the stories and encourages engagement.\n3. Personalized Adventures: The app customizes each story based on the user's interests, needs, and personal growth areas. By collecting user input, the app tailors the storylines and learning outcomes to ensure that each adventure resonates with the child's unique experiences.\n4. Parent-Child Cooperative Play: The app encourages parents and children to collaborate and engage in the stories together. Through a split-screen mode or multi-device connectivity, parents and children can actively participate in the stories and learn essential life skills together.\n5. Skill-Building Mini-Games: Interspersed throughout the stories are educational mini-games that reinforce the lessons and themes. These games are designed to be fun, engaging, and accessible to children, encouraging them to practice and develop various skills.\n6. Progress Tracking and Rewards: The app tracks the user's progress through the stories and rewards them with badges, trophies, and other collectibles. This gamification aspect encourages continued engagement and fosters a sense of accomplishment and growth.\n7. Community Features: The app provides a safe, moderated online community where parents can share their experiences, discuss the stories and lessons, and connect with other families who are also using the app. This community fosters support, inspiration, and learning.\n8. Expert Resources: The app includes access to resources from experts in child development, psychology, and education, providing parents with additional tools and guidance on fostering the growth and well-being of their children.\n\nEggy's Interactive Adventure World is a one-of-a-kind, mind-blowing app that revolutionizes the way parents and children engage with stories and learn essential life skills. Combining the power of storytelling, cutting-edge technology, and expert resources, this app is a must-have for families looking to strengthen their bonds and support their children's personal growth.", + "generated_text": "The 1999\u20132000 season was the 10th season of the National Hockey League (NHL). The 1999\u20132000 season was the 10th season of the National Hockey League (NHL). The 1999\u20132000 season was the 10th season of the National Hockey League (NHL). The 1999\u20132000 season was the 10th season was the 10th season was the 10 season was the 10 season was the 10", + "generated_tokens": [ + 2, + 1784, + 1032, + 1049, + 1057, + 1057, + 1057, + 1882, + 1050, + 1048, + 1048, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 1411, + 5526, + 1307, + 1278, + 5805, + 46982, + 8525, + 1319, + 19644, + 1076, + 1577, + 1531, + 1032, + 1049, + 1057, + 1057, + 1057, + 1882, + 1050, + 1048, + 1048, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 1411, + 5526, + 1307, + 1278, + 5805, + 46982, + 8525, + 1319, + 19644, + 1076, + 1577, + 1531, + 1032, + 1049, + 1057, + 1057, + 1057, + 1882, + 1050, + 1048, + 1048, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 1411, + 5526, + 1307, + 1278, + 5805, + 46982, + 8525, + 1319, + 19644, + 1076, + 1577, + 1531, + 1032, + 1049, + 1057, + 1057, + 1057, + 1882, + 1050, + 1048, + 1048, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 1411, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 1411, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048, + 5526, + 1486, + 1278, + 1032, + 1049, + 1048 + ], + "latency": 85.16301679611206, + "cuda_graph_request_count_map": null, + "step_count": 6144, + "logprobs": [ + -12.107745170593262, + -2.9727728366851807, + -3.720092535018921, + -5.592433929443359, + -10.964235305786133, + -3.654498338699341, + -9.33439826965332, + -4.833785057067871, + -5.187321662902832, + -2.6944785118103027, + -6.9262237548828125, + -0.654232919216156, + -0.5550781488418579, + -0.21346639096736908, + -0.0134271876886487, + -0.010840100236237049, + -1.3878544569015503, + -0.6296291351318359, + -7.9766011238098145, + -0.4393192231655121, + -5.639142036437988, + -3.277270793914795, + -1.0206468105316162, + -11.703084945678711, + -0.7100943922996521, + -0.2809169888496399, + -2.771284818649292, + -7.190817832946777, + -4.048691749572754, + -0.012056218460202217, + -3.3802318572998047, + -0.6807184815406799, + -3.4844107627868652, + -3.312331199645996, + -0.5001641511917114, + -2.61255145072937, + -4.243694305419922, + -4.333778381347656, + -6.0625810623168945, + -0.011777156963944435, + -0.37577226758003235, + -0.9490834474563599, + -3.5450198650360107, + -2.1778035163879395, + -0.45957911014556885, + -3.00771164894104, + -1.7600425481796265, + -0.09766030311584473, + -2.467618942260742, + -1.329679012298584, + -0.8384320735931396, + -1.1864604949951172, + -3.628342866897583, + -0.2470003068447113, + -1.8938640356063843, + -5.168431282043457, + -0.05005566030740738, + -2.258014678955078, + -2.449028968811035, + -0.0034086955711245537, + -3.9485883712768555, + -1.6201664209365845, + -5.139942646026611, + -4.859354496002197, + -0.23686674237251282, + -0.5541543364524841, + -2.5826025009155273, + -6.114635467529297, + -4.3380208015441895, + -0.7412900924682617, + -0.3221715986728668, + -0.13805493712425232, + -4.1797332763671875, + -7.3456268310546875, + -0.13762745261192322, + -2.0905232429504395, + -1.0178627967834473, + -4.108260631561279, + -0.6007124185562134, + -1.0410642623901367, + -4.122039794921875, + -0.35905471444129944, + -1.4274661540985107, + -4.139932155609131, + -0.4237431585788727, + -1.6294409036636353, + -0.9811424016952515, + -4.132790565490723, + -1.1318120956420898, + -6.8258256912231445, + -1.5455098152160645, + -0.6984409093856812, + -13.664215087890625, + -0.1166313961148262, + -1.6347849369049072, + -0.28875046968460083, + -0.03130083531141281, + -1.5293006896972656, + -1.6488375663757324, + -4.224111557006836, + -4.760683059692383, + -1.9758747816085815, + -1.5828256607055664, + -2.8463857173919678, + -0.2620386481285095, + -1.7243889570236206, + -1.7945923805236816, + -0.8884308338165283, + -0.3766394555568695, + -0.34033581614494324, + -9.05566692352295, + -0.22754782438278198, + -0.033802058547735214, + -0.34108465909957886, + -0.5644669532775879, + -2.0925779342651367, + -4.547505855560303, + -10.870464324951172, + -1.1072022914886475, + -5.503787994384766, + -3.259672164916992, + -0.007964519783854485, + -3.0111639499664307, + -4.246737480163574, + -0.7813188433647156, + -3.331031322479248, + -4.485962867736816, + -0.9492117166519165, + -2.6757047176361084, + -1.1591349840164185, + -1.122117519378662, + -2.629878044128418, + -5.986321926116943, + -0.2146703153848648, + -0.002392764901742339, + -7.372479438781738, + -0.007077385671436787, + -0.06599216908216476, + -0.0970711037516594, + -3.2874932289123535, + -0.0019583588000386953, + -0.9122000336647034, + -4.930907249450684, + -0.019508399069309235, + -0.308611661195755, + -0.07778516411781311, + -3.8497893810272217, + -0.46124517917633057, + -0.38821348547935486, + -2.668412208557129, + -1.845987319946289, + -0.06470083445310593, + -0.006619549356400967, + -1.2610487937927246, + -0.13015533983707428, + -3.365312099456787, + -0.0014690094394609332, + -1.6789823770523071, + -1.2499005794525146, + -3.3992111682891846, + -5.563300132751465, + -0.823418140411377, + -4.24124813079834, + -1.6597849130630493, + -0.6941139698028564, + -1.5637556314468384, + -0.5482053756713867, + -0.9507225751876831, + -3.764758586883545, + -0.0006518622976727784, + -0.7540555000305176, + -5.058262825012207, + -0.3302401602268219, + -2.8130555152893066, + -0.17079885303974152, + -2.871047019958496, + -0.3991694450378418, + -3.1476998329162598, + -0.3488404452800751, + -2.0545666217803955, + -4.201597690582275, + -5.164614677429199, + -0.0271432027220726, + -0.0009785869624465704, + -3.3444161415100098, + -1.3117046356201172, + -6.375423431396484, + -0.05535568296909332, + -0.3919340968132019, + -0.060594215989112854, + -6.507473468780518, + -0.0023910999298095703, + -2.143423318862915, + -3.335618257522583, + -2.953970432281494, + -0.0013383012264966965, + -0.8080525398254395, + -0.29526084661483765, + -0.04036511853337288, + -3.231475353240967, + -1.0585589408874512, + -6.136373043060303, + -0.006182829383760691, + -0.035548023879528046, + -5.509808540344238, + -1.8490750789642334, + -9.83314037322998, + -0.07037576287984848, + -3.1621387004852295, + -6.762360095977783, + -1.3490527868270874, + -3.601043462753296, + -1.176393985748291, + -0.4342959523200989, + -0.06266004592180252, + -5.464046001434326, + -0.017946599051356316, + -1.0416009426116943, + -1.6117159128189087, + -12.289417266845703, + -1.5004339218139648, + -5.76563835144043, + -4.038386821746826, + -0.20812086760997772, + -3.6306562423706055, + -1.3901070356369019, + -1.087137222290039, + -2.423213243484497, + -4.503086090087891, + -0.0008031480247154832, + -0.03627370297908783, + -0.1653430461883545, + -7.958648681640625, + -1.1018548011779785, + -1.290948748588562, + -3.8049263954162598, + -1.8253734111785889, + -0.059022851288318634, + -0.0013984196120873094, + -4.698851585388184, + -2.5421664714813232, + -0.024493809789419174, + -4.828659534454346, + -3.0295286178588867, + -3.550312042236328, + -0.1185273677110672, + -0.22595760226249695, + -0.10782183706760406, + -1.4033282995224, + -0.4485701024532318, + -0.2889708876609802, + -0.05471855774521828, + -0.007632025051862001, + -2.1156554222106934, + -0.6249589323997498, + -4.198577404022217, + -0.14178156852722168, + -4.284021377563477, + -2.227515935897827, + -3.5022120475769043, + -0.19575819373130798, + -15.964509963989258, + -4.055960655212402, + -11.125024795532227, + -0.7681724429130554, + -3.0436902046203613, + -7.030262470245361, + -4.376729488372803, + -5.476145267486572, + -0.4219042658805847, + -3.7689766883850098, + -0.060010604560375214, + -0.8134393692016602, + -0.11386934667825699, + -0.025473715737462044, + -0.09736856073141098, + -4.357361793518066, + -0.3670865297317505, + -0.08063744008541107, + -0.1311480849981308, + -1.0903867483139038, + -1.2705107927322388, + -1.5076212882995605, + -4.295275688171387, + -0.04185756668448448, + -0.19810955226421356, + -1.9645220041275024, + -0.9597910642623901, + -0.13429655134677887, + -0.002283110748976469, + -7.066074371337891, + -3.639211654663086, + -1.0263917446136475, + -8.124760627746582, + -1.132537841796875, + -0.09160765260457993, + -0.08996370434761047, + -10.165366172790527, + -3.501585006713867, + -0.0019847711082547903, + -0.05309417471289635, + -0.31209683418273926, + -0.15089339017868042, + -1.23564875125885, + -1.2685208320617676, + -7.832758903503418, + -0.19271136820316315, + -0.014305183663964272, + -0.0007532381569035351, + -0.44688940048217773, + -2.6239724159240723, + -1.738666296005249, + -1.6480977535247803, + -0.46753185987472534, + -8.656959533691406, + -3.79868483543396, + -0.9281394481658936, + -2.2381181716918945, + -1.7654449939727783, + -0.4948798418045044, + -0.025028761476278305, + -1.5435361862182617, + -1.6390818357467651, + -1.4962153434753418, + -0.3425217270851135, + -0.013077914714813232, + -0.038474079221487045, + -5.3364362716674805, + -0.42365288734436035, + -1.884093999862671, + -3.510357618331909, + -6.198029518127441, + -0.44375038146972656, + -0.0008789013954810798, + -3.6025230884552, + -1.419615626335144, + -2.6723289489746094, + -5.775190830230713, + -1.1380761861801147, + -2.6683366298675537, + -0.43395891785621643, + -0.003145867260172963, + -8.63144302368164, + -1.646262764930725, + -1.732487678527832, + -4.561546802520752, + -0.5277953147888184, + -0.07333153486251831, + -0.5624169707298279, + -0.12201295047998428, + -2.6561455726623535, + -1.1071691513061523, + -2.6895060539245605, + -0.040864069014787674, + -0.04126371443271637, + -1.8294739723205566, + -0.09022177755832672, + -0.3154001832008362, + -0.46215569972991943, + -2.2462844848632812, + -0.30149081349372864, + -0.52588951587677, + -8.288043975830078, + -0.0002057340752799064, + -0.8021711707115173, + -4.4546098709106445, + -0.0001565095444675535, + -0.0015961299650371075, + -0.15216240286827087, + -0.3677564561367035, + -5.018707275390625, + -0.7850045561790466, + -1.9582659006118774, + -1.0046892166137695, + -10.0401029586792, + -0.16878114640712738, + -5.944240570068359, + -1.5523078441619873, + -5.7253522872924805, + -0.47948503494262695, + -0.44009655714035034, + -5.671053886413574, + -0.003280022880062461, + -0.7937742471694946, + -0.9639376401901245, + -0.00030048147891648114, + -1.0747740268707275, + -0.8839919567108154, + -3.416811466217041, + -1.6602673530578613, + -0.2706959843635559, + -0.0024333172477781773, + -4.478696823120117, + -6.20179557800293, + -0.11359559744596481, + -0.202009916305542, + -0.022310219705104828, + -2.367263078689575, + -1.0405994653701782, + -5.984308242797852, + -2.105138063430786, + -9.583202362060547, + -0.0004957877099514008, + -3.0655455589294434, + -0.0669412910938263, + -0.8977450728416443, + -2.2271294593811035, + -2.6617536544799805, + -1.8184051513671875, + -0.8291114568710327, + -0.4864235818386078, + -0.7993525862693787, + -3.51106858253479, + -2.1530935764312744, + -0.257144957780838, + -1.3934082984924316, + -1.3137131929397583, + -0.3384077548980713, + -0.1697217971086502, + -2.353395938873291, + -0.03406282886862755, + -0.39059701561927795, + -3.422821044921875, + -1.7117210626602173, + -0.7018465399742126, + -1.5995906591415405, + -3.6218395233154297, + -0.12497704476118088, + -0.16966234147548676, + -0.7313685417175293, + -0.4956285357475281, + -1.0840849876403809, + -5.042126655578613, + -0.00031704644788987935, + -7.683258056640625, + -0.9210801720619202, + -4.687852382659912, + -0.0028814247343689203, + -0.043382611125707626, + -4.1948652267456055, + -2.66593337059021, + -0.06153333932161331, + -0.0023110604379326105, + -6.729236602783203, + -5.777127742767334, + -0.08932067453861237, + -0.09890018403530121, + -0.009886111132800579, + -3.1145148277282715, + -3.725565195083618, + -0.0021998509764671326, + -3.9927196502685547, + -2.753793239593506, + -1.6037236452102661, + -0.17461130023002625, + -4.804804801940918, + -0.2311229705810547, + -0.30256444215774536, + -2.235363006591797, + -0.006614102050662041, + -0.34757524728775024, + -1.4946835041046143, + -1.222062587738037, + -3.658839225769043, + -1.356170892715454, + -0.5371109843254089, + -3.7580835819244385, + -4.54621696472168, + -0.31577637791633606, + -3.677156925201416, + -2.7181396484375, + -7.4674882888793945, + -0.00019369633810129017, + -2.3798398971557617, + -2.5452184677124023, + -0.2858496308326721, + -4.315659523010254, + -0.025835415348410606, + -0.000603493710514158, + -0.2546294331550598, + -0.12032663822174072, + -2.006908655166626, + -5.990736961364746, + -7.146596908569336, + -0.23356498777866364, + -0.2201036810874939, + -0.01235415879637003, + -0.011248741298913956, + -1.4155778884887695, + -0.40242519974708557, + -5.877886772155762, + -0.7865053415298462, + -0.03231288120150566, + -0.004864405374974012, + -0.0050629740580916405, + -2.7049152851104736, + -6.822089195251465, + -0.39252761006355286, + -1.2290617227554321, + -0.007630132604390383, + -3.485461711883545, + -0.47985684871673584, + -6.1813530921936035, + -0.03757825121283531, + -0.37834712862968445, + -0.22192610800266266, + -1.165318489074707, + -0.5220151543617249, + -0.1289423257112503, + -3.216222047805786, + -1.0787583589553833, + -3.0716826915740967, + -0.6023419499397278, + -2.558605194091797, + -0.927433431148529, + -0.00364841241389513, + -0.14910078048706055, + -0.7318926453590393, + -6.159773826599121, + -0.0015301911626011133, + -1.8908276557922363, + -1.9641315937042236, + -0.021651331335306168, + -2.1648828983306885, + -2.2700207233428955, + -7.833290100097656, + -0.03397307172417641, + -0.8344621658325195, + -0.02225659228861332, + -0.06639260798692703, + -2.3780317306518555, + -3.180129051208496, + -0.09030630439519882, + -2.4138312339782715, + -1.3445552587509155, + -1.848326325416565, + -0.9726964831352234, + -2.851792335510254, + -0.0630769282579422, + -0.0011394681641831994, + -0.05843213573098183, + -2.6616668701171875, + -1.575437068939209, + -0.180197611451149, + -5.552371501922607, + -0.26108410954475403, + -2.529611587524414, + -0.37780019640922546, + -5.141795635223389, + -0.5921107530593872, + -0.2474975287914276, + -0.10687454044818878, + -4.891775131225586, + -0.25011152029037476, + -2.4100728034973145, + -1.358667016029358, + -2.790961503982544, + -3.8654675483703613, + -1.0076243877410889, + -0.7456949949264526, + -1.5575554370880127, + -2.05328631401062, + -1.6538066864013672, + -0.0558217354118824, + -0.0001817776501411572, + -0.0011643542675301433, + -0.038359593600034714, + -1.4208931922912598, + -0.542127251625061, + -0.3162364959716797, + -0.3966117799282074, + -1.1765563488006592, + -1.7920958995819092, + -0.18425509333610535, + -0.1092008650302887, + -0.46676987409591675, + -0.24977745115756989, + -1.0375996828079224, + -0.5268858671188354, + -0.008942908607423306, + -0.6404479146003723, + -0.0033111530356109142, + -5.3165931603871286e-05, + -0.5154370665550232, + -0.39286962151527405, + -1.401839256286621, + -0.6232213973999023, + -0.02168831042945385, + -0.004282470792531967, + -0.005199837032705545, + -0.09748794883489609, + -0.040823787450790405, + -0.00014852374442853034, + -0.0005832401220686734, + -0.005303124897181988, + -0.6537013053894043, + -0.38026049733161926, + -0.04189129173755646, + -0.010385753586888313, + -0.008756335824728012, + -0.013362848199903965, + -0.000504723924677819, + -0.002797620603814721, + -0.0014512732159346342, + -0.0013321106089279056, + -0.010883613489568233, + -0.005159396678209305, + -0.004701037425547838, + -0.01591104455292225, + -0.001474246964789927, + -1.2278481335670222e-05, + -0.010548785328865051, + -0.08341525495052338, + -0.03858809545636177, + -0.056062061339616776, + -0.0009532198309898376, + -0.0005789510905742645, + -0.0008986725588329136, + -0.00710969977080822, + -0.0006561510381288826, + -1.4781842764932662e-05, + -5.578839045483619e-05, + -0.0006398299592547119, + -0.0028786908369511366, + -0.0034092895220965147, + -0.008268529549241066, + -0.006602259818464518, + -0.004517706111073494, + -0.02233586646616459, + -0.0006323245470412076, + -0.009195122867822647, + -0.0029284947086125612, + -0.004457537550479174, + -0.017873765900731087, + -0.008801711723208427, + -0.0036383166443556547, + -0.08078611642122269, + -0.006347495596855879, + -0.0002177716523874551, + -0.04688572511076927, + -0.2718890309333801, + -0.07996802777051926, + -0.04674842208623886, + -0.009984076954424381, + -0.010000954382121563, + -0.050126753747463226, + -0.5864179730415344, + -0.2915390133857727, + -0.008090462535619736, + -0.032190412282943726, + -0.03461571782827377, + -0.2785419523715973, + -0.05830562859773636, + -0.02893950417637825, + -0.12241066247224808, + -0.02711048536002636, + -0.16450686752796173, + -0.09852994978427887, + -0.2651047706604004, + -0.35559725761413574, + -0.12606258690357208, + -0.32793670892715454, + -0.20878805220127106, + -0.7587923407554626, + -0.4730657637119293, + -1.496794581413269, + -0.2486363798379898, + -0.20723387598991394, + -0.1872958242893219, + -0.19151091575622559, + -0.3350580036640167, + -1.3085839748382568, + -0.6109102964401245, + -0.2947172224521637, + -0.37130236625671387, + -0.19888469576835632, + -0.18297068774700165, + -0.9978674054145813, + -0.5471905469894409, + -0.4379975199699402, + -0.407988041639328, + -0.17325688898563385, + -0.16938896477222443 + ] + }, + "96": { + "input_prompt": "what is the concept of double materiality in sustainability?", + "generated_text": " What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double materiality in sustainability? What is the concept of double material", + "generated_tokens": [ + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318, + 1532, + 1294, + 60187, + 1063, + 5675, + 1395, + 1278, + 7401, + 1307, + 5558, + 4318 + ], + "latency": 123.03724575042725, + "cuda_graph_request_count_map": null, + "step_count": 6144, + "logprobs": [ + -4.917365074157715, + -0.9960631132125854, + -7.875392913818359, + -0.2993181347846985, + -7.760880470275879, + -10.308395385742188, + -2.1807961463928223, + -1.6412583589553833, + -9.521512985229492, + -1.627489447593689, + -1.8410861492156982, + -0.9285702705383301, + -0.2576955556869507, + -0.9641067981719971, + -0.02314644306898117, + -0.6696561574935913, + -0.07035009562969208, + -0.004622488282620907, + -0.025748632848262787, + -0.06276137381792068, + -0.17385317385196686, + -0.3285445272922516, + -0.0592009499669075, + -0.007940039038658142, + -0.22664028406143188, + -0.0017957051750272512, + -0.022929180413484573, + -0.005733947269618511, + -0.0012996093137189746, + -0.006419987417757511, + -0.02376849390566349, + -0.27800270915031433, + -0.4650723934173584, + -0.04936715215444565, + -0.003972141072154045, + -0.01477995328605175, + -0.0012044801842421293, + -0.014891182072460651, + -0.002709767082706094, + -0.0009939497103914618, + -0.0028436246793717146, + -0.006759870797395706, + -0.15416178107261658, + -0.20121537148952484, + -0.016414370387792587, + -0.0015769677702337503, + -0.008138825185596943, + -0.0007713441736996174, + -0.013819841668009758, + -0.003826678032055497, + -0.0005918181850574911, + -0.0014938872773200274, + -0.00485716899856925, + -0.081083282828331, + -0.09642580896615982, + -0.009630884043872356, + -0.0010948146227747202, + -0.007085552904754877, + -0.0006310140597634017, + -0.013073914684355259, + -0.0039152647368609905, + -0.000364713923772797, + -0.001292108790948987, + -0.004158303141593933, + -0.044283974915742874, + -0.05722038820385933, + -0.006369172595441341, + -0.0007976687629707158, + -0.005993015132844448, + -0.0004935238393954933, + -0.011310506612062454, + -0.002951553324237466, + -0.000387831823900342, + -0.000977038755081594, + -0.0036971091758459806, + -0.030511993914842606, + -0.04246694967150688, + -0.004863100592046976, + -0.0006927236099727452, + -0.005206122528761625, + -0.0005129451747052372, + -0.00894621666520834, + -0.0028565814718604088, + -0.00041333239641971886, + -0.0009002208826132119, + -0.0033131728414446115, + -0.021188799291849136, + -0.03330245241522789, + -0.0038543473929166794, + -0.0006504327175207436, + -0.004474864806979895, + -0.00048029806930571795, + -0.009718249551951885, + -0.0030443770810961723, + -0.0003743662964552641, + -0.0009439303539693356, + -0.003729770192876458, + -0.016505014151334763, + -0.0290373582392931, + -0.003315192647278309, + -0.0005821678787469864, + -0.004148805979639292, + -0.00042489083716645837, + -0.006856840569525957, + -0.0028660909738391638, + -0.00032574593205936253, + -0.0006986799417063594, + -0.003671098267659545, + -0.012792548164725304, + -0.02553274855017662, + -0.002730690874159336, + -0.0005067494930699468, + -0.0036923582665622234, + -0.0004451475979294628, + -0.007225453853607178, + -0.002746859099715948, + -0.0003323002893012017, + -0.0008517451351508498, + -0.003630714723840356, + -0.011284693144261837, + -0.02186688780784607, + -0.0025819556321948767, + -0.0004931663861498237, + -0.0031721293926239014, + -0.00040725519647821784, + -0.0062383925542235374, + -0.00238693761639297, + -0.00040749352774582803, + -0.0007970731821842492, + -0.003604583442211151, + -0.010075542144477367, + -0.022386692464351654, + -0.0025295186787843704, + -0.00046302087139338255, + -0.003138143103569746, + -0.0004233417857903987, + -0.006178564392030239, + -0.0021037368569523096 + ] + }, + "127": { + "input_prompt": "as before we talk about using , i usually do if else() to in case the property would be null before.\nif today i public a property and i want to use using to instead , will code be like this?\n\npublic Material Mat;\n\nusing(Mat){\n// Do something;\n}\n\nis that right?", + "generated_text": " i am not sure if i am using the wrong syntax or if i am using the syntax correctly but i am not sure if i am using the syntax correctly or if i am using the syntax incorrectly.\n\nI am not sure if i am using the wrong syntax or if i am using the syntax correctly but i am not sure if i am using the syntax correctly or if i am using the syntax incorrectly.\n\nI am not sure if i am using the wrong syntax or if i am using the syntax correctly but i am not sure if i am using the syntax correctly or if i am using the syntax incorrectly.\n\nI am not sure if i am using", + "generated_tokens": [ + 1623, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 8462, + 22692, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1809, + 1623, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 73751, + 1338, + 1073, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 8462, + 22692, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1809, + 1623, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 73751, + 1338, + 1073, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 8462, + 22692, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1809, + 1623, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 17047, + 1505, + 1693, + 1623, + 1855, + 2505, + 1278, + 22692, + 73751, + 1338, + 1073, + 1855, + 1605, + 5257, + 1693, + 1623, + 1855, + 2505 + ], + "latency": 151.56505846977234, + "cuda_graph_request_count_map": null, + "step_count": 6144, + "logprobs": [ + -7.793755531311035, + -4.224940776824951, + -5.446587562561035, + -0.267395555973053, + -6.118141174316406, + -11.105525970458984, + -6.67517614364624, + -6.902530670166016, + -3.2218151092529297, + -5.99644660949707, + -8.431673049926758, + -8.04836368560791, + -3.992594003677368, + -7.550079345703125, + -2.873685836791992, + -3.3877997398376465, + -7.9306745529174805, + -7.005484580993652, + -1.0481306314468384, + -2.686237335205078, + -6.131283760070801, + -6.2994704246521, + -7.931419372558594, + -11.3147554397583, + -8.544670104980469, + -12.01729679107666, + -3.89847469329834, + -1.7964364290237427, + -2.952878952026367, + -1.9217232465744019, + -2.272329330444336, + -0.37552154064178467, + -1.667820692062378, + -7.510344505310059, + -3.498040199279785, + -7.980632305145264, + -7.672002792358398, + -4.4999470710754395, + -7.155375003814697, + -2.4486124515533447, + -4.785946846008301, + -1.153855800628662, + -2.3994438648223877, + -4.250652313232422, + -12.24446964263916, + -8.344388008117676, + -2.608186721801758, + -5.200589179992676, + -8.25888442993164, + -3.6245617866516113, + -7.689338207244873, + -7.345355033874512, + -1.2661759853363037, + -7.265620231628418, + -1.9884108304977417, + -6.269482612609863, + -2.41705584526062, + -1.8929681777954102, + -1.8259913921356201, + -2.0997350215911865, + -2.323200225830078, + -1.3998825550079346, + -0.8789899945259094, + -1.082053542137146, + -1.1831339597702026, + -1.4462857246398926, + -1.6481035947799683, + -1.4408715963363647, + -1.2603964805603027, + -1.5267670154571533, + -1.6345772743225098, + -1.3796477317810059, + -0.7609691023826599, + -0.3548354506492615, + -0.7552334666252136, + -0.44776833057403564, + -1.1078286170959473, + -1.3036658763885498, + -0.5214896202087402, + -0.8486822843551636, + -0.22470997273921967, + -0.4705755412578583, + -0.5639711022377014, + -0.5388108491897583, + -0.6052999496459961, + -0.1002030223608017, + -0.286334365606308, + -0.45798981189727783, + -1.0107953548431396, + -0.11875647306442261, + -0.6969441771507263, + -0.4609107971191406, + -0.07614769786596298, + -0.5035472512245178, + -0.1682187020778656, + -0.10476160794496536, + -0.6586751341819763, + -0.35806939005851746, + -1.5364394187927246, + -2.4093759059906006, + -1.977368950843811, + -1.6216907501220703, + -0.27647316455841064, + -0.2991848587989807, + -0.2783535420894623, + -0.05913994088768959, + -0.03023873083293438, + -0.043339803814888, + -0.7320341467857361, + -0.0030677898321300745, + -0.0332595594227314, + -0.012804670259356499, + -0.004041599575430155, + -0.0014899593079462647, + -0.001948602613992989, + -0.0029070996679365635, + -0.040939707309007645, + -0.013942227698862553, + -0.04897322878241539, + -0.011005887761712074, + -0.0044113704934716225, + -0.0013179434463381767, + -0.003658389439806342, + -0.009758152067661285, + -0.0014104428701102734, + -0.0016671819612383842, + -0.000771939754486084, + -0.0015519729349762201, + -0.003720743814483285, + -0.004249115474522114, + -0.00485657574608922, + -0.005053604021668434, + -0.002336274366825819, + -0.0009155849111266434, + -0.0004978132783435285, + -0.0005953923100605607, + -0.0011395872570574284, + -0.001485078944824636, + -0.3072909712791443, + -1.7295066118240356, + -0.4807289242744446, + -0.1245415136218071, + -0.011858444660902023, + -0.020613837987184525, + -0.011020978912711143, + -0.003106294432654977, + -0.0009966888464987278, + -0.0019349202048033476, + -0.037407051771879196, + -0.0003496989083942026, + -0.005922981072217226, + -0.007394562941044569, + -0.0006037319544702768, + -0.0008836655179038644, + -0.0002884448622353375, + -0.00047600860125385225, + -0.0024947968777269125, + -0.00442774873226881, + -0.004059052560478449, + -0.0018594847060739994, + -0.0006179092451930046, + -0.00022635281493421644, + -0.0006730675231665373, + -0.003022746881470084, + -0.0002343380037928, + -0.00047791501856409013, + -9.440929716220126e-05, + -0.00021550717065110803, + -0.0013523490633815527, + -0.0032202552538365126, + -0.001157686347141862, + -0.004449942149221897, + -0.0016590891173109412, + -0.00101062236353755, + -0.0003079893649555743, + -0.00048375347978435457, + -0.0021734442561864853, + -0.00423036003485322, + -0.11514264345169067, + -0.8658493757247925, + -0.084366075694561, + -0.02140468917787075, + -0.0060798698104918, + -0.008638513274490833, + -0.003212531330063939, + -0.0009598892065696418, + -0.00032085992279462516 + ] + }, + "throughput": 107.66332959870442 +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..fe938d51e4f --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt_dynamic_inference_tp8_pp1_583m_logitsmatch/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,161 @@ +{ + "0": { + "input_prompt": "Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies.", + "generated_text": " And that this is the place where you can be yourself, and be yourself in the most beautiful way. And that this is the place where you can", + "generated_tokens": [ + 3060, + 1455, + 1593, + 1395, + 1278, + 3535, + 2478, + 1636, + 1710, + 1402, + 14019, + 1044, + 1321, + 1402, + 14019, + 1294, + 1278, + 2725, + 15568, + 3039, + 1046, + 3060, + 1455, + 1593, + 1395, + 1278, + 3535, + 2478, + 1636, + 1710 + ], + "latency": 2.1402676105499268, + "cuda_graph_request_count_map": null, + "step_count": 240, + "logprobs": [ + -9.358587265014648, + -2.7594826221466064, + -4.608366012573242, + -1.4093360900878906, + -0.6152952313423157, + -1.7217562198638916, + -2.496668815612793, + -2.0547454357147217, + -2.441960573196411, + -6.280838966369629, + -1.5643692016601562, + -3.462346076965332, + -4.428728103637695, + -3.8633861541748047, + -1.9936373233795166, + -1.8929449319839478, + -3.796365737915039, + -6.8360137939453125, + -0.2901247441768646, + -0.9246833324432373, + -6.633338928222656, + -7.166708469390869, + -12.771251678466797, + -2.198296308517456, + -3.7778120040893555, + -0.4983733296394348, + -4.381269454956055, + -0.0666784718632698, + -0.09580295532941818, + -3.2437636852264404, + -10.079947471618652, + -1.172220230102539, + -5.977442741394043, + -5.046236038208008, + -3.855658531188965, + -2.5585858821868896, + -3.356245994567871, + -5.557229518890381, + -1.6787731647491455, + -5.483290672302246, + -12.218501091003418, + -12.61402702331543, + -0.09662941098213196, + -2.5431432723999023, + -1.4071024656295776, + -2.9154715538024902, + -1.1964417695999146, + -0.006458481773734093, + -3.3625335693359375, + -13.262511253356934, + -4.314079761505127, + -2.617699146270752, + -5.987792015075684, + -0.778266429901123, + -0.048888545483350754, + -1.548882007598877, + -1.1381981372833252, + -5.627166748046875, + -0.4078553318977356, + -4.958505630493164, + -0.6187160611152649, + -0.7174848914146423, + -2.469533920288086, + -13.620073318481445, + -0.09088654816150665, + -3.526974678039551, + -1.4195809364318848, + -6.402483940124512, + -0.5898402333259583, + -3.565917491912842, + -0.8561318516731262, + -1.6140165328979492, + -5.370549201965332, + -17.159223556518555, + -6.583524703979492, + -0.8855001926422119, + -4.19431209564209, + -1.2012220621109009, + -2.2563133239746094, + -1.7674944400787354, + -0.22064533829689026, + -9.292220115661621, + -0.12445646524429321, + -7.29617977142334, + -2.526529312133789, + -4.071560859680176, + -3.5568013191223145, + -1.926215410232544, + -2.349026918411255, + -2.2132363319396973, + -0.3125414550304413, + -1.4718132019042969, + -2.149106740951538, + -1.0855519771575928, + -1.631832242012024, + -1.3751734495162964, + -1.9396103620529175, + -1.5293723344802856, + -0.8444125056266785, + -1.2414811849594116, + -1.9522171020507812, + -2.4338042736053467, + -1.5651824474334717, + -0.9498789310455322, + -1.8044980764389038, + -2.356677770614624, + -1.247452974319458, + -1.550165057182312, + -0.5635553598403931, + -0.6177330017089844, + -0.4778785705566406, + -0.020452087745070457, + -0.48500269651412964, + -0.23854275047779083, + -0.06543659418821335, + -0.11837350577116013, + -0.0585334412753582 + ] + }, + "throughput": 13.93210545115292 +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_eos.json index b3668b31178..2ef31d337c6 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_eos.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_memory_speed/golden_values_dev_dgxh100_eos.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.04748, - "2": 11.03561, - "3": 9.58773, - "4": 9.25819, - "5": 9.52742, - "6": 9.87911, - "7": 9.48366, - "8": 8.93879, - "9": 8.6551, - "10": 9.10915, - "11": 8.51806, - "12": 8.54732, - "13": 8.48144, - "14": 8.05312, - "15": 8.10118, - "16": 8.10344, - "17": 8.08878, - "18": 7.78589, - "19": 8.15794, - "20": 7.88069, - "21": 7.58542, - "22": 7.54895, - "23": 7.4296, - "24": 7.41901, - "25": 7.67277, - "26": 7.07835, - "27": 7.61157, - "28": 7.31513, - "29": 7.49487, - "30": 7.64287, - "31": 7.39102, - "32": 7.59148, - "33": 7.6393, - "34": 7.70086, - "35": 7.2119, - "36": 7.08623, - "37": 7.43064, - "38": 7.18999, - "39": 7.5525, - "40": 7.54961, - "41": 7.49385, - "42": 7.25481, - "43": 7.24066, - "44": 7.42131, - "45": 7.19201, - "46": 6.90547, - "47": 7.30704, - "48": 7.15325, - "49": 7.60504, - "50": 7.04512 + "1": 11.04722, + "2": 11.03572, + "3": 9.58802, + "4": 9.25807, + "5": 9.46595, + "6": 9.99646, + "7": 9.50952, + "8": 8.97596, + "9": 8.64768, + "10": 9.40103, + "11": 8.86556, + "12": 8.63563, + "13": 8.52125, + "14": 8.08824, + "15": 8.1958, + "16": 8.22112, + "17": 8.14098, + "18": 7.8386, + "19": 8.23438, + "20": 7.95361, + "21": 7.62549, + "22": 7.60352, + "23": 7.47957, + "24": 7.46573, + "25": 7.70343, + "26": 7.10719, + "27": 7.64313, + "28": 7.34582, + "29": 7.5169, + "30": 7.67511, + "31": 7.41799, + "32": 7.61213, + "33": 7.66582, + "34": 7.73101, + "35": 7.23081, + "36": 7.10765, + "37": 7.4476, + "38": 7.21053, + "39": 7.57508, + "40": 7.5662, + "41": 7.51605, + "42": 7.27243, + "43": 7.25706, + "44": 7.44, + "45": 7.21244, + "46": 6.92421, + "47": 7.32604, + "48": 7.17147, + "49": 7.62154, + "50": 7.0624 } }, "num-zeros": { @@ -62,55 +62,55 @@ "step_interval": 1, "values": { "1": 38802612.0, - "2": 38543592.0, - "3": 38739480.0, - "4": 279954336.0, - "5": 249745312.0, - "6": 268288496.0, - "7": 604756224.0, - "8": 781485184.0, - "9": 636362112.0, - "10": 653025216.0, - "11": 668551168.0, - "12": 765583616.0, - "13": 815362944.0, - "14": 834270656.0, - "15": 755756096.0, - "16": 995153536.0, - "17": 938291584.0, - "18": 721524928.0, - "19": 756173504.0, - "20": 901129600.0, - "21": 721816384.0, - "22": 831311872.0, - "23": 803536768.0, - "24": 628253248.0, - "25": 663895680.0, - "26": 847321664.0, - "27": 828927424.0, - "28": 777678976.0, - "29": 764628608.0, - "30": 781930112.0, - "31": 771767616.0, - "32": 771755392.0, - "33": 586323648.0, - "34": 734207552.0, - "35": 690468480.0, - "36": 485982688.0, - "37": 506506336.0, - "38": 642964160.0, - "39": 661240000.0, - "40": 645048768.0, - "41": 636072704.0, - "42": 491645856.0, - "43": 601942528.0, - "44": 623448960.0, - "45": 539959424.0, - "46": 532669088.0, - "47": 529039680.0, - "48": 504121984.0, - "49": 478344480.0, - "50": 331385728.0 + "2": 38543656.0, + "3": 38739356.0, + "4": 273649600.0, + "5": 252887040.0, + "6": 255692384.0, + "7": 598483264.0, + "8": 787737984.0, + "9": 696133120.0, + "10": 505146368.0, + "11": 718888640.0, + "12": 872597184.0, + "13": 947495104.0, + "14": 1076398976.0, + "15": 856390592.0, + "16": 1048635648.0, + "17": 831370688.0, + "18": 963679552.0, + "19": 970018240.0, + "20": 935737344.0, + "21": 904189312.0, + "22": 887937280.0, + "23": 894777856.0, + "24": 703744192.0, + "25": 909232512.0, + "26": 875633216.0, + "27": 894981376.0, + "28": 919242816.0, + "29": 931351552.0, + "30": 929784768.0, + "31": 941621376.0, + "32": 885000768.0, + "33": 828484096.0, + "34": 822284800.0, + "35": 832032128.0, + "36": 787939392.0, + "37": 770719808.0, + "38": 561204672.0, + "39": 617201536.0, + "40": 695374592.0, + "41": 698978816.0, + "42": 692913728.0, + "43": 668003776.0, + "44": 673780992.0, + "45": 631182912.0, + "46": 444613312.0, + "47": 591957824.0, + "48": 617363968.0, + "49": 585295808.0, + "50": 570423872.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 6637267456.0, - "2": 6637269504.0, - "3": 6637269504.0, - "4": 6637269504.0, - "5": 6637269504.0, - "6": 6637269504.0, - "7": 6637269504.0, - "8": 6637269504.0, - "9": 6637269504.0, - "10": 6637269504.0, - "11": 6637269504.0, - "12": 6637269504.0, - "13": 6637269504.0, - "14": 6637269504.0, - "15": 6637269504.0, - "16": 6637269504.0, - "17": 6637269504.0, - "18": 6637269504.0, - "19": 6637269504.0, - "20": 6637269504.0, - "21": 6637269504.0, - "22": 6637269504.0, - "23": 6637269504.0, - "24": 6637269504.0, - "25": 6637269504.0, - "26": 6637269504.0, - "27": 6637269504.0, - "28": 6637269504.0, - "29": 6637269504.0, - "30": 6637269504.0, - "31": 6637269504.0, - "32": 6637269504.0, - "33": 6637269504.0, - "34": 6637269504.0, - "35": 6637269504.0, - "36": 6637269504.0, - "37": 6637269504.0, - "38": 6637269504.0, - "39": 6637269504.0, - "40": 6637269504.0, - "41": 6637269504.0, - "42": 6637269504.0, - "43": 6637269504.0, - "44": 6637269504.0, - "45": 6637269504.0, - "46": 6637269504.0, - "47": 6637269504.0, - "48": 6637269504.0, - "49": 6637269504.0, - "50": 6637269504.0 + "1": 6637272576.0, + "2": 6637274624.0, + "3": 6637274624.0, + "4": 6637274624.0, + "5": 6637274624.0, + "6": 6637274624.0, + "7": 6637274624.0, + "8": 6637274624.0, + "9": 6637274624.0, + "10": 6637274624.0, + "11": 6637274624.0, + "12": 6637274624.0, + "13": 6637274624.0, + "14": 6637274624.0, + "15": 6637274624.0, + "16": 6637274624.0, + "17": 6637274624.0, + "18": 6637274624.0, + "19": 6637274624.0, + "20": 6637274624.0, + "21": 6637274624.0, + "22": 6637274624.0, + "23": 6637274624.0, + "24": 6637274624.0, + "25": 6637274624.0, + "26": 6637274624.0, + "27": 6637274624.0, + "28": 6637274624.0, + "29": 6637274624.0, + "30": 6637274624.0, + "31": 6637274624.0, + "32": 6637274624.0, + "33": 6637274624.0, + "34": 6637274624.0, + "35": 6637274624.0, + "36": 6637274624.0, + "37": 6637274624.0, + "38": 6637274624.0, + "39": 6637274624.0, + "40": 6637274624.0, + "41": 6637274624.0, + "42": 6637274624.0, + "43": 6637274624.0, + "44": 6637274624.0, + "45": 6637274624.0, + "46": 6637274624.0, + "47": 6637274624.0, + "48": 6637274624.0, + "49": 6637274624.0, + "50": 6637274624.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 55055331328.0, - "2": 57809321984.0, - "3": 57919823872.0, - "4": 57919823872.0, - "5": 57919823872.0, - "6": 57919823872.0, - "7": 57919823872.0, - "8": 57919823872.0, - "9": 57919823872.0, - "10": 57919823872.0, - "11": 57919823872.0, - "12": 57919823872.0, - "13": 57932275712.0, - "14": 57932275712.0, - "15": 57932275712.0, - "16": 57932275712.0, - "17": 57932275712.0, - "18": 57932275712.0, - "19": 57932275712.0, - "20": 57932275712.0, - "21": 57932275712.0, - "22": 57932275712.0, - "23": 57932275712.0, - "24": 57932275712.0, - "25": 57932275712.0, - "26": 57932275712.0, - "27": 57932275712.0, - "28": 57932275712.0, - "29": 57932275712.0, - "30": 57932275712.0, - "31": 57932275712.0, - "32": 57932275712.0, - "33": 57932275712.0, - "34": 57932275712.0, - "35": 57932275712.0, - "36": 57932275712.0, - "37": 57932275712.0, - "38": 57932275712.0, - "39": 57932275712.0, - "40": 57932275712.0, - "41": 57932275712.0, - "42": 57932275712.0, - "43": 57932275712.0, - "44": 57932275712.0, - "45": 57932275712.0, - "46": 57932275712.0, - "47": 57932275712.0, - "48": 57932275712.0, - "49": 57932275712.0, - "50": 57932275712.0 + "1": 55056003072.0, + "2": 57810763776.0, + "3": 57920647168.0, + "4": 57920647168.0, + "5": 57920647168.0, + "6": 57920647168.0, + "7": 57920647168.0, + "8": 57920647168.0, + "9": 57920647168.0, + "10": 57920647168.0, + "11": 57920647168.0, + "12": 57920647168.0, + "13": 57920647168.0, + "14": 57920647168.0, + "15": 57920647168.0, + "16": 57920647168.0, + "17": 57920647168.0, + "18": 57920647168.0, + "19": 57920647168.0, + "20": 57920647168.0, + "21": 57920647168.0, + "22": 57920647168.0, + "23": 57920647168.0, + "24": 57920647168.0, + "25": 57920647168.0, + "26": 57920647168.0, + "27": 57920647168.0, + "28": 57920647168.0, + "29": 57920647168.0, + "30": 57920647168.0, + "31": 57920647168.0, + "32": 57920647168.0, + "33": 57920647168.0, + "34": 57961472000.0, + "35": 57961472000.0, + "36": 57961472000.0, + "37": 57961472000.0, + "38": 57961472000.0, + "39": 57961472000.0, + "40": 57961472000.0, + "41": 57961472000.0, + "42": 57961472000.0, + "43": 57961472000.0, + "44": 57961472000.0, + "45": 57961472000.0, + "46": 57961472000.0, + "47": 57961472000.0, + "48": 57961472000.0, + "49": 57961472000.0, + "50": 57961472000.0 } }, "mtp_1 loss": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 11.07654, - "2": 11.07406, - "3": 10.53883, - "4": 10.09801, - "5": 9.81156, - "6": 10.06025, - "7": 9.7962, - "8": 9.06987, - "9": 8.86879, - "10": 9.13393, - "11": 8.5017, - "12": 8.54094, - "13": 8.43678, - "14": 7.85637, - "15": 7.99846, - "16": 8.05889, - "17": 8.01134, - "18": 7.73929, - "19": 8.1188, - "20": 7.83458, - "21": 7.53103, - "22": 7.50125, - "23": 7.37135, - "24": 7.37419, - "25": 7.61596, - "26": 7.01586, - "27": 7.55739, - "28": 7.26274, - "29": 7.43991, - "30": 7.58436, - "31": 7.32289, - "32": 7.50362, - "33": 7.56884, - "34": 7.6339, - "35": 7.151, - "36": 7.01725, - "37": 7.35013, - "38": 7.12483, - "39": 7.48708, - "40": 7.47451, - "41": 7.4181, - "42": 7.17557, - "43": 7.15957, - "44": 7.34227, - "45": 7.12176, - "46": 6.82526, - "47": 7.23374, - "48": 7.07893, - "49": 7.5077, - "50": 6.97094 + "1": 11.07648, + "2": 11.07404, + "3": 10.53854, + "4": 10.09813, + "5": 9.81166, + "6": 10.09741, + "7": 9.79481, + "8": 9.0642, + "9": 8.86016, + "10": 9.34039, + "11": 8.51318, + "12": 8.59467, + "13": 8.5292, + "14": 7.95757, + "15": 8.06962, + "16": 8.11802, + "17": 8.06993, + "18": 7.80587, + "19": 8.19192, + "20": 7.8906, + "21": 7.57063, + "22": 7.55091, + "23": 7.41606, + "24": 7.42454, + "25": 7.65274, + "26": 7.05583, + "27": 7.59747, + "28": 7.29984, + "29": 7.472, + "30": 7.61908, + "31": 7.35179, + "32": 7.52979, + "33": 7.59161, + "34": 7.66287, + "35": 7.17383, + "36": 7.04133, + "37": 7.37081, + "38": 7.1443, + "39": 7.50879, + "40": 7.48921, + "41": 7.43802, + "42": 7.19405, + "43": 7.17581, + "44": 7.35785, + "45": 7.13985, + "46": 6.84014, + "47": 7.25094, + "48": 7.09407, + "49": 7.52321, + "50": 6.98987 } }, "iteration-time": { @@ -289,56 +289,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 57.80279, - "2": 1.26321, - "3": 1.18918, - "4": 2.24643, - "5": 2.25191, - "6": 1.80757, - "7": 2.09086, - "8": 1.69153, - "9": 1.81279, - "10": 1.64882, - "11": 1.03476, - "12": 1.03593, - "13": 1.04348, - "14": 1.03841, - "15": 1.04432, - "16": 1.05281, - "17": 1.04826, - "18": 1.04981, - "19": 1.05351, - "20": 1.04668, - "21": 1.05254, - "22": 1.05391, - "23": 1.04635, - "24": 1.05503, - "25": 1.04226, - "26": 1.0684, - "27": 1.04985, - "28": 1.04233, - "29": 1.05036, - "30": 1.06219, - "31": 1.044, - "32": 1.05614, - "33": 1.05729, - "34": 1.05618, - "35": 1.06289, - "36": 1.05761, - "37": 1.05956, - "38": 1.06343, - "39": 1.06848, - "40": 1.06027, - "41": 1.05493, - "42": 1.05258, - "43": 1.04879, - "44": 1.04949, - "45": 1.05964, - "46": 1.04465, - "47": 1.0491, - "48": 1.05387, - "49": 1.05218, - "50": 1.05453 + "1": 85.36715, + "2": 1.28108, + "3": 1.20997, + "4": 2.1995, + "5": 2.24721, + "6": 1.76282, + "7": 2.15873, + "8": 1.65741, + "9": 1.68158, + "10": 1.05017, + "11": 1.03578, + "12": 1.03043, + "13": 1.03186, + "14": 1.03493, + "15": 1.0319, + "16": 1.04172, + "17": 1.05419, + "18": 1.03994, + "19": 1.0369, + "20": 1.03992, + "21": 1.05243, + "22": 1.04145, + "23": 1.04202, + "24": 1.04481, + "25": 1.04991, + "26": 1.06122, + "27": 1.04535, + "28": 1.04289, + "29": 1.0367, + "30": 1.03753, + "31": 1.03324, + "32": 1.04441, + "33": 1.0476, + "34": 1.05195, + "35": 1.05343, + "36": 1.04283, + "37": 1.05165, + "38": 1.05155, + "39": 1.04776, + "40": 1.05026, + "41": 1.04796, + "42": 1.05954, + "43": 1.04773, + "44": 1.05679, + "45": 1.04821, + "46": 1.04749, + "47": 1.05428, + "48": 1.04887, + "49": 1.0568, + "50": 1.07411 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_eos.json index daa04af43dd..27fd798d121 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_eos.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph/golden_values_dev_dgxh100_eos.json @@ -4,56 +4,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 10.95004, - "2": 10.9521, - "3": 10.5115, - "4": 9.96454, - "5": 9.93941, - "6": 9.67273, - "7": 10.20975, - "8": 9.49716, - "9": 9.55902, - "10": 9.79742, - "11": 9.30109, - "12": 9.40483, - "13": 9.39546, - "14": 8.84681, - "15": 9.02444, - "16": 9.07121, - "17": 9.04574, - "18": 8.75678, - "19": 9.18159, - "20": 8.8595, - "21": 8.53503, - "22": 8.55182, - "23": 8.42441, - "24": 8.37608, - "25": 8.64304, - "26": 7.97393, - "27": 8.56806, - "28": 8.19764, - "29": 8.3928, - "30": 8.67283, - "31": 8.289, - "32": 8.43572, - "33": 8.5568, - "34": 8.66018, - "35": 8.07934, - "36": 7.94976, - "37": 8.29565, - "38": 7.98044, - "39": 8.39201, - "40": 8.35513, - "41": 8.31876, - "42": 8.0583, - "43": 8.03283, - "44": 8.24243, - "45": 8.10277, - "46": 7.61696, - "47": 8.15273, - "48": 8.00569, - "49": 8.38688, - "50": 7.81491 + "1": 10.94971, + "2": 10.95163, + "3": 10.51641, + "4": 9.9652, + "5": 9.94116, + "6": 9.67394, + "7": 10.19887, + "8": 9.50035, + "9": 9.54982, + "10": 9.79667, + "11": 9.30128, + "12": 9.40566, + "13": 9.39438, + "14": 8.84572, + "15": 9.02231, + "16": 9.06973, + "17": 9.04712, + "18": 8.75662, + "19": 9.18074, + "20": 8.86175, + "21": 8.53558, + "22": 8.55288, + "23": 8.42513, + "24": 8.37683, + "25": 8.64426, + "26": 7.9756, + "27": 8.57026, + "28": 8.1987, + "29": 8.39406, + "30": 8.67631, + "31": 8.29096, + "32": 8.43692, + "33": 8.55897, + "34": 8.66123, + "35": 8.08, + "36": 7.95214, + "37": 8.2979, + "38": 7.98177, + "39": 8.39281, + "40": 8.35852, + "41": 8.32006, + "42": 8.05954, + "43": 8.03381, + "44": 8.24236, + "45": 8.1025, + "46": 7.61814, + "47": 8.15364, + "48": 8.00693, + "49": 8.38704, + "50": 7.81592 } }, "num-zeros": { @@ -61,56 +61,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 19403624.0, - "2": 19274194.0, - "3": 19372760.0, - "4": 86525248.0, - "5": 148575568.0, - "6": 145226704.0, - "7": 171879984.0, - "8": 195785248.0, - "9": 164124752.0, - "10": 167684736.0, - "11": 221077344.0, - "12": 200384224.0, - "13": 248872528.0, - "14": 211169424.0, - "15": 214304608.0, - "16": 216075632.0, - "17": 267845984.0, - "18": 170470336.0, - "19": 176865072.0, - "20": 187955392.0, - "21": 225750704.0, - "22": 247396816.0, - "23": 211643856.0, - "24": 205638464.0, - "25": 277022272.0, - "26": 291562304.0, - "27": 225789840.0, - "28": 288202368.0, - "29": 198390384.0, - "30": 213302208.0, - "31": 227204752.0, - "32": 271112416.0, - "33": 231840432.0, - "34": 203575536.0, - "35": 191152368.0, - "36": 222566928.0, - "37": 177810112.0, - "38": 228708544.0, - "39": 211168784.0, - "40": 215603968.0, - "41": 200089440.0, - "42": 228529888.0, - "43": 198782848.0, - "44": 141902272.0, - "45": 181922816.0, - "46": 115369856.0, - "47": 170214176.0, - "48": 137292832.0, - "49": 97654936.0, - "50": 160979632.0 + "1": 19403704.0, + "2": 19274216.0, + "3": 22517470.0, + "4": 83429816.0, + "5": 139167728.0, + "6": 138921280.0, + "7": 173470304.0, + "8": 200511856.0, + "9": 165696320.0, + "10": 166120112.0, + "11": 213254416.0, + "12": 187847360.0, + "13": 231586656.0, + "14": 226879072.0, + "15": 219025920.0, + "16": 205179664.0, + "17": 280450432.0, + "18": 181477792.0, + "19": 191026096.0, + "20": 186395632.0, + "21": 233632576.0, + "22": 231696832.0, + "23": 216390688.0, + "24": 215133760.0, + "25": 233079504.0, + "26": 244437920.0, + "27": 222637584.0, + "28": 278773952.0, + "29": 253409264.0, + "30": 240036736.0, + "31": 236599008.0, + "32": 205066624.0, + "33": 263303312.0, + "34": 200444544.0, + "35": 199033824.0, + "36": 243001216.0, + "37": 151181872.0, + "38": 175301280.0, + "39": 219001024.0, + "40": 220307936.0, + "41": 217385856.0, + "42": 230074176.0, + "43": 208226784.0, + "44": 148172720.0, + "45": 141103744.0, + "46": 132664976.0, + "47": 179619392.0, + "48": 118381144.0, + "49": 86643984.0, + "50": 113798320.0 } }, "mem-allocated-bytes": { @@ -118,56 +118,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 4882187264.0, - "2": 4881607168.0, - "3": 4882283008.0, - "4": 4881322496.0, - "5": 4882174464.0, - "6": 4883177984.0, - "7": 4883252736.0, - "8": 4881774080.0, - "9": 4881443328.0, - "10": 4884319744.0, - "11": 4882319872.0, - "12": 4881232384.0, - "13": 4880836096.0, - "14": 4882124288.0, - "15": 4882108928.0, - "16": 4883384832.0, - "17": 4880466432.0, - "18": 4881518080.0, - "19": 4881734144.0, - "20": 4883215872.0, - "21": 4883534336.0, - "22": 4882774528.0, - "23": 4881818112.0, - "24": 4882441728.0, - "25": 4880546304.0, - "26": 4882178560.0, - "27": 4881892864.0, - "28": 4881869312.0, - "29": 4882979328.0, - "30": 4882715136.0, - "31": 4883084800.0, - "32": 4881436160.0, - "33": 4881766912.0, - "34": 4881406464.0, - "35": 4881531392.0, - "36": 4881479168.0, - "37": 4882455040.0, - "38": 4882054656.0, - "39": 4882005504.0, - "40": 4882743808.0, - "41": 4881211904.0, - "42": 4881378816.0, - "43": 4882133504.0, - "44": 4881860096.0, - "45": 4883165696.0, - "46": 4882168320.0, - "47": 4881526272.0, - "48": 4882125312.0, - "49": 4881533440.0, - "50": 4881598976.0 + "1": 4882326528.0, + "2": 4880379392.0, + "3": 4880237056.0, + "4": 4879834624.0, + "5": 4881078784.0, + "6": 4881928704.0, + "7": 4880591360.0, + "8": 4880542208.0, + "9": 4880547328.0, + "10": 4882171392.0, + "11": 4881006080.0, + "12": 4881802752.0, + "13": 4881517056.0, + "14": 4880953856.0, + "15": 4881994240.0, + "16": 4880894464.0, + "17": 4881596928.0, + "18": 4879665664.0, + "19": 4880534016.0, + "20": 4881038848.0, + "21": 4880376320.0, + "22": 4880297472.0, + "23": 4880386560.0, + "24": 4879292928.0, + "25": 4880342528.0, + "26": 4880798208.0, + "27": 4881144320.0, + "28": 4879861248.0, + "29": 4880552448.0, + "30": 4880334336.0, + "31": 4879297024.0, + "32": 4880034304.0, + "33": 4880325120.0, + "34": 4880040448.0, + "35": 4880456192.0, + "36": 4880591360.0, + "37": 4879780352.0, + "38": 4880071168.0, + "39": 4879615488.0, + "40": 4879229440.0, + "41": 4879632896.0, + "42": 4879910400.0, + "43": 4879630848.0, + "44": 4880927232.0, + "45": 4879393280.0, + "46": 4881033728.0, + "47": 4879720960.0, + "48": 4881100288.0, + "49": 4879365632.0, + "50": 4880260608.0 } }, "mem-max-allocated-bytes": { @@ -175,56 +175,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 41210470400.0, - "2": 41210470400.0, - "3": 41210470400.0, - "4": 41210470400.0, - "5": 41210470400.0, - "6": 41210470400.0, - "7": 41210470400.0, - "8": 41210470400.0, - "9": 41210470400.0, - "10": 41210470400.0, - "11": 41210470400.0, - "12": 41210470400.0, - "13": 41210470400.0, - "14": 41210470400.0, - "15": 41210470400.0, - "16": 41210470400.0, - "17": 41210470400.0, - "18": 41210470400.0, - "19": 41210470400.0, - "20": 41210470400.0, - "21": 41210470400.0, - "22": 41210470400.0, - "23": 41210470400.0, - "24": 41210470400.0, - "25": 41210470400.0, - "26": 41210470400.0, - "27": 41210470400.0, - "28": 41210470400.0, - "29": 41210470400.0, - "30": 41210470400.0, - "31": 41210470400.0, - "32": 41210470400.0, - "33": 41210470400.0, - "34": 41210470400.0, - "35": 41210470400.0, - "36": 41210470400.0, - "37": 41210470400.0, - "38": 41210470400.0, - "39": 41210470400.0, - "40": 41210470400.0, - "41": 41210470400.0, - "42": 41210470400.0, - "43": 41210470400.0, - "44": 41210470400.0, - "45": 41210470400.0, - "46": 41210470400.0, - "47": 41210470400.0, - "48": 41210470400.0, - "49": 41210470400.0, - "50": 41210470400.0 + "1": 41208373248.0, + "2": 41208373248.0, + "3": 41208373248.0, + "4": 41208373248.0, + "5": 41208373248.0, + "6": 41208373248.0, + "7": 41208373248.0, + "8": 41208373248.0, + "9": 41208373248.0, + "10": 41208373248.0, + "11": 41208373248.0, + "12": 41208373248.0, + "13": 41208373248.0, + "14": 41208373248.0, + "15": 41208373248.0, + "16": 41208373248.0, + "17": 41208373248.0, + "18": 41208373248.0, + "19": 41208373248.0, + "20": 41208373248.0, + "21": 41208373248.0, + "22": 41208373248.0, + "23": 41208373248.0, + "24": 41208373248.0, + "25": 41208373248.0, + "26": 41208373248.0, + "27": 41208373248.0, + "28": 41208373248.0, + "29": 41208373248.0, + "30": 41208373248.0, + "31": 41208373248.0, + "32": 41208373248.0, + "33": 41208373248.0, + "34": 41208373248.0, + "35": 41208373248.0, + "36": 41208373248.0, + "37": 41208373248.0, + "38": 41208373248.0, + "39": 41208373248.0, + "40": 41208373248.0, + "41": 41208373248.0, + "42": 41208373248.0, + "43": 41208373248.0, + "44": 41208373248.0, + "45": 41208373248.0, + "46": 41208373248.0, + "47": 41208373248.0, + "48": 41208373248.0, + "49": 41208373248.0, + "50": 41208373248.0 } }, "iteration-time": { @@ -232,56 +232,56 @@ "end_step": 50, "step_interval": 1, "values": { - "1": 96.21947, - "2": 1.10023, - "3": 0.96399, - "4": 0.91113, - "5": 1.27509, - "6": 1.00484, - "7": 1.01236, - "8": 1.1739, - "9": 0.89406, - "10": 0.88836, - "11": 0.92033, - "12": 0.88331, - "13": 0.88179, - "14": 0.88307, - "15": 0.88648, - "16": 0.88425, - "17": 0.87155, - "18": 0.87556, - "19": 0.87374, - "20": 0.8744, - "21": 0.86757, - "22": 0.87217, - "23": 0.8736, - "24": 0.86646, - "25": 0.87328, - "26": 0.87121, - "27": 0.85886, - "28": 0.86392, - "29": 0.86385, - "30": 0.86425, - "31": 0.8631, - "32": 0.8617, - "33": 0.86069, - "34": 0.86829, - "35": 0.86837, - "36": 0.86776, - "37": 0.86686, - "38": 0.86359, - "39": 0.8677, - "40": 0.86441, - "41": 0.86179, - "42": 0.86079, - "43": 0.86149, - "44": 0.86222, - "45": 0.86336, - "46": 0.85875, - "47": 0.86219, - "48": 0.86026, - "49": 0.85894, - "50": 0.8544 + "1": 93.50581, + "2": 1.10311, + "3": 0.97651, + "4": 0.91679, + "5": 1.42607, + "6": 1.09159, + "7": 1.05673, + "8": 1.10123, + "9": 0.89686, + "10": 0.88361, + "11": 1.03303, + "12": 0.89241, + "13": 0.89019, + "14": 0.88818, + "15": 0.88483, + "16": 0.88301, + "17": 0.8735, + "18": 0.88523, + "19": 0.87306, + "20": 0.87169, + "21": 0.86111, + "22": 0.86812, + "23": 0.8652, + "24": 0.86412, + "25": 0.86804, + "26": 0.87119, + "27": 0.85851, + "28": 0.85867, + "29": 0.85921, + "30": 0.87123, + "31": 0.86655, + "32": 0.86406, + "33": 0.86147, + "34": 0.8603, + "35": 0.85378, + "36": 0.85478, + "37": 0.85414, + "38": 0.84972, + "39": 0.85007, + "40": 0.85059, + "41": 0.85564, + "42": 0.85309, + "43": 0.85237, + "44": 0.85578, + "45": 0.85261, + "46": 0.85507, + "47": 0.85212, + "48": 0.85143, + "49": 0.84802, + "50": 0.84388 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..5272fa38474 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.82922, + "2": 10.84163, + "3": 10.84245, + "4": 10.82, + "5": 10.85652, + "6": 10.86906, + "7": 10.83778, + "8": 10.84312, + "9": 10.84423, + "10": 10.79298, + "11": 10.86697, + "12": 10.86875, + "13": 10.86207, + "14": 10.86919, + "15": 10.8067, + "16": 10.8057, + "17": 10.77686, + "18": 10.79541, + "19": 10.78384, + "20": 10.72654, + "21": 10.69491, + "22": 10.54462, + "23": 10.6993, + "24": 10.58151, + "25": 10.53282, + "26": 10.58817, + "27": 10.601, + "28": 10.57563, + "29": 10.58022, + "30": 10.35802, + "31": 10.08769, + "32": 10.44466, + "33": 10.4477, + "34": 10.18704, + "35": 10.24483, + "36": 10.19713, + "37": 10.32294, + "38": 10.17101, + "39": 10.37026, + "40": 10.05533, + "41": 10.09491, + "42": 10.17971, + "43": 9.78263, + "44": 9.91346, + "45": 9.77951, + "46": 9.75648, + "47": 10.09647, + "48": 9.80391, + "49": 9.46649, + "50": 9.86874, + "51": 9.79428, + "52": 9.68303, + "53": 10.03314, + "54": 9.9113, + "55": 9.82995, + "56": 9.57839, + "57": 9.42377, + "58": 9.80549, + "59": 9.53292, + "60": 9.449, + "61": 9.65293, + "62": 9.95672, + "63": 9.33775, + "64": 9.74194, + "65": 8.89366, + "66": 9.67317, + "67": 9.33002, + "68": 9.76517, + "69": 9.76336, + "70": 9.71127, + "71": 9.59511, + "72": 9.54797, + "73": 9.47124, + "74": 8.89297, + "75": 9.39451, + "76": 9.04721, + "77": 10.04318, + "78": 9.70313, + "79": 9.35169, + "80": 9.38198, + "81": 9.45146, + "82": 9.67546, + "83": 9.27658, + "84": 9.39241, + "85": 9.58333, + "86": 9.04518, + "87": 9.56487, + "88": 9.72459, + "89": 9.57019, + "90": 9.79944, + "91": 9.30737, + "92": 9.3313, + "93": 9.04109, + "94": 8.80259, + "95": 9.50213, + "96": 9.5021, + "97": 9.28183, + "98": 9.64883, + "99": 8.8594, + "100": 9.37131 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 27245.0, + "2": 28958.0, + "3": 29464.0, + "4": 28046.0, + "5": 31369.0, + "6": 33287.0, + "7": 31200.0, + "8": 26921.0, + "9": 30008.0, + "10": 25870.0, + "11": 33681.0, + "12": 30344.0, + "13": 32737.0, + "14": 33315.0, + "15": 29830.0, + "16": 32475.0, + "17": 30747.0, + "18": 30381.0, + "19": 31032.0, + "20": 28243.0, + "21": 29224.0, + "22": 27340.0, + "23": 34119.0, + "24": 29049.0, + "25": 27636.0, + "26": 30662.0, + "27": 32009.0, + "28": 33355.0, + "29": 34714.0, + "30": 30387.0, + "31": 28212.0, + "32": 33411.0, + "33": 34696.0, + "34": 30053.0, + "35": 31488.0, + "36": 32943.0, + "37": 35829.0, + "38": 33740.0, + "39": 37632.0, + "40": 34779.0, + "41": 33958.0, + "42": 36396.0, + "43": 34088.0, + "44": 34090.0, + "45": 35158.0, + "46": 36174.0, + "47": 39772.0, + "48": 36516.0, + "49": 36733.0, + "50": 38234.0, + "51": 38608.0, + "52": 37030.0, + "53": 42442.0, + "54": 40944.0, + "55": 37133.0, + "56": 41001.0, + "57": 37524.0, + "58": 42317.0, + "59": 40804.0, + "60": 40450.0, + "61": 41478.0, + "62": 39766.0, + "63": 37941.0, + "64": 42197.0, + "65": 40947.0, + "66": 44094.0, + "67": 41958.0, + "68": 40060.0, + "69": 42189.0, + "70": 43436.0, + "71": 42748.0, + "72": 44280.0, + "73": 47478.0, + "74": 41456.0, + "75": 39925.0, + "76": 43490.0, + "77": 45636.0, + "78": 2141470.0, + "79": 46055.0, + "80": 51863.0, + "81": 151341.0, + "82": 49835.0, + "83": 143360.0, + "84": 2141546.0, + "85": 2145177.0, + "86": 132114.0, + "87": 2147022.0, + "88": 59899.0, + "89": 162883.0, + "90": 51330.0, + "91": 2141901.0, + "92": 44946.0, + "93": 138194.0, + "94": 2145772.0, + "95": 45247.0, + "96": 135045.0, + "97": 53170.0, + "98": 168576.0, + "99": 2141797.0, + "100": 163741.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 787516416.0, + "2": 787540992.0, + "3": 787524096.0, + "4": 787512320.0, + "5": 787547136.0, + "6": 787537920.0, + "7": 787512832.0, + "8": 787524608.0, + "9": 787528192.0, + "10": 787505152.0, + "11": 787522048.0, + "12": 787520000.0, + "13": 787529728.0, + "14": 787529216.0, + "15": 787504128.0, + "16": 787513344.0, + "17": 787503104.0, + "18": 787489280.0, + "19": 787514880.0, + "20": 787505152.0, + "21": 787479552.0, + "22": 787486208.0, + "23": 787478528.0, + "24": 787486208.0, + "25": 787451392.0, + "26": 787482112.0, + "27": 787470848.0, + "28": 787450368.0, + "29": 787458048.0, + "30": 787435008.0, + "31": 787406848.0, + "32": 787424256.0, + "33": 787435520.0, + "34": 787426304.0, + "35": 787418624.0, + "36": 787436544.0, + "37": 787428352.0, + "38": 787436544.0, + "39": 787417600.0, + "40": 787415040.0, + "41": 787405824.0, + "42": 787415040.0, + "43": 787367936.0, + "44": 787392512.0, + "45": 787399680.0, + "46": 787355136.0, + "47": 787411456.0, + "48": 787354112.0, + "49": 787374080.0, + "50": 787389440.0, + "51": 787375616.0, + "52": 787383808.0, + "53": 787379712.0, + "54": 787384832.0, + "55": 787388928.0, + "56": 787388928.0, + "57": 787351040.0, + "58": 787382784.0, + "59": 787374080.0, + "60": 787395072.0, + "61": 787405312.0, + "62": 787405824.0, + "63": 787373056.0, + "64": 787388928.0, + "65": 787351552.0, + "66": 787386880.0, + "67": 787392000.0, + "68": 787399168.0, + "69": 787383296.0, + "70": 787393024.0, + "71": 787406848.0, + "72": 787400704.0, + "73": 787401216.0, + "74": 787403264.0, + "75": 787442688.0, + "76": 787444736.0, + "77": 787445760.0, + "78": 787395072.0, + "79": 787430400.0, + "80": 787410432.0, + "81": 787412992.0, + "82": 787427840.0, + "83": 787428864.0, + "84": 787412480.0, + "85": 787412480.0, + "86": 787394560.0, + "87": 787452928.0, + "88": 787414528.0, + "89": 787404800.0, + "90": 787446784.0, + "91": 787446272.0, + "92": 787446784.0, + "93": 787430400.0, + "94": 787440128.0, + "95": 787450368.0, + "96": 787454976.0, + "97": 787427328.0, + "98": 787475968.0, + "99": 787419136.0, + "100": 787438592.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 2579673088.0, + "2": 2590714880.0, + "3": 2590714880.0, + "4": 2590714880.0, + "5": 2596039680.0, + "6": 2596039680.0, + "7": 2596039680.0, + "8": 2596039680.0, + "9": 2596039680.0, + "10": 2596039680.0, + "11": 2596039680.0, + "12": 2596039680.0, + "13": 2596039680.0, + "14": 2596039680.0, + "15": 2596039680.0, + "16": 2596039680.0, + "17": 2596039680.0, + "18": 2596039680.0, + "19": 2596039680.0, + "20": 2596039680.0, + "21": 2596039680.0, + "22": 2596039680.0, + "23": 2596039680.0, + "24": 2596039680.0, + "25": 2596039680.0, + "26": 2596039680.0, + "27": 2596039680.0, + "28": 2596039680.0, + "29": 2596039680.0, + "30": 2596039680.0, + "31": 2596039680.0, + "32": 2596039680.0, + "33": 2596039680.0, + "34": 2596039680.0, + "35": 2596039680.0, + "36": 2596039680.0, + "37": 2596039680.0, + "38": 2596039680.0, + "39": 2596039680.0, + "40": 2596039680.0, + "41": 2596039680.0, + "42": 2596039680.0, + "43": 2596039680.0, + "44": 2596039680.0, + "45": 2596039680.0, + "46": 2596039680.0, + "47": 2596039680.0, + "48": 2596039680.0, + "49": 2596039680.0, + "50": 2596039680.0, + "51": 2596039680.0, + "52": 2596039680.0, + "53": 2596039680.0, + "54": 2596039680.0, + "55": 2596039680.0, + "56": 2596039680.0, + "57": 2596039680.0, + "58": 2596039680.0, + "59": 2596039680.0, + "60": 2596039680.0, + "61": 2596039680.0, + "62": 2596039680.0, + "63": 2596039680.0, + "64": 2596039680.0, + "65": 2596039680.0, + "66": 2596039680.0, + "67": 2596039680.0, + "68": 2596039680.0, + "69": 2596039680.0, + "70": 2596039680.0, + "71": 2596039680.0, + "72": 2596039680.0, + "73": 2596039680.0, + "74": 2596039680.0, + "75": 2596039680.0, + "76": 2596039680.0, + "77": 2596039680.0, + "78": 2596039680.0, + "79": 2596039680.0, + "80": 2596039680.0, + "81": 2596039680.0, + "82": 2596039680.0, + "83": 2596039680.0, + "84": 2596039680.0, + "85": 2596039680.0, + "86": 2596039680.0, + "87": 2596039680.0, + "88": 2596039680.0, + "89": 2596039680.0, + "90": 2596039680.0, + "91": 2596039680.0, + "92": 2596039680.0, + "93": 2596039680.0, + "94": 2596039680.0, + "95": 2596039680.0, + "96": 2596039680.0, + "97": 2596039680.0, + "98": 2596039680.0, + "99": 2596039680.0, + "100": 2596039680.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.32216, + "2": 0.51152, + "3": 0.3991, + "4": 0.39384, + "5": 0.34861, + "6": 0.34066, + "7": 0.34617, + "8": 0.33486, + "9": 0.32675, + "10": 0.32667, + "11": 0.32484, + "12": 0.31668, + "13": 0.33715, + "14": 0.32412, + "15": 0.31875, + "16": 0.32114, + "17": 0.3229, + "18": 0.31808, + "19": 0.32136, + "20": 0.31859, + "21": 0.31745, + "22": 0.31017, + "23": 0.32808, + "24": 0.31401, + "25": 0.31375, + "26": 0.31997, + "27": 0.32499, + "28": 0.32994, + "29": 0.33622, + "30": 0.33243, + "31": 0.33178, + "32": 0.35562, + "33": 0.3162, + "34": 0.32935, + "35": 0.32942, + "36": 0.32747, + "37": 0.32399, + "38": 0.32853, + "39": 0.32725, + "40": 0.32666, + "41": 0.33444, + "42": 0.32666, + "43": 0.32009, + "44": 0.38316, + "45": 0.36982, + "46": 0.3282, + "47": 0.33228, + "48": 0.32173, + "49": 0.32336, + "50": 0.33092, + "51": 0.32405, + "52": 0.344, + "53": 0.31793, + "54": 0.31881, + "55": 0.32423, + "56": 0.3238, + "57": 0.32754, + "58": 0.33365, + "59": 0.3188, + "60": 0.32627, + "61": 0.32313, + "62": 0.3251, + "63": 0.32111, + "64": 0.32694, + "65": 0.32677, + "66": 0.32916, + "67": 0.32392, + "68": 0.326, + "69": 0.31823, + "70": 0.32846, + "71": 0.32194, + "72": 0.3191, + "73": 0.32552, + "74": 0.32352, + "75": 0.31973, + "76": 0.32666, + "77": 0.32946, + "78": 0.31928, + "79": 0.32534, + "80": 0.31953, + "81": 0.31781, + "82": 0.3276, + "83": 0.32328, + "84": 0.31773, + "85": 0.32013, + "86": 0.32232, + "87": 0.31793, + "88": 0.31909, + "89": 0.6397, + "90": 0.31785, + "91": 0.3271, + "92": 0.31825, + "93": 0.31968, + "94": 0.32804, + "95": 0.31746, + "96": 0.31519, + "97": 0.32525, + "98": 0.3209, + "99": 0.31591, + "100": 0.31898 + } + } +} \ No newline at end of file diff --git a/tests/test_utils/recipes/gpt-dynamic-inference.yaml b/tests/test_utils/recipes/gpt-dynamic-inference.yaml index 914d3c0a757..748e4734a6d 100644 --- a/tests/test_utils/recipes/gpt-dynamic-inference.yaml +++ b/tests/test_utils/recipes/gpt-dynamic-inference.yaml @@ -43,7 +43,7 @@ spec: "DATA_CACHE_PATH=/workspace/data/cache" "TRAINING_SCRIPT_PATH=examples/inference/gpt/gpt_dynamic_inference.py" "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" - "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json" + "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_$CLUSTER.json" "OUTPUT_PATH={assets_dir}" "TENSORBOARD_PATH={assets_dir}/generations_{environment}_$CLUSTER.json" "N_REPEAT={n_repeat}" @@ -74,4 +74,3 @@ products: - environment: [dev] scope: [mr] platforms: [dgx_h100] -