@@ -1082,7 +1082,9 @@ def add_instance_rate_stats_to_benchmark_metrics(
10821082
10831083 if not benchmark_metric_stats :
10841084 benchmark_metric_stats = []
1085- benchmark_metric_stats .append (JumpStartBenchmarkStat (instance_type_rate ))
1085+ benchmark_metric_stats .append (
1086+ JumpStartBenchmarkStat ({"concurrency" : None , ** instance_type_rate })
1087+ )
10861088
10871089 final_benchmark_metrics [instance_type ] = benchmark_metric_stats
10881090 except ClientError as e :
@@ -1127,43 +1129,94 @@ def get_metrics_from_deployment_configs(
11271129 if not deployment_configs :
11281130 return {}
11291131
1130- data = {"Instance Type" : [], "Config Name" : []}
1132+ data = {"Instance Type" : [], "Config Name" : [], "Concurrent Users" : [] }
11311133 instance_rate_data = {}
11321134 for index , deployment_config in enumerate (deployment_configs ):
11331135 benchmark_metrics = deployment_config .benchmark_metrics
11341136 if not deployment_config .deployment_args or not benchmark_metrics :
11351137 continue
11361138
1137- for inner_index , current_instance_type in enumerate (benchmark_metrics ):
1138- current_instance_type_metrics = benchmark_metrics [current_instance_type ]
1139-
1140- data ["Config Name" ].append (deployment_config .deployment_config_name )
1141- instance_type_to_display = (
1142- f"{ current_instance_type }
1143- if index == 0
1144- and current_instance_type == deployment_config .deployment_args .default_instance_type
1145- else current_instance_type
1139+ for current_instance_type , current_instance_type_metrics in benchmark_metrics .items ():
1140+ instance_type_rate , concurrent_users = _normalize_benchmark_metrics (
1141+ current_instance_type_metrics
11461142 )
1147- data ["Instance Type" ].append (instance_type_to_display )
1148-
1149- for metric in current_instance_type_metrics :
1150- column_name = f"{ metric .name } { metric .unit }
1151-
1152- if metric .name .lower () == "instance rate" :
1153- if column_name not in instance_rate_data :
1154- instance_rate_data [column_name ] = []
1155- instance_rate_data [column_name ].append (metric .value )
1156- else :
1157- if column_name not in data :
1158- data [column_name ] = []
1159- for _ in range (len (data [column_name ]), inner_index ):
1160- data [column_name ].append (" - " )
1143+
1144+ for concurrent_user , metrics in concurrent_users .items ():
1145+ instance_type_to_display = (
1146+ f"{ current_instance_type }
1147+ if index == 0
1148+ and int (concurrent_user ) == 1
1149+ and current_instance_type
1150+ == deployment_config .deployment_args .default_instance_type
1151+ else current_instance_type
1152+ )
1153+
1154+ data ["Config Name" ].append (deployment_config .deployment_config_name )
1155+ data ["Instance Type" ].append (instance_type_to_display )
1156+ data ["Concurrent Users" ].append (concurrent_user )
1157+
1158+ if instance_type_rate :
1159+ instance_rate_column_name = (
1160+ f"{ instance_type_rate .name } { instance_type_rate .unit }
1161+ )
1162+ instance_rate_data [instance_rate_column_name ] = instance_rate_data .get (
1163+ instance_rate_column_name , []
1164+ )
1165+ instance_rate_data [instance_rate_column_name ].append (instance_type_rate .value )
1166+
1167+ for metric in metrics :
1168+ column_name = _normalize_benchmark_metric_column_name (metric .name )
1169+ data [column_name ] = data .get (column_name , [])
11611170 data [column_name ].append (metric .value )
11621171
11631172 data = {** data , ** instance_rate_data }
11641173 return data
11651174
11661175
1176+ def _normalize_benchmark_metric_column_name (name : str ) -> str :
1177+ """Normalizes benchmark metric column name.
1178+
1179+ Args:
1180+ name (str): Name of the metric.
1181+ Returns:
1182+ str: Normalized metric column name.
1183+ """
1184+ if "latency" in name .lower ():
1185+ name = "Latency for each user (TTFT in ms)"
1186+ elif "throughput" in name .lower ():
1187+ name = "Throughput per user (token/seconds)"
1188+ return name
1189+
1190+
1191+ def _normalize_benchmark_metrics (
1192+ benchmark_metric_stats : List [JumpStartBenchmarkStat ],
1193+ ) -> Tuple [JumpStartBenchmarkStat , Dict [str , List [JumpStartBenchmarkStat ]]]:
1194+ """Normalizes benchmark metrics dict.
1195+
1196+ Args:
1197+ benchmark_metric_stats (List[JumpStartBenchmarkStat]):
1198+ List of benchmark metrics stats.
1199+ Returns:
1200+ Tuple[JumpStartBenchmarkStat, Dict[str, List[JumpStartBenchmarkStat]]]:
1201+ Normalized benchmark metrics dict.
1202+ """
1203+ instance_type_rate = None
1204+ concurrent_users = {}
1205+ for current_instance_type_metric in benchmark_metric_stats :
1206+ if current_instance_type_metric .name .lower () == "instance rate" :
1207+ instance_type_rate = current_instance_type_metric
1208+ elif current_instance_type_metric .concurrency not in concurrent_users :
1209+ concurrent_users [current_instance_type_metric .concurrency ] = [
1210+ current_instance_type_metric
1211+ ]
1212+ else :
1213+ concurrent_users [current_instance_type_metric .concurrency ].append (
1214+ current_instance_type_metric
1215+ )
1216+
1217+ return instance_type_rate , concurrent_users
1218+
1219+
11671220def deployment_config_response_data (
11681221 deployment_configs : Optional [List [DeploymentConfigMetadata ]],
11691222) -> List [Dict [str , Any ]]:
0 commit comments