diff --git a/tests/performance_tests/test_cases/gpt/gpt_16b_perf/model_config.yaml b/tests/performance_tests/test_cases/gpt/gpt_16b_perf/model_config.yaml index 9d016044481..c283965d0f7 100644 --- a/tests/performance_tests/test_cases/gpt/gpt_16b_perf/model_config.yaml +++ b/tests/performance_tests/test_cases/gpt/gpt_16b_perf/model_config.yaml @@ -20,9 +20,11 @@ BATCH_SIZES: - 8 - 32 TOLERANCE_PCT: 10 +# p99 omitted on purpose: with NUM_TIMED_ITERS=5 it is the max of 5 samples, +# not a real percentile, so it produces flaky regressions even when throughput +# / avg / p50 are stable. p99 is still recorded in results.json for visibility. METRICS: - throughput_tok_per_sec - avg_latency_ms - p50_latency_ms - - p99_latency_ms - tpot_ms_per_tok diff --git a/tests/performance_tests/test_cases/gpt/gpt_583m_perf/model_config.yaml b/tests/performance_tests/test_cases/gpt/gpt_583m_perf/model_config.yaml index b97c56bb0b3..fee8b9a8208 100644 --- a/tests/performance_tests/test_cases/gpt/gpt_583m_perf/model_config.yaml +++ b/tests/performance_tests/test_cases/gpt/gpt_583m_perf/model_config.yaml @@ -22,9 +22,11 @@ BATCH_SIZES: - 32 - 128 TOLERANCE_PCT: 10 +# p99 omitted on purpose: with NUM_TIMED_ITERS=5 it is the max of 5 samples, +# not a real percentile, so it produces flaky regressions even when throughput +# / avg / p50 are stable. p99 is still recorded in results.json for visibility. METRICS: - throughput_tok_per_sec - avg_latency_ms - p50_latency_ms - - p99_latency_ms - tpot_ms_per_tok diff --git a/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/model_config.yaml b/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/model_config.yaml index 04303bd2663..d884f4ef057 100644 --- a/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/model_config.yaml +++ b/tests/performance_tests/test_cases/hybrid/hybrid_2b_perf/model_config.yaml @@ -18,9 +18,11 @@ BATCH_SIZES: - 32 - 128 TOLERANCE_PCT: 10 +# p99 omitted on purpose: with NUM_TIMED_ITERS=5 it is the max of 5 samples, +# not a real percentile, so it produces flaky regressions even when throughput +# / avg / p50 are stable. p99 is still recorded in results.json for visibility. METRICS: - throughput_tok_per_sec - avg_latency_ms - p50_latency_ms - - p99_latency_ms - tpot_ms_per_tok