Speedup speculative decoding by implementing fr-spec

outputs/20250919_175633/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "100"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}

outputs/20250919_175633/stats.jsonl

@@ -0,0 +1,19 @@
+{"input_tokens": 9468}
+{"output_tokens": 21363}
+{"input_time": 6.011072126999466}
+{"output_time": 452.5689067610002}
+{"total_time": 458.5799788879997}
+{"drafter_prefill_forward_time": 0.000760874999969019}
+{"target_prefill_forward_time": 0.02968344500004605}
+{"prefill_forward_ratio": 0.025632974877674696}
+{"drafter_decode_forward_time": 7.910397590991124}
+{"target_decode_forward_time": 172.45367566900495}
+{"decode_forward_ratio": 0.04586969550114876}
+{"input_throughput": 1575.093394317017}
+{"output_throughput": 47.203861513362234}
+{"total_throughput": 67.2314567128757}
+{"drafts": 12482}
+{"draft_tokens": 12482}
+{"draft_utilization_rate": 70.70181060727447}
+{"accepted_tokens": 8825}
+{"acceptance_length": 1.7070181060727447}

outputs/20250919_181208/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "100"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}

outputs/20250919_181227/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "3"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}

outputs/20250919_181518/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "3"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}

outputs/20250919_181951/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "2"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}

outputs/20250919_182203/args.jsonl

@@ -0,0 +1,56 @@
+{"seed": "0"}
+{"request_id_prefix": ""}
+{"num_prompts": "2"}
+{"dataset_name": "hf"}
+{"no_stream": "False"}
+{"dataset_path": "philschmid/mt-bench"}
+{"custom_output_len": "256"}
+{"custom_skip_chat_template": "True"}
+{"spec_bench_output_len": "256"}
+{"spec_bench_category": "None"}
+{"sonnet_input_len": "550"}
+{"sonnet_output_len": "150"}
+{"sonnet_prefix_len": "200"}
+{"sharegpt_output_len": "None"}
+{"blazedit_min_distance": "0.0"}
+{"blazedit_max_distance": "1.0"}
+{"random_input_len": "1024"}
+{"random_output_len": "128"}
+{"random_range_ratio": "0.0"}
+{"random_prefix_len": "0"}
+{"random_batch_size": "1"}
+{"random_mm_base_items_per_request": "1"}
+{"random_mm_num_mm_items_range_ratio": "0.0"}
+{"random_mm_limit_mm_per_prompt": "{'image': 255, 'video': 0}"}
+{"random_mm_bucket_config": "{(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}"}
+{"hf_subset": "None"}
+{"hf_split": "train"}
+{"hf_name": "None"}
+{"hf_output_len": "None"}
+{"prefix_repetition_prefix_len": "256"}
+{"prefix_repetition_suffix_len": "256"}
+{"prefix_repetition_num_prefixes": "10"}
+{"prefix_repetition_output_len": "128"}
+{"method": "eagle"}
+{"num_spec_tokens": "1"}
+{"spec_token_tree": "None"}
+{"spec_token_tree_depth": "None"}
+{"spec_token_tree_branching": "None"}
+{"prompt_lookup_max": "5"}
+{"prompt_lookup_min": "2"}
+{"tp": "1"}
+{"enforce_eager": "False"}
+{"enable_chunked_prefill": "False"}
+{"temp": "0"}
+{"top_p": "1.0"}
+{"top_k": "-1"}
+{"print_output": "False"}
+{"max_num_seqs": "1"}
+{"output_len": "256"}
+{"model_dir": "None"}
+{"eagle_dir": "None"}
+{"custom_mm_prompts": "False"}
+{"draft_vocab_frequency_path": "None"}
+{"draft_vocab_frequency_keep_threshold": "None"}
+{"compilation_config": "{\"level\": \"0\"}"}
+{"endpoint_type": "openai-chat"}