diff --git a/python/sglang/benchmark/datasets/common.py b/python/sglang/benchmark/datasets/common.py index 29bafff6b30c..e606e9ed8827 100644 --- a/python/sglang/benchmark/datasets/common.py +++ b/python/sglang/benchmark/datasets/common.py @@ -67,11 +67,11 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int @lru_cache(maxsize=1) def get_available_tokens(tokenizer): """Get valid token ids from the tokenizer vocabulary.""" - return [ + return sorted( token_id for token_id in tokenizer.get_vocab().values() if isinstance(token_id, int) - ] + ) def gen_prompt(tokenizer, token_num): diff --git a/python/sglang/benchmark/datasets/image.py b/python/sglang/benchmark/datasets/image.py index 5deffe0a4ea5..f44a0a786384 100644 --- a/python/sglang/benchmark/datasets/image.py +++ b/python/sglang/benchmark/datasets/image.py @@ -1,4 +1,5 @@ import io +import random import warnings from argparse import Namespace from dataclasses import dataclass @@ -30,6 +31,7 @@ class ImageDataset(BaseDataset): image_resolution: str backend: str random_image_count: bool + seed: int @classmethod def from_args(cls, args: Namespace) -> "ImageDataset": @@ -44,10 +46,15 @@ def from_args(cls, args: Namespace) -> "ImageDataset": image_resolution=args.image_resolution, backend=args.backend, random_image_count=args.random_image_count, + seed=args.seed, ) def load(self, tokenizer=None, model_id=None) -> List[DatasetRow]: processor = get_processor(model_id) + # Processor initialization may consume global RNG state. Reset it here so + # --seed fixes the generated prompts, image sizes, and image contents. + random.seed(self.seed) + np.random.seed(self.seed) return sample_image_requests( num_requests=self.num_requests, image_count=self.image_count, diff --git a/test/registered/bench_fn/test_benchmark_datasets_api.py b/test/registered/bench_fn/test_benchmark_datasets_api.py index 4c7c3d7f4f8f..b25a337d786d 100644 --- a/test/registered/bench_fn/test_benchmark_datasets_api.py +++ b/test/registered/bench_fn/test_benchmark_datasets_api.py @@ -34,6 +34,7 @@ sample_generated_shared_prefix_requests, ) from sglang.benchmark.datasets.image import ( + ImageDataset, parse_random_image_resolution, sample_image_requests, ) @@ -479,6 +480,43 @@ def test_image_sampler_random_resolution(self): self.assertGreaterEqual(height, 8) self.assertLessEqual(height, 16) + def test_image_dataset_seed_is_independent_of_processor_initialization(self): + dataset = ImageDataset.from_args( + make_args( + num_prompts=3, + image_resolution="random:8x16-16x32", + seed=20260717, + ) + ) + processor_init_count = 0 + + def get_processor_with_rng_side_effects(_model_id): + nonlocal processor_init_count + processor_init_count += 1 + random.random() + np.random.random(processor_init_count) + return self.processor + + with patch( + "sglang.benchmark.datasets.image.get_processor", + side_effect=get_processor_with_rng_side_effects, + ): + first = dataset.load(model_id="test-model") + random.seed(999) + np.random.seed(999) + second = dataset.load(model_id="test-model") + + self.assertEqual( + [ + (row.prompt, row.prompt_len, row.output_len, row.image_data) + for row in first + ], + [ + (row.prompt, row.prompt_len, row.output_len, row.image_data) + for row in second + ], + ) + def test_parse_random_image_resolution(self): self.assertEqual( parse_random_image_resolution("random:256x384-1024x1536"), @@ -520,6 +558,30 @@ def fake_choices(population, k): self.assertFalse(special_token_ids & sampled_pool) self.assertTrue(sampled_pool) + def test_gen_mm_prompt_is_independent_of_vocab_order(self): + class OrderedVocabTokenizer: + all_special_ids = [] + + def __init__(self, items): + self.vocab = dict(items) + + def get_vocab(self): + return self.vocab + + def decode(self, token_ids): + return " ".join(map(str, token_ids)) + + items = [(f"token_{token_id}", token_id) for token_id in range(32)] + first = OrderedVocabTokenizer(items) + second = OrderedVocabTokenizer(reversed(items)) + + random.seed(20260717) + first_prompt = gen_mm_prompt(first, image_pad_id=None, token_num=16) + random.seed(20260717) + second_prompt = gen_mm_prompt(second, image_pad_id=None, token_num=16) + + self.assertEqual(first_prompt, second_prompt) + def test_mmmu_sampler(self): fake_records = [ {"image_1": Image.new("RGB", (4, 4), color="white"), "question": "q1"},