Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ def test_swa_offload_window_covers_unaligned_hit(
assert len(sched._jobs) == 1
job = next(iter(sched._jobs.values()))
if needs_left:
assert {left_key, swa[end - 1]} <= job.keys
assert {left_key, swa[end - 1]} <= set(job.keys)
manager.complete_load(job.keys, state.req_context)


Expand Down Expand Up @@ -3685,7 +3685,7 @@ def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
# First store: fires when block 0 is fully populated.
runner.run(decoded_tokens=[0, 0], expected_stored=(0, 1, 2))
assert runner.manager.complete_store.call_count == 1
first_call_keys = set(runner.manager.complete_store.call_args.args[0])
first_call_keys = runner.manager.complete_store.call_args.args[0]
assert len(first_call_keys) == 1
runner.manager.complete_store.reset_mock()

Expand All @@ -3695,7 +3695,7 @@ def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
expected_stored=(3, 4, 5),
)
assert runner.manager.complete_store.call_count == 1
second_call_keys = set(runner.manager.complete_store.call_args.args[0])
second_call_keys = runner.manager.complete_store.call_args.args[0]
assert first_call_keys != second_call_keys
runner.manager.complete_store.reset_mock()

Expand All @@ -3704,6 +3704,47 @@ def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
assert runner.manager.complete_store.call_count == 0


@pytest.mark.parametrize("async_scheduling", [True, False])
def test_store_job_preserves_order_and_deduplicates_keys(
request_runner, async_scheduling: bool
):
tokens_per_block = 4
blocks_per_chunk = 3
tokens_per_chunk = tokens_per_block * blocks_per_chunk
runner = request_runner(
blocks_per_chunk=blocks_per_chunk,
block_size=tokens_per_block,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
)
runner.new_request(token_ids=[0] * (tokens_per_chunk * 2))
returned_keys = []

def prepare_store(keys, req_context):
keys = list(keys)
returned_keys.extend(keys)
output = generate_store_output(keys)
output.keys_to_store.append(keys[0])
return output

runner.manager.prepare_store.side_effect = prepare_store
job_keys = {}

def capture_job_keys():
job_keys.update(runner.connector_scheduler._jobs.items())

runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=tuple(range(6)),
post_step_fn=capture_job_keys,
)

assert len(job_keys) == 1
keys = next(iter(job_keys.values())).keys
assert keys == tuple(returned_keys)
assert runner.manager.complete_store.call_args.args[0] == keys


@pytest.mark.parametrize("async_scheduling", [True, False])
def test_complete_store_waits_for_all_worker_acks(
request_runner, async_scheduling: bool
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,9 @@ class TransferJobStatus:
# Number of workers still pending. Starts at num_workers,
# decremented as each worker reports completion. Job is done at 0.
pending_count: int
# Offload keys this job covers; passed to manager.complete_*().
keys: set[OffloadKey]
# Offload keys this job covers, in prefix order; passed to
# manager.complete_*().
keys: tuple[OffloadKey, ...]
is_store: bool
# Unlike an async load, store jobs may be created next to it.
is_sync_load: bool = False
Expand Down Expand Up @@ -1584,7 +1585,7 @@ def update_state_after_alloc(
self._jobs[load_job_id] = TransferJobStatus(
req_id=request.request_id,
pending_count=self.config.num_workers,
keys=set(keys_to_load),
keys=tuple(dict.fromkeys(keys_to_load)),
is_store=False,
is_sync_load=sync_load,
)
Expand Down Expand Up @@ -1691,7 +1692,7 @@ def _build_aligned_boundary_store_jobs(
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys={key},
keys=(key,),
is_store=True,
fenced_block_ids=[block_id],
)
Expand Down Expand Up @@ -1838,7 +1839,7 @@ def _build_partial_tail_store_jobs(
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys=set(store_output.keys_to_store),
keys=tuple(dict.fromkeys(store_output.keys_to_store)),
is_store=True,
fenced_block_ids=source_blocks,
)
Expand Down Expand Up @@ -2099,7 +2100,8 @@ def _build_store_jobs(
req_status.advance_stored_idx(num_offloadable_tokens)
continue

keys_to_store = set(store_output.keys_to_store)
ordered_keys_to_store = tuple(dict.fromkeys(store_output.keys_to_store))
keys_to_store = set(ordered_keys_to_store)

group_sizes: list[int] = []
block_indices: list[int] = []
Expand Down Expand Up @@ -2173,7 +2175,7 @@ def _build_store_jobs(
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys=set(keys_to_store),
keys=ordered_keys_to_store,
is_store=True,
deferred_fence_block_ids=deferred_fence_block_ids,
fenced_block_ids=fenced_block_ids or None,
Expand Down