Skip to content

Speed up set_lora_info by eliminating unnecessary H2D transfers #6960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 9, 2025
Merged

Speed up set_lora_info by eliminating unnecessary H2D transfers #6960

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
51d9447
Eliminate cudaStreamSynchronize in QKVParallelLinearWithLoRA.
lifuhuang Jun 7, 2025
21d2aaa
Merge branch 'main' into lifuhuang/lora-sync
lifuhuang Jun 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 12 additions & 15 deletions python/sglang/srt/lora/layers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ def set_lora_info(
self.A_buffer_gate_up = A_buffer
if self.lora_backend.fuse_stacked_lora_b:
# B_buffer_gate_up: (num_lora, 2 * output_dim, r)
if not hasattr(self, "B_buffer_gate_up") or self.B_buffer_gate_up is None:
if getattr(self, "B_buffer_gate_up", None) is None:
self.B_buffer_gate_up = torch.empty(
(
B_buffer[0].shape[0],
Expand Down Expand Up @@ -202,7 +202,7 @@ def set_lora_info(
output_dim_q, output_dim_kv = B_buffer_q.shape[-2], B_buffer_kv.shape[-2]

# B_buffer_qkv: (num_lora, output_dim_q + 2 * output_dim_kv, r)
if not hasattr(self, "B_buffer_qkv") or self.B_buffer_qkv is None:
if getattr(self, "B_buffer_qkv", None) is None:
self.B_buffer_qkv = torch.empty(
(
B_buffer_q[0].shape[0],
Expand All @@ -221,20 +221,17 @@ def set_lora_info(
)

# Offsets of q/k/v in output dimension
if not hasattr(self, "output_offset") or self.output_offset is None:
self.output_offset = torch.empty(
4, dtype=torch.int32, device=B_buffer_q.device
if getattr(self, "output_offset", None) is None:
self.output_offset = torch.tensor(
[
0,
output_dim_q,
output_dim_q + output_dim_kv,
output_dim_q + 2 * output_dim_kv,
],
dtype=torch.int32,
device=B_buffer_q.device,
)
self.output_offset[:4] = torch.tensor(
[
0,
output_dim_q,
output_dim_q + output_dim_kv,
output_dim_q + 2 * output_dim_kv,
],
dtype=torch.int32,
device=B_buffer_q.device,
)
# For computing number of launched blocks
self.max_qkv_out_dim = max(output_dim_q, output_dim_kv)
else:
Expand Down
Loading