diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml
new file mode 100644
index 0000000000..f2636cab5b
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml
@@ -0,0 +1,18 @@
+sources:
+ - /mnt/hdfs/user_dir/dataset/imagenet1k_train
+ - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data
+ - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k
+ - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn
+names:
+ - imagenet1k
+ - tulu-3-sft-mixture
+ - sharegpt4v_cap_100k
+ - seed_edit_p23_multi_turn
+schedule:
+ - schedule_type: const
+ weights: [0.4, 0.2, 0.2, 0.2]
+# Heterogeneous schemas (imagenet string / tulu dict / sharegpt4v list) — keep
+# them in the VeOmni weighted multisource sampler.
+level: token
+stopping_strategy: all_exhausted
+upstream_sharded: true
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml
new file mode 100644
index 0000000000..e8082463e0
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml
@@ -0,0 +1,37 @@
+# BAGEL-7B-MoT inference - text+image to image edit.
+# Official edit builds VAE/ViT/text context, then calls image generation directly.
+initial: prompt_encode
+
+states:
+ prompt_encode:
+ body:
+ - { from: bagel_text_encoder, to: bagel_qwen2_mot }
+ - { from: bagel_siglip_navit, to: bagel_qwen2_mot }
+ - { from: bagel_vae.encode_context, to: bagel_flow_connector.embed_context_latents }
+ - { from: bagel_flow_connector.embed_context_latents, to: bagel_qwen2_mot }
+ - { from: bagel_qwen2_mot, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: query_denoise }
+
+ query_denoise:
+ body:
+ - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers }
+ - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch }
+ - { from: bagel_qwen2_mot.denoise_branch, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: velocity_collect }
+
+ velocity_collect:
+ body:
+ - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity }
+ - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise }
+ - { from: bagel_flow_connector.advance_denoise, to: end }
+ transitions:
+ - { condition: { type: module_signal, key: image_complete }, next_state: image_decode }
+ - { condition: { type: default }, next_state: query_denoise }
+
+ image_decode:
+ body:
+ - { from: bagel_vae.decode_generated, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: done }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml
new file mode 100644
index 0000000000..74f1864075
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml
@@ -0,0 +1,34 @@
+# BAGEL-7B-MoT inference - text to image.
+initial: prompt_encode
+
+states:
+ prompt_encode:
+ body:
+ - { from: bagel_text_encoder, to: bagel_qwen2_mot }
+ - { from: bagel_siglip_navit, to: bagel_qwen2_mot }
+ - { from: bagel_qwen2_mot, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: query_denoise }
+
+ query_denoise:
+ body:
+ - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers }
+ - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch }
+ - { from: bagel_qwen2_mot.denoise_branch, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: velocity_collect }
+
+ velocity_collect:
+ body:
+ - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity }
+ - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise }
+ - { from: bagel_flow_connector.advance_denoise, to: end }
+ transitions:
+ - { condition: { type: module_signal, key: image_complete }, next_state: image_decode }
+ - { condition: { type: default }, next_state: query_denoise }
+
+ image_decode:
+ body:
+ - { from: bagel_vae.decode_generated, to: end }
+ transitions:
+ - { condition: { type: default }, next_state: done }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml
new file mode 100644
index 0000000000..29ba541d85
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml
@@ -0,0 +1,19 @@
+# BAGEL-7B-MoT inference - understanding to text (text/image -> text).
+initial: prompt_encode
+
+states:
+ prompt_encode:
+ body:
+ - { from: bagel_text_encoder, to: bagel_qwen2_mot }
+ - { from: bagel_siglip_navit, to: bagel_qwen2_mot }
+ - { from: bagel_qwen2_mot, to: end }
+ transitions:
+ - { condition: { type: module_signal, key: text_done }, next_state: done }
+ - { condition: { type: default }, next_state: text_ar }
+
+ text_ar:
+ body:
+ - { from: bagel_qwen2_mot, to: bagel_text_encoder }
+ - { from: bagel_text_encoder, to: end }
+ transitions:
+ - { condition: { type: module_signal, key: text_done }, next_state: done }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml
new file mode 100644
index 0000000000..8c062c522a
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml
@@ -0,0 +1,42 @@
+# Per-module INFERENCE overrides for BAGEL-7B-MoT — all eager.
+#
+# Single-process load (`python tasks/omni/infer_omni.py ...`): every module
+# loads via `from_pretrained(device_map='auto')`, spread across visible GPUs.
+
+bagel_vae:
+ model_path: bagel_vae
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+ fsdp_config:
+ fsdp_mode: eager
+
+bagel_siglip_navit:
+ model_path: bagel_siglip_navit
+ accelerator:
+ fsdp_config:
+ fsdp_mode: eager
+
+bagel_text_encoder:
+ model_path: bagel_text_encoder
+ accelerator:
+ fsdp_config:
+ fsdp_mode: eager
+ gradient_checkpointing:
+ enable: false
+
+bagel_flow_connector:
+ model_path: bagel_flow_connector
+ accelerator:
+ fsdp_config:
+ fsdp_mode: eager
+ gradient_checkpointing:
+ enable: false
+
+bagel_qwen2_mot:
+ model_path: bagel_qwen2_mot
+ ops_implementation:
+ attn_implementation: flex_attention
+ accelerator:
+ fsdp_config:
+ fsdp_mode: eager
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml
new file mode 100644
index 0000000000..b6b65e8555
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml
@@ -0,0 +1,33 @@
+bagel_vae:
+ model_path: bagel_vae
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+ fsdp_config:
+ fsdp_mode: fsdp2
+bagel_siglip_navit:
+ model_path: bagel_siglip_navit
+ accelerator:
+ fsdp_config:
+ fsdp_mode: fsdp2
+bagel_text_encoder:
+ model_path: bagel_text_encoder
+ accelerator:
+ fsdp_config:
+ fsdp_mode: fsdp2
+ gradient_checkpointing:
+ enable: false
+bagel_flow_connector:
+ model_path: bagel_flow_connector
+ accelerator:
+ fsdp_config:
+ fsdp_mode: fsdp2
+ gradient_checkpointing:
+ enable: false
+bagel_qwen2_mot:
+ model_path: bagel_qwen2_mot
+ ops_implementation:
+ attn_implementation: flex_attention
+ accelerator:
+ fsdp_config:
+ fsdp_mode: fsdp2
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml
new file mode 100644
index 0000000000..78b9c32181
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml
@@ -0,0 +1,83 @@
+# VeOmni offline VAE posterior cache config for OmniModel - BAGEL-7B-MoT.
+
+model:
+ model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT
+ model_config:
+ modules: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml
+ train_graph: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml
+ infer_graph:
+ infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml
+ infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml
+ infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml
+ infer_type: infer_und
+ ops_implementation:
+ attn_implementation: flash_attention_2
+
+ accelerator:
+ init_device: meta
+ gradient_checkpointing:
+ enable: false
+ enable_reentrant: false
+ fsdp_config:
+ fsdp_mode: fsdp2
+ reshard_after_forward: true
+ reshard_after_backward: true
+ forward_prefetch: true
+ offload: false
+ max_load_broadcast_size: 20.0
+ mixed_precision:
+ enable: true
+ param_dtype: bfloat16
+ reduce_dtype: float32
+ offload_config:
+ enable_activation: false
+ activation_gpu_limit: 0.0
+
+ optimizer:
+ type: adamw
+ lr: 1.0e-4
+ lr_min: 1.0e-6
+ lr_start: 0.0
+ lr_warmup_ratio: 0.03
+ lr_decay_style: cosine
+ lr_decay_ratio: 1.0
+ weight_decay: 0.01
+ max_grad_norm: 1.0
+ no_decay_params: []
+ no_decay_modules: []
+
+data:
+ data_type: seedomni
+ max_seq_len: 4096
+ train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml
+ datasets_type: iterable
+ multisource_datasets_type: veomni_weighted_multisource
+ train_sample: 1000000
+ dataloader:
+ type: native
+ num_workers: 4
+ drop_last: false
+ pin_memory: true
+ prefetch_factor: 2
+
+train:
+ training_task: offline_embedding
+ offline_cache_dir: outputs/bagel_vae_cached_dataset
+ seed: 42
+ num_train_epochs: 1
+ global_batch_size: 32
+ micro_batch_size: 2
+ dyn_bsz: false
+
+ checkpoint:
+ output_dir: outputs/bagel_7b_mot_offline_cache
+ save_steps: 0
+ save_epochs: 0
+ save_hf_weights: false
+ hf_save_steps: 0
+ hf_save_epochs: 0
+
+ wandb:
+ enable: false
+ project: VeOmni_omni
+ name: bagel_7b_mot_offline_cache
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml
new file mode 100644
index 0000000000..82f52f7241
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml
@@ -0,0 +1,2 @@
+# BAGEL-7B-MoT VAE posterior cache production DAG.
+- { from: bagel_vae.offline_encode, to: end }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml
new file mode 100644
index 0000000000..b1894fca61
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml
@@ -0,0 +1,5 @@
+bagel_vae:
+ model_path: bagel_vae
+ accelerator:
+ gradient_checkpointing:
+ enable: false
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml
new file mode 100644
index 0000000000..41ba85608c
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml
@@ -0,0 +1,83 @@
+# VeOmni training / inference config for OmniModel - BAGEL-7B-MoT.
+
+model:
+ model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT
+ model_config:
+ modules: configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml
+ train_graph: configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml
+ infer_graph:
+ infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml
+ infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml
+ infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml
+ infer_type: infer_und
+ ops_implementation:
+ attn_implementation: flash_attention_2
+
+ accelerator:
+ init_device: meta
+ gradient_checkpointing:
+ enable: true
+ enable_reentrant: false
+ fsdp_config:
+ fsdp_mode: fsdp2
+ reshard_after_forward: true
+ reshard_after_backward: true
+ forward_prefetch: true
+ offload: false
+ max_load_broadcast_size: 20.0
+ mixed_precision:
+ enable: true
+ param_dtype: bfloat16
+ reduce_dtype: float32
+ offload_config:
+ enable_activation: false
+ activation_gpu_limit: 0.0
+
+ optimizer:
+ type: adamw
+ lr: 1.0e-4
+ lr_min: 1.0e-6
+ lr_start: 0.0
+ lr_warmup_ratio: 0.03
+ lr_decay_style: cosine
+ lr_decay_ratio: 1.0
+ weight_decay: 0.01
+ max_grad_norm: 1.0
+ no_decay_params: []
+ no_decay_modules: []
+
+data:
+ data_type: seedomni
+ max_seq_len: 4096
+ train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml
+ # Streaming + VeOmni weighted multisource: no HF schema merge across sources.
+ datasets_type: iterable
+ multisource_datasets_type: veomni_weighted_multisource
+ # Iterable has no __len__; train_steps = ceil(train_sample / global_batch_size).
+ train_sample: 1000000
+ dataloader:
+ type: native
+ num_workers: 4
+ drop_last: true
+ pin_memory: true
+ prefetch_factor: 2
+
+train:
+ seed: 42
+ num_train_epochs: 1
+ global_batch_size: 32
+ micro_batch_size: 2
+ dyn_bsz: false
+
+ checkpoint:
+ output_dir: outputs/bagel_7b_mot_omni_sft
+ save_steps: 500
+ save_epochs: 0
+ save_hf_weights: true
+ hf_save_steps: 500
+ hf_save_epochs: 0
+
+ wandb:
+ enable: true
+ project: VeOmni_omni
+ name: bagel_7b_mot_omni_sft
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml
new file mode 100644
index 0000000000..3cae0e86bc
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml
@@ -0,0 +1,9 @@
+# BAGEL-7B-MoT training DAG.
+- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot }
+- { from: bagel_siglip_navit, to: bagel_qwen2_mot }
+- { from: bagel_vae.encode, to: bagel_flow_connector.embed_latent }
+- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot }
+- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode }
+- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity }
+- { from: bagel_text_encoder.decode, to: end }
+- { from: bagel_flow_connector.decode_velocity, to: end }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml
new file mode 100644
index 0000000000..c03909f5bd
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml
@@ -0,0 +1,23 @@
+bagel_vae:
+ model_path: bagel_vae
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_siglip_navit:
+ model_path: bagel_siglip_navit
+bagel_text_encoder:
+ model_path: bagel_text_encoder
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_flow_connector:
+ model_path: bagel_flow_connector
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_qwen2_mot:
+ model_path: bagel_qwen2_mot
+ ops_implementation:
+ attn_implementation: flex_attention
+ accelerator:
+ enable_async: true
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml
new file mode 100644
index 0000000000..517489071e
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml
@@ -0,0 +1,81 @@
+# VeOmni cached VAE posterior training config for OmniModel - BAGEL-7B-MoT.
+
+model:
+ model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT
+ model_config:
+ modules: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml
+ train_graph: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml
+ infer_graph:
+ infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml
+ infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml
+ infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml
+ infer_type: infer_und
+ ops_implementation:
+ attn_implementation: flash_attention_2
+
+ accelerator:
+ init_device: meta
+ gradient_checkpointing:
+ enable: true
+ enable_reentrant: false
+ fsdp_config:
+ fsdp_mode: fsdp2
+ reshard_after_forward: true
+ reshard_after_backward: true
+ forward_prefetch: true
+ offload: false
+ max_load_broadcast_size: 20.0
+ mixed_precision:
+ enable: true
+ param_dtype: bfloat16
+ reduce_dtype: float32
+ offload_config:
+ enable_activation: false
+ activation_gpu_limit: 0.0
+
+ optimizer:
+ type: adamw
+ lr: 1.0e-4
+ lr_min: 1.0e-6
+ lr_start: 0.0
+ lr_warmup_ratio: 0.03
+ lr_decay_style: cosine
+ lr_decay_ratio: 1.0
+ weight_decay: 0.01
+ max_grad_norm: 1.0
+ no_decay_params: []
+ no_decay_modules: []
+
+data:
+ data_type: seedomni_cached
+ max_seq_len: 4096
+ train_path: outputs/bagel_vae_cached_dataset
+ datasets_type: iterable
+ train_sample: 1000000
+ dataloader:
+ type: native
+ num_workers: 4
+ drop_last: true
+ pin_memory: true
+ prefetch_factor: 2
+
+train:
+ training_task: offline_training
+ seed: 42
+ num_train_epochs: 1
+ global_batch_size: 32
+ micro_batch_size: 2
+ dyn_bsz: false
+
+ checkpoint:
+ output_dir: outputs/bagel_7b_mot_omni_sft_from_cache
+ save_steps: 500
+ save_epochs: 0
+ save_hf_weights: true
+ hf_save_steps: 500
+ hf_save_epochs: 0
+
+ wandb:
+ enable: true
+ project: VeOmni_omni
+ name: bagel_7b_mot_omni_sft_from_cache
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml
new file mode 100644
index 0000000000..71951d199d
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml
@@ -0,0 +1,9 @@
+# BAGEL-7B-MoT training DAG from cached VAE posterior carrier items.
+- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot }
+- { from: bagel_siglip_navit, to: bagel_qwen2_mot }
+- { from: bagel_vae.online_process, to: bagel_flow_connector.embed_latent }
+- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot }
+- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode }
+- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity }
+- { from: bagel_text_encoder.decode, to: end }
+- { from: bagel_flow_connector.decode_velocity, to: end }
diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml
new file mode 100644
index 0000000000..c03909f5bd
--- /dev/null
+++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml
@@ -0,0 +1,23 @@
+bagel_vae:
+ model_path: bagel_vae
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_siglip_navit:
+ model_path: bagel_siglip_navit
+bagel_text_encoder:
+ model_path: bagel_text_encoder
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_flow_connector:
+ model_path: bagel_flow_connector
+ accelerator:
+ gradient_checkpointing:
+ enable: false
+bagel_qwen2_mot:
+ model_path: bagel_qwen2_mot
+ ops_implementation:
+ attn_implementation: flex_attention
+ accelerator:
+ enable_async: true
diff --git a/docs/index.md b/docs/index.md
index 4de17b3691..b83380c0fb 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -120,6 +120,7 @@ seed_omni/mixins/offline_encoding.md
seed_omni/models/qwen3.md
seed_omni/models/qwen3vl.md
+seed_omni/models/bagel.md
```
```{toctree}
diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md
new file mode 100644
index 0000000000..eb249318d0
--- /dev/null
+++ b/docs/seed_omni/models/bagel.md
@@ -0,0 +1,360 @@
+# BAGEL-7B-MoT (SeedOmni)
+
+End-to-end recipe for training and inferring **BAGEL-7B-MoT** as a SeedOmni
+graph model. The upstream BAGEL checkpoint is split into five OmniModules: text
+embedding / LM head, SigLIP-NaViT understanding tower, VAE codec, flow connector,
+and Qwen2-MoT backbone.
+
+All paths below assume the upstream BAGEL checkpoint lives at
+`/mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT`. Adjust to your
+own storage.
+
+Config dir: `configs/seed_omni/Bagel/bagel_7b_mot/`.
+
+| Module | Holds | Role |
+|--------|-------|------|
+| `bagel_text_encoder` | tokenizer, token embedding, LM head | text/template markers, text logits, image-marker embeddings |
+| `bagel_siglip_navit` | SigLIP-NaViT tower + connector | user image context for understanding and edit prompts |
+| `bagel_vae` | VAE encoder/decoder | assistant image training latents, edit context latents, generated latent decode |
+| `bagel_flow_connector` | VAE↔LLM projections, timestep embedding | latent patch embedding, velocity prediction, denoise state |
+| `bagel_qwen2_mot` | Qwen2-MoT decoder backbone | text AR and flow-denoise hidden states |
+
+The [config layout](../usage/training_and_inference.md#1-config-layout) uses the
+same `base.yaml` for training and inference. The training block references `train/modules_train.yaml` and `train/graph_train.yaml`; the
+inference block maps each scenario to a separate generation graph.
+
+| File | Role |
+|------|------|
+| `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, and train blocks. |
+| `train/modules_train.yaml` | Per-module training paths. `bagel_qwen2_mot` is the accelerated class with `flex_attention`. |
+| `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses FlexAttention. |
+| `infer/modules_infer_fsdp.yaml` | Distributed inference: every module uses FSDP2; MoT uses FlexAttention. |
+| `train/graph_train.yaml` | Training DAG. |
+| `infer/graph_infer_und.yaml` | Image/text understanding to text. |
+| `infer/graph_infer_gen.yaml` | Text to image generation. |
+| `infer/graph_infer_edit.yaml` | Text+image to image edit. |
+| `data.yaml` | Weighted multisource data list. |
+
+The BAGEL graphs cover understanding, generation, and edit.
+
+---
+
+## 1. Convert the checkpoint
+
+The converter reads upstream `llm_config.json`, `vit_config.json`,
+`ema.safetensors`, and `ae.safetensors`, then writes one sub-checkpoint per
+module:
+
+```bash
+python scripts/seed_omni/convert_model.py \
+ --model_path /mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT \
+ --output_dir /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT
+```
+
+The family is picked from the source `config.json` (`model_type: bagel`). The
+output root becomes `model.model_path` in `base.yaml`. It contains:
+
+```text
+BAGEL-7B-MoT/
+├── config.json # module entries, infer_type, generation_kwargs
+├── training_graph.yaml
+├── generation_graph.yaml # infer_und / infer_gen / infer_edit
+├── bagel_text_encoder/
+├── bagel_siglip_navit/
+├── bagel_vae/
+├── bagel_flow_connector/
+└── bagel_qwen2_mot/
+```
+
+`bagel_text_encoder` also stores the tokenizer, re-saved from the upstream
+checkpoint via `AutoTokenizer`; SigLIP-NaViT and VAE save their processors next
+to their weights.
+
+---
+
+## 2. Prepare data
+
+`data.yaml` lists a weighted multisource mixture:
+
+```yaml
+sources:
+ - /mnt/hdfs/user_dir/dataset/imagenet1k_train
+ - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data
+ - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k
+ - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn
+names:
+ - imagenet1k
+ - tulu-3-sft-mixture
+ - sharegpt4v_cap_100k
+ - seed_edit_p23_multi_turn
+schedule:
+ - { schedule_type: const, weights: [0.4, 0.2, 0.2, 0.2] }
+```
+
+`ConversationItem`s carry no module ownership: each Bagel module selects its
+rows by `type` / `role` / `meta` tags. Bagel defines two private meta keys in
+`veomni/models/seed_omni/modules/bagel/sources.py`:
+
+| Key | Values | Meaning |
+|---|---|---|
+| `BAGEL_CONTEXT_KEY` (`"bagel_context"`) | `BAGEL_SIGLIP_CONTEXT` (`"siglip"`), `BAGEL_VAE_CONTEXT` (`"vae"`) | Which image copy a row is: SigLIP-NaViT understanding features, or VAE latent context/target. The vision start/end marker rows around each copy carry the same tag. |
+| `BAGEL_PHASE_KEY` (`"bagel_phase"`) | `start_token`, `flow_query`, `flow_hidden`, `flow_velocity`, `generated_latent` | Inference-only step of a `type="output"` row (see §4). |
+
+The VAE CPU preprocessor (`route_image_contexts`) expands each raw training
+image by its data-layer `_img_tag` into context copies:
+
+- `und` → one `BAGEL_SIGLIP_CONTEXT` copy for SigLIP-NaViT.
+- `gen` → one `BAGEL_VAE_CONTEXT` copy, the flow-matching target.
+- `edit` → a `BAGEL_VAE_CONTEXT` copy followed by a `BAGEL_SIGLIP_CONTEXT`
+ copy of the same image. `_img_tag` alone cannot tell these two apart, which
+ is why the context tag exists.
+
+At inference, raw user images have no `_img_tag`; they become SigLIP context,
+plus a VAE context copy when `infer_type` is `infer_edit`.
+
+A sample with no image for SigLIP-NaViT or the VAE gets an FSDP placeholder
+flagged `is_dummy=True`. It keeps the context tag (and a representative
+`role` / `_img_tag`: `user`/`und` for SigLIP, `assistant`/`gen` for the VAE),
+so the encoder selects it with the same filter as real rows; packing and loss
+code skip it via `item.is_dummy`.
+
+---
+
+## 3. Train
+
+Training uses the accelerated Qwen2-MoT class from `train/modules_train.yaml`. The
+default packed attention backend is FlexAttention.
+
+On H20 (SM90), eligible BF16/FP16 calls automatically use Flex's FlashAttention-4
+backend from the standard `gpu` extra. The MoT training module enables async
+Ulysses when sequence parallelism is active. A single-node H20 baseline is:
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.accelerator.ulysses_size 4 \
+ --train.micro_batch_size 1
+```
+
+The online and cached-VAE training module configs both set
+`bagel_qwen2_mot.accelerator.enable_async: true`. For a matched synchronous
+control, override
+`--model.model_config.modules.bagel_qwen2_mot.accelerator.enable_async false`.
+With SP disabled, the fused synchronous QKV path is used regardless of this
+flag. Both paths retain the same split-QKV external checkpoint schema.
+
+The async path dispatches through the `("async_ulysses_qkv", "bagel")` and
+`("async_ulysses_o", "bagel")` compound ops, overlapping Q/K exchanges with
+later routed projections and inverse exchanges with backward projection work.
+It forwards the module's configured `rms_norm` and `rope` handles into the
+compound ops and passes `skip_ulysses=True` to attention because Q/K/V have
+already been redistributed. Inference continues to use its existing packed
+prefill and FlashAttention-2 denoise/decode paths.
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml
+```
+
+The training DAG is:
+
+```text
+bagel_text_encoder.encode -> bagel_qwen2_mot
+bagel_siglip_navit -> bagel_qwen2_mot
+bagel_vae.encode -> bagel_flow_connector.embed_latent
+bagel_flow_connector.embed_latent -> bagel_qwen2_mot
+bagel_qwen2_mot -> bagel_text_encoder.decode
+bagel_qwen2_mot -> bagel_flow_connector.decode_velocity
+```
+
+`bagel_flow_connector.embed_latent` patchifies VAE latents, samples the training
+noise/timestep, writes the velocity target, and projects noised latent patches
+into LLM space. `bagel_flow_connector.decode_velocity` consumes the MoT hidden
+states for those latent query positions and computes the flow velocity loss.
+
+Quick smoke run:
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \
+ --train.max_steps 10 \
+ --train.global_batch_size 8 \
+ --train.micro_batch_size 1 \
+ --train.wandb.enable false
+```
+
+### 3.1 Attention backends
+
+`bagel_qwen2_mot` keeps one packed visibility metadata and materializes it as
+Flex, Magi, or SDPA. Visibility is the same in all three:
+`same_document & (causal | same_full_span) & ~foreign_noise_key`. Packed MoT
+training does not use FlashAttention-2.
+
+| Backend | Class | Config | Use |
+|---------|-------|--------|-----|
+| FlexAttention | accelerated / eager | `train/modules_train.yaml`, `infer/modules_infer_eager.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training and inference (only supported packed backend on SM90) |
+| MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM100+ CUTE DSL/JIT fused training |
+| SDPA | eager | CLI override onto `bagel_qwen2_mot` | Dense-mask oracle |
+
+Leave the YAML on `flex_attention` unless you are opting into Magi on SM100+.
+Enable Magi from the CLI (the launcher rewrites it to `veomni_magi_attention`):
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.model_config.modules.bagel_qwen2_mot.ops_implementation.attn_implementation magi_attention \
+ --train.micro_batch_size 1
+```
+
+Magi requires physical batch size 1, `cp_size == 1`, and NVIDIA SM100 or newer
+(with CUTE DSL/JIT installed via `uv sync --extra gpu --extra magi --dev`).
+SM90 and older GPUs only support FlexAttention. Ulysses still works. On 8 ranks,
+`--model.accelerator.ulysses_size 4` gives `dp_shard=2` and SP4. Packed
+Magi/Flex is the training (and FSDP prefill) path; the denoise loop still uses
+FlashAttention-2. The Magi adapter contract is in
+[`docs/transformers_v5/veomni_attention_interface.md`](../../transformers_v5/veomni_attention_interface.md).
+
+### 3.2 Offline VAE posterior cache (two stages)
+
+Caching is not a special framework mode — it is a different `train_graph` plus a
+different dataset type. Each stage has its own modules / graph YAML, and
+`train.training_task` tells the trainer which stage it runs. `bagel_vae` can be
+cached because it mixes in `OfflineEncodingMixin`; no config flag declares it (see
+[Offline Encoding](../mixins/offline_encoding.md#how-a-run-builds-a-cached-module)).
+
+**Stage 1 — produce the cache** (`train.training_task: offline_embedding`). `offline_cache/modules_train.yaml` declares only
+`bagel_vae`, and the DAG is a single edge, so nothing else is built:
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml
+```
+
+```text
+bagel_vae.offline_encode -> end
+```
+
+Posteriors are written to `train.offline_cache_dir`
+(`outputs/bagel_vae_cached_dataset` by default), reading normal `seedomni` data.
+Samples without a VAE image get the preprocessor's dummy row encoded too, so
+every cached sample carries a VAE posterior.
+
+**Stage 2 — train from the cache** (`train.training_task: offline_training`).
+`bagel_vae` mixes in `OfflineEncodingMixin`, so it is built on meta and never
+loads its weights; the graph calls its `online_process`. `with_cache/modules_train.yaml` is the same file as
+`train/modules_train.yaml`. `data.data_type`
+becomes `seedomni_cached` and `data.train_path` points at the stage-1 output
+directory. The cached items are marked as posteriors, so the VAE preprocessor
+leaves them alone:
+
+```bash
+bash train.sh tasks/omni/train_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml
+```
+
+The DAG matches §3 except that `bagel_vae.online_process` replaces
+`bagel_vae.encode` as the latent source — it rehydrates the cached posterior
+instead of encoding pixels.
+
+---
+
+## 4. Inference
+
+`tasks/omni/infer_omni.py` selects a generation graph with `--model.model_config.infer_type`.
+Use `infer/modules_infer_eager.yaml` for a single-process run or
+`infer/modules_infer_fsdp.yaml` with `bash train.sh` for a torchrun/FSDP2 run.
+
+### 4.1 Understanding
+
+```bash
+python tasks/omni/infer_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.model_config.infer_type infer_und \
+ --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \
+ --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \
+ --infer.images /path/to/image.jpg \
+ --infer.prompt "Describe this image." \
+ --infer.output_dir bagel_out
+```
+
+`infer_und` runs SigLIP-NaViT for the prompt image, inserts image marker
+embeddings via the text encoder, and then uses Qwen2-MoT + text encoder AR
+decode until `text_done`.
+
+### 4.2 Text-to-image generation
+
+```bash
+python tasks/omni/infer_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.model_config.infer_type infer_gen \
+ --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \
+ --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \
+ --infer.prompt "A watercolor painting of a small cabin beside a lake." \
+ --infer.output_dir bagel_out \
+ --infer.generation_kwargs.num_timesteps 50 \
+ --infer.generation_kwargs.timestep_shift 3.0
+```
+
+`infer_gen` runs prompt prefill, then loops:
+
+```text
+flow_connector.prepare_denoise_query
+ -> text_encoder.encode_image_markers
+ -> qwen2_mot.denoise_branch
+ -> flow_connector.decode_velocity_from_hidden
+ -> qwen2_mot.collect_velocity
+ -> flow_connector.advance_denoise
+```
+
+Each step moves the tail `output` row through `BAGEL_PHASE_KEY` values
+`flow_query` → `flow_hidden` → `flow_velocity`. When the denoise state emits
+`image_complete`, the row is re-tagged `generated_latent` and
+`bagel_vae.decode_generated` decodes it.
+
+### 4.3 Image edit
+
+```bash
+python tasks/omni/infer_omni.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \
+ --model.model_config.infer_type infer_edit \
+ --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \
+ --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \
+ --infer.images /path/to/source.jpg \
+ --infer.prompt "Make it look like a snowy evening." \
+ --infer.output_dir bagel_out
+```
+
+Edit first builds context from both image branches: VAE encodes the
+`BAGEL_VAE_CONTEXT` copy of the edit image, SigLIP-NaViT encodes the
+`BAGEL_SIGLIP_CONTEXT` copy as visual
+context, and `flow_connector.embed_context_latents` projects the VAE context into
+the denoise prompt. The downstream denoise loop is shared with `infer_gen`.
+
+---
+
+## 5. Visualize the graphs
+
+```bash
+python scripts/seed_omni/visualize_graph.py \
+ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml
+# -> graphs/bagel_7b_mot_base/{training,infer_edit,infer_gen,infer_und}.mmd
+```
+
+The visualized graphs are generated from the same config loader path as training
+and inference, so regenerate them after changing module entrypoints or graph
+YAML.
+
+---
+
+## 6. Contract checks
+
+The Bagel module and graph contracts cover carrier context-tag routing, generation
+state transitions, packing/cache behavior, and graph config structure. Packed
+MoT also compares Flex and Magi against eager SDPA on toy CE / MSE / gradients
+in `tests/seed_omni/modules/bagel/test_bagel_accel_align.py`. Magi cases skip unless an
+NVIDIA SM100+ GPU with the CUTE DSL/JIT backend is present.
+
+```bash
+.venv/bin/python -m pytest -q tests/seed_omni/modules/bagel
+```
diff --git a/docs/seed_omni/usage/training_and_inference.md b/docs/seed_omni/usage/training_and_inference.md
index 17a9bfc963..9d28cf60c4 100644
--- a/docs/seed_omni/usage/training_and_inference.md
+++ b/docs/seed_omni/usage/training_and_inference.md
@@ -206,7 +206,9 @@ flowchart TB
`generate()` always lives on the native `modeling.py`, so an all-eager Inferencer
run builds the same object a native user gets from `OmniModel.from_pretrained`;
-the launcher only projects the YAML onto `OmniConfig` first.
+the launcher only projects the YAML onto `OmniConfig` first. When the checkpoint
+declares `generation_kwargs` defaults, `infer.generation_kwargs` overrides them
+per key rather than replacing the bag.
Inference does not use sequence or context parallelism: keep `ulysses_size` and
`cp_size` at 1 on every module. The generation loop decodes one token at a time
diff --git a/tasks/omni/infer_omni_native.py b/tasks/omni/infer_omni_native.py
index ef43d4d1f2..776cc04b00 100644
--- a/tasks/omni/infer_omni_native.py
+++ b/tasks/omni/infer_omni_native.py
@@ -77,7 +77,7 @@ def main() -> None:
torch_dtype="auto",
**scenario,
).eval()
- processor = OmniProcessor.from_pretrained(args.model_path)
+ processor = OmniProcessor.from_pretrained(args.model_path, **scenario)
model_input = processor(
text=args.prompt,
images=args.image or None,
diff --git a/tests/ops/async_ulysses/test_async_ulysses.py b/tests/ops/async_ulysses/test_async_ulysses.py
index e3d2931caf..0e1ca4a86c 100644
--- a/tests/ops/async_ulysses/test_async_ulysses.py
+++ b/tests/ops/async_ulysses/test_async_ulysses.py
@@ -42,6 +42,10 @@
("async_ulysses_o", "standard"),
("async_ulysses_o", "dit"),
)
+_BAGEL_EAGER_ROWS = (
+ ("async_ulysses_qkv", "bagel"),
+ ("async_ulysses_o", "bagel"),
+)
def test_linear_backward_matches_autograd_under_cpu_bf16_autocast():
@@ -134,7 +138,7 @@ def fake_pad(tensor: Tensor, dim, **kwargs):
def fake_unpad(tensor: Tensor, dim, size, **kwargs):
return tensor
- for op, variant in _EAGER_ROWS:
+ for op, variant in (*_EAGER_ROWS, *_BAGEL_EAGER_ROWS):
module = _eager_module(op, variant)
monkeypatch.setattr(module, "all_to_all_tensor", fake_all_to_all)
monkeypatch.setattr(module, "padding_tensor_for_seqeunce_parallel", fake_pad)
@@ -722,3 +726,193 @@ def test_dit_parity_world_size_matches_skip_gate():
assert DIT_SP_WORLD_SIZE == 4
assert AsyncUlyssesDiTSequenceParallelTest.world_size.fget(object()) == DIT_SP_WORLD_SIZE
assert callable(AsyncUlyssesDiTSequenceParallelTest._configure_repro)
+
+
+@pytest.mark.parametrize("empty_route", [None, 0, 1])
+def test_bagel_qkv_and_o_match_sequential_and_use_nested_handles(
+ monkeypatch: pytest.MonkeyPatch,
+ empty_route: int | None,
+) -> None:
+ """Verify two-route BAGEL async Ulysses QKV and O projections match sequential execution."""
+ _mock_identity_comm(monkeypatch)
+ torch.manual_seed(6201)
+ seq, hidden, num_q_heads, num_kv_heads, head_dim = 6, 16, 4, 2, 4
+ query_size = num_q_heads * head_dim
+ kv_size = num_kv_heads * head_dim
+ eps = 1e-6
+
+ if empty_route == 0:
+ route_0_indices = torch.empty(0, dtype=torch.long)
+ route_1_indices = torch.arange(seq, dtype=torch.long)
+ elif empty_route == 1:
+ route_0_indices = torch.arange(seq, dtype=torch.long)
+ route_1_indices = torch.empty(0, dtype=torch.long)
+ else:
+ route_0_indices = torch.tensor([0, 2, 4], dtype=torch.long)
+ route_1_indices = torch.tensor([1, 3, 5], dtype=torch.long)
+
+ angles = torch.randn(seq, head_dim // 2, dtype=torch.float64)
+ cos = torch.cat((angles.cos(), angles.cos()), dim=-1)
+ sin = torch.cat((angles.sin(), angles.sin()), dim=-1)
+
+ hidden_states = torch.randn(seq, hidden, dtype=torch.float64, requires_grad=True)
+ weights_0 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64)
+ weights_1 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64)
+ norm_q_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True)
+ norm_k_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True)
+ norm_q_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True)
+ norm_k_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True)
+ o_weight_0 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True)
+ o_bias_0 = torch.randn(hidden, dtype=torch.float64, requires_grad=True)
+ o_weight_1 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True)
+ o_bias_1 = torch.randn(hidden, dtype=torch.float64, requires_grad=True)
+
+ diff_tensors = (
+ hidden_states,
+ *weights_0,
+ norm_q_0,
+ norm_k_0,
+ *weights_1,
+ norm_q_1,
+ norm_k_1,
+ o_weight_0,
+ o_bias_0,
+ o_weight_1,
+ o_bias_1,
+ )
+ actual_leaves = _leaf(*diff_tensors)
+ expected_leaves = _leaf(*diff_tensors)
+
+ (
+ h_a,
+ qw0_a,
+ qb0_a,
+ kw0_a,
+ kb0_a,
+ vw0_a,
+ vb0_a,
+ nq0_a,
+ nk0_a,
+ qw1_a,
+ qb1_a,
+ kw1_a,
+ kb1_a,
+ vw1_a,
+ vb1_a,
+ nq1_a,
+ nk1_a,
+ ow0_a,
+ ob0_a,
+ ow1_a,
+ ob1_a,
+ ) = actual_leaves
+ q_a, k_a, v_a = VeomniOp("async_ulysses_qkv", "bagel")(
+ h_a,
+ route_0_indices,
+ route_1_indices,
+ cos,
+ sin,
+ qw0_a,
+ qb0_a,
+ kw0_a,
+ kb0_a,
+ vw0_a,
+ vb0_a,
+ nq0_a,
+ nk0_a,
+ qw1_a,
+ qb1_a,
+ kw1_a,
+ kb1_a,
+ vw1_a,
+ vb1_a,
+ nq1_a,
+ nk1_a,
+ head_dim=head_dim,
+ rms_norm_eps=eps,
+ unpadded_dim_size=seq,
+ group=object(),
+ rms_norm=VeomniOp("rms_norm", "standard", "eager"),
+ rope=VeomniOp("rope", "full", "eager"),
+ )
+ attn_a = (
+ q_a
+ + k_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1)
+ + v_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1)
+ )
+ out_a = VeomniOp("async_ulysses_o", "bagel")(
+ attn_a,
+ route_0_indices,
+ route_1_indices,
+ ow0_a,
+ ob0_a,
+ ow1_a,
+ ob1_a,
+ group=object(),
+ )
+
+ (
+ h_e,
+ qw0_e,
+ qb0_e,
+ kw0_e,
+ kb0_e,
+ vw0_e,
+ vb0_e,
+ nq0_e,
+ nk0_e,
+ qw1_e,
+ qb1_e,
+ kw1_e,
+ kb1_e,
+ vw1_e,
+ vb1_e,
+ nq1_e,
+ nk1_e,
+ ow0_e,
+ ob0_e,
+ ow1_e,
+ ob1_e,
+ ) = expected_leaves
+ rms = VeomniOp("rms_norm", "standard", "eager")
+ rope = VeomniOp("rope", "full", "eager")
+
+ def routed_qkv(w0, b0, w1, b1, heads, n0=None, n1=None):
+ p0 = F.linear(h_e.index_select(0, route_0_indices), w0, b0).reshape(-1, heads, head_dim)
+ p1 = F.linear(h_e.index_select(0, route_1_indices), w1, b1).reshape(-1, heads, head_dim)
+ if n0 is not None and n1 is not None:
+ p0 = rms(p0, n0, eps=eps)
+ p1 = rms(p1, n1, eps=eps)
+ combined = p0.new_zeros((seq, heads, head_dim))
+ combined = combined.index_copy(0, route_0_indices, p0)
+ combined = combined.index_copy(0, route_1_indices, p1)
+ return combined
+
+ q_e = routed_qkv(qw0_e, qb0_e, qw1_e, qb1_e, num_q_heads, nq0_e, nq1_e)
+ k_e = routed_qkv(kw0_e, kb0_e, kw1_e, kb1_e, num_kv_heads, nk0_e, nk1_e)
+ v_e = routed_qkv(vw0_e, vb0_e, vw1_e, vb1_e, num_kv_heads)
+ q_e, k_e = rope(
+ q_e.transpose(0, 1).unsqueeze(0),
+ k_e.transpose(0, 1).unsqueeze(0),
+ cos.unsqueeze(0),
+ sin.unsqueeze(0),
+ unsqueeze_dim=1,
+ )
+ q_e = q_e.squeeze(0).transpose(0, 1).contiguous()
+ k_e = k_e.squeeze(0).transpose(0, 1).contiguous()
+ attn_e = (
+ q_e
+ + k_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1)
+ + v_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1)
+ )
+ flat_e = attn_e.reshape(seq, query_size)
+ out_e = flat_e.new_zeros((seq, hidden))
+ out_e = out_e.index_copy(0, route_0_indices, F.linear(flat_e.index_select(0, route_0_indices), ow0_e, ob0_e))
+ out_e = out_e.index_copy(0, route_1_indices, F.linear(flat_e.index_select(0, route_1_indices), ow1_e, ob1_e))
+
+ torch.testing.assert_close(out_a, out_e)
+ grad_out = torch.randn_like(out_a)
+ out_a.backward(grad_out)
+ out_e.backward(grad_out)
+ for actual, expected in zip(actual_leaves, expected_leaves, strict=True):
+ torch.testing.assert_close(actual.grad, expected.grad)
diff --git a/tests/ops/base/test_builtin_entries.py b/tests/ops/base/test_builtin_entries.py
index c981ab5e4e..c849cf76f4 100644
--- a/tests/ops/base/test_builtin_entries.py
+++ b/tests/ops/base/test_builtin_entries.py
@@ -53,8 +53,10 @@
("moe_unpermute", "standard", "triton", "cuda", _GPU_NVIDIA_SM70, ("triton",)),
("async_ulysses_qkv", "standard", "eager", "any", _ANY, ()),
("async_ulysses_qkv", "dit", "eager", "any", _ANY, ()),
+ ("async_ulysses_qkv", "bagel", "eager", "any", _ANY, ()),
("async_ulysses_o", "standard", "eager", "any", _ANY, ()),
("async_ulysses_o", "dit", "eager", "any", _ANY, ()),
+ ("async_ulysses_o", "bagel", "eager", "any", _ANY, ()),
("attention", "standard", "eager", "any", _ANY, ()),
("attention", "standard", "sdpa", "any", _ANY, ()),
("attention", "standard", "flash_attention_2", "cuda", _GPU_SM80, ("flash_attn",)),
diff --git a/tests/seed_omni/arguments/test_module_args.py b/tests/seed_omni/arguments/test_module_args.py
index 9de23dd8a4..21c9c6546a 100644
--- a/tests/seed_omni/arguments/test_module_args.py
+++ b/tests/seed_omni/arguments/test_module_args.py
@@ -509,6 +509,39 @@ def test_a_launcher_infer_graph_does_not_inherit_the_checkpoint_infer_type(tmp_p
assert build_omni_model_runtime_args(args, for_inference=True).infer_type == "understanding"
+def test_build_model_runtime_args_merges_checkpoint_generation_kwargs(tmp_path):
+ """Checkpoint ``generation_kwargs`` supply the defaults and ``infer.generation_kwargs`` overrides per key,
+ even when the launcher YAML also declares ``model.model_config.infer_graph``."""
+ infer_dir = _cfg_dir() / "infer"
+ runtime_cfg = _model_runtime(
+ model_path=str(tmp_path),
+ infer_graph={"infer_und": str(infer_dir / "graph_infer_und.yaml")},
+ infer_type="infer_und",
+ )
+ runtime_cfg.generation_kwargs = {"num_timesteps": 50, "timestep_shift": 3.0}
+ export_root = tmp_path / "exported"
+ _with_module_configs(runtime_cfg.to_hf_config()).save_pretrained(export_root)
+
+ args = OmniArguments(
+ model=OmniModelRuntimeArguments(model_path=str(export_root)),
+ data=OmniDataArguments(train_path=""),
+ infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}),
+ )
+ resolved = build_omni_model_runtime_args(args, for_inference=True)
+ assert resolved.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0}
+
+ args_with_launcher_graph = OmniArguments(
+ model=OmniModelRuntimeArguments(
+ model_path=str(export_root),
+ model_config={"infer_graph": {"understanding": str(infer_dir / "graph_infer_und.yaml")}},
+ ),
+ data=OmniDataArguments(train_path=""),
+ infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}),
+ )
+ resolved_with_launcher_graph = build_omni_model_runtime_args(args_with_launcher_graph, for_inference=True)
+ assert resolved_with_launcher_graph.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0}
+
+
def test_build_model_runtime_args_keeps_every_training_scenario_from_omni_checkpoint(tmp_path):
"""The checkpoint stores the whole ``training_graphs`` map and its ``train_type``;
reading back only the active graph would rename it ``default`` and drop the rest."""
diff --git a/tests/seed_omni/model/test_processor.py b/tests/seed_omni/model/test_processor.py
index d14ec82a2c..8b69afe393 100644
--- a/tests/seed_omni/model/test_processor.py
+++ b/tests/seed_omni/model/test_processor.py
@@ -155,6 +155,7 @@ def test_omni_processor_from_pretrained_collects_module_preprocessors(
mock_from_pretrained.assert_called_once_with(tmp_path, infer_type="infer_gen")
fake_mod_cls.preprocessor_class.from_pretrained.assert_called_once()
assert len(processor._preprocessors) == 1
+ assert processor.infer_type == "infer_gen"
@patch("veomni.models.seed_omni.processing_omni.OMNI_MODEL_REGISTRY")
diff --git a/tests/seed_omni/modules/bagel/__init__.py b/tests/seed_omni/modules/bagel/__init__.py
new file mode 100644
index 0000000000..54ca978df6
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/__init__.py
@@ -0,0 +1 @@
+"""BAGEL SeedOmni tests."""
diff --git a/tests/seed_omni/modules/bagel/helpers.py b/tests/seed_omni/modules/bagel/helpers.py
new file mode 100644
index 0000000000..59fcb3239d
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/helpers.py
@@ -0,0 +1,382 @@
+"""Shared helpers for BAGEL tests."""
+
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+from typing import Any, Literal
+
+import torch
+import torch.nn.functional as F
+
+from veomni.arguments.omni_arguments_types import (
+ OmniArguments,
+ OmniDataArguments,
+ OmniInferArguments,
+ OmniModelRuntimeArguments,
+ build_omni_model_runtime_args,
+ build_omni_module_runtime_args,
+)
+from veomni.models.seed_omni import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_MODEL_REGISTRY
+from veomni.models.seed_omni.configuration_omni import OmniConfig
+from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import PackedConversation, preprocess_mot_inputs
+from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT
+from veomni.models.seed_omni.modules.bagel.text_encoder.chat_template import BagelChatTemplate
+from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem
+
+
+ToyCase = Literal["ce_only", "vit_ce", "mse_only", "mixed"]
+
+ALIGN_ATOL = 2e-2
+ALIGN_RTOL = 2e-2
+ALIGN_GRAD_ATOL = 5e-2
+ALIGN_GRAD_RTOL = 5e-2
+
+
+class FakeBagelTokenizer:
+ """Char-ordinal tokenizer with the marker tokens/ids that ``BagelChatTemplate`` resolves."""
+
+ bos_token = ""
+ eos_token = ""
+ boi_token = ""
+ eoi_token = ""
+ bos_token_id = 1
+ eos_token_id = 2
+ boi_token_id = 3
+ eoi_token_id = 4
+ pad_token_id = 0
+ unk_token_id = -1
+ special_tokens = {
+ "": 1,
+ "": 2,
+ "": 3,
+ "": 4,
+ "<|im_start|>": 5,
+ "<|im_end|>": 2,
+ "<|vision_start|>": 7,
+ "<|vision_end|>": 8,
+ }
+
+ def convert_tokens_to_ids(self, token):
+ return self.special_tokens.get(token, self.unk_token_id)
+
+ def decode(self, token_ids, skip_special_tokens=True):
+ del skip_special_tokens
+ special_by_id = {v: k for k, v in self.special_tokens.items()}
+ return "".join(special_by_id.get(int(t), chr(max(32, int(t)))) for t in token_ids)
+
+ def __call__(self, text, add_special_tokens=False):
+ del add_special_tokens
+ input_ids = []
+ index = 0
+ while index < len(text):
+ matched = False
+ for token, token_id in sorted(self.special_tokens.items(), key=lambda item: len(item[0]), reverse=True):
+ if text.startswith(token, index):
+ input_ids.append(token_id)
+ index += len(token)
+ matched = True
+ break
+ if matched:
+ continue
+ input_ids.append(ord(text[index]))
+ index += 1
+ return {"input_ids": input_ids}
+
+
+def bagel_template() -> BagelChatTemplate:
+ return BagelChatTemplate(FakeBagelTokenizer())
+
+
+def bagel_cfg_dir() -> Path:
+ return Path(__file__).resolve().parents[4] / "configs" / "seed_omni" / "Bagel" / "bagel_7b_mot"
+
+
+def config_cls(model_type: str):
+ return OMNI_CONFIG_REGISTRY[model_type]()
+
+
+def model_cls(model_type: str):
+ """Return the VeOmni-accelerated module class (training / graph hooks)."""
+ return OMNI_ACCELERATED_MODEL_REGISTRY[model_type]()
+
+
+def native_model_cls(model_type: str):
+ """Return the HF-native module class (eager inference weights)."""
+ return OMNI_MODEL_REGISTRY[model_type]()
+
+
+def load_module_runtime_args(
+ *,
+ model_path: str = "",
+ modules_path: Path,
+ for_inference: bool = False,
+) -> dict:
+ """Build per-module ``OmniModuleRuntimeArguments`` from ``modules_path``."""
+ base = OmniArguments(
+ model=OmniModelRuntimeArguments(
+ model_path=model_path or ".",
+ model_config={"modules": str(modules_path)},
+ ),
+ data=OmniDataArguments(train_path=""),
+ )._to_module_global_args()
+ return build_omni_module_runtime_args(
+ global_args=base,
+ model_path=model_path,
+ modules=str(modules_path),
+ for_inference=for_inference,
+ )
+
+
+def load_omni_config(
+ *,
+ model_path: str = "",
+ modules_path: Path,
+ train_graph_path: Path | None = None,
+ infer_graph_path: Path | None = None,
+ generation_kwargs: dict | None = None,
+) -> OmniConfig:
+ """Resolve and project an ``OmniConfig`` from BAGEL module and graph YAML files."""
+ model_path = model_path or "."
+ model_config = {"modules": str(modules_path)}
+ if train_graph_path is not None:
+ model_config["train_graph"] = str(train_graph_path)
+ if infer_graph_path is not None:
+ scenario = infer_graph_path.stem.removeprefix("graph_")
+ model_config["infer_graph"] = {scenario: str(infer_graph_path)}
+ model_config["infer_type"] = scenario
+ args = OmniArguments(
+ model=OmniModelRuntimeArguments(
+ model_path=model_path,
+ model_config=model_config,
+ ),
+ data=OmniDataArguments(train_path="."),
+ infer=OmniInferArguments(generation_kwargs=dict(generation_kwargs or {})),
+ )
+ return build_omni_model_runtime_args(args).to_hf_config()
+
+
+def tiny_bagel_qwen2_cfg() -> dict:
+ return dict(
+ vocab_size=128,
+ hidden_size=64,
+ intermediate_size=128,
+ num_hidden_layers=1,
+ num_attention_heads=4,
+ num_key_value_heads=4,
+ max_position_embeddings=64,
+ bos_token_id=1,
+ eos_token_id=2,
+ ops_implementation={
+ "attn_implementation": "sdpa",
+ "rms_norm_implementation": "eager",
+ "swiglu_mlp_implementation": "eager",
+ "rotary_pos_emb_implementation": "eager",
+ },
+ )
+
+
+def tiny_align_qwen2_cfg() -> dict:
+ config = tiny_bagel_qwen2_cfg()
+ config["num_key_value_heads"] = 2
+ return config
+
+
+def _clone_meta(meta: dict[str, Any]) -> dict[str, Any]:
+ """Deep-copy a ``ConversationItem.meta`` dict while detaching and cloning tensor values."""
+ cloned: dict[str, Any] = {}
+ for key, value in meta.items():
+ cloned[key] = value.detach().clone() if torch.is_tensor(value) else copy.deepcopy(value)
+ return cloned
+
+
+def clone_conversation(conversation: list[list[ConversationItem]]) -> list[list[ConversationItem]]:
+ """Deep-copy a packed conversation so eager and accelerated can mutate independently."""
+ cloned: list[list[ConversationItem]] = []
+ for sample in conversation:
+ cloned_sample: list[ConversationItem] = []
+ for item in sample:
+ value = item.value
+ if torch.is_tensor(value):
+ requires_grad = value.requires_grad
+ value = value.detach().clone()
+ if requires_grad:
+ value = value.requires_grad_()
+ cloned_sample.append(
+ ConversationItem(
+ type=item.type,
+ value=value,
+ role=item.role,
+ is_dummy=item.is_dummy,
+ meta=_clone_meta(item.meta),
+ )
+ )
+ cloned.append(cloned_sample)
+ return cloned
+
+
+def _embed(
+ generator: torch.Generator,
+ length: int,
+ hidden_size: int,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+ requires_grad: bool = True,
+) -> torch.Tensor:
+ """Generate a random ``(length, hidden_size)`` leaf embedding tensor."""
+ value = torch.randn(length, hidden_size, generator=generator, device=device, dtype=dtype)
+ if requires_grad:
+ value = value.requires_grad_()
+ return value
+
+
+def build_toy_conversation(
+ case: ToyCase,
+ *,
+ hidden_size: int,
+ vocab_size: int,
+ patch_latent_dim: int,
+ device: torch.device,
+ dtype: torch.dtype,
+ seed: int = 17,
+) -> list[list[ConversationItem]]:
+ """Build one packed sample covering CE-only, vit-CE, MSE-only, or mixed."""
+ generator = torch.Generator(device=device).manual_seed(seed)
+ items: list[ConversationItem] = []
+
+ if case in {"vit_ce", "mixed"}:
+ items.append(
+ ConversationItem(
+ type="image",
+ value=_embed(generator, 4, hidden_size, device=device, dtype=dtype),
+ role="user",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "und"},
+ )
+ )
+
+ if case in {"ce_only", "vit_ce", "mixed"}:
+ labels = torch.randint(3, vocab_size, (8,), generator=generator, device=device)
+ items.append(
+ ConversationItem(
+ type="text",
+ value=_embed(generator, 8, hidden_size, device=device, dtype=dtype),
+ role="assistant",
+ meta={"labels": labels},
+ )
+ )
+
+ if case in {"mse_only", "mixed"}:
+ items.append(
+ ConversationItem(
+ type="image",
+ value=_embed(generator, 6, hidden_size, device=device, dtype=dtype),
+ role="assistant",
+ meta={
+ BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT,
+ _IMG_TAG_KEY: "gen",
+ "flow_velocity_target": torch.randn(
+ 6,
+ patch_latent_dim,
+ generator=generator,
+ device=device,
+ dtype=dtype,
+ ),
+ },
+ )
+ )
+
+ return [items]
+
+
+def scatter_mot_hidden(
+ conversation: list[list[ConversationItem]],
+ packed: PackedConversation,
+ hidden_states: torch.Tensor,
+) -> None:
+ """Write packed MoT hidden states back onto the original carrier items."""
+ for span in packed.spans:
+ span_hidden = hidden_states[span.start : span.start + span.length]
+ offset = 0
+ for item, length in zip(span.items, span.lengths, strict=True):
+ item.value = span_hidden[offset : offset + length]
+ offset += length
+
+
+def run_eager_mot(
+ model: torch.nn.Module,
+ conversation: list[list[ConversationItem]],
+) -> torch.Tensor:
+ """Pack ``conversation``, run eager ``BagelQwen2MoT``, and scatter hidden states back."""
+ packed = preprocess_mot_inputs(
+ conversation,
+ device=model.device,
+ dtype=model.dtype,
+ hidden_size=int(model.config.hidden_size),
+ )
+ if packed is None:
+ raise ValueError("toy conversation produced no packable tokens")
+ outputs = model(
+ packed_sequence=packed.packed_sequence,
+ packed_position_ids=packed.packed_position_ids,
+ packed_token_type_ids=packed.packed_token_type_ids,
+ packed_attention_metadata=packed.packed_attention_metadata,
+ )
+ hidden_states = outputs["hidden_states"]
+ scatter_mot_hidden(conversation, packed, hidden_states)
+ return hidden_states
+
+
+def run_accelerated_mot(
+ model: torch.nn.Module,
+ conversation: list[list[ConversationItem]],
+) -> torch.Tensor:
+ inputs = model.forward_pre(conversation_list=conversation)
+ outputs = model(**inputs)
+ hidden_states = outputs["hidden_states"]
+ model.forward_post(**outputs)
+ return hidden_states
+
+
+def conversation_ce_loss(
+ conversation: list[list[ConversationItem]],
+ lm_head: torch.nn.Linear,
+) -> torch.Tensor | None:
+ """Compute shifted causal LM cross-entropy loss over text items in ``conversation``."""
+ hidden_parts: list[torch.Tensor] = []
+ label_parts: list[torch.Tensor] = []
+ for item in (item for sample in conversation for item in sample):
+ labels = item.meta.get("labels")
+ if item.type != "text" or not torch.is_tensor(item.value) or not torch.is_tensor(labels):
+ continue
+ hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value
+ hidden_parts.append(hidden)
+ label_parts.append(labels.to(device=hidden.device))
+ if not hidden_parts:
+ return None
+ hidden = torch.cat(hidden_parts, dim=0)
+ labels = torch.cat(label_parts, dim=0)
+ shift_labels = F.pad(labels[..., 1:].contiguous(), (0, 1), value=-100)
+ logits = lm_head(hidden.float())
+ return F.cross_entropy(logits, shift_labels, ignore_index=-100)
+
+
+def conversation_mse_loss(
+ conversation: list[list[ConversationItem]],
+ llm2vae: torch.nn.Linear,
+) -> torch.Tensor | None:
+ """Compute flow-matching MSE velocity loss over target image items in ``conversation``."""
+ velocity_parts: list[torch.Tensor] = []
+ target_parts: list[torch.Tensor] = []
+ for item in (item for sample in conversation for item in sample):
+ target = item.meta.get("flow_velocity_target")
+ if not torch.is_tensor(item.value) or not torch.is_tensor(target):
+ continue
+ hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value
+ velocity_parts.append(llm2vae(hidden.float()))
+ target_parts.append(target.to(device=hidden.device, dtype=torch.float32))
+ if not velocity_parts:
+ return None
+ mse = (torch.cat(velocity_parts, dim=0) - torch.cat(target_parts, dim=0)).square()
+ return mse.mean(dim=-1).mean()
diff --git a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py
new file mode 100644
index 0000000000..b3fe186308
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py
@@ -0,0 +1,261 @@
+"""Accelerated BAGEL Qwen2-MoT alignment: single-GPU eager parity and 4-GPU FSDP2 + Async Ulysses."""
+
+from __future__ import annotations
+
+import copy
+import importlib.util
+from dataclasses import replace
+
+import pytest
+import torch
+import torch.distributed as dist
+from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard
+
+from scripts.ci.accelerator.launch_plan import LaunchSpec
+from tests.seed_omni.modules.bagel.helpers import (
+ ALIGN_ATOL,
+ ALIGN_GRAD_ATOL,
+ ALIGN_GRAD_RTOL,
+ ALIGN_RTOL,
+ build_toy_conversation,
+ clone_conversation,
+ config_cls,
+ conversation_ce_loss,
+ conversation_mse_loss,
+ native_model_cls,
+ run_accelerated_mot,
+ run_eager_mot,
+ tiny_align_qwen2_cfg,
+)
+from tests.tools.launch_utils import torchrun
+from veomni.distributed.parallel_state import _init_parallel_state, use_parallel_state
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import (
+ build_mot_attention_metadata,
+ pad_mot_attention_metadata,
+)
+from veomni.ops.kernels.attention.standard import flex as flex_backend
+from veomni.ops.kernels.attention.standard.magi import _kernel as magi_kernel
+from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type, get_torch_device
+from veomni.utils.helper import enable_high_precision_for_bf16
+
+
+_PATCH_LATENT_DIM = 4
+_MAGI_IMPLEMENTATION = "veomni_magi_attention"
+_FLEX_IMPLEMENTATION = "veomni_flex_attention"
+
+
+def _is_magi_ffa_available() -> bool:
+ """Return whether a runnable MagiAttention CUTE DSL/JIT backend is installed on this GPU."""
+ if not IS_CUDA_AVAILABLE or importlib.util.find_spec("magi_attention") is None:
+ return False
+
+ device = torch.device(get_device_type())
+ return magi_kernel.get_kernel_mode(device) == magi_kernel.KERNEL_CUTE_JIT and (
+ importlib.util.find_spec("flash_attn_cute") is not None
+ )
+
+
+_MAGI_FFA_AVAILABLE = _is_magi_ffa_available()
+_MAGI_FFA_REASON = "MagiAttention alignment requires an NVIDIA SM100+ GPU with the CUTE DSL/JIT backend"
+
+
+def _build_aligned_models(device: torch.device, dtype: torch.dtype, *, attn_implementation: str):
+ torch.manual_seed(29)
+ cfg = tiny_align_qwen2_cfg()
+ cfg.pop("ops_implementation", None)
+ eager_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation="sdpa")
+ accel_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation=attn_implementation)
+ eager = native_model_cls("bagel_qwen2_mot")(eager_cfg).to(device=device, dtype=dtype).train()
+ accel = BagelQwen2MoTAccelerated(accel_cfg).to(device=device, dtype=dtype).train()
+ accel.load_state_dict(copy.deepcopy(eager.state_dict()))
+ return eager, accel
+
+
+def _assert_accelerated_matches_eager(attn_implementation: str) -> None:
+ """Verify ``BagelQwen2MoTAccelerated`` matches eager SDPA outputs, CE/MSE losses, and gradients on mixed data."""
+ device = torch.device(get_device_type())
+ dtype = torch.bfloat16
+ eager, accel = _build_aligned_models(device, dtype, attn_implementation=attn_implementation)
+ hidden_size = int(eager.config.hidden_size)
+ vocab_size = int(eager.config.vocab_size)
+ conversation = build_toy_conversation(
+ "mixed",
+ hidden_size=hidden_size,
+ vocab_size=vocab_size,
+ patch_latent_dim=_PATCH_LATENT_DIM,
+ device=device,
+ dtype=dtype,
+ )
+ eager_conversation = clone_conversation(conversation)
+ accel_conversation = clone_conversation(conversation)
+ lm_head = torch.nn.Linear(hidden_size, vocab_size, bias=False).to(device=device, dtype=torch.float32)
+ llm2vae = torch.nn.Linear(hidden_size, _PATCH_LATENT_DIM, bias=False).to(device=device, dtype=torch.float32)
+
+ eager_hidden = run_eager_mot(eager, eager_conversation)
+ accel_hidden = run_accelerated_mot(accel, accel_conversation)
+ assert torch.isfinite(eager_hidden).all() and torch.isfinite(accel_hidden).all()
+ assert eager_hidden.shape == accel_hidden.shape
+
+ eager_ce = conversation_ce_loss(eager_conversation, lm_head)
+ accel_ce = conversation_ce_loss(accel_conversation, lm_head)
+ eager_mse = conversation_mse_loss(eager_conversation, llm2vae)
+ accel_mse = conversation_mse_loss(accel_conversation, llm2vae)
+ assert eager_ce is not None and accel_ce is not None
+ assert eager_mse is not None and accel_mse is not None
+ torch.testing.assert_close(accel_ce, eager_ce, atol=ALIGN_ATOL, rtol=ALIGN_RTOL)
+ torch.testing.assert_close(accel_mse, eager_mse, atol=ALIGN_ATOL, rtol=ALIGN_RTOL)
+
+ eager_loss = eager_hidden.float().square().mean() + eager_ce + eager_mse
+ accel_loss = accel_hidden.float().square().mean() + accel_ce + accel_mse
+ eager_loss.backward()
+ accel_loss.backward()
+
+ for name in (
+ "model.layers.0.self_attn.o_proj.weight",
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.norm.weight",
+ ):
+ eager_grad = dict(eager.named_parameters())[name].grad
+ accel_grad = dict(accel.named_parameters())[name].grad
+ assert eager_grad is not None and accel_grad is not None
+ torch.testing.assert_close(
+ accel_grad,
+ eager_grad,
+ atol=ALIGN_GRAD_ATOL,
+ rtol=ALIGN_GRAD_RTOL,
+ msg=f"gradient mismatch for {name}",
+ )
+
+
+@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1))
+@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="FlexAttention alignment requires CUDA")
+def test_accelerated_ce_mse_matches_eager_on_toy_data() -> None:
+ _assert_accelerated_matches_eager(_FLEX_IMPLEMENTATION)
+
+
+@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1))
+@pytest.mark.skipif(not _MAGI_FFA_AVAILABLE, reason=_MAGI_FFA_REASON)
+def test_magi_ce_mse_matches_eager_on_toy_data() -> None:
+ _assert_accelerated_matches_eager(_MAGI_IMPLEMENTATION)
+
+
+def _bagel_fa4_async_worker() -> None:
+ """Compare synchronous and asynchronous Ulysses in ``BagelQwen2MoTAccelerated`` under FSDP2 + Flex-FA4 over 2 steps."""
+ rank = dist.get_rank()
+ device = torch.device(f"{get_device_type()}:{rank}")
+ dtype = torch.bfloat16
+ torch.manual_seed(6321)
+ enable_high_precision_for_bf16()
+ state = _init_parallel_state(dp_size=1, ulysses_size=4, name="bagel_sync_test")
+ async_state = replace(state, async_enabled=True)
+
+ # SP4 leaves seven Q heads and one KV head per rank, matching BAGEL's
+ # non-power-of-two GQA geometry while exercising FA4 + Liger + checkpointing + FSDP2.
+ config = BagelQwen2MoTConfig(
+ vocab_size=128,
+ hidden_size=896,
+ intermediate_size=1024,
+ num_hidden_layers=1,
+ num_attention_heads=28,
+ num_key_value_heads=4,
+ max_position_embeddings=1024,
+ bos_token_id=1,
+ eos_token_id=2,
+ ops_implementation={
+ "attn_implementation": "flex_attention",
+ "rms_norm_implementation": "liger_kernel",
+ "rotary_pos_emb_implementation": "liger_kernel",
+ "swiglu_mlp_implementation": "eager",
+ },
+ )
+ reference = accelerated.BagelQwen2MoTAccelerated(config).to(device=device, dtype=dtype).train()
+ candidate = copy.deepcopy(reference)
+ for model in (reference, candidate):
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
+ model.float()
+ policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32)
+ for layer in model.model.layers:
+ fully_shard(layer, mesh=state.fsdp_mesh, mp_policy=policy)
+ fully_shard(model.model, mesh=state.fsdp_mesh, mp_policy=policy)
+
+ selected_backends = []
+ original_resolve = flex_backend.resolve_flex_attention
+
+ def capture_backend(*args, **kwargs):
+ interface = original_resolve(*args, **kwargs)
+ selected_backends.append(interface)
+ return interface
+
+ flex_backend.resolve_flex_attention = capture_backend
+ sequence_length, padded_length = 259, 260
+ metadata = build_mot_attention_metadata(
+ [[65, 64, 64], [33, 33]],
+ [["causal", "noise", "noise"], ["full", "causal"]],
+ device=device,
+ )
+ metadata = pad_mot_attention_metadata(metadata, padded_length=padded_length)
+ hidden = torch.randn(padded_length, config.hidden_size, device=device, dtype=dtype)
+ token_types = torch.zeros(padded_length, device=device, dtype=torch.long)
+ token_types[65:193] = 1
+ token_types[sequence_length:] = -1
+ hidden[sequence_length:] = 0
+ output_gradient = torch.randn_like(hidden)
+ output_gradient[sequence_length:] = 0
+ positions = torch.arange(padded_length, device=device)
+ start, stop = rank * 65, (rank + 1) * 65
+ inputs = {
+ "packed_position_ids": positions[start:stop],
+ "packed_attention_metadata": metadata,
+ "packed_und_token_indexes": torch.where(token_types[start:stop] == 0)[0],
+ "packed_gen_token_indexes": torch.where(token_types[start:stop] == 1)[0],
+ }
+ optimizers = [torch.optim.SGD(model.parameters(), lr=1e-3) for model in (reference, candidate)]
+ try:
+ for _ in range(2):
+ outputs, input_gradients = [], []
+ for model, model_state in ((reference, state), (candidate, async_state)):
+ local_input = hidden[start:stop].detach().clone().requires_grad_()
+ with use_parallel_state(model_state):
+ output = model.model(packed_sequence=local_input, **inputs).packed_query_sequence
+ output.backward(output_gradient[start:stop])
+ outputs.append(output.detach())
+ input_gradients.append(local_input.grad)
+ torch.testing.assert_close(outputs[1], outputs[0], atol=3e-2, rtol=3e-2)
+ torch.testing.assert_close(input_gradients[1], input_gradients[0], atol=3e-2, rtol=3e-2)
+ for (name, expected), (actual_name, actual) in zip(
+ reference.named_parameters(), candidate.named_parameters(), strict=True
+ ):
+ assert name == actual_name
+ assert expected.grad is not None and actual.grad is not None, name
+ expected_grad, actual_grad = expected.grad.to_local(), actual.grad.to_local()
+ assert torch.isfinite(actual_grad).all(), name
+ error = (actual_grad.float() - expected_grad.float()).norm()
+ scale = expected_grad.float().norm()
+ assert error <= 0.01 * scale + 1e-7, (
+ f"{name}: relative gradient L2 error {error / scale.clamp_min(1e-12)}"
+ )
+
+ for optimizer in optimizers:
+ optimizer.step()
+ optimizer.zero_grad(set_to_none=True)
+ assert selected_backends and all(
+ selected is flex_backend._flex_attention_fa4 for selected in selected_backends
+ )
+ finally:
+ flex_backend.resolve_flex_attention = original_resolve
+
+
+@pytest.mark.launch_plan(provider=LaunchSpec(nproc=4))
+@pytest.mark.skipif(
+ not IS_CUDA_AVAILABLE or get_torch_device().device_count() < 4,
+ reason="BAGEL FA4 + async Ulysses integration requires four SM90+ GPUs",
+)
+def test_bagel_fa4_async_matches_sync_forward_backward_and_updates() -> None:
+ """Verify BAGEL async Ulysses matches synchronous Ulysses under Flex-FA4, Liger, checkpointing, and FSDP2."""
+ device = torch.device(get_device_type())
+ if flex_backend.get_gpu_compute_capability(device) < 90 or not flex_backend._fa4._flash_attn_cute_available():
+ pytest.skip("Flex-FA4 integration requires SM90+ and flash-attn-4")
+ torchrun(_bagel_fa4_async_worker, world_size=4)
diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py
new file mode 100644
index 0000000000..ff15caea08
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/test_generation.py
@@ -0,0 +1,237 @@
+"""End-to-end BAGEL generation tests across ``infer_und``, ``infer_gen``, and ``infer_edit``."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import torch
+from PIL import Image
+
+from scripts.ci.accelerator.launch_plan import LaunchSpec
+from tests.seed_omni.modules.bagel.helpers import (
+ FakeBagelTokenizer,
+ bagel_cfg_dir,
+ bagel_template,
+ config_cls,
+ load_omni_config,
+ native_model_cls,
+ tiny_bagel_qwen2_cfg,
+)
+from veomni.models.seed_omni.modeling_omni import OmniModel
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated
+from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import (
+ BagelSiglipNavitPreprocessor,
+ BagelSiglipNavitProcessor,
+)
+from veomni.models.seed_omni.modules.bagel.sources import (
+ BAGEL_CONTEXT_KEY,
+ BAGEL_FLOW_HIDDEN,
+ BAGEL_FLOW_QUERY,
+ BAGEL_PHASE_KEY,
+ BAGEL_SIGLIP_CONTEXT,
+ BAGEL_START_TOKEN,
+ BAGEL_VAE_CONTEXT,
+)
+from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor
+from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEPreprocessor, BagelVAEProcessor
+from veomni.models.seed_omni.processing_omni import OmniProcessor
+from veomni.models.seed_omni.utils.conversation import ConversationItem
+
+
+pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu"))
+
+
+def _build_tiny_bagel_omni_model_and_processor(infer_graph_rel: str) -> tuple[OmniModel, OmniProcessor]:
+ """Build a tiny 5-module real BAGEL ``OmniModel`` and ``OmniProcessor`` for ``infer_graph_rel``."""
+ cfg = load_omni_config(
+ modules_path=bagel_cfg_dir() / "train/modules_train.yaml",
+ train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml",
+ infer_graph_path=bagel_cfg_dir() / infer_graph_rel,
+ )
+ cfg.training_graphs = {}
+
+ hidden_size = 8
+ qwen_kwargs = {
+ **tiny_bagel_qwen2_cfg(),
+ "hidden_size": hidden_size,
+ "intermediate_size": 16,
+ "num_attention_heads": 2,
+ "num_key_value_heads": 2,
+ }
+ text_encoder = native_model_cls("bagel_text_encoder")(
+ config_cls("bagel_text_encoder")(vocab_size=int(qwen_kwargs["vocab_size"]), hidden_size=hidden_size)
+ ).eval()
+ text_encoder._tokenizer = FakeBagelTokenizer()
+ text_encoder._chat_template = bagel_template()
+
+ siglip = native_model_cls("bagel_siglip_navit")(
+ config_cls("bagel_siglip_navit")(
+ hidden_size=hidden_size,
+ output_size=hidden_size,
+ intermediate_size=16,
+ num_hidden_layers=1,
+ num_attention_heads=2,
+ image_size=4,
+ patch_size=2,
+ rope=False,
+ ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"},
+ )
+ ).eval()
+
+ qwen_mot = native_model_cls("bagel_qwen2_mot")(config_cls("bagel_qwen2_mot")(**qwen_kwargs)).eval()
+
+ flow = native_model_cls("bagel_flow_connector")(
+ config_cls("bagel_flow_connector")(
+ hidden_size=hidden_size,
+ z_channels=2,
+ latent_patch_size=1,
+ patch_latent_dim=2,
+ max_latent_size=4,
+ timestep_frequency_embedding_size=4,
+ resolution=4,
+ )
+ ).eval()
+
+ vae_cfg = config_cls("bagel_vae")(
+ resolution=4,
+ ch=32,
+ ch_mult=[1],
+ num_res_blocks=1,
+ z_channels=2,
+ max_image_size=4,
+ min_image_size=4,
+ image_stride=2,
+ max_pixels=16,
+ downsample=1,
+ )
+ vae = native_model_cls("bagel_vae")(vae_cfg).eval()
+ vae._image_processor = BagelVAEProcessor.from_config(vae_cfg)
+
+ model = OmniModel(
+ cfg,
+ {
+ "bagel_text_encoder": text_encoder,
+ "bagel_siglip_navit": siglip,
+ "bagel_qwen2_mot": qwen_mot,
+ "bagel_flow_connector": flow,
+ "bagel_vae": vae,
+ },
+ ).eval()
+ preprocessors = {
+ "bagel_vae": BagelVAEPreprocessor(
+ BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16),
+ dtype=torch.float32,
+ ),
+ "bagel_siglip_navit": BagelSiglipNavitPreprocessor(
+ BagelSiglipNavitProcessor(
+ patch_size=2,
+ image_size=4,
+ min_image_size=2,
+ max_pixels=16,
+ vit_max_num_patch_per_side=2,
+ ),
+ dtype=torch.float32,
+ ),
+ "bagel_text_encoder": BagelTextEncoderPreprocessor(bagel_template()),
+ }
+ return model, OmniProcessor(preprocessors, infer_type=model.config.infer_type)
+
+
+def test_bagel_e2e_generation_across_und_gen_and_edit_scenarios(tmp_path: Path) -> None:
+ """Verify real 5-module BAGEL ``OmniProcessor`` + ``OmniModel.generate`` across ``infer_und``, ``infer_gen``, and ``infer_edit``."""
+ image_path = tmp_path / "sample.png"
+ Image.new("RGB", (4, 4), color=(12, 34, 56)).save(str(image_path))
+
+ # 1. infer_und: routes image to SigLIP only, appends BAGEL_START_TOKEN, prefills + decodes text
+ und_model, und_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_und.yaml")
+ assert und_model.config.infer_type == "infer_und"
+ und_request = und_processor(text="hi", images=str(image_path))
+ assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_request["conversation_list"] if item.type == "image"] == [
+ BAGEL_SIGLIP_CONTEXT
+ ]
+ assert und_request["conversation_list"][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN
+ und_model.reset()
+ und_generated = und_model.generate(
+ und_request,
+ generation_kwargs={"max_new_tokens": 2, "do_sample": False},
+ )
+ assert any(item["type"] == "text" for item in und_generated)
+ assert und_model.modules_dict["bagel_qwen2_mot"]._generation_state.main.cache is not None
+
+ # 2. infer_gen: text prompt -> CFG flow denoise loop -> decoded PIL image
+ gen_model, gen_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_gen.yaml")
+ assert gen_model.config.infer_type == "infer_gen"
+ gen_request = gen_processor(text="draw")
+ gen_model.reset()
+ gen_generated = gen_model.generate(
+ gen_request,
+ generation_kwargs={
+ "num_timesteps": 2,
+ "image_height": 4,
+ "image_width": 4,
+ "latent_downsample": 2,
+ "cfg_text_scale": 2.0,
+ "cfg_img_scale": 1.0,
+ },
+ )
+ assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in gen_generated)
+
+ # 3. infer_edit: routes user image to both VAE and SigLIP context -> 3-branch CFG denoise -> decoded PIL image
+ edit_model, edit_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_edit.yaml")
+ assert edit_model.config.infer_type == "infer_edit"
+ edit_request = edit_processor(text="edit", images=str(image_path))
+ assert [
+ item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_request["conversation_list"] if item.type == "image"
+ ] == [BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT]
+ edit_model.reset()
+ edit_generated = edit_model.generate(
+ edit_request,
+ generation_kwargs={
+ "num_timesteps": 2,
+ "image_height": 4,
+ "image_width": 4,
+ "latent_downsample": 2,
+ "cfg_text_scale": 2.0,
+ "cfg_img_scale": 1.5,
+ },
+ )
+ assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in edit_generated)
+
+
+def test_bagel_qwen2_mot_accelerated_denoise_branch_packs_cfg_branches(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Verify accelerated ``denoise_branch`` stacks active CFG branches into one packed ``forward_inference`` call."""
+ model = BagelQwen2MoTAccelerated(config_cls("bagel_qwen2_mot")(**tiny_bagel_qwen2_cfg()))
+ state = model._generation_state
+ for branch, length, next_pos in ((state.main, 2, 3), (state.cfg_text, 0, 7), (state.cfg_img, 3, 11)):
+ cache = model._new_empty_cache()
+ if length > 0:
+ cache.key_cache[0] = torch.ones(length, 1, 1)
+ cache.value_cache[0] = torch.ones(length, 1, 1)
+ branch.install_cache(
+ cache=cache,
+ cache_len=length,
+ next_position_id=torch.tensor(next_pos),
+ device=model.device,
+ )
+
+ captured: dict[str, object] = {}
+
+ def _capture_forward_inference(self, **kwargs):
+ del self
+ captured.update(kwargs)
+ return {"hidden_states": kwargs["packed_query_sequence"]}
+
+ monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference)
+ tail = ConversationItem(
+ type="output",
+ value=torch.zeros(5, int(model.config.hidden_size)),
+ role="assistant",
+ meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5},
+ )
+ model.denoise_branch([tail], generation_kwargs={"cfg_text_scale": 2.0, "cfg_img_scale": 1.5})
+
+ assert captured["query_lens"].tolist() == [5, 5, 5]
+ assert int(captured["packed_query_sequence"].shape[0]) == 15
+ assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN
+ assert int(tail.value.shape[0]) == 15
diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py
new file mode 100644
index 0000000000..ac48a1f003
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/test_processing.py
@@ -0,0 +1,298 @@
+"""BAGEL preprocessing, MoT packing, and offline VAE cache round-trip tests."""
+
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from scripts.ci.accelerator.launch_plan import LaunchSpec
+from tests.seed_omni.modules.bagel.helpers import (
+ bagel_template,
+ config_cls,
+ model_cls,
+ tiny_bagel_qwen2_cfg,
+)
+from veomni.data.dataset import get_length_by_labels_fn, get_length_fn
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated as mot_acc
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import TrainingMixin
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import preprocess_mot_inputs
+from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import (
+ _OMNI_TOKEN_LEN as BAGEL_SIGLIP_TOKEN_LEN,
+)
+from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import (
+ BagelSiglipNavitPreprocessor,
+ BagelSiglipNavitProcessor,
+)
+from veomni.models.seed_omni.modules.bagel.sources import (
+ BAGEL_CONTEXT_KEY,
+ BAGEL_PHASE_KEY,
+ BAGEL_SIGLIP_CONTEXT,
+ BAGEL_START_TOKEN,
+ BAGEL_VAE_CONTEXT,
+)
+from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor
+from veomni.models.seed_omni.modules.bagel.vae.processing import (
+ BAGEL_VAE_PIXEL_SHAPE,
+ BAGEL_VAE_POSTERIOR,
+ BagelVAEPreprocessor,
+ BagelVAEProcessor,
+)
+from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem
+
+
+pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu"))
+
+
+def _tiny_vae(meta: bool = False, **config_overrides):
+ """Construct a tiny ``BagelVAE`` on CPU or meta device with a bound processor."""
+ BagelVAE = model_cls("bagel_vae")
+ BagelVAEConfig = config_cls("bagel_vae")
+ config_kwargs = dict(
+ resolution=8,
+ ch=32,
+ ch_mult=[1],
+ num_res_blocks=1,
+ z_channels=2,
+ max_image_size=8,
+ min_image_size=4,
+ image_stride=4,
+ max_pixels=64,
+ downsample=1,
+ )
+ config_kwargs.update(config_overrides)
+ with torch.device("meta" if meta else "cpu"):
+ model = BagelVAE(BagelVAEConfig(**config_kwargs))
+ model._image_processor = BagelVAEProcessor.from_config(model.config)
+ return model
+
+
+def _build_preprocessors():
+ return (
+ BagelVAEPreprocessor(
+ BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16),
+ dtype=torch.bfloat16,
+ dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16),
+ dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long),
+ ),
+ BagelSiglipNavitPreprocessor(
+ BagelSiglipNavitProcessor(
+ patch_size=2,
+ image_size=4,
+ min_image_size=2,
+ max_pixels=16,
+ vit_max_num_patch_per_side=2,
+ ),
+ dtype=torch.bfloat16,
+ dummy_pixel_values=torch.zeros(1, 12, dtype=torch.bfloat16),
+ ),
+ BagelTextEncoderPreprocessor(bagel_template()),
+ )
+
+
+def test_bagel_preprocessors_route_training_and_inference_contexts() -> None:
+ """Verify training (``edit``/``gen`` + text-only dummy injection + ``dyn_bsz`` token lens) and inference routing."""
+ vae_pre, siglip_pre, text_pre = _build_preprocessors()
+ user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8)
+ assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8)
+
+ # 1. Training batch: sample 0 has edit + gen images; sample 1 is text-only (receives both VAE and SigLIP dummies)
+ train_batch = [
+ [
+ ConversationItem(type="text", value="edit", role="user"),
+ ConversationItem(type="image", value=user_image.clone(), role="user", meta={_IMG_TAG_KEY: "edit"}),
+ ConversationItem(
+ type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"}
+ ),
+ ],
+ [
+ ConversationItem(type="text", value="hi", role="user"),
+ ConversationItem(type="text", value="ok", role="assistant"),
+ ],
+ ]
+ for pre in (vae_pre, siglip_pre, text_pre):
+ pre({"conversation_list": train_batch})
+
+ edit_sample, text_sample = train_batch
+ assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_sample if item.type == "image"] == [
+ BAGEL_VAE_CONTEXT,
+ BAGEL_SIGLIP_CONTEXT,
+ BAGEL_VAE_CONTEXT,
+ ]
+ assert [item.meta.get(_IMG_TAG_KEY) for item in edit_sample if item.type == "image"] == ["edit", "edit", "gen"]
+ dummies = [item for item in text_sample if item.is_dummy and item.type == "image"]
+ assert {item.meta.get(BAGEL_CONTEXT_KEY) for item in dummies} == {BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT}
+ for sample in train_batch:
+ assert all(
+ isinstance(item.meta.get(_OMNI_TOKEN_LEN), int) and item.meta[_OMNI_TOKEN_LEN] > 0 for item in sample
+ )
+ assert (
+ get_length_fn({"conversation_list": edit_sample})
+ > get_length_by_labels_fn({"conversation_list": edit_sample})
+ > 0
+ )
+
+ # 2. Inference routing: infer_und routes to SigLIP + appends BAGEL_START_TOKEN; infer_edit expands to VAE + SigLIP
+ und_batch = [[ConversationItem(type="image", value=user_image.clone(), role="user")]]
+ for pre in (vae_pre, siglip_pre, text_pre):
+ pre({"conversation_list": und_batch}, inference=True, generation_kwargs={"infer_type": "infer_und"})
+ assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_batch[0] if item.type == "image"] == [
+ BAGEL_SIGLIP_CONTEXT
+ ]
+ assert und_batch[0][1].meta[BAGEL_SIGLIP_TOKEN_LEN] == 4
+ assert und_batch[0][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN
+
+
+def test_bagel_mot_packing_sp_padding_and_scatter(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Verify ``preprocess_mot_inputs`` expert routing, ``forward_pre`` SP padding, and ``forward_post`` triplet scatter."""
+ BagelQwen2MoT = model_cls("bagel_qwen2_mot")
+ BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot")
+ model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train()
+ hidden_size = int(model.config.hidden_size)
+
+ monkeypatch.setattr(
+ mot_acc,
+ "get_parallel_state",
+ lambda: SimpleNamespace(sp_size=4, cp_size=1, ulysses_size=4, sp_group=object()),
+ )
+
+ def _fake_sp_pad(tensor, dim, pad_value):
+ pad_shape = list(tensor.shape)
+ pad_shape[dim] = 3
+ padding = torch.full(pad_shape, pad_value, device=tensor.device, dtype=tensor.dtype)
+ return torch.cat((tensor, padding), dim=dim)
+
+ monkeypatch.setattr(mot_acc, "sp_pad", _fake_sp_pad)
+ monkeypatch.setattr(mot_acc, "slice_input_tensor", lambda tensor, **kwargs: tensor)
+
+ text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user")
+ image = ConversationItem(
+ type="image",
+ value=torch.ones(3, hidden_size),
+ role="assistant",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"},
+ )
+ inputs = model.forward_pre(conversation_list=[[text, image]])
+ assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1, -1, -1, -1]))
+ assert torch.equal(inputs["packed_attention_metadata"][2, 5:], torch.tensor([-1, -1, -1], dtype=torch.int32))
+
+ # Triplet scatter in forward_post
+ def _marker(context: str) -> ConversationItem:
+ return ConversationItem(
+ type="text",
+ value=torch.zeros(1, 4),
+ role="user",
+ meta={"labels": torch.full((1,), -100), BAGEL_CONTEXT_KEY: context},
+ )
+
+ latent_item = ConversationItem(
+ type="image",
+ value=torch.zeros(2, 4),
+ role="assistant",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"},
+ )
+ siglip_item = ConversationItem(
+ type="image",
+ value=torch.zeros(3, 4),
+ role="user",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "edit"},
+ )
+ sample = [
+ _marker(BAGEL_VAE_CONTEXT),
+ latent_item,
+ _marker(BAGEL_VAE_CONTEXT),
+ _marker(BAGEL_SIGLIP_CONTEXT),
+ siglip_item,
+ _marker(BAGEL_SIGLIP_CONTEXT),
+ ]
+ packed = preprocess_mot_inputs([sample], device=torch.device("cpu"), dtype=torch.float32, hidden_size=4)
+ assert packed is not None
+ assert torch.equal(packed.packed_token_type_ids, torch.tensor([0, 1, 1, 0, 0, 0, 0, 0, 0]))
+ hidden_states = torch.arange(9 * 4, dtype=torch.float32).reshape(9, 4)
+ monkeypatch.setattr(
+ mot_acc,
+ "get_parallel_state",
+ lambda: SimpleNamespace(sp_size=1, cp_size=1, ulysses_size=1, sp_group=None),
+ )
+ stub = SimpleNamespace(device=torch.device("cpu"), _conversation_carrier=[sample], _packed_training=packed)
+ TrainingMixin.forward_post(stub, hidden_states)
+ assert torch.equal(latent_item.value, hidden_states[1:3])
+ assert torch.equal(siglip_item.value, hidden_states[5:8])
+
+
+def test_bagel_vae_encode_and_offline_cache_roundtrip(tmp_path: Path) -> None:
+ """Verify ragged-shape VAE ``encode``/``offline_encode`` and parquet cache replay into ``BagelFlowConnector``."""
+ from datasets import load_dataset
+
+ from veomni.data.seed_omni.seedomni_transform import process_seedomni_cached_example
+ from veomni.models.seed_omni.utils.offline_cache import SeedOmniOfflineCacheWriter
+
+ vae = _tiny_vae(ch_mult=[1, 1], downsample=2)
+ meta_vae = _tiny_vae(meta=True, ch_mult=[1, 1], downsample=2)
+
+ edit_img = ConversationItem(
+ type="image",
+ value=torch.zeros(3, 4, 8),
+ role="assistant",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 8])},
+ )
+ gen_img = ConversationItem(
+ type="image",
+ value=torch.ones(3, 8, 4),
+ role="assistant",
+ meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([8, 4])},
+ )
+
+ # 1. Ragged micro-batch offline_encode matches single-item posteriors without spatial padding
+ single_edit = copy.deepcopy([[edit_img]])
+ vae.post_forward(
+ "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_edit))
+ )
+ single_gen = copy.deepcopy([[gen_img]])
+ vae.post_forward(
+ "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_gen))
+ )
+
+ batch = [[copy.deepcopy(edit_img), copy.deepcopy(gen_img)]]
+ pre = vae.pre_forward("offline_encode", conversation_list=batch)
+ encoded = vae.post_forward("offline_encode", **vae.offline_encode(**pre))["conversation_list"]
+ torch.testing.assert_close(encoded[0][0].value, single_edit[0][0].value)
+ torch.testing.assert_close(encoded[0][1].value, single_gen[0][0].value)
+
+ # 2. Parquet round-trip -> meta-built online_process -> BagelFlowConnector.embed_latent_pre
+ writer = SeedOmniOfflineCacheWriter(str(tmp_path))
+ writer.open()
+ writer.save_conversation_list(encoded)
+ writer.flush()
+ dataset = load_dataset("parquet", data_files=[str(tmp_path / "shard_000000.parquet")], split="train")
+ replayed = [process_seedomni_cached_example(dataset[0])[0]["conversation_list"]]
+ assert all(item.meta[BAGEL_VAE_POSTERIOR] for item in replayed[0])
+
+ assert meta_vae.encoder.conv_in.weight.is_meta
+ processed = meta_vae.post_forward(
+ "online_process",
+ **meta_vae.online_process(**meta_vae.pre_forward("online_process", conversation_list=replayed)),
+ )["conversation_list"]
+ assert [item.meta[_IMG_TAG_KEY] for item in processed[0]] == ["edit", "gen"]
+
+ flow = model_cls("bagel_flow_connector")(
+ config_cls("bagel_flow_connector")(
+ hidden_size=4,
+ z_channels=2,
+ latent_patch_size=1,
+ patch_latent_dim=2,
+ max_latent_size=4,
+ timestep_frequency_embedding_size=4,
+ timestep_shift=3.0,
+ )
+ )
+ edit_item, gen_item = processed[0]
+ inputs = flow.embed_latent_pre(conversation_list=processed)
+ assert flow._embed_lengths == [8, 8]
+ assert torch.equal(inputs["timesteps"][:8], torch.zeros(8))
+ assert "flow_velocity_target" not in edit_item.meta
+ assert gen_item.meta["flow_velocity_target"].shape == (8, 2)
+ assert torch.all(gen_item.meta["timestep"] == gen_item.meta["timestep"][0])
diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py
new file mode 100644
index 0000000000..d0a8bf2dc6
--- /dev/null
+++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py
@@ -0,0 +1,361 @@
+"""BAGEL module registry, YAML graph configs, MoT attention masks, QKV checkpoint conversion, and ops."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import torch
+from safetensors import safe_open
+from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict
+from torch.nn.attention.flex_attention import BlockMask
+from transformers.utils import SAFE_WEIGHTS_NAME
+
+from scripts.ci.accelerator.launch_plan import LaunchSpec
+from tests.models.conftest import available_nvidia_ops # noqa: F401
+from tests.seed_omni.modules.bagel.helpers import (
+ bagel_cfg_dir,
+ load_module_runtime_args,
+ load_omni_config,
+ tiny_bagel_qwen2_cfg,
+)
+from veomni.models.checkpoint.weights import load_model_weights
+from veomni.models.seed_omni import (
+ OMNI_ACCELERATED_MODEL_REGISTRY,
+ OMNI_CONFIG_REGISTRY,
+ OMNI_MODEL_REGISTRY,
+ OMNI_PROCESSOR_REGISTRY,
+)
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAttentionAccelerated
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import (
+ build_mot_attention_metadata,
+ build_mot_magi_mask,
+ build_mot_sdpa_mask,
+ pad_mot_attention_metadata,
+)
+from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT, BagelQwen2RotaryEmbedding
+from veomni.models.seed_omni.modules.bagel.siglip_navit.accelerated.accelerated import BagelSiglipNavitAccelerated
+from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig
+from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit
+from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN
+from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT
+from veomni.models.seed_omni.utils.conversation import ConversationItem
+from veomni.ops.mask import MagiAttentionMask
+from veomni.utils.save_safetensor_utils import get_model_save_state
+
+
+pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu"))
+
+
+def _flex_config() -> BagelQwen2MoTConfig:
+ return BagelQwen2MoTConfig(
+ vocab_size=128,
+ hidden_size=64,
+ intermediate_size=128,
+ num_hidden_layers=1,
+ num_attention_heads=4,
+ num_key_value_heads=1,
+ max_position_embeddings=64,
+ bos_token_id=1,
+ eos_token_id=2,
+ ops_implementation={
+ "attn_implementation": "veomni_flex_attention",
+ "rms_norm_implementation": "eager",
+ "swiglu_mlp_implementation": "eager",
+ "rotary_pos_emb_implementation": "eager",
+ },
+ )
+
+
+def _expand_magi_mask(mask: MagiAttentionMask, sequence_length: int) -> torch.Tensor:
+ visible = torch.zeros(sequence_length, sequence_length, dtype=torch.bool, device=mask.q_ranges.device)
+ attn_types = (
+ mask.attn_type_map
+ if mask.attn_type_map is not None
+ else torch.zeros(mask.q_ranges.shape[0], dtype=torch.int32, device=mask.q_ranges.device)
+ )
+ for range_index in range(mask.q_ranges.shape[0]):
+ q_start, q_end = int(mask.q_ranges[range_index, 0]), int(mask.q_ranges[range_index, 1])
+ k_start, k_end = int(mask.k_ranges[range_index, 0]), int(mask.k_ranges[range_index, 1])
+ if int(attn_types[range_index]) == 0:
+ visible[q_start:q_end, k_start:k_end] = True
+ else:
+ q_idx = torch.arange(q_start, q_end, device=visible.device)
+ k_idx = torch.arange(k_start, k_end, device=visible.device)
+ visible[q_start:q_end, k_start:k_end] |= q_idx[:, None] >= k_idx[None, :]
+ return visible
+
+
+@pytest.mark.parametrize(
+ "infer_graph",
+ ["infer/graph_infer_und.yaml", "infer/graph_infer_gen.yaml", "infer/graph_infer_edit.yaml"],
+)
+def test_bagel_registry_and_yaml_graphs_load(infer_graph: str) -> None:
+ """Verify all five BAGEL modules are registered and train + infer YAML configs resolve cleanly."""
+ expected = {
+ "bagel_text_encoder": ("BagelTextEncoder", "BagelTextEncoderAccelerated"),
+ "bagel_siglip_navit": ("BagelSiglipNavit", "BagelSiglipNavitAccelerated"),
+ "bagel_qwen2_mot": ("BagelQwen2MoT", "BagelQwen2MoTAccelerated"),
+ "bagel_flow_connector": ("BagelFlowConnector", "BagelFlowConnectorAccelerated"),
+ "bagel_vae": ("BagelVAE", "BagelVAEAccelerated"),
+ }
+ for model_type, (native_name, accel_name) in expected.items():
+ assert model_type in OMNI_CONFIG_REGISTRY.valid_keys()
+ assert OMNI_MODEL_REGISTRY[model_type]().__name__ == native_name
+ assert OMNI_ACCELERATED_MODEL_REGISTRY[model_type]().__name__ == accel_name
+ assert {"bagel_siglip_navit", "bagel_vae"}.issubset(set(OMNI_PROCESSOR_REGISTRY.valid_keys()))
+
+ cfg = load_omni_config(
+ modules_path=bagel_cfg_dir() / "train/modules_train.yaml",
+ train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml",
+ infer_graph_path=bagel_cfg_dir() / infer_graph,
+ )
+ assert set(cfg._module_entries) == set(expected)
+ assert cfg.generation_graph["initial"] == "prompt_encode"
+ assert any(
+ t.get("next_state") == "done"
+ for state in cfg.generation_graph["states"].values()
+ for t in state.get("transitions", [])
+ )
+ runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml")
+ assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == "veomni_flex_attention"
+ assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True
+
+
+def test_mot_attention_masks_and_sp_padding() -> None:
+ """Verify SDPA, Magi, and Flex masks agree on multi-sample causal/full/noise visibility and SP padding."""
+ metadata = build_mot_attention_metadata(
+ [[2, 3, 2], [4, 2]],
+ [["full", "causal", "noise"], ["causal", "noise"]],
+ device=torch.device("cpu"),
+ )
+ padded = pad_mot_attention_metadata(metadata, padded_length=16)
+ sdpa = build_mot_sdpa_mask(padded)
+ magi = build_mot_magi_mask(padded)
+ visible = _expand_magi_mask(magi, 16)
+ torch.testing.assert_close(visible[None, None], sdpa)
+ assert not bool(visible[:13, 13:].any())
+ assert bool(visible[13:, 13:].all())
+ assert isinstance(
+ BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_flex_attention"),
+ BlockMask,
+ )
+ assert isinstance(
+ BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_magi_attention"),
+ MagiAttentionMask,
+ )
+
+
+def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path: Path) -> None:
+ """Verify fused runtime QKV parameters keep fused keys in ``state_dict`` / DCP and split on HF safetensors export."""
+ torch.manual_seed(3014)
+ eager_cfg = BagelQwen2MoTConfig(**{**tiny_bagel_qwen2_cfg(), "num_key_value_heads": 1})
+ eager_ref = BagelQwen2MoT(eager_cfg)
+ eager_split_state = eager_ref.state_dict()
+
+ reference = accelerated.BagelQwen2MoTAccelerated(_flex_config())
+ reference.load_state_dict(eager_split_state)
+ runtime_state = reference.state_dict()
+ dcp_state = get_model_state_dict(reference)
+ assert "model.layers.0.self_attn.qkv_proj_und.weight" in runtime_state
+ assert "model.layers.0.self_attn.qkv_proj_gen.weight" in runtime_state
+ assert "model.layers.0.self_attn.q_proj.weight" not in runtime_state
+ assert set(dcp_state) == set(runtime_state)
+
+ dcp_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config())
+ set_model_state_dict(dcp_loaded, dcp_state)
+ for name in ("qkv_proj_und", "qkv_proj_gen"):
+ expected = getattr(reference.model.layers[0].self_attn, name)
+ actual = getattr(dcp_loaded.model.layers[0].self_attn, name)
+ torch.testing.assert_close(actual.weight, expected.weight)
+ torch.testing.assert_close(actual.bias, expected.bias)
+
+ reverted_hf_state = get_model_save_state(reference, None)
+ for key in (
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.0.self_attn.k_proj.weight",
+ "model.layers.0.self_attn.v_proj.weight",
+ "model.layers.0.self_attn.q_proj_moe_gen.weight",
+ ):
+ torch.testing.assert_close(reverted_hf_state[key], eager_split_state[key].to(torch.bfloat16))
+
+ reference.save_pretrained(tmp_path)
+ with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint:
+ checkpoint_keys = set(checkpoint.keys())
+ assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_keys
+ assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_keys
+
+ hf_loaded = accelerated.BagelQwen2MoTAccelerated.from_pretrained(tmp_path, config=_flex_config())
+ veomni_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config())
+ load_model_weights(veomni_loaded, str(tmp_path), init_device="cpu")
+ for name in ("qkv_proj_und", "qkv_proj_gen"):
+ expected = getattr(reference.model.layers[0].self_attn, name)
+ torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).weight, expected.weight)
+ torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).weight, expected.weight)
+
+
+def test_bagel_modules_ops_dispatch_meta_rotary_and_flops(available_nvidia_ops) -> None: # noqa: F811
+ """Verify module-level ``VeomniOp`` binding, ``meta -> to_empty`` FP32 rotary table, and multi-image SigLIP FLOPs."""
+ default_cfg = BagelQwen2MoTConfig(
+ vocab_size=128,
+ hidden_size=64,
+ intermediate_size=128,
+ num_hidden_layers=1,
+ num_attention_heads=4,
+ num_key_value_heads=1,
+ max_position_embeddings=64,
+ bos_token_id=1,
+ eos_token_id=2,
+ )
+ assert default_cfg._attn_implementation == "veomni_flex_attention"
+
+ with torch.device("meta"):
+ meta_model = accelerated.BagelQwen2MoTAccelerated(_flex_config())
+ materialized = meta_model.to_empty(device="cpu").to(dtype=torch.bfloat16)
+ layer = materialized.model.layers[0]
+ assert layer.self_attn.veomni_async_ulysses_qkv.variant == "bagel"
+ assert layer.self_attn.veomni_async_ulysses_o.variant == "bagel"
+ assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention"
+ assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2"
+
+ ref_rotary = BagelQwen2RotaryEmbedding(_flex_config())
+ rotary = materialized.model.rotary_emb
+ assert rotary.inv_freq.dtype == torch.float32
+ assert rotary.original_inv_freq.dtype == torch.float32
+ torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq)
+
+ # BagelFlowConnector meta -> to_empty -> init_weights preserves zero-initialized llm2vae velocity head
+ flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()(
+ hidden_size=16,
+ z_channels=2,
+ latent_patch_size=1,
+ patch_latent_dim=2,
+ max_latent_size=4,
+ timestep_frequency_embedding_size=4,
+ )
+ eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg)
+ with torch.device("meta"):
+ meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg)
+ materialized_flow = meta_flow.to_empty(device="cpu")
+ materialized_flow.init_weights()
+ assert torch.count_nonzero(materialized_flow.llm2vae.weight) == 0
+ assert torch.count_nonzero(materialized_flow.llm2vae.bias) == 0
+ assert torch.count_nonzero(materialized_flow.vae2llm.weight) > 0
+ torch.testing.assert_close(materialized_flow.latent_pos_embed.pos_embed, eager_flow.latent_pos_embed.pos_embed)
+
+ siglip_cfg = BagelSiglipNavitConfig(
+ hidden_size=32,
+ intermediate_size=64,
+ num_hidden_layers=1,
+ num_attention_heads=4,
+ image_size=28,
+ patch_size=14,
+ ops_implementation={"attn_implementation": "sdpa", "rotary_pos_emb_vision_implementation": "eager"},
+ )
+ siglip = BagelSiglipNavit(siglip_cfg).eval()
+ assert siglip.vision_model.encoder.layers[0].self_attn.veomni_attn.impl == "veomni_sdpa"
+
+ # Multi-image SigLIP FLOPs meter uses per-image lengths while keeping per-sample token counts
+ acc_siglip = BagelSiglipNavitAccelerated(siglip_cfg)
+ acc_siglip.config.num_hidden_layers = 26
+ patch_dim = siglip_cfg.num_channels * siglip_cfg.patch_size * siglip_cfg.patch_size
+ items = [
+ ConversationItem(
+ type="image",
+ role="user",
+ value=torch.zeros(1024, patch_dim, dtype=torch.bfloat16),
+ meta={
+ BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT,
+ _OMNI_TOKEN_LEN: 1024,
+ _OMNI_POSITION_IDS: torch.zeros(1024, dtype=torch.long),
+ },
+ )
+ for _ in range(2)
+ ]
+ acc_siglip.forward_pre(conversation_list=[items])
+ acc_siglip.metric_meter_add("forward")
+ flops, seqlens = acc_siglip.metric_meter_collect()
+ assert seqlens == [2048]
+ assert flops == pytest.approx(acc_siglip.estimate_flops([1024, 1024]))
+ assert flops < acc_siglip.estimate_flops([2048])
+
+
+def _two_rank_fsdp2_pos_embed_materialize_worker(rank: int, rendezvous: str) -> None:
+ """Verify two-rank FSDP2 ``meta -> fully_shard -> to_empty -> init_weights`` materializes sharded ``PositionEmbedding``."""
+ import torch.distributed as dist
+ from torch.distributed.fsdp import fully_shard
+ from torch.distributed.tensor import DTensor
+
+ from veomni.distributed.parallel_state import _init_parallel_state, clear_parallel_state
+
+ dist.init_process_group("gloo", init_method=f"file://{rendezvous}", world_size=2, rank=rank)
+ try:
+ ps = _init_parallel_state(dp_size=2, dp_shard_size=2, ulysses_size=1, dp_mode="fsdp2", device_type="cpu")
+
+ flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()(
+ hidden_size=16,
+ z_channels=2,
+ latent_patch_size=1,
+ patch_latent_dim=2,
+ max_latent_size=4,
+ timestep_frequency_embedding_size=4,
+ )
+ eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg)
+ with torch.device("meta"):
+ meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg)
+ fully_shard(meta_flow, mesh=ps.fsdp_mesh)
+ meta_flow.to_empty(device="cpu")
+ meta_flow.init_weights()
+
+ assert isinstance(meta_flow.latent_pos_embed.pos_embed.data, DTensor)
+ assert meta_flow.latent_pos_embed.pos_embed.to_local().shape == (8, 16)
+ torch.testing.assert_close(
+ meta_flow.latent_pos_embed.pos_embed.full_tensor(),
+ eager_flow.latent_pos_embed.pos_embed,
+ )
+ assert torch.count_nonzero(meta_flow.llm2vae.weight.full_tensor()) == 0
+ assert torch.count_nonzero(meta_flow.llm2vae.bias.full_tensor()) == 0
+ assert torch.count_nonzero(meta_flow.vae2llm.weight.full_tensor()) > 0
+
+ siglip_cfg = BagelSiglipNavitConfig(
+ hidden_size=16,
+ output_size=16,
+ intermediate_size=32,
+ num_hidden_layers=1,
+ num_attention_heads=2,
+ image_size=8,
+ patch_size=4,
+ vit_max_num_patch_per_side=4,
+ ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"},
+ )
+ eager_siglip = BagelSiglipNavit(siglip_cfg)
+ with torch.device("meta"):
+ meta_siglip = BagelSiglipNavit(siglip_cfg)
+ fully_shard(meta_siglip, mesh=ps.fsdp_mesh)
+ meta_siglip.to_empty(device="cpu")
+ meta_siglip.init_weights()
+
+ assert isinstance(meta_siglip.vit_pos_embed.pos_embed.data, DTensor)
+ assert meta_siglip.vit_pos_embed.pos_embed.to_local().shape == (8, 16)
+ torch.testing.assert_close(
+ meta_siglip.vit_pos_embed.pos_embed.full_tensor(),
+ eager_siglip.vit_pos_embed.pos_embed,
+ )
+ finally:
+ clear_parallel_state()
+ dist.destroy_process_group()
+
+
+@pytest.mark.launch_plan(provider=LaunchSpec(nproc=2, device="cpu"))
+def test_bagel_pos_embed_two_rank_fsdp2_meta_materialization(tmp_path: Path) -> None:
+ """Verify two-rank FSDP2 config-only initialization distributes ``PositionEmbedding.pos_embed`` across shards."""
+ import torch.multiprocessing as mp
+
+ mp.spawn(
+ _two_rank_fsdp2_pos_embed_materialize_worker,
+ args=(str(tmp_path / "rendezvous"),),
+ nprocs=2,
+ join=True,
+ )
diff --git a/tests/seed_omni/trainer/test_omni_inferencer.py b/tests/seed_omni/trainer/test_omni_inferencer.py
index 258ced4713..d9a3b10f96 100644
--- a/tests/seed_omni/trainer/test_omni_inferencer.py
+++ b/tests/seed_omni/trainer/test_omni_inferencer.py
@@ -88,15 +88,20 @@ def generate(request, generation_kwargs=None):
seen["input_ids"] = request["input_ids"]
return []
+ def processor(**kwargs):
+ seen["processor_kwargs"] = kwargs
+ return {"input_ids": torch.ones(2, dtype=torch.long)}
+
inferencer = _inferencer_with("infer_gen", {})
inferencer._distributed = distributed
inferencer.device = torch.device("meta")
- inferencer.processor = lambda **kwargs: {"input_ids": torch.ones(2, dtype=torch.long)}
+ inferencer.processor = processor
inferencer.model = SimpleNamespace(reset=lambda: None, generate=generate)
- inferencer._run(InferenceRequest(prompt="hi"))
+ inferencer._run(InferenceRequest(prompt="hi", generation_kwargs={"infer_type": "infer_edit"}))
assert seen["input_ids"].device.type == expected_device
+ assert seen["processor_kwargs"]["infer_type"] == "infer_edit"
def test_extract_generated_text_keeps_only_filled_text_items():
diff --git a/veomni/arguments/omni_arguments_types.py b/veomni/arguments/omni_arguments_types.py
index f9eb8595d3..10b414e146 100644
--- a/veomni/arguments/omni_arguments_types.py
+++ b/veomni/arguments/omni_arguments_types.py
@@ -290,7 +290,10 @@ def build_omni_model_runtime_args(args: "OmniArguments", *, for_inference: bool
generation_graphs=generation_graphs,
train_type=train_type,
infer_type=infer_type,
- generation_kwargs=dict(args.infer.generation_kwargs),
+ generation_kwargs={
+ **(omni_cfg.generation_kwargs or {} if omni_cfg is not None else {}),
+ **args.infer.generation_kwargs,
+ },
)
diff --git a/veomni/models/checkpoint/convert.py b/veomni/models/checkpoint/convert.py
index d4b4149478..f5bbffd78f 100644
--- a/veomni/models/checkpoint/convert.py
+++ b/veomni/models/checkpoint/convert.py
@@ -223,6 +223,8 @@ def get_checkpoint_tensor_converter(
model: Union["nn.Module", "PreTrainedModel"],
) -> Optional["CheckpointTensorConverter"]:
"""Return the checkpoint tensor converter registered on *model*, or ``None``."""
+ while isinstance(getattr(model, "module", None), nn.Module):
+ model = model.module
factory = getattr(model, "_create_checkpoint_tensor_converter", None)
if factory is None:
return None
diff --git a/veomni/models/seed_omni/configuration_omni.py b/veomni/models/seed_omni/configuration_omni.py
index 323b8a30ca..a02d2f45b0 100644
--- a/veomni/models/seed_omni/configuration_omni.py
+++ b/veomni/models/seed_omni/configuration_omni.py
@@ -68,11 +68,10 @@ def select_graph(
empty_hint: str = "",
unknown_hint: str = "scenario",
) -> Any:
- """Pick the active graph out of a scenario map; unset ``scenario`` takes the first."""
+ """Pick the active graph out of a scenario map."""
if not graphs:
raise ValueError(f"No graph scenarios are declared. {empty_hint}".strip())
- if scenario is None:
- return next(iter(graphs.values()))
+ assert scenario is not None, f"`{unknown_hint}` must be normalized before calling `select_graph`."
if scenario not in graphs:
known = ", ".join(graphs)
raise KeyError(f"Unknown {unknown_hint} {scenario!r}; expected one of: {known}.")
@@ -220,6 +219,7 @@ def __init__(
check_infer_type_names(generation_graphs or {})
self.train_type = train_type
self.infer_type = infer_type
+ self._resolve_graph_types()
self.generation_kwargs = generation_kwargs
# Model-level kernels: this composed model's defaults for every module
# it holds. Sparse — only what was explicitly selected. A loaded module
@@ -454,6 +454,17 @@ def load_checkpoint_sidecars(self, checkpoint_root: Union[str, os.PathLike]) ->
base_ops_implementation=self.ops_implementation,
)
+ def _resolve_graph_types(self) -> None:
+ """Bind the active scenario in each graph map once the map is final."""
+ for graphs, attr in ((self.training_graphs, "train_type"), (self.generation_graphs, "infer_type")):
+ if not graphs:
+ continue
+ selected = getattr(self, attr)
+ if selected is None:
+ setattr(self, attr, next(iter(graphs)))
+ elif selected not in graphs:
+ raise KeyError(f"Unknown {attr} {selected!r}; expected one of: {', '.join(graphs)}.")
+
def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike]) -> None:
"""Load ``training_graphs`` and ``generation_graphs`` from YAML sidecars when present.
@@ -469,6 +480,7 @@ def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike])
if not self.generation_graphs:
self.generation_graphs = self._read_graph_sidecar(root, DEFAULT_GENERATION_GRAPH_FILE, dict)
check_infer_type_names(self.generation_graphs)
+ self._resolve_graph_types()
@classmethod
def _read_graph_sidecar(cls, root: str, filename: str, payload_type: type) -> Dict:
diff --git a/veomni/models/seed_omni/modules/__init__.py b/veomni/models/seed_omni/modules/__init__.py
index 0aa5efc852..9d98923cad 100644
--- a/veomni/models/seed_omni/modules/__init__.py
+++ b/veomni/models/seed_omni/modules/__init__.py
@@ -110,8 +110,8 @@ def read_model_type(model_path: str) -> str:
return model_type
-# Side-effect only: attach @register factories under base/, fake_model/, qwen3/, qwen3vl/, text_encoder/.
-from . import base, fake_model, qwen3, qwen3vl, text_encoder # noqa: F401 E402
+# Side-effect only: attach @register factories under bagel/, base/, fake_model/, qwen3/, qwen3vl/, text_encoder/.
+from . import bagel, base, fake_model, qwen3, qwen3vl, text_encoder # noqa: F401 E402
__all__ = [
diff --git a/veomni/models/seed_omni/modules/bagel/__init__.py b/veomni/models/seed_omni/modules/bagel/__init__.py
new file mode 100644
index 0000000000..07280c870b
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/__init__.py
@@ -0,0 +1,25 @@
+"""BAGEL SeedOmni modules.
+
+The SeedOmni graph keeps BAGEL modules split by producer/consumer boundaries:
+
+- training: text/SigLIP/VAE/flow hooks write embedded carrier items, then
+ ``bagel_qwen2_mot.forward`` packs them into the MoT backbone and scatters
+ hidden states back to the same items.
+- prompt inference: ``bagel_text_encoder`` and optional VAE/SigLIP context
+ feed ``bagel_qwen2_mot.generate`` for prompt prefill or text decoding.
+- image denoise: ``flow_connector.prepare_denoise_query`` writes
+ ``bagel_flow_query``; text marker wrapping feeds ``qwen2_mot.denoise_branch``;
+ ``flow_connector.decode_velocity_from_hidden`` writes ``bagel_flow_velocity``;
+ ``qwen2_mot.collect_velocity`` strips branch markers and handles CFG merge; finally
+ ``flow_connector.advance_denoise`` either loops or emits generated latents
+ for VAE decode.
+"""
+
+from . import (
+ convert_model, # noqa: F401
+ flow_connector, # noqa: F401
+ qwen2_mot, # noqa: F401
+ siglip_navit, # noqa: F401
+ text_encoder, # noqa: F401
+ vae, # noqa: F401
+)
diff --git a/veomni/models/seed_omni/modules/bagel/convert_model.py b/veomni/models/seed_omni/modules/bagel/convert_model.py
new file mode 100644
index 0000000000..cb5923f115
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/convert_model.py
@@ -0,0 +1,299 @@
+"""Split a BAGEL checkpoint into SeedOmni modules.
+
+Registered under ``OMNI_CONVERT_REGISTRY["bagel"]``; :func:`convert_checkpoint`
+writes the omni checkpoint. Graphs come from ``configs/seed_omni/Bagel/bagel_7b_mot/``.
+"""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Callable
+
+import torch
+from safetensors import safe_open
+from transformers import AutoTokenizer
+from transformers.initialization import no_init_weights
+
+from veomni.models.checkpoint.weights import init_empty_weights
+from veomni.models.seed_omni.utils.convert_registry import (
+ OMNI_CONVERT_REGISTRY,
+ attach_module_assets,
+ load_family_graphs,
+)
+
+
+BAGEL_GENERATION_KWARGS = {
+ "max_new_tokens": 2048,
+ "temperature": 0.0,
+ "top_p": 1.0,
+ "do_sample": False,
+ "image_height": 1024,
+ "image_width": 1024,
+ "latent_downsample": 16,
+ "num_timesteps": 50,
+ "timestep_shift": 3.0,
+ "cfg_text_scale": 1.0,
+ "cfg_img_scale": 1.0,
+ "cfg_interval": [0.0, 1.0],
+ "cfg_renorm_min": 0.0,
+ "cfg_renorm_type": "global",
+ "enable_taylorseer": False,
+}
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+ """Read and parse a UTF-8 JSON file."""
+ with path.open("r", encoding="utf-8") as f:
+ return json.load(f)
+
+
+def _load_prefixed_safetensors(
+ path: Path,
+ remap: Callable[[str], str | None],
+ consumed_keys: set[str] | None = None,
+) -> dict[str, Any]:
+ """Load safetensors keys selected by ``remap`` onto CPU."""
+ state_dict: dict[str, Any] = {}
+ with safe_open(path, framework="pt", device="cpu") as f:
+ for key in f.keys():
+ target_key = remap(key)
+ if target_key is None:
+ continue
+ if consumed_keys is not None:
+ consumed_keys.add(key)
+ state_dict[target_key] = f.get_tensor(key)
+ return state_dict
+
+
+def _safetensor_keys(path: Path) -> set[str]:
+ """Return the set of tensor keys stored in ``path``."""
+ with safe_open(path, framework="pt", device="cpu") as f:
+ return set(f.keys())
+
+
+def _assert_no_unhandled_keys(path: Path, consumed_keys: set[str], ignored_keys: set[str]) -> None:
+ """Raise if any safetensors key in ``path`` is neither consumed nor ignored."""
+ unhandled = sorted(_safetensor_keys(path) - consumed_keys - ignored_keys)
+ if unhandled:
+ preview = ", ".join(unhandled[:20])
+ raise RuntimeError(f"Unhandled source keys in {path}: {preview}")
+
+
+def _instantiate(model_type: str, config: Any):
+ """Instantiate an empty-weight SeedOmni module for ``model_type``."""
+ from veomni.models.seed_omni.modules import OMNI_MODEL_REGISTRY
+
+ model_cls = OMNI_MODEL_REGISTRY[model_type]()
+ with no_init_weights(), init_empty_weights():
+ return model_cls._from_config(config)
+
+
+def _materialize_allowed_missing_parameters(model: Any, missing: list[str]) -> None:
+ """Materialize and initialize deterministic missing position-embedding parameters."""
+ for key in missing:
+ if not key.endswith(".pos_embed"):
+ continue
+ module_name, _, param_name = key.rpartition(".")
+ module = model.get_submodule(module_name)
+ parameter = getattr(module, param_name)
+ setattr(
+ module,
+ param_name,
+ torch.nn.Parameter(torch.empty(tuple(parameter.shape)), requires_grad=parameter.requires_grad),
+ )
+ if not hasattr(module, "_init_weights"):
+ raise RuntimeError(f"Cannot initialize deterministic missing parameter: {key}")
+ module._init_weights()
+
+
+def _build_module(
+ model_type: str,
+ config: Any,
+ state_dict: dict[str, Any],
+ *,
+ allowed_missing: set[str] | None = None,
+) -> Any:
+ """Build ``model_type`` from ``config`` and assign ``state_dict`` tensors."""
+ model = _instantiate(model_type, config)
+ allowed_missing = set() if allowed_missing is None else allowed_missing
+ missing, unexpected = model.load_state_dict(state_dict, strict=False, assign=True)
+ unexpected = list(unexpected)
+ missing = list(missing)
+ unexpected_missing = [key for key in missing if key not in allowed_missing]
+ if unexpected_missing or unexpected:
+ raise RuntimeError(f"{model_type} load mismatch: missing={unexpected_missing}, unexpected={unexpected}")
+ _materialize_allowed_missing_parameters(model, [key for key in missing if key in allowed_missing])
+ print(f"[bagel] extracted {model_type}")
+ return model
+
+
+def convert_bagel_checkpoint(
+ model_path: str,
+ *,
+ max_latent_size: int = 64,
+ **kwargs: Any,
+) -> dict[str, Any]:
+ """Split an upstream BAGEL checkpoint into five SeedOmni modules."""
+ training_graphs, generation_graphs = load_family_graphs(
+ "configs/seed_omni/Bagel/bagel_7b_mot",
+ training="train/graph_train.yaml",
+ generation={
+ "infer_edit": "infer/graph_infer_edit.yaml",
+ "infer_gen": "infer/graph_infer_gen.yaml",
+ "infer_und": "infer/graph_infer_und.yaml",
+ },
+ training_graph=kwargs.pop("training_graph", None),
+ generation_graph=kwargs.pop("generation_graph", None),
+ )
+ del kwargs
+ max_latent_size = int(max_latent_size)
+
+ model_root = Path(model_path)
+
+ ema_path = model_root / "ema.safetensors"
+ ae_path = model_root / "ae.safetensors"
+ if not ema_path.exists():
+ raise FileNotFoundError(f"Missing BAGEL EMA weights: {ema_path}")
+ if not ae_path.exists():
+ raise FileNotFoundError(f"Missing BAGEL VAE weights: {ae_path}")
+
+ consumed_ema_keys: set[str] = set()
+ consumed_ae_keys: set[str] = set()
+
+ llm_dict = _read_json(model_root / "llm_config.json")
+ vit_dict = _read_json(model_root / "vit_config.json")
+ hidden_size = int(llm_dict["hidden_size"])
+ vit_hidden_size = int(vit_dict["hidden_size"])
+
+ from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY
+
+ text_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_text_encoder"]()
+ text_cfg = text_cfg_cls(
+ vocab_size=int(llm_dict["vocab_size"]),
+ hidden_size=hidden_size,
+ tie_word_embeddings=False,
+ lm_head_bias=False,
+ )
+ text_state = _load_prefixed_safetensors(
+ ema_path,
+ lambda key: {
+ "language_model.model.embed_tokens.weight": "embed_tokens.weight",
+ "language_model.lm_head.weight": "lm_head.weight",
+ }.get(key),
+ consumed_ema_keys,
+ )
+ text_encoder = _build_module("bagel_text_encoder", text_cfg, text_state)
+ attach_module_assets(text_encoder, tokenizer=AutoTokenizer.from_pretrained(model_path))
+
+ qwen_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_qwen2_mot"]()
+ qwen_cfg = qwen_cfg_cls(
+ **llm_dict,
+ qk_norm=True,
+ layer_module="Qwen2MoTDecoderLayer",
+ freeze_und=False,
+ )
+ qwen_state = _load_prefixed_safetensors(
+ ema_path,
+ lambda key: (
+ key.removeprefix("language_model.")
+ if key.startswith("language_model.")
+ and not key.startswith("language_model.model.embed_tokens.")
+ and not key.startswith("language_model.lm_head.")
+ else None
+ ),
+ consumed_ema_keys,
+ )
+ qwen2_mot = _build_module("bagel_qwen2_mot", qwen_cfg, qwen_state)
+
+ siglip_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_siglip_navit"]()
+ siglip_cfg = siglip_cfg_cls(
+ hidden_size=vit_hidden_size,
+ output_size=hidden_size,
+ image_size=int(vit_dict["image_size"]),
+ intermediate_size=int(vit_dict["intermediate_size"]),
+ num_attention_heads=int(vit_dict["num_attention_heads"]),
+ # Official inference/training uses one fewer active layer than vit_config.json.
+ num_hidden_layers=int(vit_dict["num_hidden_layers"]) - 1,
+ patch_size=int(vit_dict["patch_size"]),
+ connector_act="gelu_pytorch_tanh",
+ vit_max_num_patch_per_side=int(vit_dict["image_size"]) // int(vit_dict["patch_size"]),
+ rope=False,
+ )
+ siglip_state = _load_prefixed_safetensors(
+ ema_path,
+ lambda key: (
+ key.removeprefix("vit_model.")
+ if key.startswith("vit_model.")
+ else key
+ if key.startswith("connector.")
+ else None
+ ),
+ consumed_ema_keys,
+ )
+ siglip = _build_module(
+ "bagel_siglip_navit",
+ siglip_cfg,
+ siglip_state,
+ allowed_missing={"vit_pos_embed.pos_embed"},
+ )
+ from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import BagelSiglipNavitProcessor
+
+ attach_module_assets(siglip, image_processor=BagelSiglipNavitProcessor.from_config(siglip_cfg))
+
+ flow_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()
+ patch_latent_dim = 2 * 2 * 16
+ flow_cfg = flow_cfg_cls(
+ hidden_size=hidden_size,
+ patch_latent_dim=patch_latent_dim,
+ max_latent_size=max_latent_size,
+ timestep_shift=1.0,
+ )
+ flow_prefixes = ("vae2llm.", "llm2vae.", "time_embedder.")
+ flow_state = _load_prefixed_safetensors(
+ ema_path,
+ lambda key: key if key.startswith(flow_prefixes) else None,
+ consumed_ema_keys,
+ )
+ flow_connector = _build_module(
+ "bagel_flow_connector",
+ flow_cfg,
+ flow_state,
+ allowed_missing={"latent_pos_embed.pos_embed"},
+ )
+
+ vae_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_vae"]()
+ vae_cfg = vae_cfg_cls()
+ vae_state = _load_prefixed_safetensors(ae_path, lambda key: key, consumed_ae_keys)
+ vae = _build_module("bagel_vae", vae_cfg, vae_state)
+ from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEProcessor
+
+ attach_module_assets(vae, image_processor=BagelVAEProcessor.from_config(vae_cfg))
+
+ # Position embeddings are deterministic sin-cos buffers regenerated from config.
+ ignored_ema_keys = {"latent_pos_embed.pos_embed", "vit_pos_embed.pos_embed"}
+ _assert_no_unhandled_keys(ema_path, consumed_ema_keys, ignored_ema_keys)
+ _assert_no_unhandled_keys(ae_path, consumed_ae_keys, set())
+
+ return {
+ "modules": {
+ "bagel_vae": vae,
+ "bagel_siglip_navit": siglip,
+ "bagel_text_encoder": text_encoder,
+ "bagel_flow_connector": flow_connector,
+ "bagel_qwen2_mot": qwen2_mot,
+ },
+ "training_graphs": training_graphs,
+ "generation_graphs": generation_graphs,
+ "infer_type": "infer_und",
+ "generation_kwargs": deepcopy(BAGEL_GENERATION_KWARGS),
+ }
+
+
+@OMNI_CONVERT_REGISTRY.register("bagel")
+def _register_bagel_convert():
+ return convert_bagel_checkpoint
+
+
+__all__ = ["convert_bagel_checkpoint"]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py
new file mode 100644
index 0000000000..a84b9c7f23
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py
@@ -0,0 +1,24 @@
+"""BAGEL VAE/LLM flow connector module."""
+
+from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY
+
+
+@OMNI_CONFIG_REGISTRY.register("bagel_flow_connector")
+def register_bagel_flow_connector_config():
+ from .configuration import BagelFlowConnectorConfig
+
+ return BagelFlowConnectorConfig
+
+
+@OMNI_MODEL_REGISTRY.register("bagel_flow_connector")
+def register_bagel_flow_connector_model():
+ from .modeling import BagelFlowConnector
+
+ return BagelFlowConnector
+
+
+@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_flow_connector")
+def register_bagel_flow_connector_accelerated_model():
+ from .accelerated import BagelFlowConnectorAccelerated
+
+ return BagelFlowConnectorAccelerated
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py
new file mode 100644
index 0000000000..d1c9c0a21c
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py
@@ -0,0 +1,10 @@
+"""Accelerated runtime for the bagel/flow_connector OmniModule (conversation graph)."""
+
+from .accelerated import (
+ BagelFlowConnectorAccelerated,
+)
+
+
+__all__ = [
+ "BagelFlowConnectorAccelerated",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py
new file mode 100644
index 0000000000..25c9037283
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py
@@ -0,0 +1,381 @@
+"""SeedOmni carrier hooks for BAGEL's flow connector — training-graph hooks only.
+
+``embed_context_latents()`` / ``prepare_denoise_query()`` / ``decode_velocity_from_hidden()``
+/ ``advance_denoise()`` and the shared :class:`~.generation_state.FlowGenerationState`
+live on the native :class:`~.modeling.BagelFlowConnector` — this file only carries
+the training pre/forward/post hooks (carrier selection, SP token slicing, dummy
+alignment).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from .......distributed.parallel_state import get_parallel_state
+from .......distributed.sequence_parallel import (
+ gather_outputs,
+ reduce_sequence_parallel_loss,
+ slice_input_tensor,
+ sp_pad,
+)
+from .....mixins.base_mixin import BaseMixin
+from .....mixins.metric_meter_mixin import MetricMeterMixin
+from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward
+from .....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items
+from ...sources import BAGEL_FLOW_HIDDEN, BAGEL_FLOW_VELOCITY, BAGEL_PHASE_KEY, bagel_phase
+from ..configuration import BagelFlowConnectorConfig
+from ..modeling import BagelFlowConnector, scatter_flow_latent_embeds, select_vae_context_latent_items
+from ..processing import preprocess_context_latent_embed, preprocess_decode_velocity, preprocess_latent_embed
+
+
+def slice_sp_token_inputs(method: str, **inputs: torch.Tensor) -> tuple[int, dict[str, torch.Tensor]]:
+ """Pad and slice token-level ``inputs`` across the active Ulysses SP group."""
+ ps = get_parallel_state()
+ if ps.cp_size != 1:
+ raise ValueError(f"BAGEL flow connector {method} supports Ulysses groups only; got cp_size={ps.cp_size}.")
+
+ lengths = {name: int(tensor.shape[0]) for name, tensor in inputs.items()}
+ if not lengths or next(iter(lengths.values())) == 0:
+ raise ValueError(f"BAGEL flow connector {method} SP requires at least one token.")
+ if len(set(lengths.values())) != 1:
+ raise ValueError(f"BAGEL flow connector {method} SP inputs must have matching token lengths: {lengths}.")
+
+ full_length = next(iter(lengths.values()))
+ local_inputs = {
+ name: slice_input_tensor(
+ sp_pad(tensor, dim=0, pad_value=0),
+ dim=0,
+ padding=False,
+ group=ps.sp_group,
+ )
+ for name, tensor in inputs.items()
+ }
+ return full_length, local_inputs
+
+
+def gather_sp_token_output(method: str, output: torch.Tensor, full_length: int | None) -> torch.Tensor:
+ """All-gather token-level ``output`` across the SP group and trim SP padding."""
+ if full_length is None:
+ raise RuntimeError(f"BAGEL flow connector {method} SP token length was not initialized.")
+ output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group)
+ return output.narrow(0, 0, full_length)
+
+
+class TrainingMixin(TrainingModuleMixin):
+ """Training-graph carrier hooks — depends on :class:`BagelFlowConnector` modeling APIs."""
+
+ config: BagelFlowConnectorConfig
+ device: torch.device
+ dtype: torch.dtype
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self._conversation_carrier: list[list[ConversationItem]] | None = None
+ self._embed_items: list[ConversationItem] = []
+ self._embed_lengths: list[int] = []
+ self._sp_embed_length: int | None = None
+ self._decode_target_groups: list[list[ConversationItem]] = []
+ self._decode_lengths: list[int] = []
+ self._decode_target: torch.Tensor | None = None
+ self._sp_decode_length: int | None = None
+
+ @pre_forward("embed_latent")
+ def embed_latent_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Prepare patchified latent tokens, position IDs, and timesteps for ``embed_latent``."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+ self._embed_lengths = []
+ self._sp_embed_length = None
+
+ self._embed_items = select_vae_context_latent_items(conversation_list)
+ if not self._embed_items:
+ raise ValueError("BAGEL flow connector requires per-sample VAE context carriers before embed_latent.")
+
+ parts: list[dict[str, torch.Tensor]] = []
+ meter_lengths: list[int] = []
+ for item in self._embed_items:
+ if item.is_dummy:
+ inputs, lengths = preprocess_context_latent_embed(
+ [item],
+ config=self.config,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ anchor = item.value.to(device=self.device, dtype=self.dtype).sum() * 0.0
+ inputs["latents"] = inputs["latents"] + anchor
+ parts.append(inputs)
+ self._embed_lengths.extend(lengths)
+ meter_lengths.extend(int(v) for v in lengths)
+ continue
+
+ tag = item.meta.get(_IMG_TAG_KEY)
+ if tag == "gen":
+ inputs, lengths = preprocess_latent_embed(
+ [item],
+ config=self.config,
+ device=self.device,
+ dtype=self.dtype,
+ timestep_shift=float(self.config.timestep_shift),
+ )
+ elif tag == "edit":
+ inputs, lengths = preprocess_context_latent_embed(
+ [item],
+ config=self.config,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ else:
+ raise ValueError(
+ f"BAGEL flow connector training expects VAE image {_IMG_TAG_KEY} to be 'edit' or 'gen', got {tag!r}."
+ )
+
+ parts.append(inputs)
+ self._embed_lengths.extend(lengths)
+ meter_lengths.extend(int(v) for v in lengths)
+
+ self.metric_meter_set_seqlens("embed_latent", meter_lengths)
+
+ inputs = {
+ "latents": torch.cat([part["latents"] for part in parts], dim=0),
+ "position_ids": torch.cat([part["position_ids"] for part in parts], dim=0),
+ "timesteps": torch.cat([part["timesteps"] for part in parts], dim=0),
+ }
+ ps = get_parallel_state()
+ if ps.sp_size == 1:
+ self._sp_embed_length = None
+ return inputs
+
+ self._sp_embed_length, inputs = slice_sp_token_inputs("embed_latent", **inputs)
+ return inputs
+
+ @post_forward("embed_latent")
+ def embed_latent_post(self, latent_embeds: torch.Tensor) -> dict[str, Any]:
+ """Gather SP shards if needed and scatter ``latent_embeds`` back onto conversation items."""
+ if get_parallel_state().sp_size > 1:
+ latent_embeds = gather_sp_token_output("embed_latent", latent_embeds, self._sp_embed_length)
+
+ conversation = self._conversation_carrier
+ embed_items = self._embed_items
+ embed_lengths = self._embed_lengths
+ self._conversation_carrier = None
+ self._embed_items = []
+ self._embed_lengths = []
+ self._sp_embed_length = None
+
+ scatter_flow_latent_embeds(
+ embed_items,
+ embed_lengths,
+ latent_embeds,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": conversation}
+
+ @pre_forward("decode_velocity")
+ def decode_velocity_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Collect latent-token hidden states and velocity targets for ``decode_velocity``."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+ self._decode_lengths = []
+ self._decode_target = None
+ self._sp_decode_length = None
+
+ self._decode_target_groups = self._select_velocity_target_groups(conversation_list)
+
+ inputs_parts: list[dict[str, torch.Tensor]] = []
+ target_parts: list[torch.Tensor] = []
+ for sample, group in zip(conversation_list, self._decode_target_groups, strict=True):
+ if not group:
+ # No velocity target for this sample; run an anchored dummy decode
+ # so the decode head stays in the graph with zero loss.
+ dummy = self._anchor_dummy_decode_velocity_inputs([sample])
+ inputs_parts.append(dummy)
+ self._decode_lengths.append(1)
+ continue
+
+ inputs, lengths, target = preprocess_decode_velocity(
+ group,
+ config=self.config,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ inputs_parts.append(inputs)
+ self._decode_lengths.extend(lengths)
+ target_parts.append(target)
+ if target_parts:
+ self._decode_target = torch.cat(target_parts, dim=0)
+
+ inputs = {"hidden_states": torch.cat([part["hidden_states"] for part in inputs_parts], dim=0)}
+ ps = get_parallel_state()
+ if ps.sp_size == 1:
+ self._sp_decode_length = None
+ return inputs
+
+ self._sp_decode_length, inputs = slice_sp_token_inputs("decode_velocity", **inputs)
+ return inputs
+
+ @post_forward("decode_velocity")
+ def decode_velocity_post(self, velocity: torch.Tensor) -> dict[str, Any]:
+ """Gather predicted velocity tokens, scatter onto items, and compute MSE flow loss."""
+ if get_parallel_state().sp_size > 1:
+ velocity = gather_sp_token_output("decode_velocity", velocity, self._sp_decode_length)
+
+ conversation = self._conversation_carrier
+ decode_target_groups = self._decode_target_groups
+ decode_lengths = self._decode_lengths
+ target = self._decode_target
+ self._conversation_carrier = None
+ self._decode_target_groups = []
+ self._decode_lengths = []
+ self._decode_target = None
+ self._sp_decode_length = None
+
+ real_velocity_parts = self._scatter_velocity(decode_target_groups, decode_lengths, velocity)
+ loss = velocity.float().sum() * 0.0
+ if target is not None and real_velocity_parts:
+ real_velocity = torch.cat(real_velocity_parts, dim=0)
+ mse = (real_velocity - target.to(device=velocity.device, dtype=velocity.dtype)).float().square()
+ num_valid_tokens = torch.tensor(mse.shape[0], device=velocity.device, dtype=torch.long)
+ loss = loss + mse.mean(dim=-1).sum() / num_valid_tokens.to(dtype=loss.dtype)
+ else:
+ num_valid_tokens = torch.zeros((), device=velocity.device, dtype=torch.long)
+
+ ps = get_parallel_state()
+ loss_reduction_group = (
+ ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None)
+ )
+ if loss_reduction_group is not None:
+ loss = reduce_sequence_parallel_loss(loss, num_valid_tokens, group=loss_reduction_group)
+ return {"conversation_list": conversation, "_loss": loss}
+
+ def _anchor_dummy_decode_velocity_inputs(
+ self,
+ conversation_list: list[list[ConversationItem]] | None,
+ ) -> dict[str, torch.Tensor]:
+ """Tie dummy flow loss to MoT hidden states without changing its value."""
+ dummy = {
+ "hidden_states": torch.zeros(
+ 1,
+ int(self.config.hidden_size),
+ device=self.device,
+ dtype=self.dtype,
+ )
+ }
+
+ if conversation_list is None:
+ return dummy
+
+ anchor = None
+ for item in iter_desired_items(
+ conversation_list,
+ types=["text", "image", "output"],
+ roles=["user", "assistant"],
+ ):
+ if item.is_dummy:
+ continue
+ value = item.value
+ if not torch.is_tensor(value):
+ continue
+ if value.dim() == 3 and value.shape[0] == 1:
+ value = value.squeeze(0)
+ if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size):
+ anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0
+ break
+ if anchor is None:
+ return dummy
+
+ return {"hidden_states": dummy["hidden_states"] + anchor}
+
+ def _select_velocity_target_groups(
+ self,
+ conversation_list: list[list[ConversationItem]] | None,
+ ) -> list[list[ConversationItem]]:
+ """Group non-dummy image items carrying ``flow_velocity_target`` per sample."""
+ if conversation_list is None:
+ raise ValueError("BAGEL flow connector requires conversation_list to select velocity targets.")
+
+ target_groups: list[list[ConversationItem]] = []
+ for sample in conversation_list:
+ # Keep one target group per sample. An empty group means the hook
+ # should run a sample-level dummy decode anchored to this sample's
+ # MoT hidden states.
+ sample_target_items: list[ConversationItem] = []
+ for item in iter_desired_items([sample], types=["image"]):
+ if not item.is_dummy and torch.is_tensor(item.meta.get("flow_velocity_target")):
+ sample_target_items.append(item)
+ target_groups.append(sample_target_items)
+ return target_groups
+
+ def _scatter_velocity(
+ self,
+ target_groups: list[list[ConversationItem]],
+ decode_lengths: list[int],
+ velocity: torch.Tensor,
+ ) -> list[torch.Tensor]:
+ """Write predicted velocity spans onto target items and return non-dummy spans."""
+ real_velocity_parts: list[torch.Tensor] = []
+ offset = 0
+ length_iter = iter(decode_lengths)
+ for group in target_groups:
+ if not group:
+ length = next(length_iter)
+ offset += length
+ continue
+ for item in group:
+ length = next(length_iter)
+ span = velocity[offset : offset + length]
+ offset += length
+ item.value = span.to(device=self.device, dtype=self.dtype)
+ if bagel_phase(item) == BAGEL_FLOW_HIDDEN:
+ item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY
+ real_velocity_parts.append(span)
+ if offset != int(velocity.shape[0]):
+ raise RuntimeError("BAGEL flow connector token count mismatch during velocity scatter.")
+ return real_velocity_parts
+
+
+class MeterMixin(MetricMeterMixin):
+ """Per-module training meter for BAGEL's flow connector."""
+
+ config: BagelFlowConnectorConfig
+
+ def estimate_flops(self, seqlens: list[int]) -> float:
+ # Only ``embed_latent`` stashes lengths. Per latent token it runs vae2llm
+ # and the per-token timestep MLP (freq -> hidden -> hidden); the hidden x
+ # hidden layer dominates. ``decode_velocity``'s llm2vae is folded in per
+ # embedded token, which slightly over-counts edit-context tokens that are
+ # never decoded (llm2vae is ~2% of the per-token cost).
+ cfg = self.config
+ hidden = int(cfg.hidden_size)
+ latent = int(cfg.patch_latent_dim)
+ freq = int(cfg.timestep_frequency_embedding_size)
+ per_token_n = latent * hidden + freq * hidden + hidden * hidden + hidden * latent
+ return 6 * per_token_n * sum(seqlens) / 1e12
+
+
+class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin):
+ """Carrier hooks for latent embedding and velocity projection.
+
+ ``embed_context_latents()`` / ``prepare_denoise_query()`` /
+ ``decode_velocity_from_hidden()`` / ``advance_denoise()`` and the shared
+ flow-matching generation-FSM state already live on the native
+ :class:`~.modeling.BagelFlowConnector` (via its own :class:`~.modeling.InferenceMixin`),
+ so no ``InferenceMixin`` is needed here.
+ """
+
+
+class BagelFlowConnectorAccelerated(VeOmniMixin, BagelFlowConnector):
+ pass
+
+
+__all__ = ["BagelFlowConnectorAccelerated"]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py
new file mode 100644
index 0000000000..5d6eb4000f
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py
@@ -0,0 +1,34 @@
+"""Configuration for BAGEL's VAE/LLM flow connector."""
+
+from ...module_configuration_base import OmniModuleConfig
+
+
+class BagelFlowConnectorConfig(OmniModuleConfig):
+ """BAGEL latent-token connector config."""
+
+ model_type = "bagel_flow_connector"
+
+ def __init__(
+ self,
+ hidden_size: int = 3584,
+ z_channels: int = 16,
+ latent_patch_size: int = 2,
+ patch_latent_dim: int | None = None,
+ max_latent_size: int = 64,
+ timestep_frequency_embedding_size: int = 256,
+ timestep_shift: float = 1.0,
+ **kwargs,
+ ) -> None:
+ self.hidden_size = hidden_size
+ self.z_channels = z_channels
+ self.latent_patch_size = latent_patch_size
+ self.patch_latent_dim = (
+ latent_patch_size * latent_patch_size * z_channels if patch_latent_dim is None else patch_latent_dim
+ )
+ self.max_latent_size = max_latent_size
+ self.timestep_frequency_embedding_size = timestep_frequency_embedding_size
+ self.timestep_shift = timestep_shift
+ super().__init__(**kwargs)
+
+
+__all__ = ["BagelFlowConnectorConfig"]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py
new file mode 100644
index 0000000000..fb483a7408
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py
@@ -0,0 +1,156 @@
+"""Generation-time denoise state for BAGEL flow connector."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+
+
+@dataclass
+class FlowGenerationState:
+ """Flow-side latent denoise state.
+
+ The flow connector owns the numerical denoise state: noisy latent tokens,
+ timesteps, dt values, and step progress. Branch/cache/controller state
+ belongs to the MoT module and is intentionally not represented here.
+ """
+
+ _latents: torch.Tensor | None = None
+ _timestep_values: torch.Tensor | None = None
+ _dt_values: torch.Tensor | None = None
+ _step_index: int = 0
+ _image_shape: tuple[int, int] | None = None
+ _grid_shape: tuple[int, int] | None = None
+ _token_count: int = 0
+
+ def reset(self) -> None:
+ self._latents = None
+ self._timestep_values = None
+ self._dt_values = None
+ self._step_index = 0
+ self._image_shape = None
+ self._grid_shape = None
+ self._token_count = 0
+
+ def initialize(
+ self,
+ generation_kwargs: dict[str, Any],
+ *,
+ resolution: int,
+ patch_latent_dim: int,
+ device: torch.device,
+ ) -> None:
+ """Initialize initial Gaussian latent tokens and shifted Euler timestep schedule."""
+ if bool(generation_kwargs.get("enable_taylorseer", False)):
+ raise NotImplementedError("BAGEL infer_gen TaylorSeer is not implemented in the SeedOmni graph path.")
+
+ image_height = int(generation_kwargs.get("image_height", resolution))
+ image_width = int(generation_kwargs.get("image_width", image_height))
+ latent_downsample = int(generation_kwargs.get("latent_downsample", 16))
+ if image_height % latent_downsample != 0 or image_width % latent_downsample != 0:
+ raise ValueError("BAGEL infer_gen image size must be divisible by latent_downsample.")
+
+ grid_shape = (image_height // latent_downsample, image_width // latent_downsample)
+ token_count = int(grid_shape[0] * grid_shape[1])
+ num_timesteps = int(generation_kwargs.get("num_timesteps", 50))
+ if num_timesteps < 2:
+ raise ValueError("BAGEL infer_gen num_timesteps must be at least 2.")
+
+ timestep_shift = float(generation_kwargs.get("timestep_shift", 1.0))
+ timesteps = torch.linspace(1.0, 0.0, num_timesteps, device=device, dtype=torch.float32)
+ timesteps = timestep_shift * timesteps / (1.0 + (timestep_shift - 1.0) * timesteps)
+
+ self._timestep_values = timesteps[:-1]
+ self._dt_values = timesteps[:-1] - timesteps[1:]
+ self._latents = torch.randn(
+ token_count,
+ int(patch_latent_dim),
+ device=device,
+ dtype=torch.float32,
+ )
+ self._step_index = 0
+ self._image_shape = (image_height, image_width)
+ self._grid_shape = grid_shape
+ self._token_count = token_count
+
+ @property
+ def initialized(self) -> bool:
+ return self._latents is not None
+
+ @property
+ def step_index(self) -> int:
+ return self._step_index
+
+ @property
+ def image_shape(self) -> tuple[int, int] | None:
+ return self._image_shape
+
+ @property
+ def token_count(self) -> int:
+ return self._token_count
+
+ @property
+ def latents(self) -> torch.Tensor:
+ """Return the current noisy latent tokens ``x_t``."""
+ if self._latents is None:
+ raise RuntimeError("BAGEL flow denoise latent state is not initialized.")
+ return self._latents
+
+ @property
+ def timestep_values(self) -> torch.Tensor:
+ """Return the shifted timestep schedule tensor."""
+ if self._timestep_values is None:
+ raise RuntimeError("BAGEL flow denoise timesteps are not initialized.")
+ return self._timestep_values
+
+ @property
+ def dt_values(self) -> torch.Tensor:
+ """Return the step size deltas ``dt`` between consecutive timesteps."""
+ if self._dt_values is None:
+ raise RuntimeError("BAGEL flow denoise dt values are not initialized.")
+ return self._dt_values
+
+ @property
+ def grid_shape(self) -> tuple[int, int]:
+ """Return the ``(height, width)`` patch grid dimensions."""
+ if self._grid_shape is None:
+ raise RuntimeError("BAGEL flow denoise latent grid shape is not initialized.")
+ return self._grid_shape
+
+ def current_timestep(self) -> torch.Tensor:
+ """Return the scalar timestep tensor for the current Euler step."""
+ timesteps = self.timestep_values
+ if self._step_index >= int(timesteps.numel()):
+ raise RuntimeError("BAGEL flow denoise timestep index is past the final step.")
+ return timesteps[self._step_index]
+
+ def current_timestep_tokens(self) -> torch.Tensor:
+ x_t = self.latents
+ return self.current_timestep().reshape(1).expand(int(x_t.shape[0]))
+
+ def is_complete(self) -> bool:
+ timesteps = self.timestep_values
+ return self._step_index >= int(timesteps.numel())
+
+ def advance(self, velocity: torch.Tensor) -> bool:
+ """Apply one Euler step ``x_t <- x_t - v_t * dt`` and return whether denoising finished."""
+ x_t = self.latents
+ dts = self.dt_values
+ # Official generate_image does `x_t - v_t.to(x_t.device) * dts[i]` without
+ # pre-casting velocity to fp32. bf16 * fp32 is not always the same as
+ # fp32(v) * fp32(dt); keep the mixed-dtype multiply.
+ velocity = velocity.to(device=x_t.device)
+ if velocity.shape != x_t.shape:
+ raise ValueError(
+ f"BAGEL flow velocity shape mismatch: got {tuple(velocity.shape)}, expected {tuple(x_t.shape)}."
+ )
+
+ dt = dts[self._step_index].to(device=x_t.device)
+ self._latents = x_t - velocity * dt
+ self._step_index += 1
+ return self.step_index >= int(dts.numel())
+
+
+__all__ = ["FlowGenerationState"]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py
new file mode 100644
index 0000000000..4d052c526f
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py
@@ -0,0 +1,403 @@
+"""BAGEL flow connector.
+
+``embed_latent`` projects patchified VAE latent tokens, timestep embeddings,
+and latent position embeddings to MoT hidden width. ``decode_velocity`` projects
+MoT hidden states back to patch-latent velocity tokens. Carrier selection,
+dummy alignment, and loss computation live in the SeedOmni module mixin.
+
+The flow-matching FSM ``generate``-family endpoints (``embed_context_latents``
+/ ``prepare_denoise_query`` / ``decode_velocity_from_hidden`` / ``advance_denoise``)
+and their shared :class:`~.generation_state.FlowGenerationState` live here too —
+pure inference over an already-loaded ``BagelFlowConnector`` needs no VeOmni
+training-graph machinery. ``accelerated.py`` only carries the training-graph
+pre/forward/post hooks (carrier selection, SP token slicing, dummy alignment).
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import torch
+import torch.nn as nn
+
+from ....graphs.generation_graph import FSM_SIGNAL_KEY
+from ....utils.conversation import ConversationItem, iter_desired_items
+from ...module_modeling_base import PretrainedOmniModule
+from ..position_embedding import PositionEmbedding
+from ..sources import (
+ BAGEL_CONTEXT_KEY,
+ BAGEL_FLOW_HIDDEN,
+ BAGEL_FLOW_QUERY,
+ BAGEL_FLOW_VELOCITY,
+ BAGEL_GENERATED_LATENT,
+ BAGEL_PHASE_KEY,
+ BAGEL_VAE_CONTEXT,
+ get_tail_phase_item,
+)
+from .configuration import BagelFlowConnectorConfig
+from .generation_state import FlowGenerationState
+from .processing import flattened_position_ids, preprocess_context_latent_embed, unpatchify_latent_tokens
+
+
+SIGNAL_IMAGE_COMPLETE = "image_complete"
+
+
+def select_vae_context_latent_items(
+ conversation_list: list[list[ConversationItem]] | None,
+) -> list[ConversationItem]:
+ """Select per-sample VAE context latents shared by training and inference hooks."""
+ if conversation_list is None:
+ raise ValueError("BAGEL flow connector requires conversation_list to select VAE context latents.")
+
+ # VAE preprocessing already makes BAGEL_VAE_CONTEXT per-sample by
+ # appending a dummy carrier only for samples without real VAE context.
+ # Reuse those existing carriers here; do not append another dummy.
+ return list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}))
+
+
+def scatter_flow_latent_embeds(
+ embed_items: list[ConversationItem],
+ embed_lengths: list[int],
+ latent_embeds: torch.Tensor,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> None:
+ """Write flow latent embeds back onto carrier items (training + inference)."""
+ offset = 0
+ for item, length in zip(embed_items, embed_lengths, strict=True):
+ item.value = latent_embeds[offset : offset + length].to(device=device, dtype=dtype)
+ offset += length
+
+ # Hand the upstream VAE dummy carrier off as a flow dummy anchor;
+ # downstream MoT should not treat it as another VAE image span. Only
+ # the context tag survives: it is how MoT recognizes the flow anchor.
+ if item.is_dummy:
+ item.type = "output"
+ item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}
+
+ if offset != int(latent_embeds.shape[0]):
+ raise RuntimeError("BAGEL flow connector latent count mismatch during embed scatter.")
+
+
+class InferenceMixin:
+ """Flow-matching denoise FSM (``embed_context_latents`` / ``prepare_denoise_query``
+ / ``decode_velocity_from_hidden`` / ``advance_denoise``) — HF ``GenerationMixin``
+ analog.
+
+ Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in
+ :class:`BagelFlowConnector`'s bases: ``PretrainedOmniModule`` ships a no-op
+ ``reset_local_inference_state`` default (kept as a safety net for modules
+ that don't need real inference state), and MRO resolves left-to-right —
+ put second, that no-op would shadow the real one below.
+ """
+
+ def reset_local_inference_state(self) -> None:
+ self._generation_state.reset()
+
+ def embed_context_latents(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Embed clean VAE context latents into LLM hidden space during prompt encoding."""
+ del generation_kwargs, kwargs
+ if conversation_list is None:
+ return {"conversation_list": conversation_list}
+
+ batched = [conversation_list]
+ embed_items = [item for item in select_vae_context_latent_items(batched) if not item.is_dummy]
+ if not embed_items:
+ return {"conversation_list": conversation_list}
+
+ inputs, embed_lengths = preprocess_context_latent_embed(
+ embed_items,
+ config=self.config,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ outputs = self.embed_latent(**inputs)
+ scatter_flow_latent_embeds(
+ embed_items,
+ embed_lengths,
+ outputs["latent_embeds"],
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": conversation_list}
+
+ def prepare_denoise_query(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Embed the current noisy latent tokens and write a ``flow_query`` output item."""
+ del kwargs
+ if conversation_list is None:
+ raise ValueError("BAGEL flow inference requires conversation_list.")
+
+ state = self._generation_state
+ if not state.initialized:
+ self._generation_state.initialize(
+ generation_kwargs or {},
+ resolution=int(getattr(self.config, "resolution", 1024)),
+ patch_latent_dim=int(self.config.patch_latent_dim),
+ device=self._vae2llm_device,
+ )
+
+ x_t = state.latents
+ timestep = state.current_timestep()
+ timestep_tokens = state.current_timestep_tokens()
+ position_ids = flattened_position_ids(
+ state.grid_shape,
+ max_latent_size=int(self.config.max_latent_size),
+ device=x_t.device,
+ )
+ outputs = self.embed_latent(
+ latents=x_t,
+ position_ids=position_ids,
+ timesteps=timestep_tokens,
+ )
+ query = outputs["latent_embeds"].to(device=self.device, dtype=self.dtype)
+ timestep_meta = timestep.detach().to(device=query.device, dtype=torch.float32)
+
+ item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY)
+ if item is None:
+ conversation_list.append(
+ ConversationItem(
+ type="output",
+ value=query,
+ role="assistant",
+ meta={"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY},
+ )
+ )
+ else:
+ item.type = "output"
+ item.role = "assistant"
+ item.value = query
+ item.meta = {"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY}
+ return {"conversation_list": conversation_list}
+
+ def decode_velocity_from_hidden(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Project MoT denoise hidden states to patch-latent velocity tokens."""
+ del generation_kwargs, kwargs
+ if conversation_list is None:
+ raise ValueError("BAGEL flow inference requires conversation_list.")
+
+ item = get_tail_phase_item(conversation_list, BAGEL_FLOW_HIDDEN)
+ if item is None or not torch.is_tensor(item.value):
+ raise ValueError(f"BAGEL flow decode_velocity requires an output in phase {BAGEL_FLOW_HIDDEN!r}.")
+
+ hidden = item.value
+ if hidden.dim() == 3 and hidden.shape[0] == 1:
+ hidden = hidden.squeeze(0)
+ if hidden.dim() != 2:
+ raise ValueError(f"BAGEL flow decode_velocity expected rank-2 hidden states, got {tuple(hidden.shape)}.")
+ if int(hidden.shape[-1]) != int(self.config.hidden_size):
+ raise ValueError(
+ "BAGEL flow decode_velocity hidden-size mismatch: "
+ f"got {hidden.shape[-1]}, expected {self.config.hidden_size}."
+ )
+
+ # Official llm2vae is one GEMM per CFG branch. A stacked GEMM over
+ # concatenated branches is not bitwise even when each branch hidden
+ # matches, so split back to the per-branch query length.
+ query_len = int(self._generation_state.token_count) + 2
+ if (
+ self._generation_state.initialized
+ and query_len >= 3
+ and int(hidden.shape[0]) >= query_len
+ and int(hidden.shape[0]) % query_len == 0
+ ):
+ velocity = torch.cat(
+ [self.decode_velocity(hidden_states=chunk)["velocity"] for chunk in hidden.split(query_len, dim=0)],
+ dim=0,
+ )
+ else:
+ velocity = self.decode_velocity(hidden_states=hidden)["velocity"]
+ item.type = "output"
+ item.role = "assistant"
+ item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY
+ item.value = velocity.to(device=self.device, dtype=self.dtype)
+ return {"conversation_list": conversation_list}
+
+ def advance_denoise(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Advance the Euler denoise state by one step or emit the final unpatchified latent."""
+ del generation_kwargs, kwargs
+ if conversation_list is None:
+ raise ValueError("BAGEL flow inference requires conversation_list.")
+
+ item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY)
+ if item is None or not torch.is_tensor(item.value):
+ raise ValueError(f"BAGEL flow advance requires an output in phase {BAGEL_FLOW_VELOCITY!r}.")
+
+ velocity = item.value
+ if velocity.dim() == 3 and velocity.shape[0] == 1:
+ velocity = velocity.squeeze(0)
+ complete = self._generation_state.advance(velocity)
+
+ if complete:
+ return self._emit_final_latent(conversation_list)
+ item.meta.pop("timestep", None)
+ return {"conversation_list": conversation_list}
+
+ def _emit_final_latent(
+ self,
+ conversation_list: list[ConversationItem],
+ ) -> dict[str, Any]:
+ """Unpatchify the completed latent grid and emit ``image_complete``."""
+ x_t = self._generation_state.latents
+ item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY)
+ latent = unpatchify_latent_tokens(
+ x_t,
+ self._generation_state.grid_shape,
+ z_channels=int(self.config.z_channels),
+ latent_patch_size=int(self.config.latent_patch_size),
+ )
+ if item is None:
+ conversation_list.append(
+ ConversationItem(
+ type="output",
+ value=latent,
+ role="assistant",
+ meta={BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT},
+ )
+ )
+ else:
+ item.type = "output"
+ item.role = "assistant"
+ item.value = latent.to(device=self.device, dtype=self.dtype)
+ item.meta = {BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT}
+
+ self._generation_state.reset()
+ return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: SIGNAL_IMAGE_COMPLETE}
+
+
+def _module_weight(module: nn.Module) -> torch.Tensor:
+ """Return the underlying weight parameter for plain or LoRA-wrapped linear modules."""
+ return getattr(module, "base_layer", module).weight
+
+
+class BagelFlowConnector(InferenceMixin, PretrainedOmniModule):
+ config_class = BagelFlowConnectorConfig
+ base_model_prefix = "bagel_flow_connector"
+ main_input_name = "hidden_states"
+ _no_split_modules: list[str] = []
+
+ def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs):
+ # The flow connector projects latent tokens and timesteps; it does not execute attention.
+ return attn_implementation
+
+ def __init__(self, config: BagelFlowConnectorConfig) -> None:
+ super().__init__(config)
+ self.time_embedder = TimestepEmbedder(config.hidden_size, config.timestep_frequency_embedding_size)
+ self.vae2llm = nn.Linear(config.patch_latent_dim, config.hidden_size)
+ self.llm2vae = nn.Linear(config.hidden_size, config.patch_latent_dim)
+ self.latent_pos_embed = PositionEmbedding(config.max_latent_size, config.hidden_size)
+ self._generation_state = FlowGenerationState()
+ self.post_init()
+
+ def _init_weights(self, module: nn.Module) -> None:
+ if isinstance(module, PositionEmbedding):
+ module.reset_parameters()
+ return
+ if module is self.llm2vae:
+ nn.init.constant_(module.weight, 0)
+ if module.bias is not None:
+ nn.init.constant_(module.bias, 0)
+ return
+ super()._init_weights(module)
+
+ @property
+ def _vae2llm_device(self) -> torch.device:
+ return _module_weight(self.vae2llm).device
+
+ @property
+ def _llm2vae_device(self) -> torch.device:
+ return _module_weight(self.llm2vae).device
+
+ @property
+ def _pos_embed_device(self) -> torch.device:
+ return self.latent_pos_embed.pos_embed.device
+
+ @property
+ def _time_embedder_device(self) -> torch.device:
+ return _module_weight(self.time_embedder.mlp[0]).device
+
+ def embed_latent(
+ self,
+ latents: torch.Tensor,
+ position_ids: torch.LongTensor,
+ timesteps: torch.Tensor,
+ ) -> dict[str, torch.Tensor]:
+ """Project patchified latents and add timestep and 2D sinusoidal position embeddings."""
+ vae2llm_weight = _module_weight(self.vae2llm)
+ latents = latents.to(device=vae2llm_weight.device, dtype=vae2llm_weight.dtype)
+ position_ids = position_ids.to(device=self._pos_embed_device, dtype=torch.long).reshape(-1)
+ timesteps = timesteps.to(device=self._time_embedder_device, dtype=torch.float32).reshape(-1)
+ if position_ids.numel() != latents.shape[0]:
+ raise ValueError("position_ids must have one value per latent token.")
+
+ latent_embeds = self.vae2llm(latents)
+ time_embeds = self.time_embedder(timesteps)
+ pos_embeds = self.latent_pos_embed(position_ids)
+ return {
+ "latent_embeds": latent_embeds
+ + time_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype)
+ + pos_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype)
+ }
+
+ def decode_velocity(self, hidden_states: torch.Tensor) -> dict[str, torch.Tensor]:
+ llm2vae_weight = _module_weight(self.llm2vae)
+ hidden_states = hidden_states.to(device=llm2vae_weight.device, dtype=llm2vae_weight.dtype)
+ return {"velocity": self.llm2vae(hidden_states)}
+
+
+class TimestepEmbedder(nn.Module):
+ def __init__(self, hidden_size: int, frequency_embedding_size: int = 256) -> None:
+ super().__init__()
+ self.mlp = nn.Sequential(
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+ nn.SiLU(),
+ nn.Linear(hidden_size, hidden_size, bias=True),
+ )
+ self.frequency_embedding_size = frequency_embedding_size
+
+ @staticmethod
+ def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor:
+ """Create sinusoidal timestep embeddings of width ``dim``."""
+ half = dim // 2
+ freqs = torch.exp(-math.log(max_period) * torch.arange(half, dtype=torch.float32, device=t.device) / half)
+ args = t[:, None].float() * freqs[None]
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+ if dim % 2:
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+ return embedding
+
+ def forward(self, t: torch.Tensor) -> torch.Tensor:
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+ first_weight = _module_weight(self.mlp[0])
+ return self.mlp(t_freq.to(device=first_weight.device, dtype=first_weight.dtype))
+
+
+__all__ = [
+ "BagelFlowConnector",
+ "BagelFlowConnectorConfig",
+ "InferenceMixin",
+ "SIGNAL_IMAGE_COMPLETE",
+ "select_vae_context_latent_items",
+ "scatter_flow_latent_embeds",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py
new file mode 100644
index 0000000000..dc9c0d0deb
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py
@@ -0,0 +1,216 @@
+"""Latent patch and timestep helpers for BAGEL flow connector."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from ....utils.conversation import ConversationItem
+
+
+def preprocess_latent_embed(
+ embed_items: list[ConversationItem],
+ *,
+ config: Any,
+ device: torch.device,
+ dtype: torch.dtype,
+ timestep_shift: float,
+) -> tuple[dict[str, torch.Tensor], list[int]]:
+ """Build noised latent embed inputs and training velocity targets."""
+ latents: list[torch.Tensor] = []
+ position_ids: list[torch.Tensor] = []
+ timesteps: list[torch.Tensor] = []
+ embed_lengths: list[int] = []
+ for item in embed_items:
+ clean, grid_shape = _patchify_latent_grid(
+ item.value,
+ z_channels=int(config.z_channels),
+ latent_patch_size=int(config.latent_patch_size),
+ )
+ clean = clean.to(device=device, dtype=dtype)
+ token_count = int(clean.shape[0])
+
+ noise = torch.randn_like(clean)
+ timestep = torch.sigmoid(torch.randn(1, device=device, dtype=torch.float32)).expand(token_count)
+ timestep = float(timestep_shift) * timestep / (1.0 + (float(timestep_shift) - 1.0) * timestep)
+ timestep_values = timestep.reshape(-1, 1).to(device=clean.device, dtype=torch.float32)
+ noised = (1.0 - timestep_values) * clean + timestep_values * noise
+ noised = noised.to(dtype=dtype)
+
+ item.meta["timestep"] = timestep.detach()
+ item.meta["noise"] = noise.detach()
+ item.meta["flow_velocity_target"] = (noise - clean).detach()
+ latents.append(noised)
+ position_ids.append(
+ flattened_position_ids(
+ grid_shape,
+ max_latent_size=int(config.max_latent_size),
+ device=device,
+ )
+ )
+ timesteps.append(timestep)
+ embed_lengths.append(token_count)
+
+ return (
+ {
+ "latents": torch.cat(latents, dim=0),
+ "position_ids": torch.cat(position_ids, dim=0),
+ "timesteps": torch.cat(timesteps, dim=0),
+ },
+ embed_lengths,
+ )
+
+
+def preprocess_context_latent_embed(
+ embed_items: list[ConversationItem],
+ *,
+ config: Any,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> tuple[dict[str, torch.Tensor], list[int]]:
+ """Build clean latent embed inputs for inference context latents."""
+ latents: list[torch.Tensor] = []
+ position_ids: list[torch.Tensor] = []
+ embed_lengths: list[int] = []
+ for item in embed_items:
+ clean, grid_shape = _patchify_latent_grid(
+ item.value,
+ z_channels=int(config.z_channels),
+ latent_patch_size=int(config.latent_patch_size),
+ )
+ clean = clean.to(device=device)
+ token_count = int(clean.shape[0])
+
+ latents.append(clean)
+ position_ids.append(
+ flattened_position_ids(
+ grid_shape,
+ max_latent_size=int(config.max_latent_size),
+ device=device,
+ )
+ )
+ embed_lengths.append(token_count)
+
+ return (
+ {
+ "latents": torch.cat(latents, dim=0),
+ "position_ids": torch.cat(position_ids, dim=0),
+ "timesteps": torch.zeros(sum(embed_lengths), device=device, dtype=torch.float32),
+ },
+ embed_lengths,
+ )
+
+
+def preprocess_decode_velocity(
+ decode_items: list[ConversationItem],
+ *,
+ config: Any,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> tuple[dict[str, torch.Tensor], list[int], torch.Tensor]:
+ """Build packed hidden states and velocity targets for training decode."""
+ hidden_states: list[torch.Tensor] = []
+ targets: list[torch.Tensor] = []
+ decode_lengths: list[int] = []
+ for item in decode_items:
+ hidden = item.value
+ if not torch.is_tensor(hidden):
+ raise ValueError("BAGEL flow decode_velocity expects tensor hidden states.")
+ if hidden.dim() == 3 and hidden.shape[0] == 1:
+ hidden = hidden.squeeze(0)
+ if hidden.dim() != 2:
+ raise ValueError(
+ f"BAGEL flow decode_velocity expects rank-2 hidden states, got {tuple(item.value.shape)}."
+ )
+ if int(hidden.shape[-1]) != int(config.hidden_size):
+ raise ValueError(
+ "BAGEL flow decode_velocity hidden-size mismatch: "
+ f"got {hidden.shape[-1]}, expected {config.hidden_size}."
+ )
+ hidden = hidden.to(device=device)
+ hidden_states.append(hidden)
+ decode_lengths.append(int(hidden.shape[0]))
+
+ target = item.meta.get("flow_velocity_target").to(device=device, dtype=dtype)
+ expected_shape = (hidden.shape[0], int(config.patch_latent_dim))
+ if target.shape != expected_shape:
+ raise ValueError(
+ f"BAGEL flow_velocity_target shape mismatch: got {tuple(target.shape)}, expected {expected_shape}."
+ )
+ targets.append(target)
+
+ return {"hidden_states": torch.cat(hidden_states, dim=0)}, decode_lengths, torch.cat(targets, dim=0)
+
+
+def _patchify_latent_grid(
+ value: torch.Tensor,
+ *,
+ z_channels: int,
+ latent_patch_size: int,
+) -> tuple[torch.Tensor, tuple[int, int]]:
+ """Patchify a latent grid into a list of tokens."""
+ latent = value.detach()
+ if latent.dim() == 4 and latent.shape[0] == 1:
+ latent = latent.squeeze(0)
+ if latent.dim() != 3:
+ raise ValueError(f"BAGEL flow connector expects latent grid tensors, got shape {tuple(value.shape)}.")
+ if int(latent.shape[0]) != int(z_channels):
+ raise ValueError(
+ f"BAGEL flow connector latent channel mismatch: got {int(latent.shape[0])}, expected {int(z_channels)}."
+ )
+ channels, height, width = latent.shape
+
+ patch_size = int(latent_patch_size)
+ if height % patch_size != 0 or width % patch_size != 0:
+ raise ValueError(
+ "BAGEL flow connector latent grid is not divisible by latent_patch_size: "
+ f"shape={tuple(latent.shape)}, latent_patch_size={patch_size}."
+ )
+
+ grid_h, grid_w = height // patch_size, width // patch_size
+ tokens = latent.reshape(channels, grid_h, patch_size, grid_w, patch_size)
+ tokens = torch.einsum("chpwq->hwpqc", tokens)
+ return tokens.reshape(-1, patch_size * patch_size * channels), (grid_h, grid_w)
+
+
+def unpatchify_latent_tokens(
+ tokens: torch.Tensor,
+ grid_shape: tuple[int, int],
+ *,
+ z_channels: int,
+ latent_patch_size: int,
+) -> torch.Tensor:
+ grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1])
+ patch = int(latent_patch_size)
+ channels = int(z_channels)
+ latent = tokens.reshape(grid_h, grid_w, patch, patch, channels)
+ latent = torch.einsum("hwpqc->chpwq", latent)
+ return latent.reshape(channels, grid_h * patch, grid_w * patch)
+
+
+def flattened_position_ids(
+ grid_shape: tuple[int, int],
+ *,
+ max_latent_size: int,
+ device: torch.device | None = None,
+) -> torch.Tensor:
+ """Compute flattened 2D grid position IDs bounded by ``max_latent_size``."""
+ grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1])
+ if grid_h > max_latent_size or grid_w > max_latent_size:
+ raise ValueError(
+ "BAGEL flow connector latent grid exceeds max_latent_size: "
+ f"grid={grid_shape}, max_latent_size={max_latent_size}."
+ )
+ coords_h = torch.arange(grid_h, device=device, dtype=torch.long)
+ coords_w = torch.arange(grid_w, device=device, dtype=torch.long)
+ return (coords_h[:, None] * int(max_latent_size) + coords_w).flatten()
+
+
+__all__ = [
+ "flattened_position_ids",
+ "preprocess_context_latent_embed",
+ "preprocess_decode_velocity",
+ "preprocess_latent_embed",
+ "unpatchify_latent_tokens",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/position_embedding.py b/veomni/models/seed_omni/modules/bagel/position_embedding.py
new file mode 100644
index 0000000000..39e179a417
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/position_embedding.py
@@ -0,0 +1,70 @@
+"""Shared 2D sin-cos position embedding for BAGEL vision and latent patch grids."""
+
+from __future__ import annotations
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.distributed.tensor import DTensor, distribute_tensor
+
+
+class PositionEmbedding(nn.Module):
+ """Fixed 2D sin-cos position embedding table indexed by flattened patch IDs."""
+
+ def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None:
+ super().__init__()
+ self.max_num_patch_per_side = max_num_patch_per_side
+ self.hidden_size = hidden_size
+ self.pos_embed = nn.Parameter(
+ torch.zeros(max_num_patch_per_side**2, hidden_size),
+ requires_grad=False,
+ )
+ self.reset_parameters()
+
+ def reset_parameters(self) -> None:
+ """Populate ``pos_embed`` with deterministic 2D sin-cos values."""
+ if self.pos_embed.is_meta:
+ return
+ new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side)
+ if isinstance(self.pos_embed.data, DTensor):
+ local_target = self.pos_embed.data.to_local()
+ sharded = distribute_tensor(
+ new_data.to(device=local_target.device, dtype=local_target.dtype),
+ self.pos_embed.data.device_mesh,
+ self.pos_embed.data.placements,
+ src_data_rank=None,
+ )
+ local_target.copy_(sharded.to_local().to(device=local_target.device, dtype=local_target.dtype))
+ elif hasattr(self.pos_embed.data, "to_local"):
+ self.pos_embed.data.to_local().copy_(new_data)
+ else:
+ self.pos_embed.data.copy_(new_data)
+
+ def _init_weights(self) -> None:
+ self.reset_parameters()
+
+ def forward(self, position_ids: torch.LongTensor) -> torch.Tensor:
+ return self.pos_embed[position_ids.to(device=self.pos_embed.device)]
+
+
+def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor:
+ """Build a ``[grid_size**2, embed_dim]`` 2D sin-cos position embedding tensor."""
+ grid_h = np.arange(grid_size, dtype=np.float32)
+ grid_w = np.arange(grid_size, dtype=np.float32)
+ grid = np.meshgrid(grid_w, grid_h)
+ grid = np.stack(grid, axis=0).reshape(2, 1, grid_size, grid_size)
+ emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])
+ emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])
+ return torch.from_numpy(np.concatenate([emb_h, emb_w], axis=1)).float()
+
+
+def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray:
+ """Build a 1D sin-cos position embedding array from flattened grid coordinates."""
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
+ omega /= embed_dim / 2.0
+ omega = 1.0 / 10000**omega
+ out = np.einsum("m,d->md", pos.reshape(-1), omega)
+ return np.concatenate([np.sin(out), np.cos(out)], axis=1)
+
+
+__all__ = ["PositionEmbedding"]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py
new file mode 100644
index 0000000000..8c774dee1a
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py
@@ -0,0 +1,24 @@
+"""BAGEL Qwen2 MoT backbone module."""
+
+from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY
+
+
+@OMNI_CONFIG_REGISTRY.register("bagel_qwen2_mot")
+def register_bagel_qwen2_mot_config():
+ from .configuration import BagelQwen2MoTConfig
+
+ return BagelQwen2MoTConfig
+
+
+@OMNI_MODEL_REGISTRY.register("bagel_qwen2_mot")
+def register_bagel_qwen2_mot_model():
+ from .modeling import BagelQwen2MoT
+
+ return BagelQwen2MoT
+
+
+@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_qwen2_mot")
+def register_bagel_qwen2_mot_accelerated_model():
+ from .accelerated import BagelQwen2MoTAccelerated
+
+ return BagelQwen2MoTAccelerated
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py
new file mode 100644
index 0000000000..30fc075956
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py
@@ -0,0 +1,12 @@
+"""Accelerated runtime for the bagel/qwen2_mot OmniModule (conversation graph)."""
+
+from .accelerated import (
+ BagelQwen2MoTAccelerated,
+ InferenceMixinAccelerated,
+)
+
+
+__all__ = [
+ "BagelQwen2MoTAccelerated",
+ "InferenceMixinAccelerated",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py
new file mode 100644
index 0000000000..68671c8848
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py
@@ -0,0 +1,722 @@
+"""VeOmni-accelerated BAGEL Qwen2-MoT — instance-local VeomniOp ops plus training-graph hooks.
+
+Native weights and packed ``forward`` live on :class:`BagelQwen2MoTCore`.
+This module adds packed CFG denoise, async Ulysses, fused RMSNorm / SwiGLU / RoPE,
+attention ``VeomniOp`` dispatch, and the training pre/forward/post hooks.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.distributed as dist
+from torch.nn.attention.flex_attention import BlockMask
+from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm
+
+from .......distributed.parallel_state import get_parallel_state
+from .......distributed.sequence_parallel import (
+ gather_outputs,
+ slice_input_tensor,
+ sp_pad,
+)
+from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import (
+ Qwen2MLP as PatchedQwen2MLP,
+)
+from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import (
+ Qwen2RMSNorm as PatchedQwen2RMSNorm,
+)
+from .......ops import VeomniOp
+from .......ops.mask import MagiAttentionMask
+from .....mixins.base_mixin import BaseMixin
+from .....mixins.metric_meter_mixin import MetricMeterMixin
+from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward
+from .....utils.conversation import ConversationItem, iter_desired_items
+from ...sources import (
+ BAGEL_CONTEXT_KEY,
+ BAGEL_FLOW_HIDDEN,
+ BAGEL_PHASE_KEY,
+ BAGEL_SIGLIP_CONTEXT,
+ BAGEL_VAE_CONTEXT,
+ bagel_context,
+)
+from ..checkpoint_conversion import (
+ BagelQwen2MoTCheckpointTensorConverter,
+ combine_qkv_state_dict_pre_hook,
+ register_accelerated_qkv_conversion_mapping,
+)
+from ..configuration import BagelQwen2MoTConfig
+from ..masking import build_mot_block_mask, build_mot_magi_mask, pad_mot_attention_metadata
+from ..modeling import (
+ BagelQwen2MoTAttention,
+ BagelQwen2MoTCore,
+ InferenceMixin,
+)
+from ..processing import PackedConversation, preprocess_mot_inputs
+
+
+_FLASH_ATTENTION_2 = "veomni_flash_attention_2"
+
+
+class TrainingMixin(TrainingModuleMixin):
+ """Training-graph hooks — depends on :class:`BagelQwen2MoTCore` modeling APIs."""
+
+ config: BagelQwen2MoTConfig
+ device: torch.device
+ dtype: torch.dtype
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self._conversation_carrier: list[list[ConversationItem]] | None = None
+ self._packed_training: PackedConversation | None = None
+ self._sp_full_sequence_length: int | None = None
+ self._validated_ulysses_size: int | None = None
+
+ @pre_forward("forward")
+ def forward_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Pack multimodal conversation items into MoT training tensors and slice for Ulysses SP."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+ self._packed_training = preprocess_mot_inputs(
+ conversation_list,
+ device=self.device,
+ dtype=self.dtype,
+ hidden_size=int(self.config.hidden_size),
+ )
+ if self._packed_training is None:
+ raise ValueError(
+ "BAGEL Qwen2-MoT forward requires a non-empty conversation_list with real tokens; "
+ "got no packable tokens across the whole batch."
+ )
+
+ # Meter the full replicated batch before SP slices its packed sequence.
+ self.metric_meter_set_seqlens(
+ "forward",
+ [int(sum(splits)) for splits in self._packed_training.sample_splits],
+ )
+
+ packed_sequence = self._fold_dummy_anchors(self._packed_training.packed_sequence, conversation_list)
+ sequence_length = int(packed_sequence.shape[0])
+ attention_metadata = self._packed_training.packed_attention_metadata
+ expected_metadata_shape = (3, sequence_length)
+ if tuple(attention_metadata.shape) != expected_metadata_shape:
+ raise ValueError(
+ "BAGEL Qwen2-MoT attention metadata must match the full packed sample: "
+ f"expected {expected_metadata_shape}, got {tuple(attention_metadata.shape)}."
+ )
+
+ ps = get_parallel_state()
+ if ps.sp_size == 1:
+ self._sp_full_sequence_length = None
+ return {
+ "packed_sequence": packed_sequence,
+ "packed_position_ids": self._packed_training.packed_position_ids,
+ "packed_token_type_ids": self._packed_training.packed_token_type_ids,
+ "packed_attention_metadata": attention_metadata,
+ }
+
+ if ps.cp_size != 1:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT training supports Ulysses sequence parallelism only; got cp_size={ps.cp_size}."
+ )
+ if self._validated_ulysses_size != ps.ulysses_size:
+ num_heads = int(self.config.num_attention_heads)
+ num_key_value_heads = int(self.config.num_key_value_heads)
+ if num_heads % ps.ulysses_size != 0:
+ raise ValueError(
+ "BAGEL Qwen2-MoT attention heads must be divisible by "
+ f"ulysses_size: num_heads={num_heads}, ulysses_size={ps.ulysses_size}."
+ )
+ if num_key_value_heads % ps.ulysses_size != 0:
+ raise ValueError(
+ "BAGEL Qwen2-MoT KV heads must be divisible by "
+ f"ulysses_size for native GQA: num_key_value_heads={num_key_value_heads}, "
+ f"ulysses_size={ps.ulysses_size}."
+ )
+ self._validated_ulysses_size = ps.ulysses_size
+
+ # Tensor inputs are padded then sliced for Ulysses. Attention metadata
+ # stays full because the Flex adapter reconstructs the full sequence
+ # before building/using its BlockMask.
+ self._sp_full_sequence_length = sequence_length
+ packed_sequence = sp_pad(packed_sequence, dim=0, pad_value=0)
+ packed_position_ids = sp_pad(self._packed_training.packed_position_ids, dim=0, pad_value=0)
+ packed_token_type_ids = sp_pad(self._packed_training.packed_token_type_ids, dim=0, pad_value=-1)
+ attention_metadata = pad_mot_attention_metadata(
+ attention_metadata,
+ padded_length=int(packed_sequence.shape[0]),
+ )
+ packed_sequence = slice_input_tensor(packed_sequence, dim=0, padding=False, group=ps.sp_group)
+ packed_position_ids = slice_input_tensor(
+ packed_position_ids,
+ dim=0,
+ padding=False,
+ group=ps.sp_group,
+ )
+ packed_token_type_ids = slice_input_tensor(
+ packed_token_type_ids,
+ dim=0,
+ padding=False,
+ group=ps.sp_group,
+ )
+
+ return {
+ "packed_sequence": packed_sequence,
+ "packed_position_ids": packed_position_ids,
+ "packed_token_type_ids": packed_token_type_ids,
+ # FlexAttention's generic Ulysses adapter reconstructs the complete
+ # padded sequence, so every rank keeps the same full metadata.
+ "packed_attention_metadata": attention_metadata,
+ }
+
+ @post_forward("forward")
+ def forward_post(self, hidden_states: torch.Tensor, **kwargs: Any) -> dict[str, Any]:
+ """Gather SP sequence shards if active and scatter MoT hidden states onto carrier items."""
+ del kwargs
+ ps = get_parallel_state()
+ if ps.sp_size > 1:
+ if self._sp_full_sequence_length is None:
+ raise RuntimeError("BAGEL Qwen2-MoT SP sequence length was not initialized.")
+ hidden_states = gather_outputs(hidden_states, gather_dim=0, group=ps.sp_group)
+ hidden_states = hidden_states.narrow(0, 0, self._sp_full_sequence_length)
+
+ conversation = self._conversation_carrier
+ packed = self._packed_training
+ self._conversation_carrier = None
+ self._packed_training = None
+ self._sp_full_sequence_length = None
+
+ for span in packed.spans:
+ span_hidden = hidden_states[span.start : span.start + span.length].to(device=self.device)
+ offset = 0
+ for item, length in zip(span.items, span.lengths, strict=True):
+ item.value = span_hidden[offset : offset + length]
+ offset += length
+ return {"conversation_list": conversation}
+
+ def _fold_dummy_anchors(
+ self,
+ packed_sequence: torch.Tensor,
+ conversation_list: list[list[ConversationItem]] | None,
+ ) -> torch.Tensor:
+ """Fold zero-valued FSDP dummy anchors from upstream SigLIP and flow encoders."""
+ # Dummy encoder outputs must still touch MoT on ranks where another rank
+ # has the real branch, otherwise FSDP sees different gradient buckets.
+ anchor = packed_sequence.new_zeros(())
+ has_anchor = False
+ include_siglip_dummy, include_flow_dummy = self._has_valid_upstream_embeddings(conversation_list)
+
+ for item in iter_desired_items(conversation_list or []):
+ if not item.is_dummy or not torch.is_tensor(item.value):
+ continue
+
+ # The flow connector hands its embedded VAE placeholder on as an
+ # ``output`` row; a VAE placeholder still of type ``image`` never
+ # reached the flow connector and holds no MoT-width embedding.
+ context = bagel_context(item)
+ is_flow_dummy = item.type == "output" and context == BAGEL_VAE_CONTEXT
+ if not is_flow_dummy and context != BAGEL_SIGLIP_CONTEXT:
+ continue
+ if is_flow_dummy and not include_flow_dummy:
+ continue
+ if context == BAGEL_SIGLIP_CONTEXT and not include_siglip_dummy:
+ continue
+
+ has_anchor = True
+ anchor = (
+ anchor
+ + item.value.to(
+ device=packed_sequence.device,
+ dtype=packed_sequence.dtype,
+ ).sum()
+ * 0.0
+ )
+
+ if not has_anchor:
+ return packed_sequence
+ return packed_sequence + anchor
+
+ def _has_valid_upstream_embeddings(
+ self,
+ conversation_list: list[list[ConversationItem]] | None,
+ ) -> tuple[bool, bool]:
+ """Return whether any rank in the distributed group has real SigLIP or flow embeddings."""
+ has_siglip = int(
+ self._has_valid_upstream_embedding(
+ conversation_list,
+ label="SigLIP",
+ types=["image"],
+ meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]},
+ )
+ )
+ has_flow = int(
+ self._has_valid_upstream_embedding(
+ conversation_list,
+ label="flow",
+ types=["image"],
+ meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]},
+ meta_keys=["flow_velocity_target"],
+ )
+ )
+
+ if not dist.is_available() or not dist.is_initialized():
+ return bool(has_siglip), bool(has_flow)
+ flags = torch.tensor([has_siglip, has_flow], device=self.device, dtype=torch.int32)
+ dist.all_reduce(flags, op=dist.ReduceOp.MAX)
+ return bool(flags[0].item()), bool(flags[1].item())
+
+ def _has_valid_upstream_embedding(
+ self,
+ conversation_list: list[list[ConversationItem]] | None,
+ *,
+ label: str,
+ types: list[str],
+ meta: dict[str, list[Any]] | None = None,
+ meta_keys: list[str] | None = None,
+ ) -> bool:
+ """Check whether ``conversation_list`` contains a non-dummy rank-2 embedding matching the filter."""
+ for item in iter_desired_items(
+ conversation_list or [],
+ types=types,
+ roles=["user", "assistant"],
+ meta_keys=meta_keys,
+ meta=meta,
+ ):
+ if item.is_dummy:
+ continue
+ value = item.value
+ if not torch.is_tensor(value):
+ continue
+ if value.dim() == 3 and value.shape[0] == 1:
+ value = value.squeeze(0)
+ if value.dim() != 2:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT {label} alignment expects rank-2 embeddings, got {tuple(value.shape)}."
+ )
+ if int(value.shape[-1]) != int(self.config.hidden_size):
+ raise ValueError(
+ f"BAGEL Qwen2-MoT {label} alignment hidden-size mismatch: "
+ f"got {value.shape[-1]}, expected {self.config.hidden_size}."
+ )
+ return True
+ return False
+
+
+class MeterMixin(MetricMeterMixin):
+ """Per-module training meter for BAGEL's Qwen2-MoT backbone (transformer layers only)."""
+
+ config: BagelQwen2MoTConfig
+
+ def estimate_flops(self, seqlens: list[int]) -> float:
+ """Estimate TFLOPs for the Qwen2-MoT transformer layers over ``seqlens``."""
+ cfg = self.config
+ hidden = cfg.hidden_size
+ num_layers = cfg.num_hidden_layers
+ num_heads = cfg.num_attention_heads
+ num_kv_heads = cfg.num_key_value_heads
+ # Attention uses head_dim = hidden // num_heads (see BagelQwen2MoTAttention).
+ head_dim = hidden // num_heads
+
+ # SwiGLU MLP (gate/up/down) + attention projections (q, o over num_heads;
+ # k, v over num_kv_heads). Biases are O(hidden) → negligible, ignored.
+ mlp_n = hidden * cfg.intermediate_size * 3
+ attn_linear_n = hidden * head_dim * (num_heads * 2 + num_kv_heads * 2)
+ dense_n = (mlp_n + attn_linear_n) * num_layers
+
+ tokens = sum(seqlens)
+ seqlen_sq = sum(s * s for s in seqlens)
+ dense_flops = 6 * dense_n * tokens
+ attn_flops = 12 * seqlen_sq * head_dim * num_heads * num_layers
+ return (dense_flops + attn_flops) / 1e12
+
+
+class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin):
+ """Carrier hooks and graph entrypoints for BAGEL's packed MoT backbone.
+
+ Packed CFG denoise is mixed in separately via :class:`InferenceMixinAccelerated`
+ so this mixin stays runtime-only.
+ """
+
+
+def _apply_rotary_pos_emb(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+ unsqueeze_dim: int = 1,
+ *,
+ rope_op: VeomniOp | None = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Packed RoPE dispatched through the instance-local ``VeomniOp``."""
+ if rope_op is None:
+ rope_op = VeomniOp("rope", "full", "eager")
+ if unsqueeze_dim != 1:
+ raise NotImplementedError("BAGEL packed fused RoPE requires unsqueeze_dim=1.")
+ if q.ndim != 3 or k.ndim != 3 or cos.ndim != 2 or sin.ndim != 2:
+ raise NotImplementedError(
+ "BAGEL fused RoPE requires packed q/k tensors shaped [tokens, heads, head_dim] "
+ "and cos/sin tensors shaped [tokens, head_dim]."
+ )
+ if not (q.shape[0] == k.shape[0] == cos.shape[0] == sin.shape[0]):
+ raise ValueError("BAGEL packed q/k/cos/sin tensors must share the token dimension.")
+ if not (q.shape[-1] == k.shape[-1] == cos.shape[-1] == sin.shape[-1]):
+ raise NotImplementedError("Liger full RoPE does not support partial rotary dimensions.")
+
+ # The shared full-RoPE kernels consume [batch, heads, sequence, head_dim]
+ # for LLM-dtype arithmetic (whereas 3-D [S, H, D] triggers vision FP32 promotion).
+ # BAGEL stores packed Q/K as [tokens, heads, head_dim], so adapt through
+ # a synthetic batch dimension and restore the packed layout afterwards.
+ q_embed, k_embed = rope_op(
+ q.transpose(0, 1).unsqueeze(0),
+ k.transpose(0, 1).unsqueeze(0),
+ cos.unsqueeze(0),
+ sin.unsqueeze(0),
+ unsqueeze_dim=1,
+ )
+ return (
+ q_embed.squeeze(0).transpose(0, 1).contiguous(),
+ k_embed.squeeze(0).transpose(0, 1).contiguous(),
+ )
+
+
+_PACKED_FUSED_ATTN_IMPLEMENTATIONS = (
+ "veomni_flex_attention",
+ "veomni_magi_attention",
+)
+_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"})
+_MAGI_ATTN_IMPLEMENTATIONS = frozenset({"magi_attention", "veomni_magi_attention"})
+
+
+def _normalize_packed_attn_impl(attn_implementation: str | None) -> str | None:
+ """Normalize Flex/Magi attention aliases to their canonical ``veomni_*`` names."""
+ if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS:
+ return "veomni_flex_attention"
+ if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS:
+ return "veomni_magi_attention"
+ return attn_implementation
+
+
+class BagelQwen2MoTAttentionAccelerated(BagelQwen2MoTAttention):
+ """MoT attention with instance-local RoPE and attention VeomniOps."""
+
+ def __init__(
+ self,
+ config: BagelQwen2MoTConfig,
+ layer_idx: int,
+ *,
+ rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm,
+ ):
+ super().__init__(config, layer_idx, rms_norm_cls=rms_norm_cls)
+ self.veomni_rms_norm = VeomniOp("rms_norm", "standard", config.ops_config.rms_norm_implementation)
+ self.veomni_rope = VeomniOp("rope", "full", config.ops_config.rotary_pos_emb_implementation)
+ self.veomni_async_ulysses_qkv = VeomniOp("async_ulysses_qkv", "bagel")
+ self.veomni_async_ulysses_o = VeomniOp("async_ulysses_o", "bagel")
+ attn_impl = _normalize_packed_attn_impl(
+ config.ops_config.attn_implementation
+ if "attn_implementation" in config.ops_implementation
+ else "veomni_flex_attention"
+ )
+ self.veomni_attn_impl = attn_impl
+ if attn_impl in _PACKED_FUSED_ATTN_IMPLEMENTATIONS:
+ try:
+ self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl)
+ except RuntimeError:
+ self.veomni_attn = None
+ else:
+ self.veomni_attn = None
+ try:
+ self.veomni_denoise_attn: VeomniOp | None = VeomniOp("attention", "standard", _FLASH_ATTENTION_2)
+ except RuntimeError:
+ self.veomni_denoise_attn = None
+ self.register_load_state_dict_pre_hook(combine_qkv_state_dict_pre_hook)
+
+ def _build_qkv_proj(self) -> None:
+ qkv_size = sum(self.qkv_split_sizes)
+ self.qkv_proj_und = torch.nn.Linear(self.hidden_size, qkv_size, bias=True)
+ self.qkv_proj_gen = torch.nn.Linear(self.hidden_size, qkv_size, bias=True)
+
+ def _project_qkv(
+ self,
+ hidden: torch.Tensor,
+ *,
+ is_gen: bool,
+ split: bool = False,
+ ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Project ``hidden`` through the fused und or gen QKV linear layer."""
+ projected = (self.qkv_proj_gen if is_gen else self.qkv_proj_und)(hidden)
+ if split:
+ return projected.split(self.qkv_split_sizes, dim=-1)
+ return projected
+
+ @staticmethod
+ def build_attention_mask(
+ packed_attention_metadata: torch.Tensor,
+ *,
+ attn_implementation: str | None = None,
+ ) -> BlockMask | MagiAttentionMask:
+ """Materialize a Flex ``BlockMask`` or ``MagiAttentionMask`` from ``packed_attention_metadata``."""
+ if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS:
+ return build_mot_magi_mask(packed_attention_metadata)
+ if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS or attn_implementation is None:
+ return build_mot_block_mask(packed_attention_metadata)
+ raise ValueError(
+ "BAGEL Qwen2-MoT packed fused attention requires "
+ f"{', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}, got {attn_implementation!r}."
+ )
+
+ def apply_rotary_pos_emb(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+ unsqueeze_dim: int = 1,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim, rope_op=self.veomni_rope)
+
+ def _resolve_packed_attn_op(self, *, context: str) -> VeomniOp:
+ """Resolve the packed fused attention ``VeomniOp`` bound at initialization for ``context``."""
+ if self.veomni_attn is not None:
+ return self.veomni_attn
+ if self.veomni_attn_impl not in _PACKED_FUSED_ATTN_IMPLEMENTATIONS:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT {context} requires packed fused attention "
+ f"({', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}), got "
+ f"{self.veomni_attn_impl!r}."
+ )
+ return VeomniOp("attention", "standard", self.veomni_attn_impl)
+
+ def attend(
+ self,
+ packed_query_states: torch.Tensor,
+ packed_key_states: torch.Tensor,
+ packed_value_states: torch.Tensor,
+ attention_mask: BlockMask | MagiAttentionMask,
+ ) -> torch.Tensor:
+ attn_op = self._resolve_packed_attn_op(context="training")
+ packed_attn_output, _ = attn_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ packed_key_states.transpose(0, 1).unsqueeze(0),
+ packed_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask,
+ dropout=0.0,
+ scaling=self.head_dim**-0.5,
+ )
+ packed_attn_output = packed_attn_output.squeeze(0)
+ return packed_attn_output.reshape(-1, self.num_heads * self.head_dim)
+
+ def attend_inference(
+ self,
+ packed_query_states: torch.Tensor,
+ merged_key_states: torch.Tensor,
+ merged_value_states: torch.Tensor,
+ *,
+ attention_mask: BlockMask | MagiAttentionMask | None,
+ is_causal: bool,
+ cu_seq_lens_q: torch.Tensor,
+ cu_seq_lens_k: torch.Tensor,
+ max_length_q: int,
+ max_length_k: int,
+ ) -> torch.Tensor:
+ """Run packed inference attention using Flex/Magi for prefill and FlashAttention-2 for decode/denoise."""
+ if attention_mask is not None:
+ attn_op = self._resolve_packed_attn_op(context="packed prefill")
+ packed_attn_output, _ = attn_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ merged_key_states.transpose(0, 1).unsqueeze(0),
+ merged_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask,
+ dropout=0.0,
+ scaling=self.head_dim**-0.5,
+ # Inference packs complete logical documents locally and does
+ # not enter the training-only Ulysses redistribution.
+ skip_ulysses=True,
+ )
+ return packed_attn_output.squeeze(0)
+
+ denoise_op = self.veomni_denoise_attn or VeomniOp("attention", "standard", _FLASH_ATTENTION_2)
+ packed_attn_output, _ = denoise_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ merged_key_states.transpose(0, 1).unsqueeze(0),
+ merged_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask=None,
+ dropout=0.0,
+ is_causal=is_causal,
+ cu_seq_lens_q=cu_seq_lens_q,
+ cu_seq_lens_k=cu_seq_lens_k,
+ max_length_q=max_length_q,
+ max_length_k=max_length_k,
+ # Inference owns its packed KV-cache layout and does not enter the
+ # training-only module Ulysses redistribution.
+ skip_ulysses=True,
+ )
+ return packed_attn_output.squeeze(0)
+
+ def _has_plain_linear_projections(self) -> bool:
+ """Whether async Ulysses may read this layer's projection weights directly.
+
+ A wrapped projection such as ``LoraLinear`` has no ``weight`` and must run
+ its own forward, so attention falls back to the synchronous projection path.
+ """
+ return all(
+ isinstance(getattr(self, name), torch.nn.Linear)
+ for name in ("qkv_proj_und", "qkv_proj_gen", "o_proj", "o_proj_moe_gen")
+ )
+
+ def forward_packed_train(
+ self,
+ packed_sequence: torch.Tensor,
+ attention_mask: BlockMask | MagiAttentionMask,
+ packed_position_cos: torch.Tensor,
+ packed_position_sin: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ ) -> torch.Tensor:
+ """Run packed training attention, using async Ulysses when enabled on the active parallel state."""
+ parallel_state = get_parallel_state()
+ use_async_ulysses = (
+ parallel_state.ulysses_enabled
+ and bool(parallel_state.async_enabled)
+ and self._has_plain_linear_projections()
+ )
+ if not use_async_ulysses:
+ return super().forward_packed_train(
+ packed_sequence,
+ attention_mask,
+ packed_position_cos,
+ packed_position_sin,
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ )
+ attn_op = self._resolve_packed_attn_op(context="training")
+
+ q_weight_und, k_weight_und, v_weight_und = self.qkv_proj_und.weight.split(self.qkv_split_sizes, dim=0)
+ q_bias_und, k_bias_und, v_bias_und = (
+ (None, None, None)
+ if self.qkv_proj_und.bias is None
+ else self.qkv_proj_und.bias.split(self.qkv_split_sizes, dim=0)
+ )
+ q_weight_gen, k_weight_gen, v_weight_gen = self.qkv_proj_gen.weight.split(self.qkv_split_sizes, dim=0)
+ q_bias_gen, k_bias_gen, v_bias_gen = (
+ (None, None, None)
+ if self.qkv_proj_gen.bias is None
+ else self.qkv_proj_gen.bias.split(self.qkv_split_sizes, dim=0)
+ )
+ rms_norm_op = getattr(self.q_norm, "veomni_rms_norm", self.veomni_rms_norm)
+ packed_query_states, packed_key_states, packed_value_states = self.veomni_async_ulysses_qkv(
+ hidden_states=packed_sequence,
+ route_0_indices=packed_und_token_indexes,
+ route_1_indices=packed_gen_token_indexes,
+ position_cos=packed_position_cos,
+ position_sin=packed_position_sin,
+ q_weight_0=q_weight_und,
+ q_bias_0=q_bias_und,
+ k_weight_0=k_weight_und,
+ k_bias_0=k_bias_und,
+ v_weight_0=v_weight_und,
+ v_bias_0=v_bias_und,
+ q_norm_weight_0=self.q_norm.weight,
+ k_norm_weight_0=self.k_norm.weight,
+ q_weight_1=q_weight_gen,
+ q_bias_1=q_bias_gen,
+ k_weight_1=k_weight_gen,
+ k_bias_1=k_bias_gen,
+ v_weight_1=v_weight_gen,
+ v_bias_1=v_bias_gen,
+ q_norm_weight_1=self.q_norm_moe_gen.weight,
+ k_norm_weight_1=self.k_norm_moe_gen.weight,
+ head_dim=self.head_dim,
+ rms_norm_eps=self.config.rms_norm_eps,
+ # The SP pre-hook already pads the packed sequence and builds a
+ # BlockMask for that full padded length. Keep those padding rows
+ # through attention; the module post-hook removes them after the
+ # local output shards have been gathered.
+ unpadded_dim_size=packed_sequence.shape[0] * parallel_state.ulysses_size,
+ group=parallel_state.ulysses_group,
+ rms_norm=rms_norm_op,
+ rope=self.veomni_rope,
+ )
+ packed_attn_output, _ = attn_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ packed_key_states.transpose(0, 1).unsqueeze(0),
+ packed_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask,
+ dropout=0.0,
+ scaling=self.head_dim**-0.5,
+ skip_ulysses=True,
+ )
+ return self.veomni_async_ulysses_o(
+ hidden_states=packed_attn_output.squeeze(0),
+ route_0_indices=packed_und_token_indexes,
+ route_1_indices=packed_gen_token_indexes,
+ proj_weight_0=self.o_proj.weight,
+ proj_bias_0=self.o_proj.bias,
+ proj_weight_1=self.o_proj_moe_gen.weight,
+ proj_bias_1=self.o_proj_moe_gen.bias,
+ group=parallel_state.ulysses_group,
+ )
+
+
+class InferenceMixinAccelerated(InferenceMixin):
+ """Packed CFG denoise for the FSDP/runtime MoT class.
+
+ Stacks active CFG branches into one ``forward_inference`` call so decoder
+ layers unshard once per denoise step. ``generate`` and ``collect_velocity``
+ stay on :class:`~.modeling.InferenceMixin`.
+ """
+
+ def denoise_branch(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Run one stacked CFG denoise forward across all active branches."""
+ del kwargs
+ tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs)
+ inputs = self._generation_state.preprocess_parallel_denoise_inputs(
+ query,
+ generation_kwargs or {},
+ timestep=tail.meta.get("timestep"),
+ empty_cache_factory=self._new_empty_cache,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ outputs = self.forward_inference(
+ **inputs,
+ update_past_key_values=False,
+ is_causal=False,
+ mode="gen",
+ )
+ tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN
+ tail.value = outputs["hidden_states"].to(device=self.device, dtype=self.dtype)
+ return {"conversation_list": conversation_list}
+
+
+class BagelQwen2MoTAccelerated(VeOmniMixin, InferenceMixinAccelerated, BagelQwen2MoTCore):
+ _supports_flex_attn = True
+ _supports_magi_attn = True
+ attention_cls = BagelQwen2MoTAttentionAccelerated
+ mlp_cls = PatchedQwen2MLP
+ rms_norm_cls = PatchedQwen2RMSNorm
+
+ @staticmethod
+ def _create_checkpoint_tensor_converter(
+ model: BagelQwen2MoTCore,
+ ) -> BagelQwen2MoTCheckpointTensorConverter:
+ del model
+ return BagelQwen2MoTCheckpointTensorConverter()
+
+
+register_accelerated_qkv_conversion_mapping()
+
+
+__all__ = ["BagelQwen2MoTAccelerated", "InferenceMixinAccelerated"]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py
new file mode 100644
index 0000000000..f11535eba3
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py
@@ -0,0 +1,233 @@
+"""Checkpoint compatibility for accelerated BAGEL fused QKV projections.
+
+Eager attention owns official split ``q_proj`` / ``k_proj`` / ``v_proj``
+parameters. Accelerated attention stores one merged ``qkv_proj_und`` or
+``qkv_proj_gen`` parameter per MoT branch. Existing checkpoints keep their
+separate Q/K/V keys, so only the accelerated runtime needs an adapter:
+PyTorch state-dict hooks, Transformers weight converters, or VeOmni's streaming
+checkpoint converter.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Generator
+from typing import Any
+
+import torch
+from transformers.conversion_mapping import register_checkpoint_conversion_mapping
+from transformers.core_model_loading import ConversionOps, WeightConverter
+
+from ......models.checkpoint.convert import ConvertedCheckpointTensor, export_weights
+
+
+_QKV_COMPONENTS = ("q", "k", "v")
+_QKV_BRANCHES = (
+ ("qkv_proj_und", ("q_proj", "k_proj", "v_proj")),
+ ("qkv_proj_gen", ("q_proj_moe_gen", "k_proj_moe_gen", "v_proj_moe_gen")),
+)
+_QKV_BRANCH_TARGETS = dict(_QKV_BRANCHES)
+_QKV_CHECKPOINT_RE = re.compile(
+ r"^(?P.*\.self_attn\.)"
+ r"(?P[qkv]_proj(?:_moe_gen)?)\."
+ r"(?Pweight|bias)$"
+)
+_QKV_FUSED_RE = re.compile(
+ r"^(?P.*\.self_attn\.)"
+ r"(?Pqkv_proj_(?:und|gen))\."
+ r"(?Pweight|bias)$"
+)
+
+
+def combine_qkv_state_dict_pre_hook(
+ module: torch.nn.Module,
+ state_dict: dict[str, torch.Tensor],
+ prefix: str,
+ local_metadata: dict[str, Any],
+ strict: bool,
+ missing_keys: list[str],
+ unexpected_keys: list[str],
+ error_msgs: list[str],
+) -> None:
+ """Accept the legacy split Q/K/V schema in direct PyTorch and DCP loads."""
+ del module, local_metadata, strict, missing_keys, unexpected_keys
+ for combined_name, checkpoint_names in _QKV_BRANCHES:
+ for kind in ("weight", "bias"):
+ combined_key = f"{prefix}{combined_name}.{kind}"
+ checkpoint_keys = [f"{prefix}{name}.{kind}" for name in checkpoint_names]
+ if combined_key in state_dict or not any(key in state_dict for key in checkpoint_keys):
+ continue
+ if not all(key in state_dict for key in checkpoint_keys):
+ present = [key for key in checkpoint_keys if key in state_dict]
+ error_msgs.append(
+ f"Incomplete BAGEL QKV checkpoint group for {combined_key}: found {present}, "
+ f"expected {checkpoint_keys}."
+ )
+ continue
+
+ state_dict[combined_key] = torch.cat([state_dict.pop(key) for key in checkpoint_keys], dim=0)
+
+
+class _ConcatenateQKV(ConversionOps):
+ """Concatenate checkpoint Q/K/V tensors along the projection dimension."""
+
+ @torch.no_grad()
+ def convert(
+ self,
+ input_dict: dict[str, list[torch.Tensor]],
+ source_patterns: list[str],
+ target_patterns: list[str],
+ **kwargs: Any,
+ ) -> dict[str, torch.Tensor]:
+ """Concatenate ``source_patterns`` tensors into a single fused target tensor."""
+ del kwargs
+ if len(target_patterns) != 1:
+ raise ValueError("Combined BAGEL QKV conversion requires one target pattern.")
+
+ tensors: list[torch.Tensor] = []
+ for source_pattern in source_patterns:
+ values = input_dict.pop(source_pattern)
+ tensors.extend(values if isinstance(values, list) else [values])
+ return {target_patterns[0]: torch.cat(tensors, dim=0)}
+
+ @property
+ def reverse_op(self) -> ConversionOps:
+ return _SplitQKV()
+
+
+class _SplitQKV(ConversionOps):
+ """Restore combined QKV tensors to BAGEL's external checkpoint schema."""
+
+ @torch.no_grad()
+ def convert(
+ self,
+ input_dict: dict[str, list[torch.Tensor]],
+ source_patterns: list[str],
+ target_patterns: list[str],
+ **kwargs: Any,
+ ) -> dict[str, torch.Tensor]:
+ """Split a fused QKV tensor into separate Q, K, and V checkpoint tensors."""
+ del source_patterns
+ if len(input_dict) != 1 or len(target_patterns) != 3:
+ raise ValueError("BAGEL QKV export requires one combined source and three target patterns.")
+
+ values = next(iter(input_dict.values()))
+ tensor = values[0] if isinstance(values, list) else values
+ config = kwargs["config"]
+ head_dim = config.hidden_size // config.num_attention_heads
+ split_sizes = (
+ config.num_attention_heads * head_dim,
+ config.num_key_value_heads * head_dim,
+ config.num_key_value_heads * head_dim,
+ )
+ return {
+ target: chunk.clone()
+ for target, chunk in zip(target_patterns, tensor.split(split_sizes, dim=0), strict=True)
+ }
+
+ @property
+ def reverse_op(self) -> ConversionOps:
+ return _ConcatenateQKV()
+
+
+def _qkv_weight_converters() -> list[WeightConverter]:
+ """Build Transformers ``WeightConverter`` rules for both MoT QKV branches."""
+ converters: list[WeightConverter] = []
+ for branch, source_suffix in (("und", ""), ("gen", "_moe_gen")):
+ for kind in ("weight", "bias"):
+ converters.append(
+ WeightConverter(
+ source_patterns=[
+ f"self_attn.{component}_proj{source_suffix}.{kind}" for component in _QKV_COMPONENTS
+ ],
+ target_patterns=f"self_attn.qkv_proj_{branch}.{kind}",
+ operations=[_ConcatenateQKV()],
+ )
+ )
+ return converters
+
+
+def register_accelerated_qkv_conversion_mapping() -> None:
+ """Register fused-QKV converters for the accelerated class only.
+
+ Lookups prefer class name over ``model_type``. Do not register
+ ``bagel_qwen2_mot`` here: eager ``BagelQwen2MoT`` uses native split keys.
+ """
+ register_checkpoint_conversion_mapping(
+ "BagelQwen2MoTAccelerated",
+ _qkv_weight_converters(),
+ overwrite=True,
+ )
+
+
+class BagelQwen2MoTCheckpointTensorConverter:
+ """Stream legacy Q/K/V tensors into accelerated combined runtime parameters.
+
+ VeOmni's ``load_model_weights`` assigns tensors directly instead of calling
+ ``load_state_dict``, so the PyTorch pre-hook cannot perform this merge.
+ """
+
+ def __init__(self) -> None:
+ self._pending: dict[tuple[str, str, str], dict[str, torch.Tensor]] = {}
+
+ def can_handle(self, name: str) -> bool:
+ return _QKV_CHECKPOINT_RE.fullmatch(name) is not None
+
+ def convert(self, name: str, tensor: torch.Tensor) -> ConvertedCheckpointTensor | None:
+ """Buffer split Q/K/V tensors and emit the fused ``qkv_proj_{und,gen}`` tensor once complete."""
+ match = _QKV_CHECKPOINT_RE.fullmatch(name)
+ if match is None:
+ return ConvertedCheckpointTensor(name=name, tensor=tensor)
+
+ projection = match.group("projection")
+ component = projection[0]
+ branch = "gen" if projection.endswith("_moe_gen") else "und"
+ group_key = (match.group("prefix"), branch, match.group("kind"))
+ group = self._pending.setdefault(group_key, {})
+ if component in group:
+ raise ValueError(f"Duplicate BAGEL QKV checkpoint tensor: {name}.")
+ group[component] = tensor
+ if group.keys() != set(_QKV_COMPONENTS):
+ return None
+
+ self._pending.pop(group_key)
+ combined = torch.cat([group[component] for component in _QKV_COMPONENTS], dim=0)
+ prefix, branch, kind = group_key
+ return ConvertedCheckpointTensor(
+ name=f"{prefix}qkv_proj_{branch}.{kind}",
+ tensor=combined,
+ )
+
+ def finalize(self) -> list[ConvertedCheckpointTensor]:
+ """Verify no partial QKV groups remain buffered at the end of checkpoint loading."""
+ if self._pending:
+ incomplete = ", ".join(
+ f"{prefix}qkv_proj_{branch}.{kind}" for prefix, branch, kind in sorted(self._pending)
+ )
+ raise ValueError(f"Incomplete BAGEL QKV checkpoint groups: {incomplete}.")
+ return []
+
+ def export_weights(self, model: torch.nn.Module) -> Generator[tuple[str, torch.Tensor]]:
+ """Export model parameters in HF checkpoint format with fused ``qkv_proj_{und,gen}`` split into ``{q,k,v}_proj``."""
+ base_model = model
+ while isinstance(getattr(base_model, "module", None), torch.nn.Module):
+ base_model = base_model.module
+ config = base_model.config
+ head_dim = config.hidden_size // config.num_attention_heads
+ split_sizes = (
+ config.num_attention_heads * head_dim,
+ config.num_key_value_heads * head_dim,
+ config.num_key_value_heads * head_dim,
+ )
+ for name, tensor in export_weights(base_model):
+ if tensor.dtype == torch.float32:
+ tensor = tensor.to(torch.bfloat16)
+ match = _QKV_FUSED_RE.fullmatch(name)
+ if match is None:
+ yield name, tensor
+ continue
+ prefix = match.group("prefix")
+ fused = match.group("fused")
+ kind = match.group("kind")
+ for target_proj, chunk in zip(_QKV_BRANCH_TARGETS[fused], tensor.split(split_sizes, dim=0), strict=True):
+ yield f"{prefix}{target_proj}.{kind}", chunk.clone()
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py
new file mode 100644
index 0000000000..5a330813fc
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py
@@ -0,0 +1,65 @@
+"""Configuration for BAGEL's Qwen2 MoT backbone module."""
+
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+
+from ...module_configuration_base import OmniModuleConfig
+
+
+class BagelQwen2MoTConfig(OmniModuleConfig, Qwen2Config):
+ """Qwen2 config with BAGEL's runtime MoT defaults.
+
+ The upstream ``llm_config.json`` is a plain Qwen2 config. BAGEL's loader
+ mutates it at runtime with ``qk_norm=True`` and
+ ``layer_module="Qwen2MoTDecoderLayer"``; this config bakes those defaults
+ into the split module so a SeedOmni checkpoint is self-describing.
+ """
+
+ model_type = "bagel_qwen2_mot"
+
+ def __init__(
+ self,
+ vocab_size: int = 152064,
+ hidden_size: int = 3584,
+ intermediate_size: int = 18944,
+ num_hidden_layers: int = 28,
+ num_attention_heads: int = 28,
+ num_key_value_heads: int = 4,
+ max_position_embeddings: int = 32768,
+ rms_norm_eps: float = 1e-6,
+ rope_theta: float = 1000000.0,
+ bos_token_id: int = 151643,
+ eos_token_id: int = 151645,
+ tie_word_embeddings: bool = False,
+ qk_norm: bool = True,
+ layer_module: str = "Qwen2MoTDecoderLayer",
+ freeze_und: bool = False,
+ **kwargs,
+ ):
+ ops_impl = kwargs.get("ops_implementation")
+ if "attn_implementation" not in kwargs and not (
+ isinstance(ops_impl, dict) and ops_impl.get("attn_implementation") is not None
+ ):
+ kwargs["attn_implementation"] = "flex_attention"
+ super().__init__(
+ vocab_size=vocab_size,
+ hidden_size=hidden_size,
+ intermediate_size=intermediate_size,
+ num_hidden_layers=num_hidden_layers,
+ num_attention_heads=num_attention_heads,
+ num_key_value_heads=num_key_value_heads,
+ max_position_embeddings=max_position_embeddings,
+ rms_norm_eps=rms_norm_eps,
+ rope_theta=rope_theta,
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+ if self.num_attention_heads % self.num_key_value_heads != 0:
+ raise ValueError(
+ "BAGEL Qwen2-MoT query heads must be divisible by KV heads, "
+ f"got query={self.num_attention_heads}, key_value={self.num_key_value_heads}."
+ )
+ self.qk_norm = qk_norm
+ self.layer_module = layer_module
+ self.freeze_und = freeze_und
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py
new file mode 100644
index 0000000000..530cbffc54
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py
@@ -0,0 +1,516 @@
+"""Internal generation state for BAGEL Qwen2-MoT inference."""
+
+from __future__ import annotations
+
+from contextlib import nullcontext
+from dataclasses import dataclass, field
+from typing import Any, Callable, Iterator
+
+import torch
+
+from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type
+
+
+def _build_denoise_index_layout(
+ key_values_lens: torch.Tensor,
+ query_len: int,
+ *,
+ device: torch.device,
+) -> dict[str, torch.Tensor]:
+ """Build attention-space and query-space indexes for stacked CFG branches."""
+ query_indexes: list[torch.Tensor] = []
+ key_value_indexes: list[torch.Tensor] = []
+ text_indexes: list[torch.Tensor] = []
+ vae_token_indexes: list[torch.Tensor] = []
+ # Attention indexes address the merged [branch cache, branch query]
+ # layout. Expert indexes address the query-only tensor, so they use a
+ # separate cursor that does not include cached rows.
+ attention_offset = 0
+ query_offset = 0
+ for key_values_len in key_values_lens.tolist():
+ key_values_len = int(key_values_len)
+ key_value_indexes.append(
+ torch.arange(attention_offset, attention_offset + key_values_len, device=device, dtype=torch.long)
+ )
+ query_indexes.append(
+ torch.arange(
+ attention_offset + key_values_len,
+ attention_offset + key_values_len + query_len,
+ device=device,
+ dtype=torch.long,
+ )
+ )
+ text_indexes.append(
+ torch.tensor([query_offset, query_offset + query_len - 1], device=device, dtype=torch.long)
+ )
+ vae_token_indexes.append(
+ torch.arange(query_offset + 1, query_offset + query_len - 1, device=device, dtype=torch.long)
+ )
+ attention_offset += key_values_len + query_len
+ query_offset += query_len
+
+ return {
+ "packed_query_indexes": torch.cat(query_indexes, dim=0),
+ "packed_key_value_indexes": (
+ torch.cat(key_value_indexes, dim=0)
+ if key_value_indexes
+ else torch.empty(0, device=device, dtype=torch.long)
+ ),
+ "packed_text_indexes": torch.cat(text_indexes, dim=0),
+ "packed_vae_token_indexes": torch.cat(vae_token_indexes, dim=0),
+ }
+
+
+@dataclass
+class MotCacheContext:
+ """KV-cache state for one logical inference branch.
+
+ FA receives all active branches in one packed call. Each branch therefore
+ keeps its own cache plus the indexes needed to place that cache in the
+ temporary packed ``[past, query]`` attention layout.
+ """
+
+ name: str
+ _cache: Any | None = None
+ _key_values_lens: torch.Tensor | None = None
+ _packed_key_value_indexes: torch.Tensor | None = None
+ _next_position_ids: torch.Tensor | None = None
+
+ # ── Cache state ─────────────────────────────────────────────
+
+ def reset(self) -> None:
+ self._cache = None
+ self._key_values_lens = None
+ self._packed_key_value_indexes = None
+ self._next_position_ids = None
+
+ @property
+ def cache(self) -> Any | None:
+ return self._cache
+
+ @property
+ def key_values_lens(self) -> torch.Tensor | None:
+ return self._key_values_lens
+
+ @property
+ def packed_key_value_indexes(self) -> torch.Tensor | None:
+ return self._packed_key_value_indexes
+
+ @property
+ def next_position_ids(self) -> torch.Tensor | None:
+ return self._next_position_ids
+
+ # ── Cache mutation ──────────────────────────────────
+
+ def _set_cache_state(
+ self,
+ *,
+ cache: Any,
+ key_values_lens: torch.Tensor | None,
+ packed_key_value_indexes: torch.Tensor | None,
+ next_position_ids: torch.Tensor | None,
+ ) -> None:
+ self._cache = cache
+ self._key_values_lens = key_values_lens
+ self._packed_key_value_indexes = packed_key_value_indexes
+ self._next_position_ids = next_position_ids
+
+ def install_cache(
+ self,
+ *,
+ cache: Any,
+ cache_len: int,
+ next_position_id: torch.Tensor,
+ device: torch.device,
+ ) -> None:
+ """Install one contiguous cache extracted from the packed Flex prefill."""
+ if cache_len < 0:
+ raise ValueError(f"BAGEL {self.name} branch cache length must be non-negative.")
+ self._set_cache_state(
+ cache=cache,
+ key_values_lens=torch.tensor([cache_len], device=device, dtype=torch.int32),
+ packed_key_value_indexes=torch.arange(cache_len, device=device, dtype=torch.long),
+ next_position_ids=next_position_id.detach().reshape(1).to(device=device, dtype=torch.long),
+ )
+
+ def append_packed_query(
+ self,
+ *,
+ cache: Any,
+ query_lens: torch.Tensor,
+ device: torch.device,
+ ) -> None:
+ """Advance the main AR cache after a successful one-branch decode."""
+ self.require_ready()
+ next_key_values_lens = self._key_values_lens + query_lens
+
+ self._set_cache_state(
+ cache=cache,
+ key_values_lens=next_key_values_lens,
+ packed_key_value_indexes=torch.arange(
+ int(next_key_values_lens.sum().item()),
+ device=device,
+ dtype=torch.long,
+ ),
+ next_position_ids=self._next_position_ids + query_lens,
+ )
+
+ # ── Cache accessors ──────────────────────────────────
+
+ def require_ready(self) -> None:
+ """Raise if this branch's KV-cache state has not been initialized."""
+ if (
+ self._cache is None
+ or self._key_values_lens is None
+ or self._packed_key_value_indexes is None
+ or self._next_position_ids is None
+ ):
+ raise RuntimeError(f"BAGEL {self.name} branch context is not initialized.")
+
+ def cache_len(self) -> int:
+ """Return the total cached sequence length for this branch."""
+ if self._key_values_lens is None:
+ raise RuntimeError(f"BAGEL {self.name} branch cache lengths are not initialized.")
+ return int(self._key_values_lens.sum().item())
+
+ def repeated_position_ids(self, query_len: int, *, device: torch.device) -> torch.Tensor:
+ """Use one logical position for every token in a denoise image span."""
+ if self._next_position_ids is None:
+ raise RuntimeError(f"BAGEL {self.name} branch position ids are not initialized.")
+ return self._next_position_ids.reshape(1).expand(query_len).to(device=device)
+
+ def packed_query_args(
+ self,
+ query_len: int,
+ *,
+ device: torch.device,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Build sequential positions and cache-relative indexes for AR decode."""
+ self.require_ready()
+ cache_len = self.cache_len()
+ query_lens = torch.tensor([query_len], device=device, dtype=torch.int32)
+ packed_query_indexes = torch.arange(cache_len, cache_len + query_len, device=device, dtype=torch.long)
+ packed_position_ids = self._next_position_ids.reshape(1) + torch.arange(
+ query_len,
+ device=device,
+ dtype=torch.long,
+ )
+
+ return query_lens, packed_query_indexes, packed_position_ids
+
+
+@dataclass
+class MotGenerationState:
+ """Scheduling state shared by Qwen2-MoT inference graph invocations.
+
+ ``main`` is the conditional prompt, ``cfg_text`` is a prefix of ``main``,
+ and ``cfg_img`` is the text-only document used for image CFG. The state
+ also remembers the branch layout between denoise and velocity collection.
+ """
+
+ _velocity_buffer: dict[str, torch.Tensor] = field(default_factory=dict)
+ _pending_denoise_branches: tuple[str, ...] | None = None
+ _pending_denoise_query_len: int | None = None
+ _main: MotCacheContext = field(default_factory=lambda: MotCacheContext("main"))
+ _cfg_text: MotCacheContext = field(default_factory=lambda: MotCacheContext("text CFG"))
+ _cfg_img: MotCacheContext = field(default_factory=lambda: MotCacheContext("image CFG"))
+
+ def reset(self) -> None:
+ self._velocity_buffer.clear()
+ self._pending_denoise_branches = None
+ self._pending_denoise_query_len = None
+ self._main.reset()
+ self._cfg_text.reset()
+ self._cfg_img.reset()
+
+ # ── CFG cache contexts ──────────────────────────────────
+
+ @property
+ def main(self) -> MotCacheContext:
+ return self._main
+
+ @property
+ def cfg_text(self) -> MotCacheContext:
+ return self._cfg_text
+
+ @property
+ def cfg_img(self) -> MotCacheContext:
+ return self._cfg_img
+
+ def _cache_for_branch(self, branch: str) -> MotCacheContext:
+ """Return the ``MotCacheContext`` for ``branch`` (``'main'``, ``'cfg_text'``, or ``'cfg_img'``)."""
+ if branch == "main":
+ return self._main
+ if branch == "cfg_text":
+ return self._cfg_text
+ if branch == "cfg_img":
+ return self._cfg_img
+ raise RuntimeError(f"Unsupported BAGEL denoise branch {branch!r}.")
+
+ # ── CFG request policy ──────────────────────────────────
+
+ def validate_cfg_request(self, generation_kwargs: dict[str, object]) -> None:
+ """Validate that image CFG is only enabled when text CFG is also active."""
+ cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0))
+ cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0))
+ if cfg_img_scale > 1.0 and cfg_text_scale <= 1.0:
+ raise ValueError("cfg_img_scale > 1.0 requires cfg_text_scale > 1.0")
+
+ def cfg_img_requested(self, generation_kwargs: dict[str, object]) -> bool:
+ return float(generation_kwargs.get("cfg_img_scale", 1.0)) > 1.0
+
+ # ── Denoise velocity round ──────────────────────────────────
+
+ def iter_serial_denoise_inputs(
+ self,
+ query: torch.Tensor,
+ generation_kwargs: dict[str, object],
+ *,
+ timestep: object,
+ device: torch.device,
+ dtype: torch.dtype,
+ ) -> Iterator[dict[str, Any]]:
+ """Yield one ``forward_inference`` input dict per active CFG branch."""
+ query = query.to(device=device, dtype=dtype)
+ query_len = int(query.shape[0])
+ if query_len < 3:
+ raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.")
+
+ branches = self._denoise_branches_for_timestep(generation_kwargs, timestep)
+ self._pending_denoise_branches = branches
+ self._pending_denoise_query_len = query_len
+ for branch in branches:
+ context = self._cache_for_branch(branch)
+ context.require_ready()
+ cache_len = context.cache_len()
+ packed_key_value_indexes = context.packed_key_value_indexes
+ if packed_key_value_indexes is None:
+ packed_key_value_indexes = torch.empty(0, device=device, dtype=torch.long)
+ yield {
+ "packed_query_sequence": query,
+ "query_lens": torch.tensor([query_len], device=device, dtype=torch.int32),
+ "packed_query_position_ids": context.repeated_position_ids(query_len, device=device),
+ "packed_query_indexes": torch.arange(
+ cache_len, cache_len + query_len, device=device, dtype=torch.long
+ ),
+ "past_key_values": context.cache,
+ "key_values_lens": context.key_values_lens,
+ "packed_key_value_indexes": packed_key_value_indexes,
+ "packed_text_indexes": torch.tensor([0, query_len - 1], device=device, dtype=torch.long),
+ "packed_vae_token_indexes": torch.arange(1, query_len - 1, device=device, dtype=torch.long),
+ }
+
+ def preprocess_parallel_denoise_inputs(
+ self,
+ query: torch.Tensor,
+ generation_kwargs: dict[str, object],
+ *,
+ timestep: object,
+ empty_cache_factory: Callable[[], Any],
+ device: torch.device,
+ dtype: torch.dtype,
+ ) -> dict[str, Any]:
+ """Stack active CFG branches into one packed ``forward_inference`` call."""
+ query = query.to(device=device, dtype=dtype)
+ query_len = int(query.shape[0])
+ if query_len < 3:
+ raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.")
+
+ branches = self._denoise_branches_for_timestep(generation_kwargs, timestep)
+ contexts = [self._cache_for_branch(branch) for branch in branches]
+ for context in contexts:
+ context.require_ready()
+
+ query_lens = torch.full((len(branches),), query_len, device=device, dtype=torch.int32)
+ key_values_lens = torch.tensor([context.cache_len() for context in contexts], device=device, dtype=torch.int32)
+ packed_query_sequence = torch.cat([query] * len(branches), dim=0)
+ packed_query_position_ids = torch.cat(
+ [context.repeated_position_ids(query_len, device=device) for context in contexts],
+ dim=0,
+ )
+ index_layout = _build_denoise_index_layout(key_values_lens, query_len, device=device)
+
+ # The flow connector preserves row order but does not carry branch
+ # labels, so collect_velocity must replay this exact layout.
+ self._pending_denoise_branches = branches
+ self._pending_denoise_query_len = query_len
+ return {
+ "packed_query_sequence": packed_query_sequence,
+ "query_lens": query_lens,
+ "key_values_lens": key_values_lens,
+ "packed_query_position_ids": packed_query_position_ids,
+ "past_key_values": self._merged_branch_cache(contexts, key_values_lens, empty_cache_factory),
+ **index_layout,
+ }
+
+ def collect_velocity(
+ self,
+ velocity: torch.Tensor,
+ generation_kwargs: dict[str, object],
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+ ) -> torch.Tensor:
+ """Split stacked velocities by branch and apply BAGEL CFG equations."""
+ branches = self._pending_denoise_branches
+ query_len = self._pending_denoise_query_len
+ if branches is None or query_len is None:
+ raise RuntimeError("BAGEL MoT velocity collection requires a pending denoise branch layout.")
+ expected_rows = len(branches) * query_len
+ if int(velocity.shape[0]) != expected_rows:
+ raise ValueError(
+ f"BAGEL MoT velocity collection expected {expected_rows} stacked rows, got {velocity.shape[0]}."
+ )
+
+ self._velocity_buffer.clear()
+ offset = 0
+ for branch in branches:
+ branch_velocity = velocity[offset : offset + query_len].to(device=device, dtype=dtype)
+ self._velocity_buffer[branch] = self._strip_denoise_query_markers(branch_velocity)
+ offset += query_len
+
+ merged_velocity = (
+ self._merged_velocity_without_cfg(device=device)
+ if "cfg_text" not in self._velocity_buffer
+ else self._merged_velocity_with_cfg(generation_kwargs, device=device)
+ )
+ self._velocity_buffer.clear()
+ self._pending_denoise_branches = None
+ self._pending_denoise_query_len = None
+ return merged_velocity
+
+ def _denoise_branches_for_timestep(
+ self, generation_kwargs: dict[str, object], timestep: object
+ ) -> tuple[str, ...]:
+ """Return the ordered tuple of active CFG branch names for ``timestep``."""
+ self.validate_cfg_request(generation_kwargs)
+ cfg_text_active = self._cfg_text_active(generation_kwargs, timestep)
+ branches = ["main"]
+ if cfg_text_active:
+ branches.append("cfg_text")
+ if cfg_text_active and self.cfg_img_requested(generation_kwargs):
+ branches.append("cfg_img")
+ return tuple(branches)
+
+ def _cfg_text_active(self, generation_kwargs: dict[str, object], timestep: object) -> bool:
+ """Return whether text CFG guidance is active at ``timestep``."""
+ cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0))
+ if cfg_text_scale <= 1.0:
+ return False
+
+ # CFG applies only inside the configured open-left, closed-right
+ # diffusion-time interval.
+ interval = generation_kwargs.get("cfg_interval", (0.0, 1.0))
+ if interval is None:
+ lower, upper = 0.0, 1.0
+ elif isinstance(interval, (list, tuple)) and len(interval) == 2:
+ lower, upper = float(interval[0]), float(interval[1])
+ else:
+ lower, upper = float(interval), 1.0
+
+ if torch.is_tensor(timestep):
+ if timestep.numel() == 0:
+ raise ValueError("BAGEL CFG timestep tensor must not be empty.")
+ t = float(timestep.detach().reshape(-1)[0].item())
+ elif timestep is None:
+ raise ValueError("BAGEL CFG requires current-round meta['timestep'].")
+ else:
+ t = float(timestep)
+ return t > lower and t <= upper
+
+ def _strip_denoise_query_markers(self, velocity: torch.Tensor) -> torch.Tensor:
+ """Strip the leading and trailing vision marker rows from a branch velocity tensor."""
+ if int(velocity.shape[0]) < 3:
+ raise ValueError("BAGEL MoT denoise velocity must include start/end marker rows.")
+ return velocity[1:-1]
+
+ def _merged_velocity_without_cfg(self, *, device: torch.device) -> torch.Tensor:
+ """Return the unguided ``main`` branch velocity."""
+ main_velocity = self._velocity_buffer.get("main")
+ if main_velocity is None:
+ raise RuntimeError("BAGEL CFG merge requires a collected main branch velocity.")
+ return main_velocity.to(device=device)
+
+ def _merged_velocity_with_cfg(self, generation_kwargs: dict[str, object], *, device: torch.device) -> torch.Tensor:
+ """Combine ``main``, ``cfg_text``, and optional ``cfg_img`` velocities with CFG renormalization."""
+ main_velocity = self._merged_velocity_without_cfg(device=device)
+
+ cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0))
+ cfg_text_velocity = self._velocity_buffer.get("cfg_text")
+ if cfg_text_velocity is None:
+ raise RuntimeError("BAGEL CFG merge requires a collected cfg_text branch velocity.")
+ cfg_text_velocity = cfg_text_velocity.to(device=device, dtype=main_velocity.dtype)
+
+ cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0))
+ cfg_img_velocity = self._velocity_buffer.get("cfg_img")
+ if cfg_img_scale > 1.0 and cfg_img_velocity is None:
+ raise ValueError("BAGEL image CFG merge requires a collected cfg_img velocity.")
+ if cfg_img_velocity is not None:
+ cfg_img_velocity = cfg_img_velocity.to(device=device, dtype=main_velocity.dtype)
+
+ cfg_renorm_min = float(generation_kwargs.get("cfg_renorm_min", 0.0))
+ cfg_renorm_type = str(generation_kwargs.get("cfg_renorm_type", "global"))
+
+ # Official InterleaveInferencer wraps generate_image in bf16 autocast, so
+ # the CFG scale/norm/renorm math is not the same as eager bf16 * python float.
+ autocast = (
+ torch.amp.autocast(device.type, dtype=torch.bfloat16)
+ if IS_CUDA_AVAILABLE and device.type == get_device_type()
+ else nullcontext()
+ )
+ with autocast:
+ guided = cfg_text_velocity + cfg_text_scale * (main_velocity - cfg_text_velocity)
+ if cfg_renorm_type == "text_channel":
+ norm_main = torch.norm(main_velocity, dim=-1, keepdim=True)
+ norm_text = torch.norm(guided, dim=-1, keepdim=True)
+ scale = (norm_main / (norm_text + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
+ merged = guided * scale
+ if cfg_img_scale > 1.0:
+ assert cfg_img_velocity is not None
+ merged = cfg_img_velocity + cfg_img_scale * (merged - cfg_img_velocity)
+ return merged.to(device=device)
+
+ if cfg_img_scale > 1.0:
+ assert cfg_img_velocity is not None
+ guided = cfg_img_velocity + cfg_img_scale * (guided - cfg_img_velocity)
+
+ if cfg_renorm_type == "global":
+ norm_main = torch.norm(main_velocity)
+ norm_guided = torch.norm(guided)
+ elif cfg_renorm_type == "channel":
+ norm_main = torch.norm(main_velocity, dim=-1, keepdim=True)
+ norm_guided = torch.norm(guided, dim=-1, keepdim=True)
+ else:
+ raise NotImplementedError(f"BAGEL infer_gen CFG renorm type {cfg_renorm_type!r} is not implemented.")
+ scale = (norm_main / (norm_guided + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
+ return (guided * scale).to(device=device)
+
+ def _merged_branch_cache(
+ self,
+ contexts: list[MotCacheContext],
+ key_values_lens: torch.Tensor,
+ empty_cache_factory: Callable[[], Any],
+ ) -> Any:
+ """Concatenate per-branch KV caches into one packed ``NaiveCache``."""
+ # Concatenation order must match preprocess_parallel_denoise_inputs:
+ # main, then cfg_text, then cfg_img when those branches are active.
+ merged_cache = empty_cache_factory()
+ for layer_idx in merged_cache.key_cache:
+ key_parts = []
+ value_parts = []
+ for context, key_values_len in zip(contexts, key_values_lens.tolist(), strict=True):
+ key_values_len = int(key_values_len)
+ if key_values_len == 0:
+ continue
+ cache = context.cache
+ if cache is None or cache.key_cache[layer_idx] is None or cache.value_cache[layer_idx] is None:
+ raise RuntimeError(f"BAGEL {context.name} branch cache is missing layer {layer_idx}.")
+ key_parts.append(cache.key_cache[layer_idx])
+ value_parts.append(cache.value_cache[layer_idx])
+ if key_parts:
+ merged_cache.key_cache[layer_idx] = torch.cat(key_parts, dim=0)
+ merged_cache.value_cache[layer_idx] = torch.cat(value_parts, dim=0)
+ return merged_cache
+
+
+__all__ = ["MotCacheContext", "MotGenerationState"]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py
new file mode 100644
index 0000000000..52270435de
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py
@@ -0,0 +1,346 @@
+"""Attention visibility metadata and mask materializers for BAGEL Qwen2-MoT.
+
+`packed_attention_metadata` is the source of truth. `build_mot_block_mask`
+materializes a Flex `BlockMask` for accelerated training. `build_mot_magi_mask`
+materializes MagiAttention ranges. `build_mot_sdpa_mask` materializes a dense
+boolean mask for eager SDPA (``True`` means attend).
+"""
+
+from __future__ import annotations
+
+import torch
+from torch.nn.attention.flex_attention import BlockMask
+
+from ......ops.mask import MagiAttentionMask
+
+
+_MOT_BLOCK_SIZE = 128
+_MAGI_ATTN_FULL = 0
+_MAGI_ATTN_CAUSAL = 1
+
+
+def build_mot_attention_metadata(
+ sample_splits: list[list[int]],
+ sample_attn_modes: list[list[str]],
+ *,
+ device: torch.device,
+) -> torch.Tensor:
+ """Encode BAGEL packed visibility in three O(sequence) int32 rows.
+
+ Row 0 identifies independent packed documents. Row 1 identifies spans
+ whose tokens can attend bidirectionally within the span (``full`` and
+ ``noise``). Row 2 identifies noise spans so their keys remain invisible
+ outside that same noise span. ``-1`` means that a row does not apply.
+
+ Callers materialize this compact representation as a Flex ``BlockMask``,
+ MagiAttention ranges, or a dense SDPA mask, without changing the metadata.
+ """
+ total_length = sum(sum(split_lens) for split_lens in sample_splits)
+ metadata = torch.full((3, total_length), -1, device=device, dtype=torch.int32)
+ cursor = 0
+ span_id = 0
+ for document_id, (split_lens, attn_modes) in enumerate(zip(sample_splits, sample_attn_modes, strict=True)):
+ for length, mode in zip(split_lens, attn_modes, strict=True):
+ if mode not in {"causal", "full", "noise"}:
+ raise ValueError(f"Unsupported BAGEL attention mode: {mode!r}.")
+ span_end = cursor + length
+ metadata[0, cursor:span_end] = document_id
+ if mode != "causal":
+ metadata[1, cursor:span_end] = span_id
+ if mode == "noise":
+ metadata[2, cursor:span_end] = span_id
+ cursor = span_end
+ span_id += 1
+ return metadata.contiguous()
+
+
+def _validate_mot_attention_metadata(packed_attention_metadata: torch.Tensor) -> None:
+ """Validate that ``packed_attention_metadata`` is a ``[3, S]`` int32 tensor."""
+ if packed_attention_metadata.ndim != 2 or packed_attention_metadata.shape[0] != 3:
+ raise ValueError(
+ "BAGEL Qwen2-MoT attention metadata must have shape [3, sequence], "
+ f"got {tuple(packed_attention_metadata.shape)}."
+ )
+ if packed_attention_metadata.dtype != torch.int32:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT attention metadata must use torch.int32, got {packed_attention_metadata.dtype}."
+ )
+
+
+def pad_mot_attention_metadata(
+ packed_attention_metadata: torch.Tensor,
+ padded_length: int,
+) -> torch.Tensor:
+ """Pad metadata with one isolated full-attention span for SP alignment.
+
+ Giving padding its own document prevents real tokens from attending to it;
+ leaving its noise ID at ``-1`` makes the padding block safe to classify as
+ a regular clean block.
+ """
+ _validate_mot_attention_metadata(packed_attention_metadata)
+
+ sequence_length = int(packed_attention_metadata.shape[1])
+ if padded_length < sequence_length:
+ raise ValueError(
+ "BAGEL Qwen2-MoT padded attention length cannot shrink the sequence: "
+ f"sequence_length={sequence_length}, padded_length={padded_length}."
+ )
+ if padded_length == sequence_length:
+ return packed_attention_metadata
+
+ padded = torch.full(
+ (3, padded_length),
+ -1,
+ device=packed_attention_metadata.device,
+ dtype=packed_attention_metadata.dtype,
+ )
+ padded[:, :sequence_length] = packed_attention_metadata
+ padding_id = sequence_length
+ padded[0, sequence_length:] = padding_id
+ padded[1, sequence_length:] = padding_id
+ return padded.contiguous()
+
+
+def _metadata_block_ranges(
+ token_ids: torch.Tensor,
+ *,
+ block_size: int,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Summarize monotonic token IDs for conservative block classification."""
+ padding = (-int(token_ids.numel())) % block_size
+ if padding:
+ token_ids = torch.nn.functional.pad(token_ids, (0, padding), value=-1)
+ token_ids = token_ids.view(-1, block_size)
+ valid = token_ids >= 0
+ minimum = torch.where(valid, token_ids, torch.iinfo(token_ids.dtype).max).amin(dim=-1)
+ maximum = token_ids.amax(dim=-1)
+ return minimum, maximum, valid.any(dim=-1), valid.all(dim=-1)
+
+
+def _ordered_block_indices(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ """Convert a dense block-level predicate to FlexAttention ordered rows."""
+ block_count = int(blocks.shape[-1])
+ block_ids = torch.arange(block_count, device=blocks.device, dtype=torch.int32).expand(block_count, -1)
+ sentinel = torch.full_like(block_ids, block_count)
+ indices = torch.where(blocks, block_ids, sentinel).sort(dim=-1).values
+ num_blocks = blocks.sum(dim=-1, dtype=torch.int32)
+ return num_blocks.unsqueeze(0).unsqueeze(0), indices.unsqueeze(0).unsqueeze(0)
+
+
+def build_mot_block_mask(packed_attention_metadata: torch.Tensor) -> BlockMask:
+ """Build BAGEL's head-independent BlockMask from compact metadata.
+
+ Token visibility is:
+ ``same_document & (causal | same_full_span) & ~foreign_noise_key``.
+ """
+ _validate_mot_attention_metadata(packed_attention_metadata)
+
+ sequence_length = int(packed_attention_metadata.shape[1])
+ if sequence_length == 0:
+ raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.")
+ document_ids, full_span_ids, noise_span_ids = packed_attention_metadata
+
+ def mask_mod(batch_idx, head_idx, query_idx, key_idx):
+ # MoT visibility is identical across batch and attention heads, but
+ # FlexAttention requires the four-argument callback signature.
+ del batch_idx, head_idx
+ same_document = document_ids[query_idx] == document_ids[key_idx]
+ causal = query_idx >= key_idx
+ same_full_or_noise_span = (full_span_ids[query_idx] >= 0) & (
+ full_span_ids[query_idx] == full_span_ids[key_idx]
+ )
+ foreign_noise_key = (noise_span_ids[key_idx] >= 0) & (noise_span_ids[query_idx] != noise_span_ids[key_idx])
+ return same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key
+
+ doc_min, doc_max, doc_valid, doc_complete = _metadata_block_ranges(
+ document_ids,
+ block_size=_MOT_BLOCK_SIZE,
+ )
+ full_min, full_max, full_valid, full_complete = _metadata_block_ranges(
+ full_span_ids,
+ block_size=_MOT_BLOCK_SIZE,
+ )
+ _, noise_max, _, _ = _metadata_block_ranges(
+ noise_span_ids,
+ block_size=_MOT_BLOCK_SIZE,
+ )
+
+ # Range overlap produces a conservative candidate set: mixed boundary
+ # blocks may be included here and are filtered token-by-token by mask_mod.
+ same_document_block = (
+ doc_valid[:, None]
+ & doc_valid[None, :]
+ & (doc_min[:, None] <= doc_max[None, :])
+ & (doc_min[None, :] <= doc_max[:, None])
+ )
+ shared_full_span_block = (
+ full_valid[:, None]
+ & full_valid[None, :]
+ & (full_min[:, None] <= full_max[None, :])
+ & (full_min[None, :] <= full_max[:, None])
+ )
+ block_count = int(doc_min.numel())
+ block_ids = torch.arange(block_count, device=packed_attention_metadata.device)
+ causal_block = block_ids[:, None] >= block_ids[None, :]
+ candidate_blocks = same_document_block & (causal_block | shared_full_span_block)
+
+ # Full blocks bypass mask_mod. Mark a block pair full only when every token
+ # is provably in one document and visibility is unconditional: either the
+ # key block is strictly earlier and clean, or both blocks share one
+ # bidirectional span.
+ single_document_block = doc_complete & (doc_min == doc_max)
+ same_single_document = (
+ single_document_block[:, None] & single_document_block[None, :] & (doc_min[:, None] == doc_min[None, :])
+ )
+ single_full_span_block = full_complete & full_valid & (full_min == full_max)
+ same_single_full_span = (
+ single_full_span_block[:, None] & single_full_span_block[None, :] & (full_min[:, None] == full_min[None, :])
+ )
+ strictly_earlier_clean_key_block = (block_ids[:, None] > block_ids[None, :]) & (noise_max[None, :] < 0)
+ full_blocks = candidate_blocks & same_single_document & (strictly_earlier_clean_key_block | same_single_full_span)
+ partial_blocks = candidate_blocks & ~full_blocks
+
+ partial_num_blocks, partial_indices = _ordered_block_indices(partial_blocks)
+ full_num_blocks, full_indices = _ordered_block_indices(full_blocks)
+ return BlockMask.from_kv_blocks(
+ partial_num_blocks,
+ partial_indices,
+ full_num_blocks,
+ full_indices,
+ BLOCK_SIZE=_MOT_BLOCK_SIZE,
+ mask_mod=mask_mod,
+ seq_lengths=(sequence_length, sequence_length),
+ )
+
+
+def _mot_span_mode(full_span_id: int, noise_span_id: int) -> str:
+ """Classify a token span as ``'noise'``, ``'full'``, or ``'causal'`` from its span IDs."""
+ if noise_span_id >= 0:
+ return "noise"
+ if full_span_id >= 0:
+ return "full"
+ return "causal"
+
+
+def _iter_mot_spans(packed_attention_metadata: torch.Tensor) -> list[tuple[int, int, int, str]]:
+ """Recover contiguous (start, end, document_id, mode) spans from metadata."""
+ sequence_length = int(packed_attention_metadata.shape[1])
+ document_ids, full_span_ids, noise_span_ids = packed_attention_metadata
+ changed = document_ids.new_ones(sequence_length, dtype=torch.bool)
+ changed[1:] = (
+ (document_ids[1:] != document_ids[:-1])
+ | (full_span_ids[1:] != full_span_ids[:-1])
+ | (noise_span_ids[1:] != noise_span_ids[:-1])
+ )
+ starts = torch.nonzero(changed, as_tuple=False).flatten()
+ ends = torch.cat((starts[1:], starts.new_tensor([sequence_length])))
+ spans: list[tuple[int, int, int, str]] = []
+ for start, end, document_id, full_span_id, noise_span_id in zip(
+ starts.tolist(),
+ ends.tolist(),
+ document_ids[starts].tolist(),
+ full_span_ids[starts].tolist(),
+ noise_span_ids[starts].tolist(),
+ strict=True,
+ ):
+ spans.append((start, end, int(document_id), _mot_span_mode(int(full_span_id), int(noise_span_id))))
+ return spans
+
+
+def _merge_adjacent_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
+ """Coalesce contiguous ``(start, end)`` intervals into maximal ranges."""
+ if not ranges:
+ return []
+ merged = [ranges[0]]
+ for start, end in ranges[1:]:
+ last_start, last_end = merged[-1]
+ if start == last_end:
+ merged[-1] = (last_start, end)
+ else:
+ merged.append((start, end))
+ return merged
+
+
+def build_mot_magi_mask(packed_attention_metadata: torch.Tensor) -> MagiAttentionMask:
+ """Build BAGEL's MagiAttention ranges from compact metadata.
+
+ Token visibility is the same as Flex/SDPA:
+ ``same_document & (causal | same_full_span) & ~foreign_noise_key``.
+
+ Each document is encoded as:
+ - a self range per span (causal inside ``causal`` spans, full inside ``full``/``noise``)
+ - full ranges from each query span onto earlier non-noise spans in that document
+
+ Noise keys are omitted from every other span's key ranges. Metadata is
+ already the full packed sequence, including Ulysses padding, so this
+ builder does not rescale ranges by ``ulysses_size``.
+ """
+ _validate_mot_attention_metadata(packed_attention_metadata)
+ sequence_length = int(packed_attention_metadata.shape[1])
+ if sequence_length == 0:
+ raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.")
+
+ q_ranges: list[tuple[int, int]] = []
+ k_ranges: list[tuple[int, int]] = []
+ attn_types: list[int] = []
+ document_spans: list[tuple[int, int, str]] = []
+ current_document_id: int | None = None
+
+ def flush_document(spans: list[tuple[int, int, str]]) -> None:
+ """Append Magi Q/K range pairs and attention types for one document's spans."""
+ for query_index, (query_start, query_end, query_mode) in enumerate(spans):
+ earlier_clean = [
+ (key_start, key_end) for key_start, key_end, key_mode in spans[:query_index] if key_mode != "noise"
+ ]
+ for key_start, key_end in _merge_adjacent_ranges(earlier_clean):
+ q_ranges.append((query_start, query_end))
+ k_ranges.append((key_start, key_end))
+ attn_types.append(_MAGI_ATTN_FULL)
+ q_ranges.append((query_start, query_end))
+ k_ranges.append((query_start, query_end))
+ attn_types.append(_MAGI_ATTN_FULL if query_mode != "causal" else _MAGI_ATTN_CAUSAL)
+
+ for start, end, document_id, mode in _iter_mot_spans(packed_attention_metadata):
+ if current_document_id is not None and document_id != current_document_id:
+ flush_document(document_spans)
+ document_spans = []
+ current_document_id = document_id
+ document_spans.append((start, end, mode))
+ flush_document(document_spans)
+
+ device = packed_attention_metadata.device
+ return MagiAttentionMask(
+ q_ranges=torch.tensor(q_ranges, device=device, dtype=torch.int32),
+ k_ranges=torch.tensor(k_ranges, device=device, dtype=torch.int32),
+ attn_type_map=torch.tensor(attn_types, device=device, dtype=torch.int32),
+ )
+
+
+def build_mot_sdpa_mask(packed_attention_metadata: torch.Tensor) -> torch.Tensor:
+ """Materialize a dense ``[1, 1, S, S]`` boolean SDPA mask.
+
+ ``True`` means the query may attend that key. Do not store this as float32
+ ``0 / -inf``; PyTorch SDPA already treats boolean ``False`` as masked out.
+ """
+ _validate_mot_attention_metadata(packed_attention_metadata)
+ sequence_length = int(packed_attention_metadata.shape[1])
+ if sequence_length == 0:
+ raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.")
+
+ document_ids, full_span_ids, noise_span_ids = packed_attention_metadata
+ query_idx = torch.arange(sequence_length, device=packed_attention_metadata.device)
+ same_document = document_ids[:, None] == document_ids[None, :]
+ causal = query_idx[:, None] >= query_idx[None, :]
+ same_full_or_noise_span = (full_span_ids[:, None] >= 0) & (full_span_ids[:, None] == full_span_ids[None, :])
+ foreign_noise_key = (noise_span_ids[None, :] >= 0) & (noise_span_ids[:, None] != noise_span_ids[None, :])
+ visible = same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key
+ return visible.to(dtype=torch.bool)[None, None].contiguous()
+
+
+__all__ = [
+ "build_mot_attention_metadata",
+ "build_mot_block_mask",
+ "build_mot_magi_mask",
+ "build_mot_sdpa_mask",
+ "pad_mot_attention_metadata",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py
new file mode 100644
index 0000000000..e2b2975a5a
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py
@@ -0,0 +1,1568 @@
+"""BAGEL Qwen2 MoT backbone."""
+
+from contextlib import nullcontext
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+import torch
+import torch.nn as nn
+from torch.nn.attention import SDPBackend, sdpa_kernel
+from torch.nn.attention.flex_attention import BlockMask
+from torch.nn.functional import scaled_dot_product_attention
+from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm, Qwen2RotaryEmbedding
+from transformers.utils import ModelOutput
+
+from veomni.ops import VeomniOp
+from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type
+
+from ....utils.conversation import ConversationItem
+from ...module_modeling_base import PretrainedOmniModule
+from ..sources import (
+ BAGEL_FLOW_HIDDEN,
+ BAGEL_FLOW_QUERY,
+ BAGEL_FLOW_VELOCITY,
+ BAGEL_PHASE_KEY,
+ BAGEL_START_TOKEN,
+ bagel_phase,
+ get_tail_phase_item,
+)
+from .configuration import BagelQwen2MoTConfig
+from .generation_state import MotGenerationState
+from .masking import build_mot_block_mask, build_mot_sdpa_mask
+from .processing import PackedConversation, preprocess_mot_inputs
+
+
+_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"})
+
+
+class InferenceMixin:
+ """FSM ``generate`` / serial denoise / velocity-collection — HF ``GenerationMixin`` analog.
+
+ Listed *before* :class:`BagelQwen2MoTCore` (and therefore before
+ :class:`~...module_modeling_base.PretrainedOmniModule`): the pretrained
+ base ships a no-op ``reset_local_inference_state``, and MRO resolves
+ left-to-right — put second, that no-op would shadow the real one below.
+ """
+
+ def forward_inference(
+ self,
+ packed_query_sequence: torch.Tensor,
+ query_lens: torch.Tensor,
+ packed_query_position_ids: torch.Tensor,
+ packed_query_indexes: torch.Tensor,
+ past_key_values: Optional["NaiveCache"] = None,
+ key_values_lens: Optional[torch.Tensor] = None,
+ packed_key_value_indexes: Optional[torch.Tensor] = None,
+ update_past_key_values: bool = True,
+ is_causal: bool = True,
+ mode: str = "und",
+ packed_attention_metadata: Optional[torch.Tensor] = None,
+ packed_vae_token_indexes: Optional[torch.Tensor] = None,
+ packed_text_indexes: Optional[torch.Tensor] = None,
+ ) -> Dict[str, Any]:
+ """Run one packed inference step through the MoT backbone and return hidden states and KV cache."""
+ call_kwargs: Dict[str, Any] = {
+ "packed_query_sequence": packed_query_sequence,
+ "query_lens": query_lens,
+ "packed_query_position_ids": packed_query_position_ids,
+ "packed_query_indexes": packed_query_indexes,
+ "past_key_values": past_key_values,
+ "key_values_lens": key_values_lens,
+ "packed_key_value_indexes": packed_key_value_indexes,
+ "update_past_key_values": update_past_key_values,
+ "is_causal": is_causal,
+ "mode": mode,
+ "packed_attention_metadata": packed_attention_metadata,
+ }
+
+ is_gen = _check_packed_inference_mode(mode)
+ if is_gen:
+ call_kwargs["packed_vae_token_indexes"] = packed_vae_token_indexes
+ call_kwargs["packed_text_indexes"] = packed_text_indexes
+
+ output = self.model.forward_packed_inference(**call_kwargs)
+
+ return {
+ "hidden_states": output.packed_query_sequence,
+ "past_key_values": output.past_key_values,
+ }
+
+ def reset_local_inference_state(self) -> None:
+ self._generation_state.reset()
+
+ def generate(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Prefill prompt caches or decode the next understanding token in the generation FSM."""
+ del kwargs
+ if conversation_list is None:
+ raise ValueError("BAGEL Qwen2-MoT generate requires conversation_list.")
+
+ generation_kwargs = generation_kwargs or {}
+ tail = conversation_list[-1]
+ is_start_token = bagel_phase(tail) == BAGEL_START_TOKEN
+ # Prefill the full conversation_list (including the assistant start token
+ # on understanding requests). The first text_ar pass returns the prefilled
+ # start-token hidden state; subsequent AR steps decode the tail output item.
+ if self._generation_state.main.cache is None or tail.type != "output":
+ hidden_states = self._prefill_prompt(conversation_list, generation_kwargs)
+ if is_start_token:
+ if hidden_states.dim() == 3 and hidden_states.size(0) == 1:
+ hidden_states = hidden_states.squeeze(0)
+ if hidden_states.dim() != 2:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}."
+ )
+ tail.type = "output"
+ tail.value = hidden_states[-1:].contiguous()
+ return {"conversation_list": conversation_list}
+
+ if is_start_token:
+ tail.meta.pop(BAGEL_PHASE_KEY, None)
+ return {"conversation_list": conversation_list}
+
+ hidden_states = self._decode_next_token(conversation_list)
+ if hidden_states.dim() == 3 and hidden_states.size(0) == 1:
+ hidden_states = hidden_states.squeeze(0)
+ if hidden_states.dim() != 2:
+ raise ValueError(f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}.")
+ tail.value = hidden_states[-1:].contiguous()
+ return {"conversation_list": conversation_list}
+
+ def _prepare_denoise_query(
+ self,
+ conversation_list: list[ConversationItem] | None,
+ generation_kwargs: dict[str, Any] | None,
+ ) -> tuple[ConversationItem, torch.Tensor]:
+ """Validate CFG parameters and extract the tail ``flow_query`` tensor."""
+ if conversation_list is None:
+ raise ValueError("BAGEL Qwen2-MoT denoise_branch requires conversation_list.")
+
+ self._generation_state.validate_cfg_request(generation_kwargs or {})
+ self._generation_state.main.require_ready()
+ tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_QUERY)
+ if tail is None or not torch.is_tensor(tail.value):
+ raise ValueError(f"BAGEL Qwen2-MoT denoise branch requires an output in phase {BAGEL_FLOW_QUERY!r}.")
+
+ query = tail.value
+ if query.dim() == 3 and query.shape[0] == 1:
+ query = query.squeeze(0)
+ if query.dim() != 2:
+ raise ValueError(f"BAGEL Qwen2-MoT denoise branch expects rank-2 query tensor, got {tuple(query.shape)}.")
+ if int(query.shape[-1]) != int(self.config.hidden_size):
+ raise ValueError(
+ "BAGEL Qwen2-MoT denoise branch hidden-size mismatch: "
+ f"got {query.shape[-1]}, expected {self.config.hidden_size}."
+ )
+ if int(query.shape[0]) < 3:
+ raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.")
+ return tail, query
+
+ def denoise_branch(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Run serial CFG denoise forwards over active branches and write ``flow_hidden``."""
+ del kwargs
+ tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs)
+
+ hiddens: list[torch.Tensor] = []
+ for inputs in self._generation_state.iter_serial_denoise_inputs(
+ query,
+ generation_kwargs or {},
+ timestep=tail.meta.get("timestep"),
+ device=self.device,
+ dtype=self.dtype,
+ ):
+ outputs = self.forward_inference(
+ **inputs,
+ update_past_key_values=False,
+ is_causal=False,
+ mode="gen",
+ )
+ hiddens.append(outputs["hidden_states"])
+ if not hiddens:
+ raise RuntimeError("BAGEL Qwen2-MoT denoise branch produced no branch hidden states.")
+
+ tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN
+ tail.value = torch.cat(hiddens, dim=0).to(device=self.device, dtype=self.dtype)
+ return {"conversation_list": conversation_list}
+
+ def collect_velocity(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Strip marker rows from ``flow_velocity`` and apply CFG combination."""
+ del kwargs
+ if conversation_list is None:
+ raise ValueError("BAGEL Qwen2-MoT collect_velocity requires conversation_list.")
+
+ self._generation_state.validate_cfg_request(generation_kwargs or {})
+ tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY)
+ if tail is None or not torch.is_tensor(tail.value):
+ raise ValueError(
+ f"BAGEL Qwen2-MoT velocity collection requires an output in phase {BAGEL_FLOW_VELOCITY!r}."
+ )
+
+ velocity = tail.value
+ if velocity.dim() == 3 and velocity.shape[0] == 1:
+ velocity = velocity.squeeze(0)
+ if velocity.dim() != 2:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT velocity collection expects rank-2 velocity, got {tuple(velocity.shape)}."
+ )
+
+ # Restore branch identity from the layout recorded by denoise_branch,
+ # strip marker rows, and replace the stacked tensor with guided velocity.
+ tail.value = self._generation_state.collect_velocity(
+ velocity,
+ generation_kwargs or {},
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": conversation_list}
+
+ def _prefill_prompt(
+ self,
+ conversation_list: list[ConversationItem],
+ generation_kwargs: dict[str, Any],
+ ) -> torch.Tensor:
+ """Pack and prefill the main prompt and optional CFG branch caches."""
+ main_packed = preprocess_mot_inputs(
+ [conversation_list],
+ device=self.device,
+ dtype=self.dtype,
+ hidden_size=int(self.config.hidden_size),
+ )
+ if main_packed is None:
+ raise ValueError("BAGEL Qwen2-MoT generate requires at least one embedded text/image item.")
+
+ state = self._generation_state
+ state.main.reset()
+ state.cfg_text.reset()
+ state.cfg_img.reset()
+
+ # Text CFG reuses the latest valid prefix of the main prompt: its cache
+ # boundary is before a text span and after an image span.
+ cfg_text_slice, cfg_text_next_position_id = self._cfg_text_prefix_slice(main_packed)
+ use_cfg_img = state.cfg_img_requested(generation_kwargs)
+
+ # Image CFG keeps logical text spans only. Inspect the grouped spans so
+ # marker tokens belonging to image triplets are not treated as text.
+ cfg_img_items = []
+ if use_cfg_img:
+ cfg_img_items = [item for span in main_packed.spans if span.item.type == "text" for item in span.items]
+
+ # Pack main and cfg_img as independent logical documents. The attention
+ # metadata prevents cross-document attention inside the Flex kernel.
+ packed = main_packed
+ if cfg_img_items:
+ packed = preprocess_mot_inputs(
+ [conversation_list, cfg_img_items],
+ device=self.device,
+ dtype=self.dtype,
+ hidden_size=int(self.config.hidden_size),
+ )
+ if packed is None:
+ raise RuntimeError("BAGEL Qwen2-MoT CFG prompt packing produced no tokens.")
+
+ main_length = sum(main_packed.sample_splits[0])
+ total_length = int(packed.packed_sequence.shape[0])
+ main_slice = slice(0, main_length)
+ cfg_img_slice = slice(main_length, total_length)
+
+ # This is a fresh-cache prefill: forward_inference allocates the cache,
+ # while packed attention metadata selects FlexAttention.
+ outputs = self.forward_packed_prefill(packed)
+ packed_cache = outputs["past_key_values"]
+
+ # Detach main and cfg_img from the temporary packed cache allocation.
+ # cfg_text is a read-only prefix view of main and can share its storage.
+ main_cache = self._slice_prefill_cache(packed_cache, main_slice, clone=True)
+ cfg_text_cache = self._slice_prefill_cache(main_cache, cfg_text_slice, clone=False)
+
+ state.main.install_cache(
+ cache=main_cache,
+ cache_len=main_length,
+ next_position_id=self._next_position_id(packed.packed_position_ids[main_slice]),
+ device=self.device,
+ )
+ state.cfg_text.install_cache(
+ cache=cfg_text_cache,
+ cache_len=cfg_text_slice.stop,
+ next_position_id=cfg_text_next_position_id,
+ device=self.device,
+ )
+ if use_cfg_img:
+ cfg_img_cache = self._slice_prefill_cache(packed_cache, cfg_img_slice, clone=True)
+ state.cfg_img.install_cache(
+ cache=cfg_img_cache,
+ cache_len=cfg_img_slice.stop - cfg_img_slice.start,
+ next_position_id=self._next_position_id(packed.packed_position_ids[cfg_img_slice]),
+ device=self.device,
+ )
+
+ # cfg_img is an auxiliary packed document; graph consumers should only
+ # observe hidden states from the main prompt.
+ hidden_states = outputs["hidden_states"]
+ if int(hidden_states.shape[0]) != total_length:
+ raise RuntimeError(
+ "BAGEL Qwen2-MoT FlexAttention prefill returned an unexpected sequence length: "
+ f"expected {total_length}, got {hidden_states.shape[0]}."
+ )
+ return hidden_states[main_slice]
+
+ def forward_packed_prefill(self, packed: PackedConversation) -> dict[str, Any]:
+ """Run one fresh-cache Flex prefill over all logical documents."""
+ total_length = int(packed.packed_sequence.shape[0])
+ document_lens = [sum(split_lens) for split_lens in packed.sample_splits]
+ query_lens = torch.tensor(document_lens, device=self.device, dtype=torch.int32)
+
+ # A packed prefill may contain both understanding and generation expert
+ # tokens even though it executes as one model forward.
+ packed_gen_token_indexes = torch.nonzero(
+ packed.packed_token_type_ids == 1,
+ as_tuple=False,
+ ).flatten()
+ mode = "gen" if int(packed_gen_token_indexes.numel()) > 0 else "und"
+ call_kwargs: dict[str, Any] = {
+ "packed_query_sequence": packed.packed_sequence,
+ "query_lens": query_lens,
+ "packed_query_position_ids": packed.packed_position_ids,
+ "packed_query_indexes": torch.arange(total_length, device=self.device, dtype=torch.long),
+ "update_past_key_values": True,
+ "is_causal": False,
+ "mode": mode,
+ "packed_attention_metadata": packed.packed_attention_metadata,
+ }
+ if mode == "gen":
+ call_kwargs["packed_text_indexes"] = torch.nonzero(
+ packed.packed_token_type_ids == 0,
+ as_tuple=False,
+ ).flatten()
+ call_kwargs["packed_vae_token_indexes"] = packed_gen_token_indexes
+ return self.forward_inference(**call_kwargs)
+
+ def _decode_next_token(self, conversation_list: list[ConversationItem]) -> torch.Tensor:
+ """Run one AR decode step against the cached ``main`` prompt context."""
+ # AR has one query token and an existing contiguous cache, so the
+ # dedicated FlashAttention path is preferable to rebuilding a packed mask.
+ main_context = self._generation_state.main
+ main_context.require_ready()
+ tail = conversation_list[-1]
+ if tail.type != "output":
+ raise ValueError(f"BAGEL Qwen2-MoT decode expects tail output item, got {tail.type!r}.")
+
+ packed_query_sequence = tail.value
+ if not torch.is_tensor(packed_query_sequence):
+ raise ValueError("BAGEL Qwen2-MoT decode expects tail output.value to be an embedding tensor.")
+ if packed_query_sequence.dim() == 3 and packed_query_sequence.shape[0] == 1:
+ packed_query_sequence = packed_query_sequence.squeeze(0)
+ if packed_query_sequence.dim() != 2:
+ raise ValueError(
+ f"BAGEL Qwen2-MoT expected tail output embedding rank 2, got {tuple(packed_query_sequence.shape)}."
+ )
+ packed_query_sequence = packed_query_sequence[-1:].contiguous().to(device=self.device, dtype=self.dtype)
+
+ query_lens, packed_query_indexes, packed_position_ids = main_context.packed_query_args(
+ 1,
+ device=self.device,
+ )
+ outputs = self.forward_inference(
+ packed_query_sequence=packed_query_sequence,
+ query_lens=query_lens,
+ packed_query_position_ids=packed_position_ids,
+ packed_query_indexes=packed_query_indexes,
+ past_key_values=main_context.cache,
+ key_values_lens=main_context.key_values_lens,
+ packed_key_value_indexes=main_context.packed_key_value_indexes,
+ update_past_key_values=True,
+ is_causal=True,
+ mode="und",
+ )
+ main_context.append_packed_query(
+ cache=outputs["past_key_values"],
+ query_lens=query_lens,
+ device=self.device,
+ )
+
+ return outputs["hidden_states"]
+
+ def _cfg_text_prefix_slice(self, packed: PackedConversation) -> tuple[slice, torch.Tensor]:
+ """Compute the ``main`` cache prefix slice and next position ID for text CFG."""
+ # Moving across an image keeps the image in text-CFG conditioning;
+ # encountering text moves the boundary to just before that text span.
+ prefix_end = 0
+ next_position_id = torch.zeros(1, device=self.device, dtype=torch.long)
+ for span in packed.spans:
+ span_end = span.start + span.length
+ if span.item.type == "text":
+ prefix_end = span.start
+ next_position_id = packed.packed_position_ids[span.start].reshape(1)
+ elif span.item.type == "image":
+ prefix_end = span_end
+ next_position_id = self._next_position_id(packed.packed_position_ids[:span_end])
+ return slice(0, prefix_end), next_position_id
+
+ def _slice_prefill_cache(self, packed_cache: Any, rows: slice, *, clone: bool) -> Any:
+ """Extract a row slice from ``packed_cache`` into a standalone ``NaiveCache``."""
+ # main/cfg_img outlive the temporary packed cache and own their storage.
+ # cfg_text is an immutable prefix of main, so it may use cheap views.
+ cache = self._new_empty_cache()
+ if rows.stop <= rows.start:
+ return cache
+ for layer_idx in cache.key_cache:
+ key = packed_cache.key_cache[layer_idx][rows]
+ value = packed_cache.value_cache[layer_idx][rows]
+ cache.key_cache[layer_idx] = key.clone() if clone else key
+ cache.value_cache[layer_idx] = value.clone() if clone else value
+ return cache
+
+ def _next_position_id(self, position_ids: torch.Tensor) -> torch.Tensor:
+ """Return ``max(position_ids) + 1`` or ``0`` when ``position_ids`` is empty."""
+ if int(position_ids.numel()) == 0:
+ return torch.zeros(1, device=self.device, dtype=torch.long)
+ return position_ids.max().reshape(1).to(device=self.device, dtype=torch.long) + 1
+
+ def _new_empty_cache(self) -> Any:
+ return NaiveCache(len(self.model.layers))
+
+
+class BagelQwen2MoTCore(PretrainedOmniModule):
+ """Shared weights, packed train/infer forward, and generation-state shell.
+
+ Public eager and accelerated classes mix their inference mixins onto this
+ core. It is not registered on ``OMNI_*_REGISTRY``.
+ """
+
+ config_class = BagelQwen2MoTConfig
+ base_model_prefix = "bagel_qwen2_mot"
+ main_input_name = "inputs_embeds"
+ _no_split_modules = ["BagelQwen2MoTDecoderLayer"]
+ supports_gradient_checkpointing = True
+ _supports_sdpa = True
+ _supports_flex_attn = True
+
+ def __init__(self, config: BagelQwen2MoTConfig):
+ super().__init__(config)
+ self.model = BagelQwen2MoTBackbone(
+ config,
+ attention_cls=type(self).attention_cls,
+ mlp_cls=type(self).mlp_cls,
+ rms_norm_cls=type(self).rms_norm_cls,
+ )
+ self._generation_state = MotGenerationState()
+ self.post_init()
+
+ def forward( # type: ignore[override]
+ self,
+ packed_sequence: torch.Tensor,
+ packed_position_ids: torch.Tensor,
+ packed_token_type_ids: torch.Tensor,
+ packed_attention_metadata: torch.Tensor,
+ ) -> Dict[str, Any]:
+ packed_und_token_indexes = torch.nonzero(packed_token_type_ids == 0, as_tuple=False).flatten()
+ packed_gen_token_indexes = torch.nonzero(packed_token_type_ids == 1, as_tuple=False).flatten()
+ output = self.model(
+ packed_sequence=packed_sequence,
+ packed_position_ids=packed_position_ids,
+ packed_attention_metadata=packed_attention_metadata,
+ packed_und_token_indexes=packed_und_token_indexes,
+ packed_gen_token_indexes=packed_gen_token_indexes,
+ )
+ return {"hidden_states": output.packed_query_sequence}
+
+
+class BagelQwen2MoT(InferenceMixin, BagelQwen2MoTCore):
+ """HF-native eager Qwen2-MoT with serial CFG denoise."""
+
+
+class NaiveCache:
+ """Official BAGEL packed KV cache."""
+
+ def __init__(self, num_layers: int):
+ self.key_cache = dict.fromkeys(range(num_layers))
+ self.value_cache = dict.fromkeys(range(num_layers))
+
+
+@dataclass
+class BaseNavitOutputWithPast(ModelOutput):
+ packed_query_sequence: torch.FloatTensor | None = None
+ past_key_values: Optional[NaiveCache] = None
+
+
+def _rotate_half(x: torch.Tensor) -> torch.Tensor:
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def _apply_rotary_pos_emb(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+ unsqueeze_dim: int = 1,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Eager packed RoPE. Fused dispatch lives on the accelerated attention class."""
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ q_embed = (q * cos) + (_rotate_half(q) * sin)
+ k_embed = (k * cos) + (_rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+def _sdpa_context(device: torch.device):
+ """Pin mem-efficient SDPA on CUDA, matching official dense-train Bagel.
+
+ Official 2D-mask training forces ``SDPBackend.EFFICIENT_ATTENTION`` so the
+ dispatcher cannot pick MATH (OOM) or a newer CUDNN kernel. CPU/NPU keep the
+ default dispatcher because Efficient is CUDA-only.
+ """
+ if IS_CUDA_AVAILABLE and device.type == get_device_type():
+ return sdpa_kernel(backends=[SDPBackend.EFFICIENT_ATTENTION])
+ return nullcontext()
+
+
+def _repeat_kv_heads(packed_states: torch.Tensor, num_query_heads: int) -> torch.Tensor:
+ """Expand packed KV from ``[tokens, kv_heads, dim]`` to query heads.
+
+ Official dense-train Bagel repeats KV before Efficient SDPA. The CUDA
+ Efficient kernel does not accept ``enable_gqa`` with a 2D mask.
+ """
+ num_kv_heads = int(packed_states.shape[1])
+ if num_kv_heads == num_query_heads:
+ return packed_states
+ if num_query_heads % num_kv_heads != 0:
+ raise ValueError(
+ "BAGEL Qwen2-MoT GQA requires query heads divisible by KV heads: "
+ f"num_query_heads={num_query_heads}, num_kv_heads={num_kv_heads}."
+ )
+ groups = num_query_heads // num_kv_heads
+ packed_states = packed_states[:, :, None, :].repeat(1, 1, groups, 1)
+ return packed_states.reshape(-1, num_query_heads, packed_states.shape[-1])
+
+
+def _sdpa_packed_attention(
+ packed_query_states: torch.Tensor,
+ packed_key_states: torch.Tensor,
+ packed_value_states: torch.Tensor,
+ *,
+ attention_mask: Optional[torch.Tensor] = None,
+ is_causal: bool = False,
+ cu_seq_lens_q: Optional[torch.Tensor] = None,
+ cu_seq_lens_k: Optional[torch.Tensor] = None,
+ scale: float,
+ enable_gqa: bool,
+) -> torch.Tensor:
+ """Packed SDPA returning ``[tokens, heads, head_dim]``.
+
+ The 2D-mask path pins Efficient SDPA on CUDA. The varlen / ``is_causal``
+ path is left unpinned: official inference uses ``flash_attn_varlen``, not
+ Efficient.
+ """
+ if packed_query_states.shape[0] == 0:
+ return packed_query_states.new_zeros(packed_query_states.shape)
+
+ if attention_mask is not None:
+ if enable_gqa:
+ num_query_heads = int(packed_query_states.shape[1])
+ packed_key_states = _repeat_kv_heads(packed_key_states, num_query_heads)
+ packed_value_states = _repeat_kv_heads(packed_value_states, num_query_heads)
+ query = packed_query_states.transpose(0, 1).unsqueeze(0)
+ key = packed_key_states.transpose(0, 1).unsqueeze(0)
+ value = packed_value_states.transpose(0, 1).unsqueeze(0)
+ with _sdpa_context(packed_query_states.device):
+ output = scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ attn_mask=attention_mask,
+ dropout_p=0.0,
+ is_causal=False,
+ scale=scale,
+ )
+ return output.squeeze(0).transpose(0, 1).contiguous()
+
+ if cu_seq_lens_q is None or cu_seq_lens_k is None:
+ raise ValueError("BAGEL Qwen2-MoT SDPA without an attention mask requires cu_seq_lens_q/k.")
+
+ outputs: list[torch.Tensor] = []
+ for seq_idx in range(int(cu_seq_lens_q.shape[0]) - 1):
+ query_start = int(cu_seq_lens_q[seq_idx].item())
+ query_end = int(cu_seq_lens_q[seq_idx + 1].item())
+ key_start = int(cu_seq_lens_k[seq_idx].item())
+ key_end = int(cu_seq_lens_k[seq_idx + 1].item())
+ if query_end <= query_start:
+ continue
+
+ query = packed_query_states[query_start:query_end].transpose(0, 1).unsqueeze(0)
+ key = packed_key_states[key_start:key_end].transpose(0, 1).unsqueeze(0)
+ value = packed_value_states[key_start:key_end].transpose(0, 1).unsqueeze(0)
+
+ query_length = query_end - query_start
+ key_length = key_end - key_start
+ causal_mask = None
+ sdpa_is_causal = is_causal
+ if is_causal and query_length != key_length:
+ # FlashAttention varlen uses bottom-right causal alignment. PyTorch
+ # SDPA's ``is_causal=True`` is upper-left aligned when Q and K have
+ # different lengths, which makes a one-token cached decode see only
+ # the first key. The common Q=1 case needs no mask because the final
+ # logical query can see the entire cache; use an explicit lower-right
+ # mask only for multi-token cached decode.
+ sdpa_is_causal = False
+ if query_length != 1:
+ causal_mask = torch.ones(
+ (query_length, key_length),
+ dtype=torch.bool,
+ device=query.device,
+ ).tril(diagonal=key_length - query_length)
+
+ output = scaled_dot_product_attention(
+ query,
+ key,
+ value,
+ attn_mask=causal_mask,
+ dropout_p=0.0,
+ is_causal=sdpa_is_causal,
+ scale=scale,
+ enable_gqa=enable_gqa,
+ )
+ outputs.append(output.squeeze(0).transpose(0, 1))
+ if not outputs:
+ return packed_query_states.new_zeros(packed_query_states.shape)
+ return torch.cat(outputs, dim=0)
+
+
+def _fold_zero_anchors(target: torch.Tensor, *anchors: torch.Tensor) -> torch.Tensor:
+ """Add ``0 * sum(anchor)`` to ``target`` so empty-route parameters remain in the autograd graph."""
+ # When a packed batch has no generation tokens, the MoT "gen" expert weights still
+ # run on an empty slice and produce zero-sized outputs. Folding ``sum() * 0.0`` of
+ # those outputs into ``target`` keeps the gen-expert parameters in the autograd graph
+ # (with zero gradient) so FSDP/DP gradient reduction sees the same parameter set on
+ # every rank regardless of which modalities a micro-batch contains.
+ anchor = target.new_zeros(())
+ has_anchor = False
+ for value in anchors:
+ if torch.is_tensor(value):
+ anchor = anchor + value.sum() * 0.0
+ has_anchor = True
+ if not has_anchor:
+ return target
+ return target + anchor
+
+
+def _scatter_moe(
+ packed: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ und_value: torch.Tensor,
+ gen_value: torch.Tensor,
+ *,
+ extra_und_anchors: tuple[torch.Tensor, ...] = (),
+ extra_gen_anchors: tuple[torch.Tensor, ...] = (),
+) -> torch.Tensor:
+ """Write MoT und/gen expert outputs into a packed buffer and keep empty-route FSDP anchors."""
+ packed[packed_und_token_indexes] = und_value
+ packed[packed_gen_token_indexes] = gen_value
+ if int(packed_und_token_indexes.numel()) == 0:
+ packed = _fold_zero_anchors(packed, und_value, *extra_und_anchors)
+ if int(packed_gen_token_indexes.numel()) == 0:
+ packed = _fold_zero_anchors(packed, gen_value, *extra_gen_anchors)
+ return packed
+
+
+def _check_packed_inference_mode(
+ mode: str,
+) -> bool:
+ """Validate ``mode`` (``'und'`` or ``'gen'``) and return ``True`` for ``'gen'``."""
+ if mode == "und":
+ return False
+ if mode == "gen":
+ return True
+ raise ValueError(f"Unsupported BAGEL Qwen2 MoT inference mode: {mode!r}")
+
+
+class BagelQwen2RotaryEmbedding(Qwen2RotaryEmbedding):
+ """Qwen2 RoPE preserving fp32 inverse frequencies across dtype casts and meta materialization."""
+
+ def _apply(self, fn: Any, recurse: bool = True) -> nn.Module:
+ # Rebuild the true fp32 table after ``fn`` so ``.to(bfloat16)`` does not
+ # quantize rotary buffers and ``to_empty(device=...)`` from ``meta``
+ # materializes valid fp32 inverse frequencies on the target device.
+ module = super()._apply(fn, recurse=recurse)
+ device = self.inv_freq.device
+ if device.type == "meta":
+ return module
+ inv_freq, _ = self.compute_default_rope_parameters(self.config, device)
+ self.inv_freq = inv_freq
+ self.original_inv_freq = inv_freq.clone()
+ return module
+
+
+def _build_rms_norm(
+ rms_norm_cls: type[Qwen2RMSNorm],
+ hidden_size: int,
+ config: BagelQwen2MoTConfig,
+) -> Qwen2RMSNorm:
+ """Instantiate ``rms_norm_cls``, forwarding ``config`` when the subclass accepts it."""
+ try:
+ return rms_norm_cls(hidden_size, eps=config.rms_norm_eps, config=config)
+ except TypeError:
+ return rms_norm_cls(hidden_size, eps=config.rms_norm_eps)
+
+
+class BagelQwen2MoTAttention(nn.Module):
+ def __init__(
+ self,
+ config: BagelQwen2MoTConfig,
+ layer_idx: int,
+ *,
+ rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm,
+ ):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+ self.hidden_size = config.hidden_size
+ self.num_heads = config.num_attention_heads
+ self.num_key_value_heads = config.num_key_value_heads
+ self.head_dim = self.hidden_size // self.num_heads
+
+ self.query_size = self.num_heads * self.head_dim
+ self.key_value_size = self.num_key_value_heads * self.head_dim
+ self.qkv_split_sizes = (self.query_size, self.key_value_size, self.key_value_size)
+
+ self._build_qkv_proj()
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+ self.q_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config)
+ self.k_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config)
+ self.o_proj_moe_gen = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+ self.q_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config)
+ self.k_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config)
+ attn_impl = (
+ config.ops_config.attn_implementation if "attn_implementation" in config.ops_implementation else "sdpa"
+ )
+ self.veomni_attn: VeomniOp | None = None
+ if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS:
+ try:
+ self.veomni_attn = VeomniOp("attention", "standard", "veomni_flex_attention")
+ except RuntimeError:
+ self.veomni_attn = None
+ attn_impl = "sdpa"
+ self.veomni_attn_impl = attn_impl
+
+ def _build_qkv_proj(self) -> None:
+ self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True)
+ self.k_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True)
+ self.v_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True)
+ self.q_proj_moe_gen = nn.Linear(self.hidden_size, self.query_size, bias=True)
+ self.k_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True)
+ self.v_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True)
+
+ def _project_qkv(
+ self,
+ hidden: torch.Tensor,
+ *,
+ is_gen: bool,
+ split: bool = False,
+ ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Project ``hidden`` through the und or gen Q/K/V linear layers."""
+ query_proj, key_proj, value_proj = (
+ (self.q_proj_moe_gen, self.k_proj_moe_gen, self.v_proj_moe_gen)
+ if is_gen
+ else (self.q_proj, self.k_proj, self.v_proj)
+ )
+ query_states = query_proj(hidden)
+ key_states = key_proj(hidden)
+ value_states = value_proj(hidden)
+ if split:
+ return query_states, key_states, value_states
+ return torch.cat((query_states, key_states, value_states), dim=-1)
+
+ @staticmethod
+ def build_attention_mask(
+ packed_attention_metadata: torch.Tensor,
+ *,
+ attn_implementation: str | None = None,
+ ) -> torch.Tensor | BlockMask:
+ if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS:
+ return build_mot_block_mask(packed_attention_metadata)
+ return build_mot_sdpa_mask(packed_attention_metadata)
+
+ def apply_rotary_pos_emb(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ cos: torch.Tensor,
+ sin: torch.Tensor,
+ unsqueeze_dim: int = 1,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim)
+
+ def project_qkv(
+ self,
+ packed_sequence: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ packed_position_cos: torch.Tensor,
+ packed_position_sin: torch.Tensor,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ packed_qkv_states = _scatter_moe(
+ packed_sequence.new_zeros((packed_sequence.shape[0], sum(self.qkv_split_sizes))),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self._project_qkv(packed_sequence[packed_und_token_indexes], is_gen=False),
+ self._project_qkv(packed_sequence[packed_gen_token_indexes], is_gen=True),
+ )
+ packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split(
+ self.qkv_split_sizes, dim=-1
+ )
+ packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim)
+ packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim)
+ packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim)
+
+ packed_query_states = _scatter_moe(
+ packed_query_states.new_zeros(packed_query_states.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.q_norm(packed_query_states[packed_und_token_indexes]),
+ self.q_norm_moe_gen(packed_query_states[packed_gen_token_indexes]),
+ )
+ packed_key_states = _scatter_moe(
+ packed_key_states.new_zeros(packed_key_states.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.k_norm(packed_key_states[packed_und_token_indexes]),
+ self.k_norm_moe_gen(packed_key_states[packed_gen_token_indexes]),
+ )
+ packed_query_states, packed_key_states = self.apply_rotary_pos_emb(
+ packed_query_states,
+ packed_key_states,
+ packed_position_cos,
+ packed_position_sin,
+ unsqueeze_dim=1,
+ )
+ return packed_query_states, packed_key_states, packed_value_states
+
+ def attend(
+ self,
+ packed_query_states: torch.Tensor,
+ packed_key_states: torch.Tensor,
+ packed_value_states: torch.Tensor,
+ attention_mask: torch.Tensor | BlockMask,
+ ) -> torch.Tensor:
+ if isinstance(attention_mask, BlockMask):
+ attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention")
+ packed_attn_output, _ = attn_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ packed_key_states.transpose(0, 1).unsqueeze(0),
+ packed_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask,
+ dropout=0.0,
+ scaling=self.head_dim**-0.5,
+ skip_ulysses=True,
+ )
+ return packed_attn_output.squeeze(0).reshape(-1, self.num_heads * self.head_dim)
+
+ packed_attn_output = _sdpa_packed_attention(
+ packed_query_states,
+ packed_key_states,
+ packed_value_states,
+ attention_mask=attention_mask,
+ scale=self.head_dim**-0.5,
+ enable_gqa=self.num_heads != self.num_key_value_heads,
+ )
+ return packed_attn_output.reshape(-1, self.num_heads * self.head_dim)
+
+ def project_o(
+ self,
+ packed_attn_output: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ ) -> torch.Tensor:
+ return _scatter_moe(
+ packed_attn_output.new_zeros(packed_attn_output.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.o_proj(packed_attn_output[packed_und_token_indexes]),
+ self.o_proj_moe_gen(packed_attn_output[packed_gen_token_indexes]),
+ )
+
+ def forward_packed_train(
+ self,
+ packed_sequence: torch.Tensor,
+ attention_mask: torch.Tensor,
+ packed_position_cos: torch.Tensor,
+ packed_position_sin: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ ) -> torch.Tensor:
+ packed_query_states, packed_key_states, packed_value_states = self.project_qkv(
+ packed_sequence,
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ packed_position_cos,
+ packed_position_sin,
+ )
+ packed_attn_output = self.attend(
+ packed_query_states,
+ packed_key_states,
+ packed_value_states,
+ attention_mask,
+ )
+ return self.project_o(
+ packed_attn_output,
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ )
+
+ def project_qkv_inference(
+ self,
+ packed_query_sequence: torch.Tensor,
+ *,
+ is_gen: bool,
+ packed_text_indexes: Optional[torch.Tensor],
+ packed_vae_token_indexes: Optional[torch.Tensor],
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Project packed inference tokens through the matching MoT attention expert."""
+ if not is_gen:
+ packed_query_states, packed_key_states, packed_value_states = self._project_qkv(
+ packed_query_sequence, is_gen=False, split=True
+ )
+ packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim)
+ packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim)
+ packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim)
+ packed_query_states = self.q_norm(packed_query_states)
+ packed_key_states = self.k_norm(packed_key_states)
+ return packed_query_states, packed_key_states, packed_value_states
+
+ packed_qkv_states = packed_query_sequence.new_zeros(
+ (packed_query_sequence.shape[0], sum(self.qkv_split_sizes))
+ )
+ packed_qkv_states[packed_text_indexes] = self._project_qkv(
+ packed_query_sequence[packed_text_indexes], is_gen=False
+ )
+ packed_qkv_states[packed_vae_token_indexes] = self._project_qkv(
+ packed_query_sequence[packed_vae_token_indexes], is_gen=True
+ )
+ packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split(
+ self.qkv_split_sizes, dim=-1
+ )
+ packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim)
+ packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim)
+ packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim)
+ normed_query_states = packed_query_states.new_zeros(packed_query_states.shape)
+ normed_query_states[packed_text_indexes] = self.q_norm(packed_query_states[packed_text_indexes])
+ normed_query_states[packed_vae_token_indexes] = self.q_norm_moe_gen(
+ packed_query_states[packed_vae_token_indexes]
+ )
+ normed_key_states = packed_key_states.new_zeros(packed_key_states.shape)
+ normed_key_states[packed_text_indexes] = self.k_norm(packed_key_states[packed_text_indexes])
+ normed_key_states[packed_vae_token_indexes] = self.k_norm_moe_gen(packed_key_states[packed_vae_token_indexes])
+ return normed_query_states, normed_key_states, packed_value_states
+
+ def merge_kv_cache(
+ self,
+ packed_key_states: torch.Tensor,
+ packed_value_states: torch.Tensor,
+ *,
+ packed_query_indexes: torch.Tensor,
+ past_key_values: Optional[NaiveCache],
+ key_values_lens: Optional[torch.Tensor],
+ packed_key_value_indexes: Optional[torch.Tensor],
+ total_key_value_tokens: int,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """Merge the current query span with this layer's packed KV cache."""
+ if past_key_values is None or past_key_values.key_cache[self.layer_idx] is None:
+ return packed_key_states, packed_value_states
+ if key_values_lens is None or packed_key_value_indexes is None:
+ raise ValueError("key_values_lens and packed_key_value_indexes are required when cache is non-empty.")
+
+ past_key_states = past_key_values.key_cache[self.layer_idx]
+ past_value_states = past_key_values.value_cache[self.layer_idx]
+ merged_key_states = past_key_states.new_zeros(
+ (total_key_value_tokens, self.num_key_value_heads, self.head_dim)
+ )
+ merged_value_states = past_key_states.new_zeros(
+ (total_key_value_tokens, self.num_key_value_heads, self.head_dim)
+ )
+ merged_key_states[packed_query_indexes] = packed_key_states
+ merged_key_states[packed_key_value_indexes] = past_key_states
+ merged_value_states[packed_query_indexes] = packed_value_states
+ merged_value_states[packed_key_value_indexes] = past_value_states
+ return merged_key_states, merged_value_states
+
+ def attend_inference(
+ self,
+ packed_query_states: torch.Tensor,
+ merged_key_states: torch.Tensor,
+ merged_value_states: torch.Tensor,
+ *,
+ attention_mask: Optional[torch.Tensor | BlockMask],
+ is_causal: bool,
+ cu_seq_lens_q: torch.Tensor,
+ cu_seq_lens_k: torch.Tensor,
+ max_length_q: int,
+ max_length_k: int,
+ ) -> torch.Tensor:
+ del max_length_q, max_length_k
+
+ if isinstance(attention_mask, BlockMask):
+ attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention")
+ packed_attn_output, _ = attn_op(
+ self,
+ packed_query_states.transpose(0, 1).unsqueeze(0),
+ merged_key_states.transpose(0, 1).unsqueeze(0),
+ merged_value_states.transpose(0, 1).unsqueeze(0),
+ attention_mask,
+ dropout=0.0,
+ scaling=self.head_dim**-0.5,
+ skip_ulysses=True,
+ )
+ return packed_attn_output.squeeze(0)
+
+ return _sdpa_packed_attention(
+ packed_query_states,
+ merged_key_states,
+ merged_value_states,
+ attention_mask=attention_mask,
+ is_causal=is_causal,
+ cu_seq_lens_q=cu_seq_lens_q,
+ cu_seq_lens_k=cu_seq_lens_k,
+ scale=self.head_dim**-0.5,
+ enable_gqa=self.num_heads != self.num_key_value_heads,
+ )
+
+ def project_o_inference(
+ self,
+ packed_attn_output: torch.Tensor,
+ *,
+ is_gen: bool,
+ packed_text_indexes: Optional[torch.Tensor],
+ packed_vae_token_indexes: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ """Project packed attention outputs through the matching MoT output expert."""
+ packed_attn_output = packed_attn_output.reshape(-1, self.hidden_size)
+ if not is_gen:
+ return self.o_proj(packed_attn_output)
+
+ output = packed_attn_output.new_zeros(packed_attn_output.shape)
+ output[packed_text_indexes] = self.o_proj(packed_attn_output[packed_text_indexes])
+ output[packed_vae_token_indexes] = self.o_proj_moe_gen(packed_attn_output[packed_vae_token_indexes])
+ return output
+
+ def forward_packed_inference(
+ self,
+ packed_query_sequence: torch.Tensor,
+ query_lens: torch.Tensor,
+ packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor],
+ packed_query_indexes: torch.Tensor,
+ past_key_values: Optional[NaiveCache] = None,
+ key_values_lens: Optional[torch.Tensor] = None,
+ packed_key_value_indexes: Optional[torch.Tensor] = None,
+ update_past_key_values: bool = True,
+ is_causal: bool = True,
+ mode: str = "und",
+ attention_mask: Optional[torch.Tensor] = None,
+ packed_vae_token_indexes: Optional[torch.Tensor] = None,
+ packed_text_indexes: Optional[torch.Tensor] = None,
+ *,
+ cu_seq_lens_q: torch.Tensor,
+ cu_seq_lens_k: torch.Tensor,
+ max_length_q: int,
+ max_length_k: int,
+ total_key_value_tokens: int,
+ ) -> tuple[torch.Tensor, Optional[NaiveCache]]:
+ is_gen = _check_packed_inference_mode(mode)
+ compute_dtype = packed_query_sequence.dtype
+ packed_query_states, packed_key_states, packed_value_states = self.project_qkv_inference(
+ packed_query_sequence,
+ is_gen=is_gen,
+ packed_text_indexes=packed_text_indexes,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ )
+
+ packed_cos, packed_sin = packed_query_position_embeddings
+ packed_query_states, packed_key_states = self.apply_rotary_pos_emb(
+ packed_query_states,
+ packed_key_states,
+ packed_cos,
+ packed_sin,
+ unsqueeze_dim=1,
+ )
+
+ packed_query_states = packed_query_states.to(compute_dtype)
+ packed_key_states = packed_key_states.to(compute_dtype)
+ packed_value_states = packed_value_states.to(compute_dtype)
+
+ merged_key_states, merged_value_states = self.merge_kv_cache(
+ packed_key_states,
+ packed_value_states,
+ packed_query_indexes=packed_query_indexes,
+ past_key_values=past_key_values,
+ key_values_lens=key_values_lens,
+ packed_key_value_indexes=packed_key_value_indexes,
+ total_key_value_tokens=total_key_value_tokens,
+ )
+ packed_attn_output = self.attend_inference(
+ packed_query_states,
+ merged_key_states,
+ merged_value_states,
+ attention_mask=attention_mask,
+ is_causal=is_causal,
+ cu_seq_lens_q=cu_seq_lens_q,
+ cu_seq_lens_k=cu_seq_lens_k,
+ max_length_q=max_length_q,
+ max_length_k=max_length_k,
+ )
+ packed_attn_output = self.project_o_inference(
+ packed_attn_output.to(compute_dtype),
+ is_gen=is_gen,
+ packed_text_indexes=packed_text_indexes,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ )
+
+ if update_past_key_values:
+ if past_key_values is None:
+ raise ValueError("past_key_values is required when update_past_key_values=True.")
+ past_key_values.key_cache[self.layer_idx] = merged_key_states
+ past_key_values.value_cache[self.layer_idx] = merged_value_states
+
+ return packed_attn_output, past_key_values
+
+ def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]:
+ if self.training:
+ return self.forward_packed_train(*args, **kwargs), None
+ return self.forward_packed_inference(*args, **kwargs)
+
+
+class BagelQwen2MoTDecoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: BagelQwen2MoTConfig,
+ layer_idx: int,
+ *,
+ attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention,
+ mlp_cls: type[Qwen2MLP] = Qwen2MLP,
+ rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm,
+ ):
+ super().__init__()
+ self.self_attn = attention_cls(config, layer_idx, rms_norm_cls=rms_norm_cls)
+ self.mlp = mlp_cls(config)
+ self.mlp_moe_gen = mlp_cls(config)
+ self.input_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+ self.input_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+ self.post_attention_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+ self.post_attention_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+
+ def _apply_moe_norm(
+ self,
+ packed_sequence: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ und_norm: nn.Module,
+ gen_norm: nn.Module,
+ ) -> torch.Tensor:
+ return _scatter_moe(
+ packed_sequence.new_zeros(packed_sequence.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ und_norm(packed_sequence[packed_und_token_indexes]),
+ gen_norm(packed_sequence[packed_gen_token_indexes]),
+ )
+
+ def forward_packed_train(
+ self,
+ packed_sequence: torch.Tensor,
+ attention_mask: torch.Tensor,
+ packed_position_cos: torch.Tensor,
+ packed_position_sin: torch.Tensor,
+ packed_und_token_indexes: torch.Tensor,
+ packed_gen_token_indexes: torch.Tensor,
+ ) -> torch.Tensor:
+ residual = packed_sequence
+ packed_sequence = self._apply_moe_norm(
+ packed_sequence,
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.input_layernorm,
+ self.input_layernorm_moe_gen,
+ )
+ packed_sequence, _ = self.self_attn(
+ packed_sequence=packed_sequence,
+ attention_mask=attention_mask,
+ packed_position_cos=packed_position_cos,
+ packed_position_sin=packed_position_sin,
+ packed_und_token_indexes=packed_und_token_indexes,
+ packed_gen_token_indexes=packed_gen_token_indexes,
+ )
+ packed_sequence = residual + packed_sequence
+
+ residual = packed_sequence
+ post_attn_und = self.post_attention_layernorm(packed_sequence[packed_und_token_indexes])
+ post_attn_gen = self.post_attention_layernorm_moe_gen(packed_sequence[packed_gen_token_indexes])
+ packed_sequence = _scatter_moe(
+ packed_sequence.new_zeros(packed_sequence.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.mlp(post_attn_und),
+ self.mlp_moe_gen(post_attn_gen),
+ extra_und_anchors=(post_attn_und,),
+ extra_gen_anchors=(post_attn_gen,),
+ )
+ return residual + packed_sequence
+
+ def _apply_inference_input_norm(
+ self,
+ packed_query_sequence: torch.Tensor,
+ *,
+ is_gen: bool,
+ packed_text_indexes: Optional[torch.Tensor],
+ packed_vae_token_indexes: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ """Normalize inference tokens with the matching MoT expert."""
+ if not is_gen:
+ return self.input_layernorm(packed_query_sequence)
+
+ normed_sequence = torch.zeros_like(packed_query_sequence)
+ normed_sequence[packed_text_indexes] = self.input_layernorm(packed_query_sequence[packed_text_indexes])
+ normed_sequence[packed_vae_token_indexes] = self.input_layernorm_moe_gen(
+ packed_query_sequence[packed_vae_token_indexes]
+ )
+ return normed_sequence
+
+ def _apply_inference_mlp(
+ self,
+ packed_query_sequence: torch.Tensor,
+ *,
+ is_gen: bool,
+ packed_text_indexes: Optional[torch.Tensor],
+ packed_vae_token_indexes: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ """Apply the inference MLP with text and VAE tokens routed independently."""
+ if not is_gen:
+ return self.mlp(self.post_attention_layernorm(packed_query_sequence))
+
+ packed_text_query_sequence = self.post_attention_layernorm(packed_query_sequence[packed_text_indexes])
+ packed_vae_query_sequence = self.post_attention_layernorm_moe_gen(
+ packed_query_sequence[packed_vae_token_indexes]
+ )
+ mlp_output = torch.zeros_like(packed_query_sequence)
+ mlp_output[packed_text_indexes] = self.mlp(packed_text_query_sequence)
+ mlp_output[packed_vae_token_indexes] = self.mlp_moe_gen(packed_vae_query_sequence)
+ return mlp_output
+
+ def forward_packed_inference(
+ self,
+ packed_query_sequence: torch.Tensor,
+ query_lens: torch.Tensor,
+ packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor],
+ packed_query_indexes: torch.Tensor,
+ past_key_values: Optional[NaiveCache] = None,
+ key_values_lens: Optional[torch.Tensor] = None,
+ packed_key_value_indexes: Optional[torch.Tensor] = None,
+ update_past_key_values: bool = True,
+ is_causal: bool = True,
+ mode: str = "und",
+ attention_mask: Optional[torch.Tensor] = None,
+ packed_vae_token_indexes: Optional[torch.Tensor] = None,
+ packed_text_indexes: Optional[torch.Tensor] = None,
+ *,
+ cu_seq_lens_q: torch.Tensor,
+ cu_seq_lens_k: torch.Tensor,
+ max_length_q: int,
+ max_length_k: int,
+ total_key_value_tokens: int,
+ ) -> tuple[torch.Tensor, Optional[NaiveCache]]:
+ is_gen = _check_packed_inference_mode(mode)
+ residual = packed_query_sequence
+ packed_query_sequence = self._apply_inference_input_norm(
+ packed_query_sequence,
+ is_gen=is_gen,
+ packed_text_indexes=packed_text_indexes,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ )
+ packed_query_sequence, past_key_values = self.self_attn(
+ packed_query_sequence=packed_query_sequence,
+ query_lens=query_lens,
+ packed_query_position_embeddings=packed_query_position_embeddings,
+ packed_query_indexes=packed_query_indexes,
+ past_key_values=past_key_values,
+ key_values_lens=key_values_lens,
+ packed_key_value_indexes=packed_key_value_indexes,
+ update_past_key_values=update_past_key_values,
+ is_causal=is_causal,
+ mode=mode,
+ attention_mask=attention_mask,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ packed_text_indexes=packed_text_indexes,
+ cu_seq_lens_q=cu_seq_lens_q,
+ cu_seq_lens_k=cu_seq_lens_k,
+ max_length_q=max_length_q,
+ max_length_k=max_length_k,
+ total_key_value_tokens=total_key_value_tokens,
+ )
+ packed_query_sequence = residual + packed_query_sequence
+
+ residual = packed_query_sequence
+ packed_query_sequence = self._apply_inference_mlp(
+ packed_query_sequence,
+ is_gen=is_gen,
+ packed_text_indexes=packed_text_indexes,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ )
+ packed_query_sequence = residual + packed_query_sequence
+
+ return packed_query_sequence, past_key_values
+
+ def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]:
+ if self.training:
+ return self.forward_packed_train(*args, **kwargs), None
+ return self.forward_packed_inference(*args, **kwargs)
+
+
+class BagelQwen2MoTBackbone(nn.Module):
+ def __init__(
+ self,
+ config: BagelQwen2MoTConfig,
+ *,
+ attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention,
+ mlp_cls: type[Qwen2MLP] = Qwen2MLP,
+ rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm,
+ ):
+ super().__init__()
+ self.config = config
+ self.gradient_checkpointing = False
+ self.layers = nn.ModuleList(
+ [
+ BagelQwen2MoTDecoderLayer(
+ config,
+ layer_idx,
+ attention_cls=attention_cls,
+ mlp_cls=mlp_cls,
+ rms_norm_cls=rms_norm_cls,
+ )
+ for layer_idx in range(config.num_hidden_layers)
+ ]
+ )
+ self.norm = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+ self.norm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config)
+ self.rotary_emb = BagelQwen2RotaryEmbedding(config=config)
+ self.use_moe = "Mo" in config.layer_module
+ self.attention_cls = attention_cls
+
+ def _bound_attn_implementation(self) -> str | None:
+ """Return the attention implementation bound onto the decoder layers at initialization."""
+ if len(self.layers) > 0:
+ bound = getattr(self.layers[0].self_attn, "veomni_attn_impl", None)
+ if bound is not None:
+ return bound
+ return self.config._attn_implementation
+
+ def _build_inference_attention_mask(
+ self,
+ packed_query_sequence: torch.Tensor,
+ packed_attention_metadata: Optional[torch.Tensor],
+ *,
+ cache_has_values: bool,
+ ) -> Any:
+ """Validate packed-prefill invariants and materialize its attention mask."""
+ if packed_attention_metadata is None:
+ return None
+ if cache_has_values:
+ raise ValueError("BAGEL packed prefill requires an empty KV cache.")
+
+ expected_metadata_shape = (3, int(packed_query_sequence.shape[0]))
+ if tuple(packed_attention_metadata.shape) != expected_metadata_shape:
+ raise ValueError(
+ "BAGEL packed prefill metadata must match the packed query sequence: "
+ f"expected {expected_metadata_shape}, got {tuple(packed_attention_metadata.shape)}."
+ )
+ return self.attention_cls.build_attention_mask(
+ packed_attention_metadata.to(device=packed_query_sequence.device),
+ attn_implementation=self._bound_attn_implementation(),
+ )
+
+ def _apply_inference_final_norm(
+ self,
+ packed_query_sequence: torch.Tensor,
+ *,
+ is_gen: bool,
+ packed_text_indexes: Optional[torch.Tensor],
+ packed_vae_token_indexes: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ """Apply the final backbone norm with the matching MoT expert."""
+ if not is_gen:
+ return self.norm(packed_query_sequence)
+
+ normed_sequence = torch.zeros_like(packed_query_sequence)
+ normed_sequence[packed_text_indexes] = self.norm(packed_query_sequence[packed_text_indexes])
+ normed_sequence[packed_vae_token_indexes] = self.norm_moe_gen(packed_query_sequence[packed_vae_token_indexes])
+ return normed_sequence
+
+ def forward_packed_train(
+ self,
+ packed_sequence: torch.Tensor,
+ packed_position_ids: torch.Tensor,
+ packed_attention_metadata: torch.Tensor,
+ packed_und_token_indexes: Optional[torch.Tensor] = None,
+ packed_gen_token_indexes: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ """Run packed training forward across all MoT decoder layers and final expert norms."""
+ attention_mask = self.attention_cls.build_attention_mask(
+ packed_attention_metadata,
+ attn_implementation=self._bound_attn_implementation(),
+ )
+ cos, sin = self.rotary_emb(packed_sequence, packed_position_ids.unsqueeze(0))
+ packed_position_cos = cos.squeeze(0)
+ packed_position_sin = sin.squeeze(0)
+
+ if self.use_moe:
+ if packed_und_token_indexes is None:
+ raise ValueError("packed_und_token_indexes is required for BAGEL MoT training.")
+ if packed_gen_token_indexes is None:
+ packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0])
+ else:
+ packed_und_token_indexes = torch.arange(packed_sequence.shape[0], device=packed_sequence.device)
+ packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0])
+
+ for decoder_layer in self.layers:
+ if self.gradient_checkpointing and self.training:
+ packed_sequence, _ = self._gradient_checkpointing_func(
+ decoder_layer.__call__,
+ packed_sequence,
+ attention_mask,
+ packed_position_cos,
+ packed_position_sin,
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ )
+ else:
+ packed_sequence, _ = decoder_layer(
+ packed_sequence=packed_sequence,
+ attention_mask=attention_mask,
+ packed_position_cos=packed_position_cos,
+ packed_position_sin=packed_position_sin,
+ packed_und_token_indexes=packed_und_token_indexes,
+ packed_gen_token_indexes=packed_gen_token_indexes,
+ )
+
+ if self.use_moe:
+ return _scatter_moe(
+ packed_sequence.new_zeros(packed_sequence.shape),
+ packed_und_token_indexes,
+ packed_gen_token_indexes,
+ self.norm(packed_sequence[packed_und_token_indexes]),
+ self.norm_moe_gen(packed_sequence[packed_gen_token_indexes]),
+ )
+ return self.norm(packed_sequence)
+
+ def forward_packed_inference(
+ self,
+ packed_query_sequence: torch.Tensor,
+ query_lens: torch.Tensor,
+ packed_query_position_ids: torch.Tensor,
+ packed_query_indexes: torch.Tensor,
+ past_key_values: Optional[NaiveCache] = None,
+ key_values_lens: Optional[torch.Tensor] = None,
+ packed_key_value_indexes: Optional[torch.Tensor] = None,
+ update_past_key_values: bool = True,
+ is_causal: bool = True,
+ mode: str = "und",
+ packed_attention_metadata: Optional[torch.Tensor] = None,
+ packed_vae_token_indexes: Optional[torch.Tensor] = None,
+ packed_text_indexes: Optional[torch.Tensor] = None,
+ ) -> BaseNavitOutputWithPast:
+ """Execute all MoT decoder layers for packed inference prefill, decode, or denoise."""
+ is_gen = _check_packed_inference_mode(mode)
+ query_device = packed_query_sequence.device
+ packed_query_indexes = packed_query_indexes.to(device=query_device)
+ packed_query_position_ids = packed_query_position_ids.to(device=query_device)
+ if packed_key_value_indexes is not None:
+ packed_key_value_indexes = packed_key_value_indexes.to(device=query_device)
+ if packed_vae_token_indexes is not None:
+ packed_vae_token_indexes = packed_vae_token_indexes.to(device=query_device)
+ if packed_text_indexes is not None:
+ packed_text_indexes = packed_text_indexes.to(device=query_device)
+ if past_key_values is None:
+ past_key_values = NaiveCache(len(self.layers))
+
+ cos, sin = self.rotary_emb(packed_query_sequence, packed_query_position_ids.unsqueeze(0))
+ packed_query_position_embeddings = (cos.squeeze(0), sin.squeeze(0))
+
+ cache_has_values = past_key_values.key_cache[0] is not None
+ if cache_has_values and key_values_lens is None:
+ raise ValueError("key_values_lens is required when cache is non-empty.")
+ attention_mask = self._build_inference_attention_mask(
+ packed_query_sequence,
+ packed_attention_metadata,
+ cache_has_values=cache_has_values,
+ )
+
+ effective_key_values_lens = key_values_lens + query_lens if cache_has_values else query_lens
+ cu_seq_lens_q = torch.nn.functional.pad(torch.cumsum(query_lens, dim=0), (1, 0)).to(torch.int32)
+ cu_seq_lens_k = torch.nn.functional.pad(torch.cumsum(effective_key_values_lens, dim=0), (1, 0)).to(torch.int32)
+ max_length_q = int(query_lens.max().item())
+ max_length_k = int(effective_key_values_lens.max().item())
+ total_key_value_tokens = int(effective_key_values_lens.sum().item())
+
+ for decoder_layer in self.layers:
+ packed_query_sequence, past_key_values = decoder_layer(
+ packed_query_sequence=packed_query_sequence,
+ query_lens=query_lens,
+ packed_query_position_embeddings=packed_query_position_embeddings,
+ packed_query_indexes=packed_query_indexes,
+ past_key_values=past_key_values,
+ key_values_lens=key_values_lens,
+ packed_key_value_indexes=packed_key_value_indexes,
+ update_past_key_values=update_past_key_values,
+ is_causal=is_causal,
+ mode=mode,
+ attention_mask=attention_mask,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ packed_text_indexes=packed_text_indexes,
+ cu_seq_lens_q=cu_seq_lens_q,
+ cu_seq_lens_k=cu_seq_lens_k,
+ max_length_q=max_length_q,
+ max_length_k=max_length_k,
+ total_key_value_tokens=total_key_value_tokens,
+ )
+
+ packed_query_sequence = self._apply_inference_final_norm(
+ packed_query_sequence,
+ is_gen=is_gen,
+ packed_text_indexes=packed_text_indexes,
+ packed_vae_token_indexes=packed_vae_token_indexes,
+ )
+ return BaseNavitOutputWithPast(
+ packed_query_sequence=packed_query_sequence,
+ past_key_values=past_key_values,
+ )
+
+ def forward(self, *args: Any, **kwargs: Any) -> BaseNavitOutputWithPast:
+ if self.training:
+ return BaseNavitOutputWithPast(packed_query_sequence=self.forward_packed_train(*args, **kwargs))
+ return self.forward_packed_inference(*args, **kwargs)
+
+
+BagelQwen2MoTCore.attention_cls = BagelQwen2MoTAttention
+BagelQwen2MoTCore.mlp_cls = Qwen2MLP
+BagelQwen2MoTCore.rms_norm_cls = Qwen2RMSNorm
+
+
+__all__ = [
+ "BaseNavitOutputWithPast",
+ "BagelQwen2MoT",
+ "BagelQwen2MoTCore",
+ "InferenceMixin",
+ "NaiveCache",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py
new file mode 100644
index 0000000000..5795f8656b
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py
@@ -0,0 +1,336 @@
+"""Conversation packing helpers for BAGEL Qwen2-MoT."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items
+from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context
+from .masking import build_mot_attention_metadata
+
+
+_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"})
+
+
+@dataclass(frozen=True)
+class PackedSpan:
+ """Describe one logical span in the packed sequence.
+
+ Most spans contain one carrier item. A VAE image span can group its leading
+ marker, image, and trailing marker; ``primary_index`` identifies the image,
+ while ``lengths`` preserves each item's subrange for routing and writeback.
+ """
+
+ start: int
+ items: tuple[ConversationItem, ...]
+ lengths: tuple[int, ...]
+ primary_index: int = 0
+
+ @property
+ def item(self) -> ConversationItem:
+ return self.items[self.primary_index]
+
+ @property
+ def length(self) -> int:
+ return sum(self.lengths)
+
+ @property
+ def primary_start(self) -> int:
+ return sum(self.lengths[: self.primary_index])
+
+ @property
+ def primary_length(self) -> int:
+ return self.lengths[self.primary_index]
+
+ @property
+ def is_image_triplet(self) -> bool:
+ return (
+ len(self.items) == 3
+ and self.primary_index == 1
+ and self.items[0].type == "text"
+ and self.items[1].type == "image"
+ and self.items[2].type == "text"
+ )
+
+
+@dataclass
+class PackedConversation:
+ """Result of packing an embedded conversation batch for MoT.
+
+ Packed tensors concatenate all samples. ``sample_splits`` preserves their
+ document/span boundaries for attention metadata. ``spans`` uses global
+ packed offsets to map model outputs back to the original carrier items.
+ Token type 0 selects the understanding expert and type 1 the generation
+ expert.
+ """
+
+ packed_sequence: torch.Tensor
+ sample_splits: list[list[int]]
+ packed_attention_metadata: torch.Tensor
+ packed_position_ids: torch.Tensor
+ packed_token_type_ids: torch.Tensor
+ spans: list[PackedSpan]
+
+
+def preprocess_mot_inputs(
+ conversation_list: list[list[ConversationItem]] | None,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+ hidden_size: int,
+) -> PackedConversation | None:
+ """Preprocess already-embedded carrier items into MoT packed inputs.
+
+ Text and SigLIP spans route through the understanding expert. VAE latent
+ spans and output spans route through the generation expert. Attention mode
+ and expert routing are deliberately computed separately: a clean VAE
+ conditioning image still uses the generation expert. Packed layout stays
+ local to MoT.
+ """
+
+ if not conversation_list:
+ raise ValueError("BAGEL Qwen2-MoT preprocessing requires a non-empty conversation_list.")
+
+ sequence_parts: list[torch.Tensor] = []
+ sample_splits_by_sample: list[list[int]] = []
+ sample_attn_modes_by_sample: list[list[str]] = []
+ position_parts: list[torch.Tensor] = []
+ token_type_parts: list[torch.Tensor] = []
+ spans: list[PackedSpan] = []
+ sequence_cursor = 0
+
+ for sample in conversation_list:
+ sample_start = sequence_cursor
+ sample_splits: list[int] = []
+ sample_attn_modes: list[str] = []
+ sample_position_cursor = 0
+
+ # Dummy carrier items are handled separately as zero-gradient anchors;
+ # only real user/assistant embeddings participate in packed attention.
+ items = [
+ item
+ for item in iter_desired_items(
+ [sample],
+ types=["text", "image", "output"],
+ roles=["user", "assistant"],
+ )
+ if not item.is_dummy
+ ]
+ item_index = 0
+ while item_index < len(items):
+ if _is_vision_marker_triplet_at(items, item_index):
+ # Treat , image tokens, and as one
+ # logical attention span while preserving three writeback
+ # ranges and routing only the image tokens to the gen expert.
+ span_items = (items[item_index], items[item_index + 1], items[item_index + 2])
+ span_values = [
+ _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size)
+ for item in span_items
+ ]
+ span_lengths = tuple(int(value.shape[0]) for value in span_values)
+ item = span_items[1]
+ value = torch.cat(span_values, dim=0)
+ length = int(value.shape[0])
+ primary_index = 1
+ item_index += 3
+ else:
+ item = items[item_index]
+ value = _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size)
+ length = int(value.shape[0])
+ span_items = (item,)
+ span_lengths = (length,)
+ primary_index = 0
+ item_index += 1
+
+ if length == 0:
+ continue
+
+ # sequence_cursor is global across the packed batch; positions
+ # restart independently for each logical document.
+ indexes = torch.arange(sequence_cursor, sequence_cursor + length, device=device, dtype=torch.long)
+ position_ids, sample_position_cursor = _mot_position_ids_for_span(
+ item,
+ start=sample_position_cursor,
+ length=length,
+ device=device,
+ )
+ mode = _mot_attn_mode_for_item(item)
+
+ sequence_parts.append(value)
+ position_parts.append(position_ids)
+ sample_splits.append(length)
+ sample_attn_modes.append(mode)
+ span = PackedSpan(
+ start=sequence_cursor,
+ items=span_items,
+ lengths=span_lengths,
+ primary_index=primary_index,
+ )
+ spans.append(span)
+ sequence_cursor += length
+
+ gen_token_indexes = _mot_gen_token_indexes_for_span(span, indexes)
+ token_type_ids = torch.zeros(length, device=device, dtype=torch.long)
+ if gen_token_indexes.numel() > 0:
+ token_type_ids[(gen_token_indexes - indexes[0]).to(device=device, dtype=torch.long)] = 1
+ token_type_parts.append(token_type_ids)
+
+ sample_len = sequence_cursor - sample_start
+ if sample_len > 0:
+ sample_splits_by_sample.append(list(sample_splits))
+ sample_attn_modes_by_sample.append(list(sample_attn_modes))
+
+ if not sequence_parts:
+ return None
+
+ packed_sequence = torch.cat(sequence_parts, dim=0).to(device=device, dtype=dtype)
+ return PackedConversation(
+ packed_sequence=packed_sequence,
+ sample_splits=sample_splits_by_sample,
+ packed_attention_metadata=build_mot_attention_metadata(
+ sample_splits_by_sample,
+ sample_attn_modes_by_sample,
+ device=device,
+ ),
+ packed_position_ids=torch.cat(position_parts, dim=0).to(device=device, dtype=torch.long),
+ packed_token_type_ids=torch.cat(token_type_parts, dim=0).to(device=device, dtype=torch.long),
+ spans=spans,
+ )
+
+
+def _mot_value_for_item(
+ item: ConversationItem,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+ hidden_size: int,
+) -> torch.Tensor:
+ """Validate and cast an item's rank-2 embedding tensor to ``(device, dtype)``."""
+ value = item.value
+ if not torch.is_tensor(value):
+ raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got {type(value).__name__}.")
+ if value.dim() == 3 and value.shape[0] == 1:
+ value = value.squeeze(0)
+ if value.dim() != 2:
+ raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got shape {tuple(item.value.shape)}.")
+ if int(value.shape[-1]) != int(hidden_size):
+ raise ValueError(f"BAGEL Qwen2-MoT item hidden-size mismatch: got {value.shape[-1]}, expected {hidden_size}.")
+ return value.to(device=device, dtype=dtype)
+
+
+def _mot_position_ids_for_span(
+ item: ConversationItem,
+ *,
+ start: int,
+ length: int,
+ device: torch.device,
+) -> tuple[torch.Tensor, int]:
+ """Assign BAGEL positions and return the next document-local position."""
+ meta_position_ids = item.meta.get("position_ids")
+ if torch.is_tensor(meta_position_ids):
+ position_ids = meta_position_ids.detach().to(device=device, dtype=torch.long).reshape(-1)
+ if int(position_ids.numel()) != length:
+ raise ValueError("BAGEL Qwen2-MoT position_ids length must match the item span length.")
+ next_start = max(start + 1, int(position_ids.max().item()) + 1) if length else start
+ return position_ids, next_start
+
+ # All latent/image tokens in one span share a position; text remains AR
+ # sequential. Thus a whole image advances the logical position only once.
+ if item.type in {"image", "output"}:
+ return torch.full((length,), start, device=device, dtype=torch.long), start + 1
+ return torch.arange(start, start + length, device=device, dtype=torch.long), start + length
+
+
+def _mot_attn_mode_for_item(item: ConversationItem) -> str:
+ """Classify visibility independently from MoT expert routing."""
+ if item.type == "output":
+ return "noise"
+ if item.type == "image":
+ if bagel_context(item) != BAGEL_VAE_CONTEXT:
+ # SigLIP context (and any non-VAE image) is clean/full-attention
+ # regardless of ``_img_tag``; the tag only disambiguates VAE roles.
+ return "full"
+ tag = item.meta.get(_IMG_TAG_KEY)
+ if tag is None:
+ # Untagged VAE images come from inference prompt routing. Training
+ # targets must carry ``_img_tag="gen"``.
+ return "full"
+ if tag not in _VALID_IMG_TAGS:
+ raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.")
+ if tag == "gen":
+ return "noise"
+ if tag == "edit":
+ # Edit VAE context is a clean conditioning image, not a noised target.
+ return "full"
+ raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.")
+ return "causal"
+
+
+def _mot_gen_token_indexes_for_span(span: PackedSpan, indexes: torch.Tensor) -> torch.Tensor:
+ """Return packed rows routed through the generation expert."""
+ item = span.item
+ if item.type == "output":
+ return indexes
+ if item.type != "image" or bagel_context(item) != BAGEL_VAE_CONTEXT:
+ return indexes.new_empty(0)
+
+ tag = item.meta.get(_IMG_TAG_KEY)
+ if tag is not None and tag not in _VALID_IMG_TAGS:
+ raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.")
+ if tag == "und":
+ raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.")
+
+ # Expert routing follows token modality, independently of whether the VAE
+ # latent is clean conditioning (untagged/edit) or a noised generation
+ # target. Marker tokens remain on the understanding path.
+ if span.is_image_triplet:
+ primary_start = span.primary_start
+ primary_end = primary_start + span.primary_length
+ return indexes[primary_start:primary_end]
+ return indexes
+
+
+def _is_vision_marker_triplet_at(items: list[ConversationItem], index: int) -> bool:
+ """Return whether ``items[index : index + 3]`` forms a ``(start, image, end)`` triplet."""
+ # Context-tag equality prevents unrelated one-token text items around an
+ # image from being mistaken for BAGEL's encoder-specific marker pair.
+ if index + 2 >= len(items):
+ return False
+ start, image, end = items[index], items[index + 1], items[index + 2]
+ context = bagel_context(image)
+ if image.type != "image" or context not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}:
+ return False
+ return _is_bagel_vision_marker(start, context=context) and _is_bagel_vision_marker(end, context=context)
+
+
+def _is_bagel_vision_marker(item: ConversationItem, *, context: str) -> bool:
+ """Return whether ``item`` is a single-token vision marker for ``context``."""
+ return item.type == "text" and bagel_context(item) == context and _text_item_length(item) == 1
+
+
+def _text_item_length(item: ConversationItem) -> int | None:
+ """Return the token length of a text ``ConversationItem``, or ``None`` if unavailable."""
+ value = item.value
+ if torch.is_tensor(value):
+ if value.dim() == 0:
+ return 1
+ if value.dim() == 1:
+ return int(value.shape[0])
+ if value.dim() == 2:
+ return int(value.shape[0])
+ if value.dim() == 3 and int(value.shape[0]) == 1:
+ return int(value.shape[1])
+ return None
+ input_ids = item.meta.get("input_ids")
+ if torch.is_tensor(input_ids):
+ return int(input_ids.reshape(-1).shape[0])
+ return None
+
+
+__all__ = [
+ "PackedConversation",
+ "PackedSpan",
+ "preprocess_mot_inputs",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py
new file mode 100644
index 0000000000..23fabe4009
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py
@@ -0,0 +1,31 @@
+"""BAGEL SigLIP NaViT vision encoder module."""
+
+from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY
+
+
+@OMNI_CONFIG_REGISTRY.register("bagel_siglip_navit")
+def register_bagel_siglip_navit_config():
+ from .configuration import BagelSiglipNavitConfig
+
+ return BagelSiglipNavitConfig
+
+
+@OMNI_MODEL_REGISTRY.register("bagel_siglip_navit")
+def register_bagel_siglip_navit_model():
+ from .modeling import BagelSiglipNavit
+
+ return BagelSiglipNavit
+
+
+@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_siglip_navit")
+def register_bagel_siglip_navit_accelerated_model():
+ from .accelerated import BagelSiglipNavitAccelerated
+
+ return BagelSiglipNavitAccelerated
+
+
+@OMNI_PROCESSOR_REGISTRY.register("bagel_siglip_navit")
+def register_bagel_siglip_navit_processor():
+ from .processing import BagelSiglipNavitProcessor
+
+ return BagelSiglipNavitProcessor
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py
new file mode 100644
index 0000000000..f4ac46ddca
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py
@@ -0,0 +1,10 @@
+"""Accelerated runtime for the bagel/siglip_navit OmniModule (conversation graph)."""
+
+from .accelerated import (
+ BagelSiglipNavitAccelerated,
+)
+
+
+__all__ = [
+ "BagelSiglipNavitAccelerated",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py
new file mode 100644
index 0000000000..607c4481b0
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py
@@ -0,0 +1,267 @@
+"""SeedOmni training-graph hooks for BAGEL's SigLIP NaViT vision encoder.
+
+``generate()`` lives natively on :class:`~.modeling.BagelSiglipNavit` — this
+file only carries the SP-aware training pre/forward/post hooks.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+
+from .......distributed.parallel_state import get_parallel_state
+from .......distributed.sequence_parallel import gather_outputs
+from .....mixins.base_mixin import BaseMixin
+from .....mixins.metric_meter_mixin import MetricMeterMixin
+from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward
+from .....utils.conversation import ConversationItem, iter_desired_items
+from ...sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT
+from ..configuration import BagelSiglipNavitConfig
+from ..modeling import (
+ BagelSiglipNavit,
+ scatter_siglip_image_embeds,
+ select_siglip_image_items,
+ siglip_inputs_from_preprocessed_items,
+)
+from ..processing import _OMNI_TOKEN_LEN
+
+
+class TrainingMixin(TrainingModuleMixin):
+ """Training-graph hooks — depends on :class:`BagelSiglipNavit` modeling APIs."""
+
+ config: BagelSiglipNavitConfig
+ device: torch.device
+ dtype: torch.dtype
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self._conversation_carrier: list[list[ConversationItem]] | None = None
+ self._image_items: list[ConversationItem] = []
+ self._sp_image_count: int | None = None
+ self._sp_token_count: int | None = None
+
+ @pre_forward("forward")
+ def forward_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Select preprocessed SigLIP image items and slice across Ulysses SP ranks if enabled."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+
+ self._image_items = select_siglip_image_items(conversation_list)
+ out = siglip_inputs_from_preprocessed_items(self._image_items, device=self.device, dtype=self.dtype)
+ self.metric_meter_set_seqlens(
+ "forward",
+ self._metric_sample_token_lens(conversation_list),
+ image_seqlens=[int(item.meta[_OMNI_TOKEN_LEN]) for item in self._image_items],
+ )
+
+ ps = get_parallel_state()
+ if ps.sp_size == 1:
+ self._sp_image_count = None
+ self._sp_token_count = None
+ return out
+
+ if ps.cp_size != 1:
+ raise ValueError(f"BAGEL SigLIP NaViT training supports Ulysses groups only; got cp_size={ps.cp_size}.")
+
+ token_lens = out["token_lens"]
+ image_count = int(token_lens.numel())
+ if image_count == 0:
+ raise ValueError("BAGEL SigLIP NaViT SP requires at least one real or dummy image.")
+ self._sp_image_count = image_count
+ self._sp_token_count = int(token_lens.sum().item())
+
+ pad_images = (-image_count) % ps.sp_size
+ if pad_images:
+ token_lens = torch.cat((token_lens, token_lens.new_ones(pad_images)), dim=0)
+ out["patchified_pixel_values"] = torch.cat(
+ (
+ out["patchified_pixel_values"],
+ out["patchified_pixel_values"].new_zeros((pad_images, *out["patchified_pixel_values"].shape[1:])),
+ ),
+ dim=0,
+ )
+ out["patchified_position_ids"] = torch.cat(
+ (
+ out["patchified_position_ids"],
+ out["patchified_position_ids"].new_zeros((pad_images, *out["patchified_position_ids"].shape[1:])),
+ ),
+ dim=0,
+ )
+
+ images_per_rank = int(token_lens.numel()) // ps.sp_size
+ image_start = ps.sp_rank * images_per_rank
+ image_end = image_start + images_per_rank
+ full_cu_seqlens = torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0))
+ token_start = int(full_cu_seqlens[image_start].item())
+ token_end = int(full_cu_seqlens[image_end].item())
+ local_token_lens = token_lens[image_start:image_end].contiguous()
+ return {
+ "patchified_pixel_values": out["patchified_pixel_values"][token_start:token_end].contiguous(),
+ "patchified_position_ids": out["patchified_position_ids"][token_start:token_end].contiguous(),
+ "cu_seqlens": torch.nn.functional.pad(torch.cumsum(local_token_lens, dim=0), (1, 0)).to(torch.int32),
+ "max_seqlen": int(local_token_lens.max().item()),
+ "token_lens": local_token_lens,
+ }
+
+ def forward(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+ """Isolate SP-sharded attention dropout RNG from downstream modules when enabled."""
+ if (
+ get_parallel_state().sp_size > 1
+ and self.training
+ and float(getattr(self.config, "attention_dropout", 0.0)) > 0.0
+ ):
+ fork_kwargs: dict[str, Any] = (
+ {"devices": []}
+ if self.device.type == "cpu"
+ else {"devices": [self.device], "device_type": self.device.type}
+ )
+ with torch.random.fork_rng(**fork_kwargs):
+ return super().forward(*args, **kwargs)
+ return super().forward(*args, **kwargs)
+
+ @post_forward("forward")
+ def forward_post(
+ self,
+ image_embeds: torch.Tensor,
+ token_lens: torch.Tensor | None = None,
+ ) -> dict[str, Any]:
+ """Gather SP image feature shards and scatter ``image_embeds`` onto conversation items."""
+ if token_lens is None:
+ raise ValueError("BagelSiglipNavit.forward_post requires token_lens for non-dummy outputs.")
+
+ ps = get_parallel_state()
+ if ps.sp_size > 1:
+ if self._sp_image_count is None or self._sp_token_count is None:
+ raise RuntimeError("BAGEL SigLIP NaViT SP image shape was not initialized.")
+ image_embeds = gather_outputs(image_embeds, gather_dim=0, group=ps.sp_group)
+ token_lens = gather_outputs(token_lens, gather_dim=0, group=ps.sp_group)
+ image_embeds = image_embeds.narrow(0, 0, self._sp_token_count)
+ token_lens = token_lens.narrow(0, 0, self._sp_image_count)
+
+ conversation = self._conversation_carrier
+ image_items = self._image_items
+ self._conversation_carrier = None
+ self._image_items = []
+ self._sp_image_count = None
+ self._sp_token_count = None
+
+ scatter_siglip_image_embeds(
+ image_items,
+ image_embeds,
+ token_lens,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": conversation}
+
+ def _metric_sample_token_lens(
+ self,
+ conversation_list: list[list[ConversationItem]],
+ ) -> list[int]:
+ """Sum SigLIP patch token lengths per sample for FLOPs metering."""
+ sample_lens: list[int] = []
+ for sample in conversation_list:
+ sample_len = 0
+ # SigLIP encodes per image, including dummy carriers, but
+ # multi-source metering needs one aggregated length per sample.
+ for item in iter_desired_items(
+ [sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}
+ ):
+ sample_len += int(item.meta[_OMNI_TOKEN_LEN])
+ sample_lens.append(sample_len)
+ return sample_lens
+
+
+class MeterMixin(MetricMeterMixin):
+ """Per-module training trace for BAGEL SigLIP NaViT."""
+
+ config: BagelSiglipNavitConfig
+
+ def metric_meter_set_seqlens(
+ self,
+ method: str,
+ seqlens: list[int],
+ *,
+ image_seqlens: list[int] | None = None,
+ ) -> None:
+ """Stash per-sample ``seqlens`` plus per-image ``image_seqlens`` for attention FLOPs."""
+ super().metric_meter_set_seqlens(method, seqlens)
+ if not hasattr(self, "_metric_full_image_seqlens"):
+ self._metric_full_image_seqlens: dict[str, list[int]] = {}
+ resolved_image_seqlens = seqlens if image_seqlens is None else image_seqlens
+ self._metric_full_image_seqlens[method] = [int(s) for s in resolved_image_seqlens]
+
+ def _metric_meter_image_seqlen_buffer(self) -> list[int]:
+ if not hasattr(self, "_metric_meter_image_seqlens"):
+ self._metric_meter_image_seqlens: list[int] = []
+ return self._metric_meter_image_seqlens
+
+ def metric_meter_add(self, method: str) -> None:
+ """Move stashed per-sample and per-image token lengths into the step buffers."""
+ super().metric_meter_add(method)
+ image_stash = getattr(self, "_metric_full_image_seqlens", None)
+ if image_stash:
+ self._metric_meter_image_seqlen_buffer().extend(image_stash.get(method, []))
+ image_stash.clear()
+
+ def metric_meter_collect(self) -> tuple[float, list[int]]:
+ """Return ``(theoretical_flops, sample_seqlens)`` using per-image lengths for attention FLOPs."""
+ seqlens = self._metric_meter_seqlen_buffer()
+ self._metric_meter_seqlens = []
+ image_seqlens = self._metric_meter_image_seqlen_buffer()
+ self._metric_meter_image_seqlens = []
+ return self.estimate_flops(seqlens, image_seqlens=image_seqlens), seqlens
+
+ def estimate_flops(
+ self,
+ seqlens: list[int],
+ *,
+ image_seqlens: list[int] | None = None,
+ ) -> float:
+ """Estimate TFLOPs for the SigLIP-NaViT encoder and MLP connector over ``seqlens``."""
+ cfg = self.config
+ dim = cfg.hidden_size
+ heads = cfg.num_attention_heads
+ head_dim = dim // heads
+ patch_embed_n = dim * cfg.num_channels * cfg.patch_size * cfg.patch_size
+ attn_linear_n = dim * 4 * dim
+ mlp_n = dim * cfg.intermediate_size * 2
+ connector_n = dim * cfg.output_size + cfg.output_size * cfg.output_size
+ dense_n = patch_embed_n + (attn_linear_n + mlp_n) * cfg.num_hidden_layers + connector_n
+ if image_seqlens is None:
+ buffered = getattr(self, "_metric_meter_image_seqlens", None)
+ stashed = getattr(self, "_metric_full_image_seqlens", {}).get("forward")
+ if buffered and sum(buffered) == sum(seqlens):
+ attn_seqlens = buffered
+ elif stashed and sum(stashed) == sum(seqlens):
+ attn_seqlens = stashed
+ else:
+ attn_seqlens = seqlens
+ else:
+ attn_seqlens = image_seqlens
+ tokens = sum(seqlens)
+ seqlen_sq = sum(length * length for length in attn_seqlens)
+ dense_flops = 6 * dense_n * tokens
+ attn_flops = 12 * seqlen_sq * head_dim * heads * cfg.num_hidden_layers
+ return (dense_flops + attn_flops) / 1e12
+
+
+class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin):
+ """Carrier hooks for BAGEL visual-understanding image features.
+
+ ``generate()`` already lives on the native :class:`~.modeling.BagelSiglipNavit`
+ (via its own :class:`~.modeling.InferenceMixin`), so no ``InferenceMixin`` is
+ needed here.
+ """
+
+
+class BagelSiglipNavitAccelerated(VeOmniMixin, BagelSiglipNavit):
+ pass
+
+
+__all__ = ["BagelSiglipNavitAccelerated"]
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py
new file mode 100644
index 0000000000..f59232e2f6
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py
@@ -0,0 +1,56 @@
+"""Configuration for :class:`BagelSiglipNavit`."""
+
+from typing import List, Optional
+
+from ...module_configuration_base import OmniModuleConfig
+
+
+class BagelSiglipNavitConfig(OmniModuleConfig):
+ """BAGEL SigLIP NaViT vision encoder config."""
+
+ model_type = "bagel_siglip_navit"
+
+ def __init__(
+ self,
+ hidden_size: int = 1152,
+ output_size: int = 3584,
+ image_size: int = 980,
+ min_image_size: int = 378,
+ max_pixels: int = 14 * 14 * 9 * 1024,
+ image_mean: Optional[List[float]] = None,
+ image_std: Optional[List[float]] = None,
+ intermediate_size: int = 4304,
+ num_attention_heads: int = 16,
+ num_hidden_layers: int = 26,
+ num_channels: int = 3,
+ patch_size: int = 14,
+ hidden_act: str = "gelu_pytorch_tanh",
+ connector_act: str = "gelu_pytorch_tanh",
+ vit_max_num_patch_per_side: int = 70,
+ layer_norm_eps: float = 1e-6,
+ attention_dropout: float = 0.0,
+ rope: bool = False,
+ **kwargs,
+ ) -> None:
+ self.hidden_size = hidden_size
+ self.output_size = output_size
+ self.image_size = image_size
+ self.min_image_size = min_image_size
+ self.max_pixels = max_pixels
+ self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean
+ self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std
+ self.intermediate_size = intermediate_size
+ self.num_attention_heads = num_attention_heads
+ self.num_hidden_layers = num_hidden_layers
+ self.num_channels = num_channels
+ self.patch_size = patch_size
+ self.hidden_act = hidden_act
+ self.connector_act = connector_act
+ self.vit_max_num_patch_per_side = vit_max_num_patch_per_side
+ self.layer_norm_eps = layer_norm_eps
+ self.attention_dropout = attention_dropout
+ self.rope = rope
+ super().__init__(**kwargs)
+
+
+__all__ = ["BagelSiglipNavitConfig"]
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py
new file mode 100644
index 0000000000..4147254847
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py
@@ -0,0 +1,514 @@
+"""BAGEL SigLIP NaViT vision encoder.
+
+The training node is ``bagel_siglip_navit.forward``. It receives local
+patchified image tensors from ``accelerated.py``, runs SigLIP/NaViT attention,
+projects features to the Bagel MoT hidden size, adds visual position embeddings,
+and returns ``{"image_embeds": ..., "token_lens": ...}``. The post-hook writes
+those feature spans back to the image ``ConversationItem.value`` fields.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.nn as nn
+from transformers.activations import ACT2FN
+
+from veomni.ops import VeomniOp
+
+from ....utils.conversation import ConversationItem, iter_desired_items
+from ...module_modeling_base import PretrainedOmniModule
+from ..position_embedding import PositionEmbedding
+from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT
+from .configuration import BagelSiglipNavitConfig
+from .processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN, BagelSiglipNavitPreprocessor, BagelSiglipNavitProcessor
+
+
+def select_siglip_image_items(
+ conversation_list: list[list[ConversationItem]] | None,
+) -> list[ConversationItem]:
+ """Select BAGEL SigLIP image carriers (training + inference)."""
+ if conversation_list is None:
+ raise ValueError("BagelSiglipNavit requires conversation_list to select image items.")
+
+ return list(
+ iter_desired_items(
+ conversation_list,
+ types=["image"],
+ meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]},
+ )
+ )
+
+
+def siglip_inputs_from_preprocessed_items(
+ image_items: list[ConversationItem],
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> dict[str, Any]:
+ """Build NaViT forward inputs from CPU-preprocessed image items."""
+ token_lens = torch.tensor(
+ [int(item.meta[_OMNI_TOKEN_LEN]) for item in image_items],
+ dtype=torch.int32,
+ device=device,
+ )
+ return {
+ "patchified_pixel_values": torch.cat([item.value for item in image_items], dim=0).to(
+ device=device, dtype=dtype, non_blocking=True
+ ),
+ "patchified_position_ids": torch.cat(
+ [item.meta[_OMNI_POSITION_IDS] for item in image_items],
+ dim=0,
+ ).to(device=device, dtype=torch.long, non_blocking=True),
+ "cu_seqlens": torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0)).to(torch.int32),
+ "max_seqlen": int(token_lens.max().item()),
+ "token_lens": token_lens,
+ }
+
+
+def scatter_siglip_image_embeds(
+ image_items: list[ConversationItem],
+ image_embeds: torch.Tensor,
+ token_lens: torch.Tensor,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> None:
+ """Scatter NaViT outputs back onto conversation image items."""
+ offset = 0
+ lengths = token_lens.detach().cpu().reshape(-1).tolist()
+ for item, length in zip(image_items, lengths, strict=True):
+ item.value = image_embeds[offset : offset + int(length)].to(device=device, dtype=dtype)
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ offset += int(length)
+
+ if offset != int(image_embeds.shape[0]):
+ raise RuntimeError("BAGEL SigLIP token count mismatch during feature scatter.")
+
+
+class InferenceMixin:
+ """FSM ``generate`` — HF ``GenerationMixin`` analog.
+
+ Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in
+ :class:`BagelSiglipNavit`'s bases for consistency with every other
+ module's native / accelerated split (see ``janus/llama/modeling.py`` for
+ the full MRO rationale where a competing no-op default exists).
+ """
+
+ def generate(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Encode SigLIP context images and scatter visual embeddings onto ``conversation_list``."""
+ del kwargs
+ batched = [conversation_list]
+ image_items = select_siglip_image_items(batched)
+ if not image_items:
+ return {"conversation_list": conversation_list}
+
+ inputs = siglip_inputs_from_preprocessed_items(image_items, device=self.device, dtype=self.dtype)
+ outputs = self.forward(**inputs)
+ token_lens = outputs.get("token_lens", inputs["token_lens"])
+ scatter_siglip_image_embeds(
+ image_items,
+ outputs["image_embeds"],
+ token_lens,
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": batched[0]}
+
+
+def _module_weight(module: nn.Module) -> torch.Tensor:
+ """Return the underlying weight parameter for plain or LoRA-wrapped modules."""
+ return getattr(module, "base_layer", module).weight
+
+
+class BagelSiglipNavit(InferenceMixin, PretrainedOmniModule):
+ config_class = BagelSiglipNavitConfig
+ image_processor_class = BagelSiglipNavitProcessor
+ preprocessor_class = BagelSiglipNavitPreprocessor
+ base_model_prefix = "bagel_siglip_navit"
+ main_input_name = "patchified_pixel_values"
+ _no_split_modules = ["BagelSiglipEncoderLayer"]
+ supports_gradient_checkpointing = True
+ _supports_sdpa = True
+ _supports_flash_attn = True
+
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__(config)
+ self.vision_model = BagelSiglipVisionTransformer(config)
+ self.connector = MLPConnector(config.hidden_size, config.output_size, config.connector_act)
+ self.vit_pos_embed = PositionEmbedding(config.vit_max_num_patch_per_side, config.output_size)
+ self._image_processor: BagelSiglipNavitProcessor = None
+ self.post_init()
+
+ def _init_weights(self, module: nn.Module) -> None:
+ if isinstance(module, PositionEmbedding):
+ module.reset_parameters()
+ return
+ super()._init_weights(module)
+
+ @property
+ def _vit_device(self) -> torch.device:
+ return _module_weight(self.vision_model.embeddings.patch_embedding).device
+
+ @property
+ def _connector_device(self) -> torch.device:
+ return _module_weight(self.connector.fc1).device
+
+ @property
+ def _pos_embed_device(self) -> torch.device:
+ return self.vit_pos_embed.pos_embed.device
+
+ def forward( # type: ignore[override]
+ self,
+ patchified_pixel_values: torch.Tensor,
+ patchified_position_ids: torch.LongTensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ token_lens: torch.IntTensor,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ del kwargs
+ vit_hidden = self._encode_patches(
+ patchified_pixel_values=patchified_pixel_values,
+ patchified_position_ids=patchified_position_ids,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ )
+ image_embeds = self._project_with_position(
+ vit_hidden=vit_hidden,
+ patchified_position_ids=patchified_position_ids,
+ )
+ return {"image_embeds": image_embeds, "token_lens": token_lens}
+
+ def _encode_patches(
+ self,
+ *,
+ patchified_pixel_values: torch.Tensor,
+ patchified_position_ids: torch.LongTensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ ) -> torch.Tensor:
+ patchified_pixel_values = patchified_pixel_values.to(device=self._vit_device, dtype=self.dtype)
+ patchified_position_ids = patchified_position_ids.to(device=self._vit_device, dtype=torch.long)
+ cu_seqlens = cu_seqlens.to(device=self._vit_device, dtype=torch.int32)
+ return self.vision_model(
+ patchified_pixel_values=patchified_pixel_values,
+ patchified_position_ids=patchified_position_ids,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ )
+
+ def _project_with_position(
+ self,
+ *,
+ vit_hidden: torch.Tensor,
+ patchified_position_ids: torch.LongTensor,
+ ) -> torch.Tensor:
+ fc1_weight = _module_weight(self.connector.fc1)
+ vit_hidden = vit_hidden.to(device=fc1_weight.device, dtype=fc1_weight.dtype)
+ patchified_position_ids = patchified_position_ids.to(device=self._pos_embed_device)
+ image_embeds = self.connector(vit_hidden)
+ return image_embeds + self.vit_pos_embed(patchified_position_ids).to(
+ device=image_embeds.device,
+ dtype=image_embeds.dtype,
+ )
+
+
+class RotaryEmbedding2D(nn.Module):
+ def __init__(self, dim: int, max_h: int, max_w: int, base: int = 10000) -> None:
+ super().__init__()
+ freq = torch.arange(0, dim, 2, dtype=torch.int64).float() / dim
+ inv_freq = 1.0 / (base**freq)
+
+ grid_h = torch.arange(0, max_h).to(inv_freq.dtype)
+ grid_h = grid_h[:, None].repeat(1, max_w)
+
+ grid_w = torch.arange(0, max_w).to(inv_freq.dtype)
+ grid_w = grid_w[None, :].repeat(max_h, 1)
+
+ cos_h, sin_h = self._forward_one_side(grid_h, inv_freq)
+ cos_w, sin_w = self._forward_one_side(grid_w, inv_freq)
+
+ self.register_buffer("cos_h", cos_h, persistent=False)
+ self.register_buffer("sin_h", sin_h, persistent=False)
+ self.register_buffer("cos_w", cos_w, persistent=False)
+ self.register_buffer("sin_w", sin_w, persistent=False)
+
+ @staticmethod
+ def _forward_one_side(grid: torch.Tensor, inv_freq: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ freqs = grid[..., None] * inv_freq[None, None, :]
+ emb = torch.cat((freqs, freqs), dim=-1).flatten(0, 1)
+ return emb.cos(), emb.sin()
+
+
+class MLPConnector(nn.Module):
+ def __init__(self, in_dim: int, out_dim: int, hidden_act: str) -> None:
+ super().__init__()
+ self.activation_fn = ACT2FN[hidden_act]
+ self.fc1 = nn.Linear(in_dim, out_dim)
+ self.fc2 = nn.Linear(out_dim, out_dim)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ return self.fc2(self.activation_fn(self.fc1(hidden_states)))
+
+
+class BagelSiglipVisionEmbeddings(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__()
+ patch_dim = config.num_channels * config.patch_size * config.patch_size
+ self.patch_embedding = nn.Linear(patch_dim, config.hidden_size, bias=True)
+ self.position_embedding = (
+ None if config.rope else nn.Embedding((config.image_size // config.patch_size) ** 2, config.hidden_size)
+ )
+
+ def forward(
+ self,
+ patchified_pixel_values: torch.Tensor,
+ patchified_position_ids: torch.LongTensor,
+ ) -> torch.Tensor:
+ patch_weight = _module_weight(self.patch_embedding)
+ patchified_pixel_values = patchified_pixel_values.to(
+ device=patch_weight.device,
+ dtype=patch_weight.dtype,
+ )
+ patch_embeds = self.patch_embedding(patchified_pixel_values)
+ if self.position_embedding is None:
+ return patch_embeds
+ position_weight = _module_weight(self.position_embedding)
+ position_ids = patchified_position_ids.to(device=position_weight.device, dtype=torch.long)
+ position_embeds = self.position_embedding(position_ids)
+ return patch_embeds + position_embeds.to(device=patch_embeds.device, dtype=patch_embeds.dtype)
+
+
+class BagelSiglipAttention(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ """Initialize NaViT self-attention projections and instance-local ``VeomniOp`` handles."""
+ super().__init__()
+ self.config = config
+ self.num_heads = config.num_attention_heads
+ self.head_dim = config.hidden_size // config.num_attention_heads
+ self.num_key_value_groups = 1
+ self.scaling = self.head_dim**-0.5
+ self.attention_dropout = config.attention_dropout
+ self.is_causal = False
+ self.q_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+ self.k_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+ self.v_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+ self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True)
+
+ rope_impl = (
+ config.ops_config.rotary_pos_emb_vision_implementation
+ if "rotary_pos_emb_vision_implementation" in config.ops_implementation
+ or "rotary_pos_emb_implementation" not in config.ops_implementation
+ else config.ops_config.rotary_pos_emb_implementation
+ )
+ self.veomni_rope = VeomniOp("rope", "full", rope_impl)
+
+ attn_impl = config.ops_config.attn_implementation
+ if "attn_implementation" not in config.ops_implementation:
+ try:
+ self.veomni_attn = VeomniOp("attention", "standard", attn_impl)
+ except RuntimeError:
+ self.veomni_attn = VeomniOp("attention", "standard", "veomni_sdpa")
+ else:
+ self.veomni_attn = VeomniOp("attention", "standard", attn_impl)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ cos_h: torch.Tensor | None = None,
+ sin_h: torch.Tensor | None = None,
+ cos_w: torch.Tensor | None = None,
+ sin_w: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ """Run varlen non-causal self-attention over packed NaViT patch tokens."""
+ total_q_len, _ = hidden_states.size()
+ query_states = self.q_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim)
+ key_states = self.k_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim)
+ value_states = self.v_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim)
+
+ if self.config.rope:
+ if cos_h is None or sin_h is None or cos_w is None or sin_w is None:
+ raise ValueError("2D RoPE tensors are required when BagelSiglipNavitConfig.rope=True.")
+ qh, qw = query_states[:, :, : self.head_dim // 2], query_states[:, :, self.head_dim // 2 :]
+ kh, kw = key_states[:, :, : self.head_dim // 2], key_states[:, :, self.head_dim // 2 :]
+ cos_h = cos_h.to(device=query_states.device, dtype=query_states.dtype)
+ sin_h = sin_h.to(device=query_states.device, dtype=query_states.dtype)
+ cos_w = cos_w.to(device=query_states.device, dtype=query_states.dtype)
+ sin_w = sin_w.to(device=query_states.device, dtype=query_states.dtype)
+ qh, kh = self.veomni_rope(qh, kh, cos_h, sin_h)
+ qw, kw = self.veomni_rope(qw, kw, cos_w, sin_w)
+ query_states = torch.cat([qh, qw], dim=-1)
+ key_states = torch.cat([kh, kw], dim=-1)
+
+ attn_dtype = (
+ torch.bfloat16
+ if value_states.dtype == torch.float32 and "flash_attention" in self.veomni_attn.impl
+ else value_states.dtype
+ )
+ query_states = query_states.to(attn_dtype).transpose(0, 1).unsqueeze(0)
+ key_states = key_states.to(attn_dtype).transpose(0, 1).unsqueeze(0)
+ value_states = value_states.to(attn_dtype).transpose(0, 1).unsqueeze(0)
+ dropout = 0.0 if not self.training else self.attention_dropout
+
+ attn_output, _ = self.veomni_attn(
+ self,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask=None,
+ scaling=self.scaling,
+ dropout=dropout,
+ cu_seq_lens_q=cu_seqlens,
+ cu_seq_lens_k=cu_seqlens,
+ max_length_q=max_seqlen,
+ max_length_k=max_seqlen,
+ is_causal=False,
+ skip_ulysses=True,
+ )
+ return self.out_proj(attn_output.reshape(total_q_len, -1).to(hidden_states.dtype))
+
+
+class BagelSiglipMLP(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__()
+ self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=True)
+ self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=True)
+ self.activation_fn = ACT2FN[config.hidden_act]
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ return self.fc2(self.activation_fn(self.fc1(hidden_states)))
+
+
+class BagelSiglipEncoderLayer(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__()
+ self.self_attn = BagelSiglipAttention(config)
+ self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+ self.mlp = BagelSiglipMLP(config)
+ self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ cos_h: torch.Tensor | None = None,
+ sin_h: torch.Tensor | None = None,
+ cos_w: torch.Tensor | None = None,
+ sin_w: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ residual = hidden_states
+ hidden_states = self.layer_norm1(hidden_states)
+ hidden_states = self.self_attn(
+ hidden_states=hidden_states,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ cos_h=cos_h,
+ sin_h=sin_h,
+ cos_w=cos_w,
+ sin_w=sin_w,
+ )
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.layer_norm2(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ return residual + hidden_states
+
+
+class BagelSiglipEncoder(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__()
+ self.gradient_checkpointing = False
+ self.layers = nn.ModuleList([BagelSiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+
+ def forward(
+ self,
+ inputs_embeds: torch.Tensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ cos_h: torch.Tensor | None = None,
+ sin_h: torch.Tensor | None = None,
+ cos_w: torch.Tensor | None = None,
+ sin_w: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ hidden_states = inputs_embeds
+ for encoder_layer in self.layers:
+ if self.gradient_checkpointing and self.training:
+ hidden_states = self._gradient_checkpointing_func(
+ encoder_layer.__call__,
+ hidden_states,
+ cu_seqlens,
+ max_seqlen,
+ cos_h,
+ sin_h,
+ cos_w,
+ sin_w,
+ )
+ else:
+ hidden_states = encoder_layer(
+ hidden_states,
+ cu_seqlens,
+ max_seqlen,
+ cos_h=cos_h,
+ sin_h=sin_h,
+ cos_w=cos_w,
+ sin_w=sin_w,
+ )
+ return hidden_states
+
+
+class BagelSiglipVisionTransformer(nn.Module):
+ def __init__(self, config: BagelSiglipNavitConfig) -> None:
+ super().__init__()
+ self.config = config
+ self.embeddings = BagelSiglipVisionEmbeddings(config)
+ if config.rope:
+ max_size = config.image_size // config.patch_size
+ dim_head = config.hidden_size // config.num_attention_heads
+ self.rope = RotaryEmbedding2D(dim_head // 2, max_size, max_size)
+ self.encoder = BagelSiglipEncoder(config)
+ self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+ def forward(
+ self,
+ patchified_pixel_values: torch.Tensor,
+ patchified_position_ids: torch.LongTensor,
+ cu_seqlens: torch.IntTensor,
+ max_seqlen: int,
+ ) -> torch.Tensor:
+ hidden_states = self.embeddings(
+ patchified_pixel_values=patchified_pixel_values,
+ patchified_position_ids=patchified_position_ids,
+ )
+ extra_inputs: dict[str, torch.Tensor] = {}
+ if self.config.rope:
+ rope_position_ids = patchified_position_ids.to(device=self.rope.cos_h.device, dtype=torch.long)
+ extra_inputs.update(
+ cos_h=self.rope.cos_h[rope_position_ids],
+ sin_h=self.rope.sin_h[rope_position_ids],
+ cos_w=self.rope.cos_w[rope_position_ids],
+ sin_w=self.rope.sin_w[rope_position_ids],
+ )
+ last_hidden_state = self.encoder(
+ inputs_embeds=hidden_states,
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ **extra_inputs,
+ )
+ return self.post_layernorm(last_hidden_state)
+
+
+__all__ = [
+ "BagelSiglipNavit",
+ "BagelSiglipNavitConfig",
+ "InferenceMixin",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py
new file mode 100644
index 0000000000..ba7ac3fe72
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py
@@ -0,0 +1,438 @@
+"""Image processor + worker-side preprocessor for BAGEL SigLIP NaViT.
+
+:class:`BagelSiglipNavitPreprocessor` is the picklable, weight-free CPU worker
+counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) —
+built straight off the checkpoint dir, with no model instance involved. Unlike
+Janus, BAGEL ships no separate ``preprocessor_config.json``: the image
+processor is fully derived from the module's own ``config.json`` via
+:meth:`BagelSiglipNavitProcessor.from_config`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from torchvision.transforms import InterpolationMode
+from torchvision.transforms import functional as TVF
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+
+from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items
+from ....utils.conversation import _OMNI_TOKEN_LEN as _CONVERSATION_TOKEN_LEN
+from ...module_processing_base import ModulePreprocessorBase
+from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT
+from .configuration import BagelSiglipNavitConfig
+
+
+class BagelSiglipNavitProcessor(BaseImageProcessor):
+ """BAGEL SigLIP NaViT image processor.
+
+ Converts raw images into flattened patch rows plus the varlen metadata that
+ the NaViT tower consumes. Carrier selection and embed scatter stay in the
+ module mixin because they are SeedOmni conversation semantics.
+ """
+
+ model_input_names = [
+ "patchified_pixel_values",
+ "patchified_position_ids",
+ "cu_seqlens",
+ "max_seqlen",
+ "token_lens",
+ ]
+
+ def __init__(
+ self,
+ patch_size: int = 14,
+ image_size: int = 980,
+ min_image_size: int = 378,
+ max_pixels: int = 14 * 14 * 9 * 1024,
+ image_mean: list[float] | None = None,
+ image_std: list[float] | None = None,
+ vit_max_num_patch_per_side: int = 70,
+ num_channels: int = 3,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.patch_size = patch_size
+ self.image_size = image_size
+ self.min_image_size = min_image_size
+ self.max_pixels = max_pixels
+ self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean
+ self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std
+ self.vit_max_num_patch_per_side = vit_max_num_patch_per_side
+ self.num_channels = num_channels
+
+ @classmethod
+ def from_config(cls, config: Any) -> BagelSiglipNavitProcessor:
+ return cls(
+ patch_size=int(config.patch_size),
+ image_size=int(config.image_size),
+ min_image_size=int(config.min_image_size),
+ max_pixels=int(config.max_pixels),
+ image_mean=list(config.image_mean),
+ image_std=list(config.image_std),
+ vit_max_num_patch_per_side=int(config.vit_max_num_patch_per_side),
+ num_channels=int(config.num_channels),
+ )
+
+ def preprocess(
+ self,
+ images: Any,
+ *,
+ return_tensors: str | None = None,
+ device: torch.device | None = None,
+ dtype: torch.dtype | None = None,
+ **kwargs: Any,
+ ) -> BatchFeature:
+ """Resize, normalize, and patchify ``images`` into NaViT varlen batch tensors."""
+ del return_tensors, kwargs
+ image_list = images if isinstance(images, list) else [images]
+
+ pixel_values: list[torch.Tensor] = []
+ position_ids: list[torch.Tensor] = []
+ token_lens: list[int] = []
+ for image in image_list:
+ image_tensor = _preprocess_image(
+ image,
+ patch_size=self.patch_size,
+ image_size=self.image_size,
+ min_image_size=self.min_image_size,
+ max_pixels=self.max_pixels,
+ image_mean=self.image_mean,
+ image_std=self.image_std,
+ )
+ patches = _patchify_image(image_tensor, self.patch_size)
+ positions = _flattened_position_ids(
+ image_tensor.shape[-2],
+ image_tensor.shape[-1],
+ patch_size=self.patch_size,
+ max_num_patches_per_side=self.vit_max_num_patch_per_side,
+ )
+ pixel_values.append(patches)
+ position_ids.append(positions)
+ token_lens.append(int(patches.shape[0]))
+
+ tensor_device = torch.device("cpu") if device is None else device
+ token_lens_tensor = torch.tensor(token_lens, dtype=torch.int32, device=tensor_device)
+ pixel_tensor = torch.cat(pixel_values, dim=0)
+ if dtype is not None:
+ pixel_tensor = pixel_tensor.to(dtype=dtype)
+ return BatchFeature(
+ data={
+ "patchified_pixel_values": pixel_tensor.to(device=tensor_device),
+ "patchified_position_ids": torch.cat(position_ids, dim=0).to(device=tensor_device, dtype=torch.long),
+ "cu_seqlens": F.pad(torch.cumsum(token_lens_tensor, dim=0), (1, 0)).to(torch.int32),
+ "max_seqlen": int(token_lens_tensor.max().item()),
+ "token_lens": token_lens_tensor,
+ }
+ )
+
+
+def _preprocess_image(
+ image: Any,
+ *,
+ patch_size: int,
+ image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ image_mean: list[float],
+ image_std: list[float],
+) -> torch.Tensor:
+ """Convert ``image`` to a normalized ``[3, H, W]`` float32 tensor aligned to ``patch_size``."""
+ if isinstance(image, (Image.Image, np.ndarray)):
+ pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image))
+ pil_image = _resize_pil(
+ pil_image,
+ patch_size=patch_size,
+ image_size=image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ tensor = _pil_to_rgb_tensor(pil_image)
+ else:
+ tensor = _to_rgb_tensor(image)
+ tensor = _resize_tensor(
+ tensor,
+ patch_size=patch_size,
+ image_size=image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1)
+ std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1)
+ return tensor.sub(mean).div(std)
+
+
+def _to_rgb_tensor(image: Any) -> torch.Tensor:
+ """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``."""
+ if isinstance(image, Image.Image):
+ pil_image = _to_rgb_pil(image)
+ array = np.array(pil_image, copy=True)
+ return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0)
+ if isinstance(image, np.ndarray):
+ return _to_rgb_tensor(Image.fromarray(image))
+ if torch.is_tensor(image):
+ tensor = image.detach().to(dtype=torch.float32)
+ if tensor.dim() != 3:
+ raise TypeError(f"BAGEL image tensor must be 3-D, got shape {tuple(tensor.shape)}.")
+ if tensor.shape[0] in (1, 3, 4):
+ pass
+ elif tensor.shape[-1] in (1, 3, 4):
+ tensor = tensor.permute(2, 0, 1).contiguous()
+ else:
+ raise TypeError(f"Unable to infer channel dimension for BAGEL image tensor {tuple(tensor.shape)}.")
+ if tensor.shape[0] == 1:
+ tensor = tensor.repeat(3, 1, 1)
+ elif tensor.shape[0] == 4:
+ alpha = tensor[3:4]
+ if alpha.max() > 1:
+ alpha = alpha / 255.0
+ rgb = tensor[:3]
+ if rgb.max() > 1:
+ rgb = rgb / 255.0
+ tensor = rgb * alpha + (1.0 - alpha)
+ if tensor.max() > 1:
+ tensor = tensor / 255.0
+ return tensor[:3].clamp(0.0, 1.0)
+ raise TypeError(f"BAGEL image item value must be PIL, numpy, or tensor, got {type(image).__name__}.")
+
+
+def _resize_tensor(
+ tensor: torch.Tensor,
+ *,
+ patch_size: int,
+ image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ img_num: int = 1,
+) -> torch.Tensor:
+ """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to the target patch grid."""
+ height, width = tensor.shape[-2:]
+ new_width, new_height = _target_size(
+ width,
+ height,
+ patch_size=patch_size,
+ image_size=image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ img_num=img_num,
+ )
+ if new_width == width and new_height == height:
+ return tensor
+ resized = F.interpolate(
+ tensor.unsqueeze(0),
+ size=(new_height, new_width),
+ mode="bicubic",
+ align_corners=False,
+ antialias=True,
+ )
+ return resized.squeeze(0).clamp(0.0, 1.0)
+
+
+def _resize_pil(
+ image: Image.Image,
+ *,
+ patch_size: int,
+ image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ img_num: int = 1,
+) -> Image.Image:
+ """Resize a PIL image with bicubic interpolation to the target patch grid."""
+ width, height = image.size
+ new_width, new_height = _target_size(
+ width,
+ height,
+ patch_size=patch_size,
+ image_size=image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ img_num=img_num,
+ )
+ if new_width == width and new_height == height:
+ return image
+ return TVF.resize(
+ image,
+ (new_height, new_width),
+ interpolation=InterpolationMode.BICUBIC,
+ antialias=True,
+ )
+
+
+def _target_size(
+ width: int,
+ height: int,
+ *,
+ patch_size: int,
+ image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ img_num: int,
+) -> tuple[int, int]:
+ """Compute patch-stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds."""
+ scale = min(image_size / max(width, height), 1.0)
+ scale = max(scale, min_image_size / min(width, height))
+ new_width, new_height = _apply_scale(width, height, scale, patch_size)
+ if new_width * new_height > max_pixels / img_num:
+ scale = max_pixels / img_num / (new_width * new_height)
+ new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size)
+ if max(new_width, new_height) > image_size:
+ scale = image_size / max(new_width, new_height)
+ new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size)
+ return new_width, new_height
+
+
+def _to_rgb_pil(image: Image.Image) -> Image.Image:
+ """Convert ``image`` to RGB, compositing transparency over a white background."""
+ if image.mode == "RGBA" or image.info.get("transparency", None) is not None:
+ rgba = image.convert("RGBA")
+ white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255))
+ white.paste(rgba, mask=rgba.split()[3])
+ return white
+ return image.convert("RGB")
+
+
+def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor:
+ array = np.array(image, copy=True)
+ return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0)
+
+
+def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]:
+ new_width = round(width * scale)
+ new_height = round(height * scale)
+ return (
+ max(stride, int(round(new_width / stride) * stride)),
+ max(stride, int(round(new_height / stride) * stride)),
+ )
+
+
+def _patchify_image(image: torch.Tensor, patch_size: int) -> torch.Tensor:
+ """Flatten a ``[C, H, W]`` image tensor into ``[(H/P)*(W/P), P*P*C]`` patch rows."""
+ channels, height, width = image.shape
+ if height % patch_size != 0 or width % patch_size != 0:
+ raise ValueError("BAGEL preprocessed image height and width must be divisible by patch_size.")
+ image = image.reshape(channels, height // patch_size, patch_size, width // patch_size, patch_size)
+ image = torch.einsum("chpwq->hwpqc", image)
+ return image.reshape(-1, patch_size**2 * channels)
+
+
+def _flattened_position_ids(
+ height: int,
+ width: int,
+ *,
+ patch_size: int,
+ max_num_patches_per_side: int,
+) -> torch.Tensor:
+ num_patches_h = height // patch_size
+ num_patches_w = width // patch_size
+ coords_h = torch.arange(0, num_patches_h, dtype=torch.long)
+ coords_w = torch.arange(0, num_patches_w, dtype=torch.long)
+ return (coords_h[:, None] * max_num_patches_per_side + coords_w).flatten()
+
+
+_OMNI_POSITION_IDS = "bagel_siglip_navit_position_ids"
+_OMNI_TOKEN_LEN = "bagel_siglip_navit_token_len"
+
+
+class BagelSiglipNavitPreprocessor(ModulePreprocessorBase):
+ """Worker-side image patchify for BAGEL SigLIP NaViT context images."""
+
+ def __init__(
+ self,
+ image_processor: Any,
+ dtype: torch.dtype | None = None,
+ dummy_pixel_values: torch.Tensor | None = None,
+ ) -> None:
+ self._image_processor = image_processor
+ self._dtype = dtype
+ self._dummy_pixel_values = dummy_pixel_values
+
+ @classmethod
+ def from_pretrained(
+ cls, module_path: str, *, config_overrides: dict[str, Any] | None = None, **kwargs: Any
+ ) -> BagelSiglipNavitPreprocessor:
+ """Build straight from the checkpoint dir — no model instance needed.
+
+ BAGEL ships no standalone ``preprocessor_config.json``; the image
+ processor is derived from the module's own ``config.json``.
+
+ ``config_overrides`` (the module's YAML ``model_config:`` block) is
+ applied on top of the on-disk default before deriving the image
+ processor's geometry, so it agrees with the live model's own config.
+ """
+ del kwargs
+ # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare.
+ config = BagelSiglipNavitConfig.from_pretrained(module_path, model_config=config_overrides)
+ return cls(BagelSiglipNavitProcessor.from_config(config))
+
+ def bind_dummy_inputs(self, config: BagelSiglipNavitConfig, dtype: torch.dtype | None = None) -> None:
+ """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model."""
+ patch_dim = int(config.num_channels) * int(config.patch_size) * int(config.patch_size)
+ self._dtype = dtype
+ self._dummy_pixel_values = torch.zeros(1, patch_dim, dtype=dtype)
+
+ def preprocess_conversations(
+ self,
+ conversation_list: list[list[ConversationItem]],
+ *,
+ inference: bool = False,
+ generation_kwargs: dict[str, Any] | None = None,
+ ) -> None:
+ """Patchify SigLIP context images in place and append per-sample dummy anchors in training."""
+ del generation_kwargs
+
+ image_items: list[ConversationItem] = []
+ for sample in conversation_list:
+ sample_image_items = list(
+ iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]})
+ )
+ if sample_image_items:
+ image_items.extend(item for item in sample_image_items if not item.is_dummy)
+ elif not inference:
+ if self._dummy_pixel_values is None:
+ raise RuntimeError(
+ f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() "
+ "before training use (pure inference never reaches this branch)."
+ )
+ sample.append(
+ ConversationItem(
+ type="image",
+ value=self._dummy_pixel_values.to(dtype=self._dtype).clone(),
+ role="user",
+ is_dummy=True,
+ meta={
+ _IMG_TAG_KEY: "und",
+ BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT,
+ _OMNI_POSITION_IDS: torch.zeros(1, dtype=torch.long),
+ _OMNI_TOKEN_LEN: 1,
+ _CONVERSATION_TOKEN_LEN: 1,
+ },
+ )
+ )
+
+ if not image_items:
+ return
+
+ inputs = self._image_processor(
+ images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype
+ )
+ lengths = inputs["token_lens"].detach().cpu().reshape(-1).tolist()
+ pixel_chunks = torch.split(inputs["patchified_pixel_values"], lengths, dim=0)
+ position_chunks = torch.split(inputs["patchified_position_ids"], lengths, dim=0)
+ for item, pixels, position_ids, length in zip(
+ image_items, pixel_chunks, position_chunks, lengths, strict=True
+ ):
+ item.value = pixels.to(dtype=self._dtype)
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ item.meta[_OMNI_POSITION_IDS] = position_ids.to(dtype=torch.long)
+ item.meta[_OMNI_TOKEN_LEN] = int(length)
+ item.meta[_CONVERSATION_TOKEN_LEN] = int(length)
+
+
+__all__ = [
+ "BagelSiglipNavitProcessor",
+ "BagelSiglipNavitPreprocessor",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/sources.py b/veomni/models/seed_omni/modules/bagel/sources.py
new file mode 100644
index 0000000000..e58bdd1f9a
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/sources.py
@@ -0,0 +1,69 @@
+"""BAGEL-local ``ConversationItem.meta`` routing tags.
+
+Items carry no module ownership (see :mod:`...utils.conversation`), so BAGEL
+routes its rows with two BAGEL-private meta keys:
+
+* ``meta[BAGEL_CONTEXT_KEY]`` — which image context copy a row belongs to:
+ ``BAGEL_SIGLIP_CONTEXT`` (understanding features, read by SigLIP-NaViT) or
+ ``BAGEL_VAE_CONTEXT`` (latent context/target, read by the VAE and flow
+ connector). The VAE preprocessor expands each raw image by its data-layer
+ ``_IMG_TAG_KEY`` into one or both copies — ``_img_tag`` alone cannot tell the
+ two copies of one ``edit`` image apart. The chat template stamps the same tag
+ on the vision start/end marker rows around each copy, and MoT reads it to
+ pick attention mode and expert routing.
+* ``meta[BAGEL_PHASE_KEY]`` — the inference/flow step an ``output`` row is in:
+ the understanding start token, then per denoise step flow query -> hidden ->
+ velocity, and finally the generated latent handed to VAE decode.
+
+An FSDP placeholder is flagged ``is_dummy=True`` and keeps the context tag of
+the rows it stands in for, so each encoder selects real and dummy rows with
+one filter.
+"""
+
+from __future__ import annotations
+
+from ...utils.conversation import ConversationItem, iter_desired_items
+
+
+BAGEL_CONTEXT_KEY = "bagel_context"
+BAGEL_SIGLIP_CONTEXT = "siglip"
+BAGEL_VAE_CONTEXT = "vae"
+
+BAGEL_PHASE_KEY = "bagel_phase"
+BAGEL_START_TOKEN = "start_token"
+BAGEL_FLOW_QUERY = "flow_query"
+BAGEL_FLOW_HIDDEN = "flow_hidden"
+BAGEL_FLOW_VELOCITY = "flow_velocity"
+BAGEL_GENERATED_LATENT = "generated_latent"
+
+
+def bagel_context(item: ConversationItem) -> str | None:
+ return item.meta.get(BAGEL_CONTEXT_KEY)
+
+
+def bagel_phase(item: ConversationItem) -> str | None:
+ return item.meta.get(BAGEL_PHASE_KEY)
+
+
+def get_tail_phase_item(conversation: list[ConversationItem], phase: str) -> ConversationItem | None:
+ """Return the latest ``output`` row in ``phase`` of a single conversation."""
+ return next(
+ iter_desired_items([conversation], types=["output"], reverse_item=True, meta={BAGEL_PHASE_KEY: [phase]}),
+ None,
+ )
+
+
+__all__ = [
+ "BAGEL_CONTEXT_KEY",
+ "BAGEL_FLOW_HIDDEN",
+ "BAGEL_FLOW_QUERY",
+ "BAGEL_FLOW_VELOCITY",
+ "BAGEL_GENERATED_LATENT",
+ "BAGEL_PHASE_KEY",
+ "BAGEL_SIGLIP_CONTEXT",
+ "BAGEL_START_TOKEN",
+ "BAGEL_VAE_CONTEXT",
+ "bagel_context",
+ "bagel_phase",
+ "get_tail_phase_item",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py
new file mode 100644
index 0000000000..f780d497c9
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py
@@ -0,0 +1,24 @@
+"""BAGEL token embedding and LM head module."""
+
+from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY
+
+
+@OMNI_CONFIG_REGISTRY.register("bagel_text_encoder")
+def register_bagel_text_encoder_config():
+ from .configuration import BagelTextEncoderConfig
+
+ return BagelTextEncoderConfig
+
+
+@OMNI_MODEL_REGISTRY.register("bagel_text_encoder")
+def register_bagel_text_encoder_model():
+ from .modeling import BagelTextEncoder
+
+ return BagelTextEncoder
+
+
+@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_text_encoder")
+def register_bagel_text_encoder_accelerated_model():
+ from .accelerated import BagelTextEncoderAccelerated
+
+ return BagelTextEncoderAccelerated
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py
new file mode 100644
index 0000000000..1a106ff116
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py
@@ -0,0 +1,10 @@
+"""Accelerated runtime for the bagel/text_encoder OmniModule (conversation graph)."""
+
+from .accelerated import (
+ BagelTextEncoderAccelerated,
+)
+
+
+__all__ = [
+ "BagelTextEncoderAccelerated",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py
new file mode 100644
index 0000000000..48ec08ae65
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py
@@ -0,0 +1,176 @@
+"""VeOmni-accelerated BagelTextEncoder — training graph hooks.
+
+FSM ``generate`` (and ``encode_image_markers``) now live natively on
+:class:`~.modeling.BagelTextEncoder`; this file only owns the training
+pre/forward/post hooks (packed-token embedding + MoT CE loss).
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+from transformers import PreTrainedTokenizerBase
+
+from .....mixins.training_module_mixin import post_forward, pre_forward
+from .....utils.conversation import ConversationItem, iter_desired_items
+from ....text_encoder.accelerated import (
+ TrainingMixin as BaseTrainingMixin,
+)
+from ....text_encoder.accelerated import (
+ VeOmniMixin as BaseVeOmniMixin,
+)
+from ..chat_template import BagelChatTemplate
+from ..configuration import BagelTextEncoderConfig
+from ..modeling import BagelTextEncoder, prepare_bagel_encode_input_ids
+
+
+class TrainingMixin(BaseTrainingMixin):
+ config: BagelTextEncoderConfig
+ device: torch.device
+ dtype: torch.dtype
+ _chat_template: BagelChatTemplate | None
+ _encode_batch_shape: torch.LongTensor | None
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self._chat_template: Optional[BagelChatTemplate] = None
+
+ def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]:
+ """Delegate to native impl — overrides ``TrainingModuleMixin.dummy_inputs``'s
+ unrelated ``(*, batch_size, device, dtype)`` signature, which otherwise wins
+ this name collision by sitting ahead of :class:`BagelTextEncoder` in MRO.
+ """
+ return BagelTextEncoder.dummy_inputs(self, kind=kind)
+
+ @pre_forward("encode")
+ def encode_pre(
+ self,
+ conversation_list: Optional[List[List[ConversationItem]]] = None,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ return super().encode_pre(conversation_list, **kwargs)
+
+ @post_forward("encode")
+ def encode_post(self, **outputs: Any) -> Dict[str, Any]:
+ return super().encode_post(**outputs)
+
+ @pre_forward("decode")
+ def decode_pre(
+ self,
+ conversation_list: Optional[List[List[ConversationItem]]] = None,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ return super().decode_pre(conversation_list, **kwargs)
+
+ @post_forward("decode")
+ def decode_post(self, **outputs: Any) -> Dict[str, Any]:
+ return super().decode_post(**outputs)
+
+ def _anchor_dummy_decode_inputs(
+ self,
+ conversation_list: Optional[List[List[ConversationItem]]],
+ dummy: Dict[str, torch.Tensor],
+ ) -> Dict[str, torch.Tensor]:
+ """Tie dummy CE loss to MoT hidden states without changing its value."""
+ if conversation_list is None:
+ return dummy
+
+ anchor = None
+ for item in iter_desired_items(
+ conversation_list, types=["text", "image", "output"], roles=["user", "assistant"]
+ ):
+ if item.is_dummy:
+ continue
+ value = item.value
+ if not torch.is_tensor(value):
+ continue
+ if value.dim() == 3 and value.shape[0] == 1:
+ value = value.squeeze(0)
+ if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size):
+ anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0
+ break
+ if anchor is None:
+ return dummy
+
+ return {
+ "hidden_states": dummy["hidden_states"] + anchor,
+ "labels": dummy["labels"],
+ }
+
+ def _prepare_encode_inputs(
+ self,
+ conversation_list: Optional[List[List[ConversationItem]]],
+ ) -> torch.Tensor:
+ """Pack non-dummy text token IDs into a flat tensor and record segment lengths."""
+ if conversation_list is None:
+ raise ValueError("BagelTextEncoder._prepare_encode_inputs requires conversation_list.")
+
+ input_ids, self._encode_batch_shape = prepare_bagel_encode_input_ids(
+ conversation_list,
+ device=self.device,
+ fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"],
+ )
+ return input_ids
+
+ def _prepare_decode_inputs(
+ self,
+ conversation_list: Optional[List[List[ConversationItem]]],
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Collect text hidden states and pre-shifted causal LM labels across ``conversation_list``."""
+ if conversation_list is None:
+ raise ValueError("BagelTextEncoder._prepare_decode_inputs requires conversation_list.")
+
+ hidden_parts: List[torch.Tensor] = []
+ shift_label_parts: List[torch.Tensor] = []
+ for item in iter_desired_items(conversation_list, types=["text"]):
+ if item.is_dummy:
+ continue
+
+ hidden_states = item.value
+ labels = item.meta["labels"]
+ if hidden_states.dim() == 3 and hidden_states.size(0) == 1:
+ hidden_states = hidden_states.squeeze(0)
+ labels = labels.reshape(-1)
+ if hidden_states.shape[0] != labels.shape[0]:
+ raise ValueError(
+ "BAGEL text decode requires hidden-state and label lengths to match: "
+ f"got {hidden_states.shape[0]} and {labels.shape[0]}."
+ )
+
+ shift_labels = torch.full_like(labels, -100, dtype=torch.long)
+ shift_labels[:-1] = labels[1:]
+ hidden_parts.append(hidden_states.to(device=self.device, dtype=self.dtype))
+ shift_label_parts.append(shift_labels)
+
+ if hidden_parts:
+ hidden_states = torch.cat(hidden_parts, dim=0)
+ shift_labels = torch.cat(shift_label_parts, dim=0).to(device=hidden_states.device, non_blocking=True)
+ if torch.any(shift_labels != -100):
+ return hidden_states, shift_labels
+
+ dummy = self._anchor_dummy_decode_inputs(conversation_list, self.dummy_inputs(kind="decode"))
+ return dummy["hidden_states"], dummy["labels"]
+
+
+class VeOmniMixin(TrainingMixin, BaseVeOmniMixin):
+ """BAGEL text-embedding accelerated wrapper — chat-template binding + training hooks.
+
+ The T2I generation FSM (``generate`` / ``encode_image_markers``) lives on
+ the native :class:`~.modeling.BagelTextEncoder`; only the chat template
+ binding and CE-loss training hooks are accelerated-specific.
+ """
+
+ @property
+ def tokenizer(self) -> PreTrainedTokenizerBase:
+ return self._tokenizer
+
+ @tokenizer.setter
+ def tokenizer(self, tokenizer: PreTrainedTokenizerBase) -> None:
+ self._tokenizer = tokenizer
+ self._chat_template = BagelChatTemplate(tokenizer)
+
+
+class BagelTextEncoderAccelerated(VeOmniMixin, BagelTextEncoder):
+ pass
+
+
+__all__ = ["BagelTextEncoderAccelerated"]
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py
new file mode 100644
index 0000000000..8c2aba26a3
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py
@@ -0,0 +1,215 @@
+"""BAGEL chat template for text/materialized image prompt rows."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+from transformers import PreTrainedTokenizerBase
+
+from ....utils.conversation import _OMNI_TOKEN_LEN, ConversationItem
+from ...text_encoder.chat_template import ChatMarkers, TextEncoderChatTemplate
+from ..sources import BAGEL_CONTEXT_KEY, BAGEL_PHASE_KEY, BAGEL_START_TOKEN, bagel_context
+from .processing import is_bagel_vision_marker
+
+
+_OMNI_TOKENIZED = "_omni_tokenized"
+
+
+@dataclass(frozen=True)
+class BagelChatMarkers(ChatMarkers):
+ vision_start_token: str
+ vision_end_token: str
+
+
+class BagelChatTemplate(TextEncoderChatTemplate):
+ """BAGEL-specific template over SeedOmni conversation rows.
+
+ The template is intentionally row-preserving: text rows are tokenized, and
+ image rows are bracketed by marker text rows, but marker-image-marker packing
+ remains a MoT responsibility.
+ """
+
+ chat_markers: BagelChatMarkers
+
+ def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None:
+ super().__init__(tokenizer)
+
+ bos_token = "<|im_start|>"
+ eos_token = "<|im_end|>"
+ vision_start_token = "<|vision_start|>"
+ vision_end_token = "<|vision_end|>"
+
+ self.bos_token_id = self._resolve_token_id_from_token(tokenizer, bos_token)
+ self.eos_token_id = self._resolve_token_id_from_token(tokenizer, eos_token)
+ self.vision_start_token_id = self._resolve_token_id_from_token(tokenizer, vision_start_token)
+ self.vision_end_token_id = self._resolve_token_id_from_token(tokenizer, vision_end_token)
+
+ self.chat_markers = BagelChatMarkers(
+ bos_token=bos_token,
+ eos_token=eos_token,
+ vision_start_token=vision_start_token,
+ vision_end_token=vision_end_token,
+ )
+
+ @staticmethod
+ def _resolve_token_id_from_token(tokenizer: PreTrainedTokenizerBase, token: str) -> int:
+ """Look up ``token`` in ``tokenizer`` and raise if it is missing or maps to ``unk_token_id``."""
+ token_id = tokenizer.convert_tokens_to_ids(token)
+ if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None):
+ raise ValueError(f"BAGEL tokenizer is missing required token: {token!r}.")
+ return int(token_id)
+
+ def tokenize_conversation(
+ self,
+ sample: list[ConversationItem],
+ *,
+ add_generation_prompt: bool = False,
+ generation_kwargs: dict[str, Any] | None = None,
+ ) -> list[ConversationItem]:
+ """Apply the BAGEL chat template, wrap image rows with vision markers, and tokenize text rows."""
+ parts = self.apply_chat_template(
+ sample,
+ inference=add_generation_prompt,
+ generation_kwargs=generation_kwargs,
+ )
+ if add_generation_prompt:
+ parts = self.apply_generation_prompt(parts, generation_kwargs=generation_kwargs)
+ self.tokenize(parts)
+ return self.merge_text_embeds(parts)
+
+ def apply_chat_template(
+ self,
+ sample: list[ConversationItem],
+ *,
+ inference: bool = False,
+ generation_kwargs: dict[str, Any] | None = None,
+ ) -> list[ConversationItem]:
+ """Wrap untokenized text items with ``<|im_start|>...<|im_end|>`` and insert image marker items."""
+ del inference, generation_kwargs
+
+ for item in sample:
+ if item.type == "text" and not is_bagel_vision_marker(item):
+ if not item.meta.get(_OMNI_TOKENIZED) and not torch.is_tensor(item.value):
+ item.value = f"{self.chat_markers.bos_token}{item.value}{self.chat_markers.eos_token}"
+ item.meta["loss_mask"] = int(item.role == "assistant")
+
+ return self._insert_prompt_context_image_markers(sample)
+
+ def apply_generation_prompt(
+ self,
+ sample: list[ConversationItem],
+ *,
+ generation_kwargs: dict[str, Any] | None = None,
+ ) -> list[ConversationItem]:
+ """Append the assistant ``<|im_start|>`` start token for understanding inference."""
+ infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type")
+ if infer_type != "infer_und":
+ return sample
+ out = list(sample)
+ out.append(
+ ConversationItem(
+ type="text",
+ value=self.chat_markers.bos_token,
+ role="assistant",
+ meta={"loss_mask": 0, BAGEL_PHASE_KEY: BAGEL_START_TOKEN},
+ )
+ )
+ return out
+
+ def tokenize(self, parts: list[ConversationItem]) -> None:
+ """Tokenize untokenized text items in place and populate ``input_ids``, ``labels``, and ``attention_mask``."""
+ for item in parts:
+ if item.type != "text" or item.is_dummy or item.meta.get(_OMNI_TOKENIZED):
+ continue
+
+ raw_value = item.value
+ token_ids = self._token_ids_for_text_item(item)
+ loss_mask = int(item.meta.pop("loss_mask", int(item.role == "assistant")))
+ item.value = token_ids
+ item.meta["input_ids"] = token_ids.detach()
+ item.meta["attention_mask"] = torch.ones_like(token_ids, dtype=torch.long)
+ item.meta["labels"] = (
+ token_ids.detach().clone() if loss_mask else torch.full_like(token_ids, -100, dtype=torch.long)
+ )
+ item.meta[_OMNI_TOKEN_LEN] = int(token_ids.numel())
+ item.meta[_OMNI_TOKENIZED] = True
+
+ if (
+ isinstance(raw_value, str)
+ and raw_value in {self.chat_markers.bos_token, self.chat_markers.eos_token}
+ and int(token_ids.numel()) != 1
+ ):
+ raise ValueError("BAGEL start/end tokens must tokenize to exactly one token.")
+
+ @staticmethod
+ def merge_text_embeds(parts: list[ConversationItem]) -> list[ConversationItem]:
+ return parts
+
+ def _token_ids_for_text_item(self, item: ConversationItem) -> torch.Tensor:
+ """Convert ``item.value`` (tensor or string) into a 1-D int64 token-id tensor."""
+ value = item.value
+ if torch.is_tensor(value):
+ token_ids = value.detach().reshape(-1).to(dtype=torch.long)
+ else:
+ token_ids = torch.tensor(
+ self.tokenizer(str(value), add_special_tokens=False)["input_ids"],
+ dtype=torch.long,
+ )
+ return token_ids
+
+ def _insert_prompt_context_image_markers(self, sample: list[ConversationItem]) -> list[ConversationItem]:
+ """Bracket each non-dummy image item with ``<|vision_start|>`` and ``<|vision_end|>`` marker items."""
+ out: list[ConversationItem] = []
+
+ index = 0
+ while index < len(sample):
+ item = sample[index]
+
+ if item.type != "image" or item.is_dummy:
+ out.append(item)
+ index += 1
+ continue
+
+ # Add vision marker for image items: + image + .
+ # Markers carry the image's context tag so MoT can regroup the triplet.
+ context = bagel_context(item)
+ marker_meta = {"loss_mask": 0} if context is None else {"loss_mask": 0, BAGEL_CONTEXT_KEY: context}
+ if not (out and bagel_context(out[-1]) == context and is_bagel_vision_marker(out[-1], context=context)):
+ out.append(
+ ConversationItem(
+ type="text",
+ value=self.chat_markers.vision_start_token,
+ role=item.role,
+ meta=dict(marker_meta),
+ )
+ )
+
+ out.append(item)
+
+ if (
+ index + 1 < len(sample)
+ and bagel_context(sample[index + 1]) == context
+ and is_bagel_vision_marker(sample[index + 1], context=context)
+ ):
+ out.append(sample[index + 1])
+ index += 2
+ else:
+ out.append(
+ ConversationItem(
+ type="text",
+ value=self.chat_markers.vision_end_token,
+ role=item.role,
+ meta=dict(marker_meta),
+ )
+ )
+ index += 1
+
+ return out
+
+
+__all__ = [
+ "BagelChatMarkers",
+ "BagelChatTemplate",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py
new file mode 100644
index 0000000000..da13aff996
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py
@@ -0,0 +1,12 @@
+"""Configuration for :class:`BagelTextEncoder`."""
+
+from ...text_encoder.configuration import TextEncoderConfig
+
+
+class BagelTextEncoderConfig(TextEncoderConfig):
+ """BAGEL text encoder config."""
+
+ model_type = "bagel_text_encoder"
+
+ def __init__(self, *args, tie_word_embeddings: bool = True, lm_head_bias: bool = False, **kwargs):
+ super().__init__(*args, tie_word_embeddings=tie_word_embeddings, lm_head_bias=lm_head_bias, **kwargs)
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py
new file mode 100644
index 0000000000..469caebc28
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py
@@ -0,0 +1,174 @@
+"""BAGEL token embedding and LM head module."""
+
+from typing import Any, Dict, List, Optional
+
+import torch
+
+from veomni.utils.tensor_utils import naflatten, unflatten
+
+from ....graphs.generation_graph import FSM_SIGNAL_KEY
+from ....utils.conversation import ConversationItem, iter_desired_items, maybe_merge_outputs
+from ...text_encoder.modeling import TextEncoder
+from ..sources import BAGEL_FLOW_QUERY, BAGEL_PHASE_KEY
+from .configuration import BagelTextEncoderConfig
+from .processing import BagelTextEncoderPreprocessor, apply_image_marker
+
+
+SIGNAL_START_IMAGE_GEN = "start_image_gen"
+SIGNAL_TEXT_DONE = "text_done"
+
+# Sentinel written by BagelTextEncoderPreprocessor onto every text item so
+# encode_pre can skip tokenizer work already completed by a DataLoader worker.
+_OMNI_TOKENIZED = "_omni_tokenized"
+
+
+def prepare_bagel_encode_input_ids(
+ conversation_list: Optional[List[List[ConversationItem]]],
+ *,
+ device: torch.device,
+ fallback_input_ids: torch.Tensor,
+) -> tuple[torch.Tensor, torch.LongTensor | None]:
+ """Pack CPU-preprocessed BAGEL text token ids (training + inference)."""
+ if conversation_list is None:
+ raise ValueError("BagelTextEncoder.prepare_bagel_encode_input_ids requires conversation_list.")
+
+ input_ids: List[torch.Tensor] = []
+ for item in iter_desired_items(conversation_list, types=["text"]):
+ if item.is_dummy:
+ continue
+ if not item.meta.get(_OMNI_TOKENIZED):
+ raise ValueError("BAGEL text encoder expects CPU-preprocessed text items.")
+ token_ids = item.meta.get("input_ids")
+ input_ids.append(token_ids.reshape(-1))
+
+ if not input_ids:
+ return fallback_input_ids, None
+
+ flat_ids, batch_shape = naflatten(input_ids)
+ return flat_ids.to(device, non_blocking=True), batch_shape
+
+
+def scatter_bagel_text_embeds(
+ conversation_list: List[List[ConversationItem]],
+ segment_embeds: List[torch.Tensor],
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> None:
+ """Write BAGEL text embeds back onto conversation text items."""
+ segment_embeds_iterator = iter(segment_embeds)
+ for item in iter_desired_items(conversation_list, types=["text"]):
+ if item.is_dummy:
+ continue
+ item.value = next(segment_embeds_iterator).to(device=device, dtype=dtype)
+ if next(segment_embeds_iterator, None) is not None:
+ raise RuntimeError("BAGEL text segment count mismatch during embed scatter.")
+
+
+class BagelTextEncoder(TextEncoder):
+ """BAGEL token embedding + LM head, plus the T2I generation FSM.
+
+ Only the T2I generation FSM (image-gen signal, marker embedding) is
+ genuinely BAGEL-specific; accelerated-only behavior lives in
+ ``accelerated.py``.
+ """
+
+ config_class = BagelTextEncoderConfig
+ preprocessor_class = BagelTextEncoderPreprocessor
+
+ def generate(
+ self,
+ conversation_list: Optional[List[ConversationItem]] = None,
+ generation_kwargs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ """Embed prompt text tokens or sample the next AR decode token."""
+ tail = conversation_list[-1]
+ batched = [conversation_list]
+
+ if tail.type == "text":
+ input_ids, batch_shape = prepare_bagel_encode_input_ids(
+ batched,
+ device=self.device,
+ fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"],
+ )
+ inputs_embeds = self.encode(input_ids)["inputs_embeds"]
+ scatter_bagel_text_embeds(
+ batched,
+ unflatten(inputs_embeds, batch_shape),
+ device=self.device,
+ dtype=self.dtype,
+ )
+ return {"conversation_list": batched[0]}
+
+ if tail.type == "output":
+ outputs: Dict[str, Any] = {"conversation_list": batched[0]}
+ hidden_states = tail.value
+ if not torch.is_tensor(hidden_states):
+ raise TypeError("BAGEL text generate expects the tail output value to be a hidden-state tensor.")
+ if hidden_states.dim() == 2:
+ hidden_states = hidden_states.unsqueeze(0)
+ sampling = self._extract_sampling_kwargs(generation_kwargs, 1.0, 1.0, kwargs)
+
+ output_token_id = self._sample_token(hidden_states, **sampling)
+ self._text_token_cache.append(output_token_id)
+
+ input_ids = self._token_id_tensor(output_token_id)
+ inputs_embeds = self.encode(input_ids)["inputs_embeds"]
+ tail.value = inputs_embeds.to(device=self.device, dtype=self.dtype)
+ tail.meta["input_ids"] = input_ids.reshape(-1).detach()
+ maybe_merge_outputs(batched[0])
+ if output_token_id == self._chat_template.eos_token_id:
+ outputs[FSM_SIGNAL_KEY] = SIGNAL_TEXT_DONE
+ outputs["generated"] = self._flush_text_generated(batched[0])
+ return outputs
+
+ raise ValueError(f"Invalid type: {tail.type}")
+
+ def encode_image_markers(
+ self,
+ conversation_list: Optional[List[ConversationItem]] = None,
+ generation_kwargs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ """Bracket ``flow_query`` output items with embedded ``<|vision_start|>`` and ``<|vision_end|>`` tokens."""
+ del generation_kwargs, kwargs
+ marker_embeds: Optional[torch.Tensor] = None
+ for item in iter_desired_items(
+ [conversation_list],
+ types=["output"],
+ meta={BAGEL_PHASE_KEY: [BAGEL_FLOW_QUERY]},
+ ):
+ if item.is_dummy:
+ continue
+
+ if marker_embeds is None:
+ marker_ids = torch.tensor(
+ [[self._chat_template.vision_start_token_id, self._chat_template.vision_end_token_id]],
+ dtype=torch.long,
+ device=self.device,
+ )
+ marker_embeds = self.encode(marker_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype)
+ marker_embeds = marker_embeds.squeeze(0)
+ apply_image_marker(item, marker_embeds, device=self.device, dtype=self.dtype)
+
+ return {"conversation_list": conversation_list}
+
+ def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]:
+ """Return minimal dummy ``encode`` or ``decode`` inputs for FSDP anchor execution."""
+ if kind == "encode":
+ return {"input_ids": torch.zeros(1, device=self.device, dtype=torch.long)}
+ return {
+ "hidden_states": torch.zeros(1, int(self.config.hidden_size), device=self.device, dtype=self.dtype),
+ "labels": torch.full((1,), -100, device=self.device, dtype=torch.long),
+ }
+
+
+__all__ = [
+ "BagelTextEncoder",
+ "BagelTextEncoderConfig",
+ "SIGNAL_START_IMAGE_GEN",
+ "SIGNAL_TEXT_DONE",
+ "prepare_bagel_encode_input_ids",
+ "scatter_bagel_text_embeds",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py
new file mode 100644
index 0000000000..9fbda4b240
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py
@@ -0,0 +1,110 @@
+"""Stateless text carrier helpers + worker-side preprocessor for BAGEL text encoder.
+
+:class:`BagelTextEncoderPreprocessor` is the picklable, weight-free CPU worker
+counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`).
+Its :meth:`from_pretrained` loads the tokenizer and builds the
+:class:`~veomni.models.seed_omni.modules.bagel.text_encoder.chat_template.BagelChatTemplate`
+straight off the checkpoint dir — no model instance involved.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import torch
+
+from .....auto import build_tokenizer
+from ....utils.conversation import ConversationItem
+from ...text_encoder.processing import TextEncoderPreprocessor
+from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context
+
+
+if TYPE_CHECKING:
+ # chat_template.py imports is_bagel_vision_marker from this module, so the
+ # reverse import must stay deferred (see from_pretrained) to avoid a cycle.
+ from .chat_template import BagelChatTemplate
+
+
+def is_bagel_vision_marker(item: ConversationItem, *, context: str | None = None) -> bool:
+ """Return whether ``item`` is a single-token text vision marker tagged with a BAGEL context."""
+ if item.type != "text":
+ return False
+ if context is not None and bagel_context(item) != context:
+ return False
+ if bagel_context(item) not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}:
+ return False
+ return _text_item_length(item) == 1
+
+
+def _text_item_length(item: ConversationItem) -> int | None:
+ """Return the token count of a text ``ConversationItem``, or ``None`` if untokenized."""
+ value = item.value
+ if torch.is_tensor(value):
+ if value.dim() == 0:
+ return 1
+ if value.dim() == 1:
+ return int(value.shape[0])
+ if value.dim() == 2:
+ return int(value.shape[0])
+ if value.dim() == 3 and int(value.shape[0]) == 1:
+ return int(value.shape[1])
+ return None
+ input_ids = item.meta.get("input_ids")
+ if torch.is_tensor(input_ids):
+ return int(input_ids.reshape(-1).shape[0])
+ return None
+
+
+def apply_image_marker(
+ item: ConversationItem,
+ marker_embeds: torch.Tensor,
+ *,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> None:
+ """Prepend and append ``marker_embeds`` to ``item.value`` if not already bracketed."""
+ image_embeds = item.value
+ if not torch.is_tensor(image_embeds):
+ return
+ if image_embeds.dim() == 3 and image_embeds.shape[0] == 1:
+ image_embeds = image_embeds.squeeze(0)
+ if image_embeds.dim() != 2:
+ return
+
+ image_embeds = image_embeds.to(device=device, dtype=dtype)
+ if (
+ image_embeds.shape[0] >= 2
+ and torch.equal(image_embeds[:1], marker_embeds[:1])
+ and torch.equal(image_embeds[-1:], marker_embeds[1:])
+ ):
+ return
+ item.value = torch.cat([marker_embeds[:1], image_embeds, marker_embeds[1:]], dim=0)
+
+
+class BagelTextEncoderPreprocessor(TextEncoderPreprocessor):
+ """Worker-side chat-template + tokenize for BAGEL text encoder inputs."""
+
+ @classmethod
+ def build_chat_template(
+ cls,
+ module_path: str,
+ *,
+ config_overrides: dict[str, Any] | None = None,
+ ) -> BagelChatTemplate:
+ del config_overrides
+ from .chat_template import BagelChatTemplate # deferred: see module-level note
+
+ return BagelChatTemplate(build_tokenizer(module_path))
+
+ def _tokenize_conversation_kwargs(self, inference: bool, **kwargs: Any) -> dict[str, Any]:
+ return {
+ "add_generation_prompt": inference,
+ "generation_kwargs": kwargs.get("generation_kwargs"),
+ }
+
+
+__all__ = [
+ "apply_image_marker",
+ "is_bagel_vision_marker",
+ "BagelTextEncoderPreprocessor",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/vae/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/__init__.py
new file mode 100644
index 0000000000..6ac7773a52
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/__init__.py
@@ -0,0 +1,31 @@
+"""BAGEL latent autoencoder module."""
+
+from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY
+
+
+@OMNI_CONFIG_REGISTRY.register("bagel_vae")
+def register_bagel_vae_config():
+ from .configuration import BagelVAEConfig
+
+ return BagelVAEConfig
+
+
+@OMNI_MODEL_REGISTRY.register("bagel_vae")
+def register_bagel_vae_model():
+ from .modeling import BagelVAE
+
+ return BagelVAE
+
+
+@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_vae")
+def register_bagel_vae_accelerated_model():
+ from .accelerated import BagelVAEAccelerated
+
+ return BagelVAEAccelerated
+
+
+@OMNI_PROCESSOR_REGISTRY.register("bagel_vae")
+def register_bagel_vae_processor():
+ from .processing import BagelVAEProcessor
+
+ return BagelVAEProcessor
diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py
new file mode 100644
index 0000000000..907701452f
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py
@@ -0,0 +1,14 @@
+"""Accelerated runtime for the bagel/vae OmniModule (conversation graph)."""
+
+from .accelerated import (
+ BAGEL_VAE_PIXEL_SHAPE,
+ BagelVAEAccelerated,
+ BagelVAEOfflineMixin,
+)
+
+
+__all__ = [
+ "BAGEL_VAE_PIXEL_SHAPE",
+ "BagelVAEAccelerated",
+ "BagelVAEOfflineMixin",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py
new file mode 100644
index 0000000000..cb0542b568
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py
@@ -0,0 +1,457 @@
+"""SeedOmni carrier hooks for BAGEL's latent VAE module — training-graph hooks only.
+
+``encode_context()`` and ``decode_generated()`` (the FSM generation nodes) live
+natively on :class:`~.modeling.BagelVAE` — this file only carries the training
+pre/forward/post hooks and offline-encoding-cache endpoints.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+import torch
+import torch.distributed as dist
+
+from veomni.utils.device import get_device_id, get_device_type
+
+from .......distributed.parallel_state import get_parallel_state
+from .......distributed.sequence_parallel import gather_outputs
+from .....mixins.base_mixin import BaseMixin
+from .....mixins.metric_meter_mixin import MetricMeterMixin
+from .....mixins.offline_encoding_mixin import OfflineEncodingMixin
+from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward
+from .....utils.conversation import ConversationItem
+from ...sources import BAGEL_CONTEXT_KEY, BAGEL_VAE_CONTEXT
+from ..configuration import BagelVAEConfig
+from ..modeling import BagelVAE, select_bagel_vae_context_items
+from ..processing import (
+ BAGEL_VAE_PIXEL_SHAPE,
+ BAGEL_VAE_POSTERIOR,
+ crop_latent_to_image_shape,
+ unpad_vae_item_pixel_values,
+)
+
+
+def _posterior_from_cache(
+ cache: torch.Tensor,
+ *,
+ z_channels: int,
+) -> tuple[tuple[torch.Tensor, torch.Tensor], bool]:
+ """Unpack ``(mean, logvar)`` from a cached ``(2, C, H, W)`` or ``(1, 2, C, H, W)`` posterior tensor."""
+ if cache.dim() == 5 and int(cache.shape[0]) == 1:
+ cache = cache.squeeze(0)
+ if cache.dim() == 4 and int(cache.shape[0]) == 2 and int(cache.shape[1]) == z_channels:
+ return (cache[0].unsqueeze(0), cache[1].unsqueeze(0)), True
+ raise ValueError(
+ "BAGEL VAE posterior cache tensor must be shaped (2, C, H, W) "
+ f"or singleton-batched (1, 2, C, H, W); got {tuple(cache.shape)}."
+ )
+
+
+class BagelVAEOfflineMixin(OfflineEncodingMixin):
+ """Offline-cache tensor endpoints and training-graph hooks for VAE encode / process."""
+
+ config: BagelVAEConfig
+ device: torch.device
+ dtype: torch.dtype
+ _conversation_carrier: list[list[ConversationItem]] | None
+ _encode_items: list[ConversationItem]
+ _sp_encode_count: int | None
+
+ def offline_encode(
+ self,
+ pixel_values: torch.Tensor | list[torch.Tensor],
+ **kwargs: object,
+ ) -> dict[str, Any]:
+ """Encode ``pixel_values`` into ``(mean, logvar)`` posterior tensor(s) for offline caching."""
+ del kwargs
+ if isinstance(pixel_values, list):
+ posteriors = self._encode_posterior_list(pixel_values)
+ return {"encoded_cache": [torch.cat(posterior, dim=0).to(dtype=self.dtype) for posterior in posteriors]}
+
+ pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype)
+ with self._runtime_context(pixel_values):
+ posterior = self._encode_posterior(pixel_values)
+ return {"encoded_cache": torch.cat(posterior, dim=1).to(dtype=self.dtype)}
+
+ def online_process(
+ self,
+ encoded_cache: torch.Tensor | list[torch.Tensor],
+ **kwargs: object,
+ ) -> dict[str, Any]:
+ """Sample scaled latent grids from cached ``(mean, logvar)`` posteriors without running the encoder."""
+ del kwargs
+ if not isinstance(encoded_cache, list):
+ encoded_cache = [encoded_cache]
+
+ latents = []
+ for cache in encoded_cache:
+ posterior, is_item_cache = _posterior_from_cache(cache, z_channels=int(self.config.z_channels))
+ latent = self._sample_scaled_latents(posterior)
+ if is_item_cache:
+ latent = latent.squeeze(0)
+ latents.append(latent.to(dtype=cache.dtype))
+ return {"latents": latents}
+
+ @post_forward("offline_encode")
+ def offline_encode_post(self, encoded_cache: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]:
+ """Gather SP shards, crop cached posteriors to each image's shape, and store on carrier items."""
+ if get_parallel_state().sp_size > 1:
+ if isinstance(encoded_cache, list):
+ encoded_cache = self._gather_encode_output_list(encoded_cache)
+ else:
+ encoded_cache = self._gather_encode_output(encoded_cache)
+
+ conversation = self._conversation_carrier
+ encode_items = self._encode_items
+ self._conversation_carrier = None
+ self._encode_items = []
+ self._sp_encode_count = None
+ self._sp_latent_shapes = None
+
+ z_channels = int(self.config.z_channels)
+ for item, cache_tensor in zip(encode_items, encoded_cache, strict=True):
+ item.type = "image"
+ if not isinstance(encoded_cache, list):
+ cache_tensor = crop_latent_to_image_shape(
+ cache_tensor,
+ item.meta.get(BAGEL_VAE_PIXEL_SHAPE),
+ downsample=int(self.config.downsample),
+ )
+ if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels:
+ cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:])
+ item.value = cache_tensor.detach().to(device=self.device, dtype=self.dtype)
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ item.meta[BAGEL_VAE_POSTERIOR] = True
+ return {"conversation_list": conversation}
+
+ @pre_forward("online_process")
+ def online_process_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Collect cached posterior tensors from VAE context items for ``online_process``."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+ self._encode_items = []
+ self._sp_encode_count = None
+
+ self._encode_items = select_bagel_vae_context_items(conversation_list)
+ encoded_cache: list[torch.Tensor] = []
+ for item in self._encode_items:
+ cache = item.value
+ if not torch.is_tensor(cache):
+ raise ValueError("BAGEL VAE online_process requires tensor posterior cache.")
+ encoded_cache.append(cache.detach().to(device=self._online_process_device()))
+ return {"encoded_cache": encoded_cache}
+
+ def _online_process_dtype(self) -> torch.dtype:
+ """Resolve the target dtype for ``online_process`` when the VAE is built on meta."""
+ config_dtype = getattr(self.config, "dtype", None) or getattr(self.config, "torch_dtype", None)
+ if isinstance(config_dtype, torch.dtype):
+ return config_dtype
+ if isinstance(config_dtype, str) and hasattr(torch, config_dtype):
+ dtype = getattr(torch, config_dtype)
+ if isinstance(dtype, torch.dtype):
+ return dtype
+ return torch.get_default_dtype()
+
+ def _online_process_device(self) -> torch.device:
+ """Resolve the target device for ``online_process`` when the VAE is built on meta."""
+ device_type = get_device_type()
+ if device_type == "cpu":
+ return torch.device("cpu")
+ return torch.device(device_type, get_device_id())
+
+
+class TrainingMixin(TrainingModuleMixin):
+ """Training-graph hooks — depends on :class:`BagelVAE` modeling APIs."""
+
+ config: BagelVAEConfig
+ device: torch.device
+ dtype: torch.dtype
+ _conversation_carrier: list[list[ConversationItem]] | None
+ _encode_items: list[ConversationItem]
+ _sp_encode_count: int | None
+
+ def __init__(self, *args, **kwargs) -> None:
+ super().__init__(*args, **kwargs)
+ self._conversation_carrier: list[list[ConversationItem]] | None = None
+ self._encode_items: list[ConversationItem] = []
+ self._sp_encode_count: int | None = None
+ self._sp_latent_shapes: list[tuple[int, int]] | None = None
+
+ @pre_forward("encode", "offline_encode")
+ def encode_pre(
+ self,
+ conversation_list: list[list[ConversationItem]] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """Collect unpadded VAE context images and slice across Ulysses SP ranks if enabled."""
+ del kwargs
+ self._conversation_carrier = conversation_list
+
+ self._encode_items = select_bagel_vae_context_items(conversation_list)
+
+ self._metric_meter_stash_latent_tokens(self._vae_latent_token_lengths())
+
+ pixel_values = [
+ unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True)
+ for item in self._encode_items
+ ]
+ ps = get_parallel_state()
+ if ps.sp_size == 1:
+ self._sp_encode_count = None
+ self._sp_latent_shapes = None
+ return {"pixel_values": pixel_values}
+
+ if ps.cp_size != 1:
+ raise ValueError(f"BAGEL VAE training supports Ulysses groups only; got cp_size={ps.cp_size}.")
+
+ self._sp_encode_count = len(pixel_values)
+ if self._sp_encode_count == 0:
+ raise ValueError("BAGEL VAE SP requires at least one real or dummy image.")
+
+ pad_count = (-self._sp_encode_count) % ps.sp_size
+ padded_pixels = pixel_values + [pixel_values[-1]] * pad_count
+ ds = max(int(self.config.downsample), 1)
+ self._sp_latent_shapes = [(int(image.shape[-2]) // ds, int(image.shape[-1]) // ds) for image in padded_pixels]
+ per_rank = len(padded_pixels) // ps.sp_size
+ start = ps.sp_rank * per_rank
+ return {"pixel_values": padded_pixels[start : start + per_rank]}
+
+ def encode(
+ self,
+ pixel_values: torch.Tensor | list[torch.Tensor],
+ **kwargs: object,
+ ) -> dict[str, Any]:
+ """Encode ``pixel_values``; under SP, defer posterior sampling until ``encode_post`` gathers all shards."""
+ if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None:
+ return {"latents": self.offline_encode(pixel_values=pixel_values, **kwargs)["encoded_cache"]}
+ return super().encode(pixel_values=pixel_values, **kwargs)
+
+ @post_forward("encode", "online_process")
+ def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]:
+ """Gather SP posterior shards, sample scaled latents after gather, and write latents onto conversation items."""
+ if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None:
+ if isinstance(latents, list):
+ gathered_cache = self._gather_encode_output_list(latents)
+ latents = self._sample_posteriors_list(gathered_cache)
+ else:
+ gathered_cache = self._gather_encode_output(latents)
+ latents = self._sample_posterior_tensor(gathered_cache, self._encode_items)
+
+ conversation = self._conversation_carrier
+ encode_items = self._encode_items
+ self._conversation_carrier = None
+ self._encode_items = []
+ self._sp_encode_count = None
+ self._sp_latent_shapes = None
+
+ if isinstance(latents, list):
+ for item, latent in zip(encode_items, latents, strict=True):
+ item.type = "image"
+ item.value = latent
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ else:
+ for item, latent in zip(encode_items, latents, strict=True):
+ item.type = "image"
+ latent = crop_latent_to_image_shape(
+ latent,
+ item.meta.get(BAGEL_VAE_PIXEL_SHAPE),
+ downsample=int(self.config.downsample),
+ )
+ item.value = latent
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ return {"conversation_list": conversation}
+
+ def _sample_posteriors_list(self, encoded_cache: list[torch.Tensor]) -> list[torch.Tensor]:
+ """Sample scaled latents in batch order from gathered ``(mean, logvar)`` posteriors."""
+ z_channels = int(self.config.z_channels)
+ latents: list[torch.Tensor] = []
+ for cache in encoded_cache:
+ if cache.dim() == 3 and int(cache.shape[0]) == 2 * z_channels:
+ cache = cache.reshape(2, z_channels, *cache.shape[-2:])
+ posterior, is_item_cache = _posterior_from_cache(cache, z_channels=z_channels)
+ with self._runtime_context(posterior[0]):
+ latent = self._sample_scaled_latents(posterior)
+ if is_item_cache:
+ latent = latent.squeeze(0)
+ latents.append(latent.to(dtype=cache.dtype))
+ return latents
+
+ def _sample_posterior_tensor(
+ self,
+ encoded_cache: torch.Tensor,
+ encode_items: list[ConversationItem],
+ ) -> list[torch.Tensor]:
+ """Crop and sample scaled latents in batch order from a gathered 4-D posterior tensor."""
+ z_channels = int(self.config.z_channels)
+ cropped_caches: list[torch.Tensor] = []
+ for item, cache_tensor in zip(encode_items, encoded_cache, strict=True):
+ cache_tensor = crop_latent_to_image_shape(
+ cache_tensor,
+ item.meta.get(BAGEL_VAE_PIXEL_SHAPE),
+ downsample=int(self.config.downsample),
+ )
+ if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels:
+ cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:])
+ cropped_caches.append(cache_tensor)
+ return self._sample_posteriors_list(cropped_caches)
+
+ def _encode_posterior_list(
+ self,
+ pixel_values: list[torch.Tensor],
+ *,
+ min_passes: int = 0,
+ ) -> list[tuple[torch.Tensor, torch.Tensor]]:
+ """Align the number of encoder passes across ``fsdp_group`` / ``sp_group`` so FSDP unshards stay in lockstep."""
+ if pixel_values and dist.is_initialized():
+ ps = get_parallel_state()
+ shard_group = ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None)
+ if shard_group is not None:
+ local_shapes = {(int(image.shape[-2]), int(image.shape[-1])) for image in pixel_values}
+ passes_tensor = torch.tensor(
+ max(len(local_shapes), min_passes),
+ device=self._encoder_device,
+ dtype=torch.int64,
+ )
+ dist.all_reduce(passes_tensor, op=dist.ReduceOp.MAX, group=shard_group)
+ min_passes = int(passes_tensor.item())
+ return super()._encode_posterior_list(pixel_values, min_passes=min_passes)
+
+ def _gather_encode_output(self, output: torch.Tensor) -> torch.Tensor:
+ """All-gather batch-sharded VAE outputs across the SP group and trim SP batch padding."""
+ if self._sp_encode_count is None:
+ raise RuntimeError("BAGEL VAE SP image count was not initialized.")
+ output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group)
+ return output.narrow(0, 0, self._sp_encode_count)
+
+ def _gather_encode_output_list(self, outputs: list[torch.Tensor]) -> list[torch.Tensor]:
+ """All-gather variable-shape per-image VAE outputs across the SP group and trim SP padding."""
+ if self._sp_encode_count is None or self._sp_latent_shapes is None:
+ raise RuntimeError("BAGEL VAE SP image shapes were not initialized.")
+ ps = get_parallel_state()
+ per_rank = len(self._sp_latent_shapes) // ps.sp_size
+ if len(outputs) != per_rank:
+ raise RuntimeError(f"BAGEL VAE SP local output count mismatch: expected {per_rank}, got {len(outputs)}.")
+ leading_shape = tuple(int(dim) for dim in outputs[0].shape[:-2])
+ leading_numel = math.prod(leading_shape)
+ rank_numels = [
+ sum(
+ leading_numel * height * width
+ for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank]
+ )
+ for rank in range(ps.sp_size)
+ ]
+ max_rank_numel = max(rank_numels)
+ flat_local = torch.cat([tensor.reshape(-1) for tensor in outputs], dim=0)
+ if int(flat_local.numel()) < max_rank_numel:
+ flat_local = torch.nn.functional.pad(flat_local, (0, max_rank_numel - int(flat_local.numel())))
+ gathered = gather_outputs(flat_local.unsqueeze(0), gather_dim=0, group=ps.sp_group)
+
+ restored: list[torch.Tensor] = []
+ for rank in range(ps.sp_size):
+ offset = 0
+ rank_flat = gathered[rank]
+ for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank]:
+ numel = leading_numel * height * width
+ restored.append(rank_flat[offset : offset + numel].reshape(*leading_shape, height, width))
+ offset += numel
+ if len(restored) == self._sp_encode_count:
+ return restored
+ return restored
+
+ def _vae_latent_token_lengths(self) -> list[int]:
+ """Compute the unpadded latent grid token count per encoded image for FLOPs metering."""
+ # Latent tokens per image = (H // ds) * (W // ds), using the real pixel shape
+ ds = max(int(self.config.downsample), 1)
+ lengths: list[int] = []
+ for item in self._encode_items:
+ shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE)
+ if torch.is_tensor(shape):
+ dims = shape.detach().reshape(-1).tolist()
+ height, width = int(dims[0]), int(dims[1])
+ else:
+ height, width = int(item.value.shape[-2]), int(item.value.shape[-1])
+ lengths.append((height // ds) * (width // ds))
+ return lengths
+
+ def _metric_meter_stash_latent_tokens(self, lengths: list[int]) -> None:
+ # currently use the same counts for encode and offline_encode
+ self.metric_meter_set_seqlens("encode", lengths)
+ self.metric_meter_set_seqlens("offline_encode", lengths)
+
+
+class MeterMixin(MetricMeterMixin):
+ """Per-module training meter for BAGEL's latent VAE codec (FLUX-style conv AE)."""
+
+ config: BagelVAEConfig
+
+ def estimate_flops(self, seqlens: list[int]) -> float:
+ """Estimate TFLOPs for the FLUX-style convolutional VAE encoder over ``seqlens``."""
+ cfg = self.config
+
+ if getattr(cfg, "freeze", True):
+ return 0.0
+
+ ch = int(cfg.ch)
+ ch_mult = [int(m) for m in cfg.ch_mult]
+ in_ch_mult = [1, *ch_mult]
+ num_res = len(ch_mult)
+ num_res_blocks = int(cfg.num_res_blocks)
+ in_channels = int(cfg.in_channels)
+ z_channels = int(cfg.z_channels)
+
+ # Conv MACs per latent token (linear term): area(level i) = N · 4**(num_res-1-i).
+ def area_factor(level: int) -> int:
+ return 4 ** (num_res - 1 - level)
+
+ lin_macs_per_token = in_channels * ch * 9 * area_factor(0) # conv_in (3×3)
+
+ for i in range(num_res):
+ af = area_factor(i)
+ block_in = ch * in_ch_mult[i]
+ block_out = ch * ch_mult[i]
+ for j in range(num_res_blocks):
+ cin = block_in if j == 0 else block_out
+ lin_macs_per_token += cin * block_out * 9 * af # ResnetBlock.conv1
+ lin_macs_per_token += block_out * block_out * 9 * af # ResnetBlock.conv2
+ if j == 0 and block_in != block_out:
+ lin_macs_per_token += block_in * block_out * af # 1×1 nin_shortcut
+ if i != num_res - 1:
+ # Strided Downsample (3×3) emits the next level's spatial area.
+ lin_macs_per_token += block_out * block_out * 9 * area_factor(i + 1)
+
+ # Mid block at the latent resolution (area factor 1): two ResnetBlocks + AttnBlock.
+ bc = ch * ch_mult[-1]
+ lin_macs_per_token += 4 * (bc * bc * 9) # block_1 + block_2 (conv1 + conv2 each)
+ lin_macs_per_token += 4 * (bc * bc) # attn q/k/v/proj_out are 1×1 convs
+ lin_macs_per_token += bc * (2 * z_channels) * 9 # conv_out (3×3)
+
+ tokens = sum(seqlens)
+ # Mid self-attention (single head, dim = bc): QK^T + attn·V ⇒ 2·N**2·bc MACs.
+ attn_macs = sum(2 * (n * n) * bc for n in seqlens)
+
+ total_macs = lin_macs_per_token * tokens + attn_macs
+ return 6 * total_macs / 1e12
+
+
+class VeOmniMixin(BaseMixin, BagelVAEOfflineMixin, TrainingMixin, MeterMixin):
+ """Carrier hooks for raw-image VAE encode and latent decode.
+
+ ``encode_context()`` / ``decode_generated()`` already live on the native
+ :class:`~.modeling.BagelVAE` (via its own :class:`~.modeling.InferenceMixin`),
+ so no ``InferenceMixin`` is needed here.
+ """
+
+ config: BagelVAEConfig
+
+
+class BagelVAEAccelerated(VeOmniMixin, BagelVAE):
+ _no_split_modules: list[str] = []
+
+
+__all__ = ["BAGEL_VAE_PIXEL_SHAPE", "BagelVAEAccelerated", "BagelVAEOfflineMixin"]
diff --git a/veomni/models/seed_omni/modules/bagel/vae/configuration.py b/veomni/models/seed_omni/modules/bagel/vae/configuration.py
new file mode 100644
index 0000000000..c7cc266ec2
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/configuration.py
@@ -0,0 +1,52 @@
+"""Configuration for BAGEL's latent VAE module."""
+
+from ...module_configuration_base import OmniModuleConfig
+
+
+class BagelVAEConfig(OmniModuleConfig):
+ """BAGEL FLUX-style latent autoencoder config."""
+
+ model_type = "bagel_vae"
+
+ def __init__(
+ self,
+ resolution: int = 256,
+ in_channels: int = 3,
+ downsample: int = 8,
+ ch: int = 128,
+ out_ch: int = 3,
+ ch_mult: list[int] | None = None,
+ num_res_blocks: int = 2,
+ z_channels: int = 16,
+ scale_factor: float = 0.3611,
+ shift_factor: float = 0.1159,
+ max_image_size: int = 1024,
+ min_image_size: int = 512,
+ image_stride: int = 16,
+ max_pixels: int = 14 * 14 * 9 * 1024,
+ image_mean: list[float] | None = None,
+ image_std: list[float] | None = None,
+ freeze: bool = True,
+ **kwargs,
+ ) -> None:
+ self.resolution = resolution
+ self.in_channels = in_channels
+ self.downsample = downsample
+ self.ch = ch
+ self.out_ch = out_ch
+ self.ch_mult = [1, 2, 4, 4] if ch_mult is None else ch_mult
+ self.num_res_blocks = num_res_blocks
+ self.z_channels = z_channels
+ self.scale_factor = scale_factor
+ self.shift_factor = shift_factor
+ self.max_image_size = max_image_size
+ self.min_image_size = min_image_size
+ self.image_stride = image_stride
+ self.max_pixels = max_pixels
+ self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean
+ self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std
+ self.freeze = freeze
+ super().__init__(**kwargs)
+
+
+__all__ = ["BagelVAEConfig"]
diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py
new file mode 100644
index 0000000000..fa518bd977
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py
@@ -0,0 +1,552 @@
+"""BAGEL latent VAE module.
+
+Tensor call-sites:
+* ``bagel_vae.encode`` maps normalized image tensors to scaled latent grids.
+* ``bagel_vae.decode`` maps generated latent grids to decoded image tensors.
+
+The VAE module is a codec boundary only. Flow timestep/noise sampling, latent
+patchification, packed MoT indexes, and conversation-carrier mutation belong to
+the Bagel VAE module mixin and downstream Bagel nodes.
+
+``encode_context()`` (image-understanding VAE encode) and ``decode_generated()``
+(final generated-latent decode) are the FSM generation-node methods, native here
+so pure HF inference (no accelerated mixins) can run BAGEL's generation graphs.
+"""
+
+from __future__ import annotations
+
+from contextlib import contextmanager, nullcontext
+from typing import Any
+
+import torch
+import torch.nn as nn
+from einops import rearrange
+from torch import Tensor
+
+from ....utils.conversation import ConversationItem, iter_desired_items
+from ...module_modeling_base import PretrainedOmniModule
+from ..sources import BAGEL_CONTEXT_KEY, BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_VAE_CONTEXT
+from .configuration import BagelVAEConfig
+from .processing import (
+ BagelVAEPreprocessor,
+ BagelVAEProcessor,
+ unpad_vae_item_pixel_values,
+)
+
+
+def select_bagel_vae_context_items(
+ conversation_list: list[list[ConversationItem]] | None,
+ *,
+ exclude_dummy: bool = False,
+) -> list[ConversationItem]:
+ """Select VAE context image items (training includes dummies; inference skips them)."""
+ if conversation_list is None:
+ raise ValueError("BagelVAE requires conversation_list to select VAE context items.")
+
+ items = list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}))
+ if exclude_dummy:
+ return [item for item in items if not item.is_dummy]
+ return items
+
+
+class InferenceMixin:
+ """FSM ``encode_context`` / ``decode_generated`` — HF ``GenerationMixin`` analog.
+
+ Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in
+ :class:`BagelVAE`'s bases for consistency with every other module's
+ native / accelerated split (see ``janus/llama/modeling.py`` for the full
+ MRO rationale where a competing no-op default exists — this module
+ doesn't override ``reset_local_inference_state`` / ``finalize`` so there
+ is nothing to shadow here).
+ """
+
+ def encode_context(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """FSM node: encode image-understanding context images into VAE latents."""
+ del generation_kwargs, kwargs
+ if conversation_list is None:
+ return {"conversation_list": conversation_list}
+
+ batched = [conversation_list]
+ image_items = select_bagel_vae_context_items(batched, exclude_dummy=True)
+ if not image_items:
+ return {"conversation_list": conversation_list}
+
+ pixel_values = [
+ unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True)
+ for item in image_items
+ ]
+ outputs = self.encode(pixel_values=pixel_values)
+ for image_item, latent in zip(image_items, outputs["latents"], strict=True):
+ image_item.value = latent.to(device=self.device, dtype=self.dtype)
+ image_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ return {"conversation_list": conversation_list}
+
+ def decode_generated(
+ self,
+ conversation_list: list[ConversationItem] | None = None,
+ generation_kwargs: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> dict[str, Any]:
+ """FSM node: decode the flow connector's completed latent into a PIL image."""
+ del generation_kwargs, kwargs
+ if conversation_list is None:
+ return {"conversation_list": conversation_list}
+
+ batched = [conversation_list]
+ decode_items = self._select_vae_decode_items(batched)
+ if not decode_items:
+ return {"conversation_list": conversation_list}
+
+ latents = []
+ for item in decode_items:
+ latents.append(
+ item.value.detach().squeeze(0)
+ if item.value.dim() == 4 and item.value.shape[0] == 1
+ else item.value.detach()
+ )
+ latents = torch.stack(latents, dim=0).to(device=self.device, dtype=self.dtype)
+
+ outputs = self.decode(latents=latents)
+ pixel_values = outputs["pixel_values"]
+ for item, image in zip(decode_items, pixel_values, strict=True):
+ item.type = "image"
+ item.value = image.to(device=self.device, dtype=self.dtype)
+ if self._image_processor is None:
+ raise RuntimeError(
+ "BagelVAE: cannot postprocess decoded image — no image processor was "
+ "loaded. Ensure `preprocessor_config.json` ships next to the weights."
+ )
+ return {
+ "conversation_list": conversation_list,
+ "generated": {"type": "image", "value": self._image_processor.postprocess(pixel_values[-1])[0]},
+ }
+
+ def _select_vae_decode_items(
+ self, conversation_list: list[list[ConversationItem]] | None
+ ) -> list[ConversationItem]:
+ """Select non-dummy ``generated_latent`` output items for VAE image decoding."""
+ if conversation_list is None:
+ raise ValueError("BagelVAE decode requires conversation_list to select latent items.")
+
+ # Final image decode consumes the completed latent emitted by the flow connector.
+ decode_items: list[ConversationItem] = []
+ for item in iter_desired_items(
+ conversation_list, types=["output"], meta={BAGEL_PHASE_KEY: [BAGEL_GENERATED_LATENT]}
+ ):
+ if not item.is_dummy:
+ decode_items.append(item)
+ return decode_items
+
+
+class BagelVAE(InferenceMixin, PretrainedOmniModule):
+ config_class = BagelVAEConfig
+ image_processor_class = BagelVAEProcessor
+ preprocessor_class = BagelVAEPreprocessor
+ base_model_prefix = "bagel_vae"
+ main_input_name = "pixel_values"
+ _no_split_modules: list[str] = ["ResnetBlock", "AttnBlock"]
+ supports_gradient_checkpointing = False
+
+ def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs):
+ # The VAE uses fixed local torch SDPA in its mid-block, with no HF backend dispatch.
+ return attn_implementation
+
+ def __init__(self, config: BagelVAEConfig, **kwargs: Any) -> None:
+ super().__init__(config, **kwargs)
+ self.encoder = Encoder(
+ resolution=config.resolution,
+ in_channels=config.in_channels,
+ ch=config.ch,
+ ch_mult=config.ch_mult,
+ num_res_blocks=config.num_res_blocks,
+ z_channels=config.z_channels,
+ )
+ self.decoder = Decoder(
+ resolution=config.resolution,
+ in_channels=config.in_channels,
+ ch=config.ch,
+ out_ch=config.out_ch,
+ ch_mult=config.ch_mult,
+ num_res_blocks=config.num_res_blocks,
+ z_channels=config.z_channels,
+ )
+ self._image_processor: BagelVAEProcessor | None = None
+ self.post_init()
+
+ def freeze_model(self) -> None:
+ """Set the VAE to eval mode and disable gradients when ``config.freeze`` is true."""
+ if self.config.freeze:
+ self.eval()
+ self.requires_grad_(False)
+
+ @property
+ def _encoder_device(self) -> torch.device:
+ return self.encoder.conv_in.weight.device
+
+ @property
+ def _decoder_device(self) -> torch.device:
+ return self.decoder.conv_in.weight.device
+
+ @contextmanager
+ def _runtime_context(self, tensor: torch.Tensor):
+ """Enter ``no_grad`` (when frozen) and autocast for VAE encoder/decoder execution."""
+ grad_context = torch.enable_grad()
+ if self.config.freeze:
+ grad_context = torch.no_grad()
+
+ autocast_context = nullcontext()
+ if self.dtype != torch.float32:
+ autocast_context = torch.amp.autocast(tensor.device.type, enabled=True, dtype=self.dtype)
+
+ with grad_context, autocast_context:
+ yield
+
+ def encode(
+ self,
+ pixel_values: torch.Tensor | list[torch.Tensor],
+ **kwargs: object,
+ ) -> dict[str, Any]:
+ """Encode ``pixel_values`` into scaled VAE latent grids."""
+ del kwargs
+ if isinstance(pixel_values, list):
+ posteriors = self._encode_posterior_list(pixel_values)
+ latents: list[torch.Tensor] = []
+ for posterior in posteriors:
+ with self._runtime_context(posterior[0]):
+ latents.append(self._sample_scaled_latents(posterior).squeeze(0).to(dtype=self.dtype))
+ return {"latents": latents}
+
+ pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype)
+ with self._runtime_context(pixel_values):
+ posterior = self._encode_posterior(pixel_values)
+ latents = self._sample_scaled_latents(posterior)
+ return {"latents": latents.to(dtype=self.dtype)}
+
+ def decode(
+ self,
+ latents: torch.Tensor,
+ **kwargs: object,
+ ) -> dict[str, Any]:
+ """Unscale ``latents`` and decode them back to normalized pixel tensors."""
+ del kwargs
+ latents = latents.to(device=self._decoder_device, dtype=self.dtype)
+ latents = latents / self.config.scale_factor + self.config.shift_factor
+ with self._runtime_context(latents):
+ pixel_values = self.decoder(latents)
+ return {"pixel_values": pixel_values.to(dtype=self.dtype)}
+
+ def _encode_posterior(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ return torch.chunk(self.encoder(pixel_values), 2, dim=1)
+
+ def _encode_posterior_list(
+ self,
+ pixel_values: list[torch.Tensor],
+ *,
+ min_passes: int = 0,
+ ) -> list[tuple[torch.Tensor, torch.Tensor]]:
+ """Encode a list of ``[C, H, W]`` images at their native shapes, batching equal shapes."""
+ if not pixel_values:
+ return []
+
+ shape_groups: dict[tuple[int, int], list[int]] = {}
+ normalized: list[torch.Tensor] = []
+ for idx, image in enumerate(pixel_values):
+ if image.dim() == 4 and int(image.shape[0]) == 1:
+ image = image.squeeze(0)
+ if image.dim() != 3:
+ raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(image.shape)}.")
+ normalized.append(image)
+ shape_key = (int(image.shape[-2]), int(image.shape[-1]))
+ shape_groups.setdefault(shape_key, []).append(idx)
+
+ group_indices = list(shape_groups.values())
+ posteriors: list[tuple[torch.Tensor, torch.Tensor] | None] = [None] * len(normalized)
+ for indices in group_indices:
+ batch = torch.stack([normalized[i] for i in indices], dim=0).to(
+ device=self._encoder_device, dtype=self.dtype
+ )
+ with self._runtime_context(batch):
+ mean_batch, logvar_batch = self._encode_posterior(batch)
+ for idx, mean_i, logvar_i in zip(indices, mean_batch, logvar_batch, strict=True):
+ posteriors[idx] = (mean_i.unsqueeze(0), logvar_i.unsqueeze(0))
+
+ if min_passes > len(group_indices):
+ dummy_batch = normalized[0].unsqueeze(0).to(device=self._encoder_device, dtype=self.dtype)
+ for _ in range(min_passes - len(group_indices)):
+ with self._runtime_context(dummy_batch):
+ mean_dummy, logvar_dummy = self._encode_posterior(dummy_batch)
+ assert posteriors[0] is not None
+ anchor = (mean_dummy.sum() + logvar_dummy.sum()) * 0.0
+ posteriors[0] = (posteriors[0][0] + anchor, posteriors[0][1])
+
+ return posteriors # type: ignore[return-value]
+
+ def _sample_scaled_latents(self, posterior: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
+ mean, logvar = posterior
+ latents = mean + torch.exp(0.5 * logvar) * torch.randn_like(mean)
+ return self.config.scale_factor * (latents - self.config.shift_factor)
+
+
+def swish(x: Tensor) -> Tensor:
+ return x * torch.sigmoid(x)
+
+
+class AttnBlock(nn.Module):
+ def __init__(self, in_channels: int) -> None:
+ super().__init__()
+ self.in_channels = in_channels
+ self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
+ self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+ self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+ self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+ self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1)
+
+ def attention(self, hidden_states: Tensor) -> Tensor:
+ hidden_states = self.norm(hidden_states)
+ query = self.q(hidden_states)
+ key = self.k(hidden_states)
+ value = self.v(hidden_states)
+
+ batch, channels, height, width = query.shape
+ query = rearrange(query, "b c h w -> b 1 (h w) c").contiguous()
+ key = rearrange(key, "b c h w -> b 1 (h w) c").contiguous()
+ value = rearrange(value, "b c h w -> b 1 (h w) c").contiguous()
+ hidden_states = nn.functional.scaled_dot_product_attention(query, key, value)
+ return rearrange(hidden_states, "b 1 (h w) c -> b c h w", h=height, w=width, c=channels, b=batch)
+
+ def forward(self, x: Tensor) -> Tensor:
+ return x + self.proj_out(self.attention(x))
+
+
+class ResnetBlock(nn.Module):
+ def __init__(self, in_channels: int, out_channels: int) -> None:
+ super().__init__()
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+ self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
+ self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True)
+ self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ if self.in_channels != self.out_channels:
+ self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
+
+ def forward(self, x: Tensor) -> Tensor:
+ hidden_states = self.norm1(x)
+ hidden_states = swish(hidden_states)
+ hidden_states = self.conv1(hidden_states)
+ hidden_states = self.norm2(hidden_states)
+ hidden_states = swish(hidden_states)
+ hidden_states = self.conv2(hidden_states)
+
+ if self.in_channels != self.out_channels:
+ x = self.nin_shortcut(x)
+ return x + hidden_states
+
+
+class Downsample(nn.Module):
+ def __init__(self, in_channels: int) -> None:
+ super().__init__()
+ self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)
+
+ def forward(self, x: Tensor) -> Tensor:
+ x = nn.functional.pad(x, (0, 1, 0, 1), mode="constant", value=0)
+ return self.conv(x)
+
+
+class Upsample(nn.Module):
+ def __init__(self, in_channels: int) -> None:
+ super().__init__()
+ self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x: Tensor) -> Tensor:
+ x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")
+ return self.conv(x)
+
+
+class Encoder(nn.Module):
+ def __init__(
+ self,
+ resolution: int,
+ in_channels: int,
+ ch: int,
+ ch_mult: list[int],
+ num_res_blocks: int,
+ z_channels: int,
+ ) -> None:
+ super().__init__()
+ self.gradient_checkpointing = False
+ self.ch = ch
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.resolution = resolution
+ self.in_channels = in_channels
+ self.conv_in = nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1)
+
+ in_ch_mult = (1,) + tuple(ch_mult)
+ self.in_ch_mult = in_ch_mult
+ self.down = nn.ModuleList()
+ block_in = self.ch
+ for i_level in range(self.num_resolutions):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_in = ch * in_ch_mult[i_level]
+ block_out = ch * ch_mult[i_level]
+ for _ in range(self.num_res_blocks):
+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+ block_in = block_out
+ down = nn.Module()
+ down.block = block
+ down.attn = attn
+ if i_level != self.num_resolutions - 1:
+ down.downsample = Downsample(block_in)
+ self.down.append(down)
+
+ self.mid = nn.Module()
+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+ self.mid.attn_1 = AttnBlock(block_in)
+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+ self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
+ self.conv_out = nn.Conv2d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, x: Tensor) -> Tensor:
+ """Encode input image tensor ``x`` into ``2 * z_channels`` mean/logvar parameters."""
+ hidden_stack = [self.conv_in(x)]
+ for i_level in range(self.num_resolutions):
+ for i_block in range(self.num_res_blocks):
+ block = self.down[i_level].block[i_block]
+ hidden_states = (
+ self._gradient_checkpointing_func(block.__call__, hidden_stack[-1])
+ if self.gradient_checkpointing and self.training
+ else block(hidden_stack[-1])
+ )
+ if len(self.down[i_level].attn) > 0:
+ attn = self.down[i_level].attn[i_block]
+ hidden_states = (
+ self._gradient_checkpointing_func(attn.__call__, hidden_states)
+ if self.gradient_checkpointing and self.training
+ else attn(hidden_states)
+ )
+ hidden_stack.append(hidden_states)
+ if i_level != self.num_resolutions - 1:
+ downsample = self.down[i_level].downsample
+ hidden_stack.append(
+ self._gradient_checkpointing_func(downsample.__call__, hidden_stack[-1])
+ if self.gradient_checkpointing and self.training
+ else downsample(hidden_stack[-1])
+ )
+
+ hidden_states = hidden_stack[-1]
+ if self.gradient_checkpointing and self.training:
+ hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states)
+ hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states)
+ hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states)
+ else:
+ hidden_states = self.mid.block_1(hidden_states)
+ hidden_states = self.mid.attn_1(hidden_states)
+ hidden_states = self.mid.block_2(hidden_states)
+ hidden_states = self.norm_out(hidden_states)
+ hidden_states = swish(hidden_states)
+ return self.conv_out(hidden_states)
+
+
+class Decoder(nn.Module):
+ def __init__(
+ self,
+ ch: int,
+ out_ch: int,
+ ch_mult: list[int],
+ num_res_blocks: int,
+ in_channels: int,
+ resolution: int,
+ z_channels: int,
+ ) -> None:
+ super().__init__()
+ self.gradient_checkpointing = False
+ self.ch = ch
+ self.num_resolutions = len(ch_mult)
+ self.num_res_blocks = num_res_blocks
+ self.resolution = resolution
+ self.in_channels = in_channels
+ self.ffactor = 2 ** (self.num_resolutions - 1)
+ block_in = ch * ch_mult[self.num_resolutions - 1]
+ curr_res = resolution // 2 ** (self.num_resolutions - 1)
+ self.z_shape = (1, z_channels, curr_res, curr_res)
+
+ self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1)
+
+ self.mid = nn.Module()
+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+ self.mid.attn_1 = AttnBlock(block_in)
+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)
+
+ self.up = nn.ModuleList()
+ for i_level in reversed(range(self.num_resolutions)):
+ block = nn.ModuleList()
+ attn = nn.ModuleList()
+ block_out = ch * ch_mult[i_level]
+ for _ in range(self.num_res_blocks + 1):
+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))
+ block_in = block_out
+ up = nn.Module()
+ up.block = block
+ up.attn = attn
+ if i_level != 0:
+ up.upsample = Upsample(block_in)
+ self.up.insert(0, up)
+
+ self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True)
+ self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1)
+
+ def forward(self, z: Tensor) -> Tensor:
+ """Decode latent tensor ``z`` into reconstructed RGB image space."""
+ hidden_states = self.conv_in(z)
+
+ if self.gradient_checkpointing and self.training:
+ hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states)
+ hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states)
+ hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states)
+ else:
+ hidden_states = self.mid.block_1(hidden_states)
+ hidden_states = self.mid.attn_1(hidden_states)
+ hidden_states = self.mid.block_2(hidden_states)
+
+ for i_level in reversed(range(self.num_resolutions)):
+ for i_block in range(self.num_res_blocks + 1):
+ block = self.up[i_level].block[i_block]
+ hidden_states = (
+ self._gradient_checkpointing_func(block.__call__, hidden_states)
+ if self.gradient_checkpointing and self.training
+ else block(hidden_states)
+ )
+ if len(self.up[i_level].attn) > 0:
+ attn = self.up[i_level].attn[i_block]
+ hidden_states = (
+ self._gradient_checkpointing_func(attn.__call__, hidden_states)
+ if self.gradient_checkpointing and self.training
+ else attn(hidden_states)
+ )
+ if i_level != 0:
+ upsample = self.up[i_level].upsample
+ hidden_states = (
+ self._gradient_checkpointing_func(upsample.__call__, hidden_states)
+ if self.gradient_checkpointing and self.training
+ else upsample(hidden_states)
+ )
+
+ hidden_states = self.norm_out(hidden_states)
+ hidden_states = swish(hidden_states)
+ return self.conv_out(hidden_states)
+
+
+__all__ = [
+ "BagelVAE",
+ "BagelVAEConfig",
+ "InferenceMixin",
+ "select_bagel_vae_context_items",
+]
diff --git a/veomni/models/seed_omni/modules/bagel/vae/processing.py b/veomni/models/seed_omni/modules/bagel/vae/processing.py
new file mode 100644
index 0000000000..879b019a94
--- /dev/null
+++ b/veomni/models/seed_omni/modules/bagel/vae/processing.py
@@ -0,0 +1,584 @@
+"""Image processor + worker-side preprocessor for BAGEL VAE.
+
+:class:`BagelVAEPreprocessor` is the picklable, weight-free CPU worker
+counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) —
+built straight off the checkpoint dir, with no model instance involved. Like
+:mod:`..siglip_navit.processing`, BAGEL ships no separate
+``preprocessor_config.json``: the image processor is derived from the
+module's own ``config.json`` via :meth:`BagelVAEProcessor.from_config`.
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from torchvision.transforms import InterpolationMode
+from torchvision.transforms import functional as TVF
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+
+from ....utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem, iter_desired_items
+from ...module_processing_base import ModulePreprocessorBase
+from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context
+from .configuration import BagelVAEConfig
+
+
+_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"})
+
+
+class BagelVAEProcessor(BaseImageProcessor):
+ """BAGEL VAE image processor.
+
+ Owns raw-image resize and normalization for VAE encode. Carrier selection,
+ context tagging, and latent scatter stay in the module mixin.
+ """
+
+ model_input_names = ["pixel_values"]
+
+ def __init__(
+ self,
+ image_stride: int = 16,
+ max_image_size: int = 1024,
+ min_image_size: int = 512,
+ max_pixels: int = 14 * 14 * 9 * 1024,
+ image_mean: list[float] | None = None,
+ image_std: list[float] | None = None,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.image_stride = image_stride
+ self.max_image_size = max_image_size
+ self.min_image_size = min_image_size
+ self.max_pixels = max_pixels
+ self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean
+ self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std
+
+ @classmethod
+ def from_config(cls, config: Any) -> BagelVAEProcessor:
+ return cls(
+ image_stride=int(config.image_stride),
+ max_image_size=int(config.max_image_size),
+ min_image_size=int(config.min_image_size),
+ max_pixels=int(config.max_pixels),
+ image_mean=list(config.image_mean),
+ image_std=list(config.image_std),
+ )
+
+ def preprocess(
+ self,
+ images: Any,
+ *,
+ return_tensors: str | None = None,
+ device: torch.device | None = None,
+ dtype: torch.dtype | None = None,
+ **kwargs: Any,
+ ) -> BatchFeature:
+ """Resize, normalize, and batch-pad ``images`` into ``pixel_values`` and ``pixel_shapes``."""
+ del return_tensors, kwargs
+ image_list = images if isinstance(images, list) else [images]
+ pixel_values = [
+ _preprocess_image(
+ image,
+ image_stride=self.image_stride,
+ max_image_size=self.max_image_size,
+ min_image_size=self.min_image_size,
+ max_pixels=self.max_pixels,
+ image_mean=self.image_mean,
+ image_std=self.image_std,
+ )
+ for image in image_list
+ ]
+ pixel_shapes = torch.tensor(
+ [[int(pixel.shape[-2]), int(pixel.shape[-1])] for pixel in pixel_values],
+ dtype=torch.long,
+ )
+ pixel_values = torch.stack(_pad_to_batch_size(pixel_values), dim=0)
+
+ if dtype is not None:
+ pixel_values = pixel_values.to(dtype=dtype)
+ if device is not None:
+ pixel_values = pixel_values.to(device=device)
+ pixel_shapes = pixel_shapes.to(device=device)
+ return BatchFeature(data={"pixel_values": pixel_values, "pixel_shapes": pixel_shapes})
+
+ def postprocess(self, images: torch.Tensor | list[torch.Tensor], **kwargs: Any) -> list[Image.Image]:
+ """Convert decoded VAE image tensors in ``[-1, 1]`` back to RGB PIL images."""
+ del kwargs
+ image_list = images if isinstance(images, list) else [images]
+ out: list[Image.Image] = []
+ for image in image_list:
+ if image.dim() == 4:
+ out.extend(_decoded_tensor_to_pil(item) for item in image)
+ else:
+ out.append(_decoded_tensor_to_pil(image))
+ return out
+
+
+def _preprocess_image(
+ image: Any,
+ *,
+ image_stride: int,
+ max_image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ image_mean: list[float],
+ image_std: list[float],
+) -> torch.Tensor:
+ """Resize and normalize ``image`` into a stride-aligned ``[3, H, W]`` float32 tensor."""
+ if isinstance(image, (Image.Image, np.ndarray)):
+ pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image))
+ pil_image = _resize_pil(
+ pil_image,
+ image_stride=image_stride,
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ tensor = _pil_to_rgb_tensor(pil_image)
+ else:
+ tensor = _to_rgb_tensor(image)
+ tensor = _resize_tensor(
+ tensor,
+ image_stride=image_stride,
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1)
+ std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1)
+ return tensor.sub(mean).div(std)
+
+
+def _to_rgb_tensor(image: Any) -> torch.Tensor:
+ """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``."""
+ if isinstance(image, Image.Image):
+ pil_image = _to_rgb_pil(image)
+ array = np.array(pil_image, copy=True)
+ return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0)
+ if isinstance(image, np.ndarray):
+ return _to_rgb_tensor(Image.fromarray(image))
+ if torch.is_tensor(image):
+ tensor = image.detach().to(dtype=torch.float32)
+ if tensor.dim() != 3:
+ raise TypeError(f"BAGEL VAE image tensor must be 3-D, got shape {tuple(tensor.shape)}.")
+ if tensor.shape[0] in (1, 3, 4):
+ pass
+ elif tensor.shape[-1] in (1, 3, 4):
+ tensor = tensor.permute(2, 0, 1).contiguous()
+ else:
+ raise TypeError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(tensor.shape)}.")
+ if tensor.shape[0] == 1:
+ tensor = tensor.repeat(3, 1, 1)
+ elif tensor.shape[0] == 4:
+ alpha = tensor[3:4]
+ if alpha.max() > 1:
+ alpha = alpha / 255.0
+ rgb = tensor[:3]
+ if rgb.max() > 1:
+ rgb = rgb / 255.0
+ tensor = rgb * alpha + (1.0 - alpha)
+ if tensor.max() > 1:
+ tensor = tensor / 255.0
+ return tensor[:3].clamp(0.0, 1.0)
+ raise TypeError(f"BAGEL VAE image item value must be PIL, numpy, or tensor, got {type(image).__name__}.")
+
+
+def _resize_tensor(
+ tensor: torch.Tensor,
+ *,
+ image_stride: int,
+ max_image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+) -> torch.Tensor:
+ """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to stride-aligned bounds."""
+ height, width = tensor.shape[-2:]
+ new_width, new_height = _target_size(
+ width,
+ height,
+ image_stride=image_stride,
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ if new_width == width and new_height == height:
+ return tensor
+ resized = F.interpolate(
+ tensor.unsqueeze(0),
+ size=(new_height, new_width),
+ mode="bicubic",
+ align_corners=False,
+ antialias=True,
+ )
+ return resized.squeeze(0).clamp(0.0, 1.0)
+
+
+def _resize_pil(
+ image: Image.Image,
+ *,
+ image_stride: int,
+ max_image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+ img_num: int = 1,
+) -> Image.Image:
+ """Resize a PIL image with bicubic interpolation to stride-aligned bounds."""
+ width, height = image.size
+ new_width, new_height = _target_size(
+ width,
+ height,
+ image_stride=image_stride,
+ max_image_size=max_image_size,
+ min_image_size=min_image_size,
+ max_pixels=max_pixels,
+ )
+ if new_width == width and new_height == height:
+ return image
+ return TVF.resize(
+ image,
+ (new_height, new_width),
+ interpolation=InterpolationMode.BICUBIC,
+ antialias=True,
+ )
+
+
+def _pad_to_batch_size(images: list[torch.Tensor]) -> list[torch.Tensor]:
+ """Zero-pad variable-size ``[C, H, W]`` image tensors on the bottom/right to a common canvas."""
+ if not images:
+ return images
+ max_height = max(int(image.shape[-2]) for image in images)
+ max_width = max(int(image.shape[-1]) for image in images)
+ padded = []
+ for image in images:
+ pad_height = max_height - int(image.shape[-2])
+ pad_width = max_width - int(image.shape[-1])
+ if pad_height or pad_width:
+ image = F.pad(image, (0, pad_width, 0, pad_height), value=0.0)
+ padded.append(image)
+ return padded
+
+
+def unpad_vae_item_pixel_values(item: ConversationItem) -> torch.Tensor:
+ """Return ``item.value`` cropped to its ``BAGEL_VAE_PIXEL_SHAPE`` if present."""
+ pixel_values = item.value
+ if not torch.is_tensor(pixel_values):
+ raise TypeError("BAGEL VAE item value must be a tensor.")
+ if pixel_values.dim() == 4 and int(pixel_values.shape[0]) == 1:
+ pixel_values = pixel_values.squeeze(0)
+ if pixel_values.dim() != 3:
+ raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(pixel_values.shape)}.")
+ pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE)
+ if torch.is_tensor(pixel_shape):
+ dims = pixel_shape.detach().reshape(-1).tolist()
+ if len(dims) != 2:
+ raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].")
+ height, width = int(dims[0]), int(dims[1])
+ pixel_values = pixel_values[..., :height, :width]
+ return pixel_values
+
+
+def crop_latent_to_image_shape(
+ latent: torch.Tensor,
+ pixel_shape: torch.Tensor | None,
+ *,
+ downsample: int,
+) -> torch.Tensor:
+ """Crop spatial batch padding off ``latent`` using the pre-pad ``pixel_shape`` metadata."""
+ if not torch.is_tensor(pixel_shape):
+ return latent
+
+ shape = pixel_shape.detach().reshape(-1).to(device="cpu", dtype=torch.long)
+ if int(shape.numel()) != 2:
+ raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].")
+
+ downsample = max(int(downsample), 1)
+ latent_height = int(shape[0].item()) // downsample
+ latent_width = int(shape[1].item()) // downsample
+ if latent_height <= 0 or latent_width <= 0:
+ raise ValueError(
+ "BAGEL VAE pixel shape metadata produces an empty latent crop: "
+ f"pixel_shape={tuple(int(v.item()) for v in shape)}, downsample={downsample}."
+ )
+ if latent_height > int(latent.shape[-2]) or latent_width > int(latent.shape[-1]):
+ raise ValueError(
+ "BAGEL VAE latent crop exceeds encoded latent shape: "
+ f"crop=({latent_height}, {latent_width}), latent_shape={tuple(latent.shape)}."
+ )
+ return latent[..., :latent_height, :latent_width]
+
+
+def _target_size(
+ width: int,
+ height: int,
+ *,
+ image_stride: int,
+ max_image_size: int,
+ min_image_size: int,
+ max_pixels: int,
+) -> tuple[int, int]:
+ """Compute stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds."""
+ scale = min(max_image_size / max(width, height), 1.0)
+ scale = max(scale, min_image_size / min(width, height))
+ new_width, new_height = _apply_scale(width, height, scale, image_stride)
+ if new_width * new_height > max_pixels:
+ scale = max_pixels / (new_width * new_height)
+ new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride)
+ if max(new_width, new_height) > max_image_size:
+ scale = max_image_size / max(new_width, new_height)
+ new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride)
+ return new_width, new_height
+
+
+def _to_rgb_pil(image: Image.Image) -> Image.Image:
+ """Convert ``image`` to RGB, compositing transparency over a white background."""
+ if image.mode == "RGBA" or image.info.get("transparency", None) is not None:
+ rgba = image.convert("RGBA")
+ white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255))
+ white.paste(rgba, mask=rgba.split()[3])
+ return white
+ return image.convert("RGB")
+
+
+def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor:
+ array = np.array(image, copy=True)
+ return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0)
+
+
+def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]:
+ new_width = round(width * scale)
+ new_height = round(height * scale)
+ return (
+ max(stride, int(round(new_width / stride) * stride)),
+ max(stride, int(round(new_height / stride) * stride)),
+ )
+
+
+def _decoded_tensor_to_pil(image: torch.Tensor) -> Image.Image:
+ """Convert a single decoded ``[-1, 1]`` tensor into an RGB ``PIL.Image``."""
+ if image.dim() == 4 and image.shape[0] == 1:
+ image = image.squeeze(0)
+ if image.dim() != 3:
+ raise ValueError(f"BAGEL VAE generated image must be rank 3, got shape {tuple(image.shape)}.")
+ image = (image.detach().to(dtype=torch.float32).clamp(-1.0, 1.0) * 0.5 + 0.5).clamp(0.0, 1.0)
+ if image.shape[0] in (1, 3, 4):
+ image = image[:3].permute(1, 2, 0)
+ elif image.shape[-1] not in (1, 3, 4):
+ raise ValueError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(image.shape)}.")
+ image = (image[..., :3] * 255.0).round().to(dtype=torch.uint8).cpu().numpy()
+ return Image.fromarray(image)
+
+
+def copy_image_item(item: ConversationItem) -> ConversationItem:
+ """Deep-copy an image ``ConversationItem`` so VAE and SigLIP context copies do not alias."""
+ value = item.value
+ if torch.is_tensor(value):
+ value = value.clone()
+ elif isinstance(value, Image.Image):
+ value = value.copy()
+ else:
+ value = copy.deepcopy(value)
+ return ConversationItem(
+ type=item.type,
+ value=value,
+ role=item.role,
+ is_dummy=item.is_dummy,
+ meta=copy.deepcopy(item.meta),
+ )
+
+
+def route_image_contexts(
+ conversation_list: list[list[ConversationItem]],
+ *,
+ inference: bool,
+ infer_type: object,
+) -> None:
+ """Expand raw images into their SigLIP and/or VAE context copies (``BAGEL_CONTEXT_KEY``)."""
+ for sample in conversation_list:
+ routed: list[ConversationItem] = []
+ for item in sample:
+ if item.type != "image" or item.is_dummy:
+ routed.append(item)
+ continue
+
+ if bagel_context(item) in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}:
+ routed.append(item)
+ continue
+
+ # Interactive inference has no dataset preprocessor, so raw user
+ # request images use ``infer_type`` in generation_kwargs for routing.
+ if inference and item.role == "user":
+ if infer_type == "infer_edit":
+ vae_item = copy_image_item(item)
+ vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ routed.extend([vae_item, item])
+ else:
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ routed.append(item)
+ continue
+
+ # Training data declares each raw image's role via ``_img_tag``.
+ tag = item.meta.get(_IMG_TAG_KEY)
+ if tag is None or tag not in _VALID_IMG_TAGS:
+ raise ValueError(f"BAGEL route_image_contexts received image with invalid {_IMG_TAG_KEY}: {tag!r}.")
+ if tag == "und":
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ routed.append(item)
+ elif tag == "gen":
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ routed.append(item)
+ else:
+ # tag == "edit": the same physical image is both VAE context and
+ # SigLIP context. Preserve the infer_edit ordering (VAE before
+ # SigLIP) so downstream marker wrapping stays stable.
+ vae_item = copy_image_item(item)
+ vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT
+ routed.extend([vae_item, item])
+
+ sample[:] = routed
+
+
+BAGEL_VAE_PIXEL_SHAPE = "bagel_vae_pixel_shape"
+# Set by ``offline_encode``: the item already holds its cached posterior, not pixels.
+BAGEL_VAE_POSTERIOR = "bagel_vae_posterior"
+
+
+class BagelVAEPreprocessor(ModulePreprocessorBase):
+ """Worker-side image normalize for BAGEL VAE context/target images."""
+
+ def __init__(
+ self,
+ image_processor: Any,
+ dtype: torch.dtype | None = None,
+ dummy_pixel_values: torch.Tensor | None = None,
+ dummy_pixel_shape: torch.Tensor | None = None,
+ ) -> None:
+ self._image_processor = image_processor
+ self._dtype = dtype
+ self._dummy_pixel_values = dummy_pixel_values
+ self._dummy_pixel_shape = dummy_pixel_shape
+
+ @classmethod
+ def from_pretrained(
+ cls,
+ module_path: str,
+ *,
+ config_overrides: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> BagelVAEPreprocessor:
+ """Build straight from the checkpoint dir — no model instance needed.
+
+ BAGEL ships no standalone ``preprocessor_config.json``; the image
+ processor is derived from the module's own ``config.json``, with
+ ``config_overrides`` (the module's YAML ``model_config:`` block) on top.
+ """
+ del kwargs
+ # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare.
+ config = BagelVAEConfig.from_pretrained(module_path, model_config=config_overrides)
+ return cls(BagelVAEProcessor.from_config(config))
+
+ def bind_dummy_inputs(self, config: BagelVAEConfig, dtype: torch.dtype | None = None) -> None:
+ """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model."""
+ size = max(int(config.image_stride), int(config.downsample) * 2)
+ self._dtype = dtype
+ self._dummy_pixel_values = torch.zeros(int(config.in_channels), size, size, dtype=dtype)
+ self._dummy_pixel_shape = torch.tensor([size, size], dtype=torch.long)
+
+ def preprocess_conversations(
+ self,
+ conversation_list: list[list[ConversationItem]],
+ *,
+ inference: bool = False,
+ generation_kwargs: dict[str, Any] | None = None,
+ ) -> None:
+ """Route image contexts, normalize VAE pixel tensors, and append per-sample dummy anchors."""
+ infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type")
+ route_image_contexts(conversation_list, inference=inference, infer_type=infer_type)
+
+ image_items: list[ConversationItem] = []
+ missing_samples: list[list[ConversationItem]] = []
+ for sample in conversation_list:
+ sample_image_items = list(
+ iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]})
+ )
+ if sample_image_items:
+ for item in sample_image_items:
+ if item.meta.get(BAGEL_VAE_POSTERIOR) and _OMNI_TOKEN_LEN not in item.meta:
+ pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE)
+ stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1)
+ if torch.is_tensor(pixel_shape) and int(pixel_shape.numel()) == 2:
+ dims = pixel_shape.detach().reshape(-1).tolist()
+ item.meta[_OMNI_TOKEN_LEN] = (int(dims[0]) // stride) * (int(dims[1]) // stride)
+ image_items.extend(
+ item for item in sample_image_items if not item.is_dummy and not item.meta.get(BAGEL_VAE_POSTERIOR)
+ )
+ elif not inference:
+ missing_samples.append(sample)
+
+ stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1)
+ if image_items:
+ inputs = self._image_processor(
+ images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype
+ )
+ for item, pixels, pixel_shape in zip(
+ image_items, inputs["pixel_values"], inputs["pixel_shapes"], strict=True
+ ):
+ dims = pixel_shape.detach().reshape(-1).tolist()
+ height, width = int(dims[0]), int(dims[1])
+ item.value = pixels[..., :height, :width].contiguous().to(dtype=self._dtype)
+ item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT
+ item.meta[BAGEL_VAE_PIXEL_SHAPE] = pixel_shape.to(dtype=torch.long)
+ item.meta[_OMNI_TOKEN_LEN] = max(1, (height // stride) * (width // stride))
+
+ if not missing_samples:
+ return
+
+ if image_items:
+ # VAE encode stacks every context item in the micro-batch, so dummies must
+ # match a real processed image shape whenever the batch has one.
+ reference_item = image_items[0]
+ dummy_pixel_values = torch.zeros_like(reference_item.value, dtype=self._dtype)
+ dummy_pixel_shape = reference_item.meta[BAGEL_VAE_PIXEL_SHAPE].to(dtype=torch.long)
+ else:
+ if self._dummy_pixel_values is None:
+ raise RuntimeError(
+ f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() "
+ "before training use (pure inference never reaches this branch)."
+ )
+ dummy_pixel_values = self._dummy_pixel_values.to(dtype=self._dtype)
+ dummy_pixel_shape = self._dummy_pixel_shape.to(dtype=torch.long)
+
+ dummy_dims = dummy_pixel_shape.detach().reshape(-1).tolist()
+ dummy_token_len = max(1, (int(dummy_dims[0]) // stride) * (int(dummy_dims[1]) // stride))
+
+ # The placeholder stands in for a generation target, the primary VAE row.
+ for sample in missing_samples:
+ sample.append(
+ ConversationItem(
+ type="image",
+ value=dummy_pixel_values.clone(),
+ role="assistant",
+ is_dummy=True,
+ meta={
+ _IMG_TAG_KEY: "gen",
+ BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT,
+ BAGEL_VAE_PIXEL_SHAPE: dummy_pixel_shape.clone(),
+ _OMNI_TOKEN_LEN: dummy_token_len,
+ },
+ )
+ )
+
+
+__all__ = [
+ "BagelVAEProcessor",
+ "BagelVAEPreprocessor",
+ "copy_image_item",
+ "crop_latent_to_image_shape",
+ "route_image_contexts",
+ "unpad_vae_item_pixel_values",
+]
diff --git a/veomni/models/seed_omni/processing_omni.py b/veomni/models/seed_omni/processing_omni.py
index 9ace3c8be2..d44926d90c 100644
--- a/veomni/models/seed_omni/processing_omni.py
+++ b/veomni/models/seed_omni/processing_omni.py
@@ -78,8 +78,14 @@ def _as_list(media: MediaInput | None) -> list[Any]:
class OmniProcessor:
"""Composed SeedOmni request preprocessor (HF ``AutoProcessor``-style API)."""
- def __init__(self, preprocessors: dict[str, ModulePreprocessorBase]) -> None:
+ def __init__(
+ self,
+ preprocessors: dict[str, ModulePreprocessorBase],
+ *,
+ infer_type: str | None = None,
+ ) -> None:
self._preprocessors: dict[str, ModulePreprocessorBase] = dict(preprocessors)
+ self.infer_type: str | None = infer_type
def __len__(self) -> int:
"""Number of worker-side preprocessors contributed by active graph modules."""
@@ -148,7 +154,7 @@ def from_config(
if preprocessor is not None:
preprocessors[name] = preprocessor
logger.info_rank0(f"OmniProcessor: module '{name}' contributes {type(preprocessor).__name__}.")
- return cls(preprocessors)
+ return cls(preprocessors, infer_type=config.infer_type)
@classmethod
def from_pretrained(
@@ -253,6 +259,8 @@ def preprocess_batch(
Training uses ``inference=False`` (default); single-request inference
uses :meth:`__call__` with ``inference=True``.
"""
+ if inference and self.infer_type is not None and generation_kwargs.get("infer_type") is None:
+ generation_kwargs["infer_type"] = self.infer_type
for preprocessor in self._preprocessors.values():
preprocessor(
batch,
diff --git a/veomni/models/seed_omni/utils/hf_layout.py b/veomni/models/seed_omni/utils/hf_layout.py
index 2206705c35..885143eece 100644
--- a/veomni/models/seed_omni/utils/hf_layout.py
+++ b/veomni/models/seed_omni/utils/hf_layout.py
@@ -308,12 +308,11 @@ def _write_hf_view(source: HFSource) -> str:
module_configs = layout.build_module_configs(source.path)
training_graphs, generation_graphs = layout.load_graphs()
- infer_type = layout.infer_type if layout.infer_type in generation_graphs else None
config = OmniConfig(
_module_entries={name: {"model_path": name} for name in module_configs},
training_graphs=training_graphs,
generation_graphs=generation_graphs,
- infer_type=infer_type,
+ infer_type=layout.infer_type,
)
config._module_configs = dict(module_configs)
config.save_pretrained(view)
diff --git a/veomni/ops/README.md b/veomni/ops/README.md
index 431534cbe0..6b4313fa4a 100644
--- a/veomni/ops/README.md
+++ b/veomni/ops/README.md
@@ -43,7 +43,7 @@ when the selected implementation runs.
| Op | Variants | Implementations |
|---|---|---|
| `attention` | `standard` | `eager`, `sdpa`, and FlashAttention/AITER/FlexAttention/MagiAttention/SageAttention names |
-| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit` | `eager` orchestration |
+| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit`, `bagel` | `eager` orchestration |
| `rms_norm` | `standard`, `deepseek_v4`, `unweighted`, `offset` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; all accept optional `group_size` (fused weighted rows delegate grouped calls to eager) |
| `layer_norm` | `standard` | `eager`, `apex` (`fused_layer_norm_cuda`); nested by async Ulysses QKV when `norm_type="layernorm"` |
| `rope` | `full`, `partial`, `interleave`, `mrope`, `deepseek_v4`, `wan` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; `full` also accepts rank-3 vision layout |
diff --git a/veomni/ops/kernels/async_ulysses/__init__.py b/veomni/ops/kernels/async_ulysses/__init__.py
index 114f4093ef..9b065a579e 100644
--- a/veomni/ops/kernels/async_ulysses/__init__.py
+++ b/veomni/ops/kernels/async_ulysses/__init__.py
@@ -14,13 +14,15 @@
"""Async Ulysses compound kernels.
-``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard`` and ``dit``
-eager rows. Nested RMSNorm and LayerNorm use raw kernel pairs.
+``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard``, ``dit``,
+and ``bagel`` eager rows. Nested RMSNorm, LayerNorm, and RoPE use raw kernel pairs.
"""
from ...registry import register_op
+from .o_proj.bagel import eager as o_bagel
from .o_proj.dit import eager as o_dit
from .o_proj.standard import eager as o_standard
+from .qkv_proj.bagel import eager as qkv_bagel
from .qkv_proj.dit import eager as qkv_dit
from .qkv_proj.standard import eager as qkv_standard
@@ -41,6 +43,14 @@
qkv_dit.backward,
description="VeOmni asynchronous Ulysses QKV projection for diffusion transformers",
)
+register_op(
+ "async_ulysses_qkv",
+ "bagel",
+ "eager",
+ qkv_bagel.forward,
+ qkv_bagel.backward,
+ description="VeOmni asynchronous Ulysses QKV projection for BAGEL two-route MoT attention",
+)
register_op(
"async_ulysses_o",
"standard",
@@ -57,3 +67,11 @@
o_dit.backward,
description="VeOmni asynchronous Ulysses output projection for diffusion transformers",
)
+register_op(
+ "async_ulysses_o",
+ "bagel",
+ "eager",
+ o_bagel.forward,
+ o_bagel.backward,
+ description="VeOmni asynchronous Ulysses output projection for BAGEL two-route MoT attention",
+)
diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py
new file mode 100644
index 0000000000..bc1d3afdf2
--- /dev/null
+++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""async_ulysses_o bagel implementations."""
diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py
new file mode 100644
index 0000000000..41cdabfdc3
--- /dev/null
+++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py
@@ -0,0 +1,158 @@
+# Copyright 2026 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""BAGEL two-route MoT async Ulysses output projection: inverse A2A then routed linear."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+
+from ......distributed.sequence_parallel.comm import get_ulysses_sequence_parallel_group
+from ......distributed.sequence_parallel.ulysses import all_to_all_tensor
+from ......distributed.sequence_parallel.utils import (
+ padding_tensor_for_seqeunce_parallel,
+ unpadding_tensor_for_seqeunce_parallel,
+)
+from .....registry import SavedState
+from ...shared.backward import linear_input_backward, linear_parameter_backward
+
+
+@dataclass
+class _BagelOMeta:
+ """Saved all-to-all group, input sequence length, head geometry, and bias flags."""
+
+ sp_group: Any
+ input_sequence_length: int
+ num_heads: int
+ head_dim: int
+ has_proj_bias_0: bool
+ has_proj_bias_1: bool
+
+
+def forward(
+ hidden_states: Tensor,
+ route_0_indices: Tensor,
+ route_1_indices: Tensor,
+ proj_weight_0: Tensor,
+ proj_bias_0: Tensor | None,
+ proj_weight_1: Tensor,
+ proj_bias_1: Tensor | None,
+ *,
+ group: Any = None,
+) -> tuple[Tensor, SavedState]:
+ """Inverse Ulysses exchange followed by a two-route MoT output projection."""
+ sp_group = get_ulysses_sequence_parallel_group() if group is None else group
+ if sp_group is None:
+ raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.")
+ input_sequence_length = hidden_states.shape[0]
+ hidden_states = padding_tensor_for_seqeunce_parallel(hidden_states, 0, group=sp_group)
+ hidden_states = all_to_all_tensor(
+ hidden_states,
+ scatter_dim=0,
+ gather_dim=1,
+ group=sp_group,
+ async_op=False,
+ )
+ num_heads = hidden_states.shape[1]
+ head_dim = hidden_states.shape[2]
+ flattened_states = hidden_states.reshape(hidden_states.shape[0], num_heads * head_dim)
+
+ output_0 = F.linear(flattened_states.index_select(0, route_0_indices), proj_weight_0, proj_bias_0)
+ output_1 = F.linear(flattened_states.index_select(0, route_1_indices), proj_weight_1, proj_bias_1)
+ output = output_0.new_zeros((flattened_states.shape[0], proj_weight_0.shape[0]))
+ output.index_copy_(0, route_0_indices, output_0)
+ output.index_copy_(0, route_1_indices, output_1)
+
+ return output, SavedState(
+ (
+ flattened_states,
+ route_0_indices,
+ route_1_indices,
+ proj_weight_0,
+ proj_weight_1,
+ ),
+ _BagelOMeta(
+ sp_group=sp_group,
+ input_sequence_length=input_sequence_length,
+ num_heads=num_heads,
+ head_dim=head_dim,
+ has_proj_bias_0=proj_bias_0 is not None,
+ has_proj_bias_1=proj_bias_1 is not None,
+ ),
+ )
+
+
+def backward(grad_output: Tensor, saved: SavedState) -> tuple[Tensor | None, ...]:
+ """Compute routed input gradients, launch reverse all-to-all, and overlap parameter gradients."""
+ meta = saved.metadata
+ assert isinstance(meta, _BagelOMeta)
+ flattened_states, route_0_indices, route_1_indices, proj_weight_0, proj_weight_1 = saved.tensors
+ grad_output_0 = grad_output.index_select(0, route_0_indices)
+ grad_output_1 = grad_output.index_select(0, route_1_indices)
+
+ flattened_states_0 = flattened_states.index_select(0, route_0_indices)
+ flattened_states_1 = flattened_states.index_select(0, route_1_indices)
+ grad_flattened_states = torch.zeros_like(flattened_states)
+ grad_flattened_states.index_copy_(
+ 0,
+ route_0_indices,
+ linear_input_backward(grad_output_0, flattened_states_0, proj_weight_0),
+ )
+ grad_flattened_states.index_copy_(
+ 0,
+ route_1_indices,
+ linear_input_backward(grad_output_1, flattened_states_1, proj_weight_1),
+ )
+ grad_hidden_states = grad_flattened_states.reshape(flattened_states.shape[0], meta.num_heads, meta.head_dim)
+ grad_hidden_wait = all_to_all_tensor(
+ grad_hidden_states,
+ scatter_dim=1,
+ gather_dim=0,
+ group=meta.sp_group,
+ async_op=True,
+ )
+
+ grad_proj_weight_0, grad_proj_bias_0 = linear_parameter_backward(
+ grad_output_0,
+ flattened_states_0,
+ proj_weight_0,
+ has_bias=meta.has_proj_bias_0,
+ )
+ grad_proj_weight_1, grad_proj_bias_1 = linear_parameter_backward(
+ grad_output_1,
+ flattened_states_1,
+ proj_weight_1,
+ has_bias=meta.has_proj_bias_1,
+ )
+
+ grad_hidden_states = unpadding_tensor_for_seqeunce_parallel(
+ grad_hidden_wait(),
+ 0,
+ meta.input_sequence_length,
+ group=meta.sp_group,
+ )
+ return (
+ grad_hidden_states,
+ None,
+ None,
+ grad_proj_weight_0,
+ grad_proj_bias_0,
+ grad_proj_weight_1,
+ grad_proj_bias_1,
+ )
diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py
new file mode 100644
index 0000000000..693dc5493e
--- /dev/null
+++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""async_ulysses_qkv bagel implementations."""
diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py
new file mode 100644
index 0000000000..1f0e0111c9
--- /dev/null
+++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py
@@ -0,0 +1,424 @@
+# Copyright 2026 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""BAGEL two-route MoT async Ulysses QKV: routed project, QK norm, RoPE, then A2A."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+from torch import Tensor
+
+from ......distributed.sequence_parallel.comm import (
+ get_ulysses_sequence_parallel_group,
+ get_ulysses_sequence_parallel_world_size,
+)
+from ......distributed.sequence_parallel.ulysses import all_to_all_tensor
+from ......distributed.sequence_parallel.utils import (
+ padding_tensor_for_seqeunce_parallel,
+ unpadding_tensor_for_seqeunce_parallel,
+)
+from .....compound import InnerHandle, append_inner, resolve_inner_op, take_inner
+from .....registry import OpEntry, SavedState
+from ...shared.backward import linear_backward, reduce_repeated_kv_gradient
+
+
+@dataclass
+class _BagelQKVMeta:
+ """Non-tensor metadata and nested-op specs for two-route MoT QKV backward."""
+
+ sp_group: Any
+ head_dim: int
+ num_q_heads: int
+ num_kv_heads: int
+ repeat_kv: int
+ has_q_bias_0: bool
+ has_k_bias_0: bool
+ has_v_bias_0: bool
+ has_q_bias_1: bool
+ has_k_bias_1: bool
+ has_v_bias_1: bool
+ rms: OpEntry
+ rope: OpEntry
+ rms_q_0: tuple[Any, int]
+ rms_q_1: tuple[Any, int]
+ rms_k_0: tuple[Any, int]
+ rms_k_1: tuple[Any, int]
+ rope_q: tuple[Any, int]
+ rope_k: tuple[Any, int]
+
+
+def _resolve_sp_group_and_size(group: Any) -> tuple[Any, int]:
+ """Return the active Ulysses process group and its world size."""
+ sp_group = get_ulysses_sequence_parallel_group() if group is None else group
+ if sp_group is None:
+ raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.")
+ ulysses_size = (
+ dist.get_world_size(sp_group)
+ if isinstance(sp_group, dist.ProcessGroup)
+ else get_ulysses_sequence_parallel_world_size()
+ )
+ return sp_group, ulysses_size
+
+
+def _project_route(
+ hidden_states: Tensor,
+ indices: Tensor,
+ weight: Tensor,
+ bias: Tensor | None,
+ num_heads: int,
+ head_dim: int,
+) -> Tensor:
+ """Project rows selected by ``indices`` and reshape to ``[tokens, heads, head_dim]``."""
+ projected = F.linear(hidden_states.index_select(0, indices), weight, bias)
+ return projected.reshape(-1, num_heads, head_dim)
+
+
+def _rotate_single_forward(
+ rope: OpEntry,
+ x: Tensor,
+ cos: Tensor,
+ sin: Tensor,
+) -> tuple[Tensor, SavedState]:
+ """Apply full RoPE to one ``[tokens, heads, head_dim]`` tensor before all-to-all."""
+ q_in = x.transpose(0, 1).unsqueeze(0)
+ dummy_k = q_in.new_zeros((1, 1, x.shape[0], x.shape[2]))
+ (q_out, _), saved = rope.forward(q_in, dummy_k, cos.unsqueeze(0), sin.unsqueeze(0), None, 1)
+ return q_out.squeeze(0).transpose(0, 1).contiguous(), saved
+
+
+def _rotate_single_backward(
+ rope: OpEntry,
+ grad_output: Tensor,
+ saved: SavedState,
+) -> Tensor:
+ """Invert ``_rotate_single_forward`` on a ``[tokens, heads, head_dim]`` gradient."""
+ grad_q = grad_output.transpose(0, 1).unsqueeze(0)
+ dummy_grad_k = grad_q.new_zeros((1, 1, grad_output.shape[0], grad_output.shape[2]))
+ dq, *_ = rope.backward((grad_q, dummy_grad_k), saved)
+ return dq.squeeze(0).transpose(0, 1).contiguous()
+
+
+def forward(
+ hidden_states: Tensor,
+ route_0_indices: Tensor,
+ route_1_indices: Tensor,
+ position_cos: Tensor,
+ position_sin: Tensor,
+ q_weight_0: Tensor,
+ q_bias_0: Tensor | None,
+ k_weight_0: Tensor,
+ k_bias_0: Tensor | None,
+ v_weight_0: Tensor,
+ v_bias_0: Tensor | None,
+ q_norm_weight_0: Tensor,
+ k_norm_weight_0: Tensor,
+ q_weight_1: Tensor,
+ q_bias_1: Tensor | None,
+ k_weight_1: Tensor,
+ k_bias_1: Tensor | None,
+ v_weight_1: Tensor,
+ v_bias_1: Tensor | None,
+ q_norm_weight_1: Tensor,
+ k_norm_weight_1: Tensor,
+ *,
+ head_dim: int,
+ rms_norm_eps: float,
+ unpadded_dim_size: int,
+ group: Any = None,
+ rms_norm: InnerHandle = None,
+ rope: InnerHandle = None,
+) -> tuple[tuple[Tensor, Tensor, Tensor], SavedState]:
+ """Two-route Q/K/V projection, per-route RMSNorm, RoPE, and async Ulysses exchange.
+
+ Inputs are token-major ``[local_sequence, hidden_size]`` with local route
+ index tensors ``route_0_indices`` and ``route_1_indices``. Outputs are
+ ``[global_sequence, local_heads, head_dim]`` for Q, K, and V.
+ """
+ num_q_heads = q_weight_0.shape[0] // head_dim
+ num_kv_heads = k_weight_0.shape[0] // head_dim
+
+ sp_group, ulysses_size = _resolve_sp_group_and_size(group)
+ if num_q_heads % ulysses_size != 0:
+ raise ValueError(f"num_query_heads ({num_q_heads}) must be divisible by ulysses_size ({ulysses_size}).")
+ if ulysses_size > num_kv_heads:
+ if ulysses_size % num_kv_heads != 0:
+ raise ValueError(
+ f"ulysses_size ({ulysses_size}) must be divisible by num_key_value_heads ({num_kv_heads})."
+ )
+ repeat_kv = ulysses_size // num_kv_heads
+ else:
+ if num_kv_heads % ulysses_size != 0:
+ raise ValueError(
+ f"num_key_value_heads ({num_kv_heads}) must be divisible by ulysses_size ({ulysses_size})."
+ )
+ repeat_kv = 1
+
+ sequence_length = hidden_states.shape[0]
+ rms_entry = resolve_inner_op(rms_norm, op="rms_norm", variant="standard")
+ rope_entry = resolve_inner_op(rope, op="rope", variant="full")
+
+ # Q projection + per-route RMSNorm + RoPE, followed by the first async
+ # exchange. K/V projection executes while earlier communication is in flight.
+ q_0 = _project_route(hidden_states, route_0_indices, q_weight_0, q_bias_0, num_q_heads, head_dim)
+ q_1 = _project_route(hidden_states, route_1_indices, q_weight_1, q_bias_1, num_q_heads, head_dim)
+ normalized_q_0, saved_rms_q_0 = rms_entry.forward(q_0, q_norm_weight_0, eps=rms_norm_eps)
+ normalized_q_1, saved_rms_q_1 = rms_entry.forward(q_1, q_norm_weight_1, eps=rms_norm_eps)
+ query = normalized_q_0.new_zeros((sequence_length, num_q_heads, head_dim))
+ query.index_copy_(0, route_0_indices, normalized_q_0)
+ query.index_copy_(0, route_1_indices, normalized_q_1)
+ query, saved_rope_q = _rotate_single_forward(rope_entry, query, position_cos, position_sin)
+ query_wait = all_to_all_tensor(
+ query,
+ scatter_dim=1,
+ gather_dim=0,
+ group=sp_group,
+ async_op=True,
+ )
+
+ k_0 = _project_route(hidden_states, route_0_indices, k_weight_0, k_bias_0, num_kv_heads, head_dim)
+ k_1 = _project_route(hidden_states, route_1_indices, k_weight_1, k_bias_1, num_kv_heads, head_dim)
+ normalized_k_0, saved_rms_k_0 = rms_entry.forward(k_0, k_norm_weight_0, eps=rms_norm_eps)
+ normalized_k_1, saved_rms_k_1 = rms_entry.forward(k_1, k_norm_weight_1, eps=rms_norm_eps)
+ key = normalized_k_0.new_zeros((sequence_length, num_kv_heads, head_dim))
+ key.index_copy_(0, route_0_indices, normalized_k_0)
+ key.index_copy_(0, route_1_indices, normalized_k_1)
+ key, saved_rope_k = _rotate_single_forward(rope_entry, key, position_cos, position_sin)
+ if repeat_kv > 1:
+ key = torch.repeat_interleave(key, dim=1, repeats=repeat_kv)
+ key_wait = all_to_all_tensor(
+ key,
+ scatter_dim=1,
+ gather_dim=0,
+ group=sp_group,
+ async_op=True,
+ )
+
+ v_0 = _project_route(hidden_states, route_0_indices, v_weight_0, v_bias_0, num_kv_heads, head_dim)
+ v_1 = _project_route(hidden_states, route_1_indices, v_weight_1, v_bias_1, num_kv_heads, head_dim)
+ value = v_0.new_zeros((sequence_length, num_kv_heads, head_dim))
+ value.index_copy_(0, route_0_indices, v_0)
+ value.index_copy_(0, route_1_indices, v_1)
+ if repeat_kv > 1:
+ value = torch.repeat_interleave(value, dim=1, repeats=repeat_kv)
+ value_wait = all_to_all_tensor(
+ value,
+ scatter_dim=1,
+ gather_dim=0,
+ group=sp_group,
+ async_op=True,
+ )
+
+ query = unpadding_tensor_for_seqeunce_parallel(query_wait(), 0, unpadded_dim_size, group=sp_group).contiguous()
+ key = unpadding_tensor_for_seqeunce_parallel(key_wait(), 0, unpadded_dim_size, group=sp_group).contiguous()
+ value = unpadding_tensor_for_seqeunce_parallel(value_wait(), 0, unpadded_dim_size, group=sp_group).contiguous()
+
+ saved: list[Tensor | None] = [
+ hidden_states,
+ route_0_indices,
+ route_1_indices,
+ q_weight_0,
+ k_weight_0,
+ v_weight_0,
+ q_weight_1,
+ k_weight_1,
+ v_weight_1,
+ ]
+ meta = _BagelQKVMeta(
+ sp_group=sp_group,
+ head_dim=head_dim,
+ num_q_heads=num_q_heads,
+ num_kv_heads=num_kv_heads,
+ repeat_kv=repeat_kv,
+ has_q_bias_0=q_bias_0 is not None,
+ has_k_bias_0=k_bias_0 is not None,
+ has_v_bias_0=v_bias_0 is not None,
+ has_q_bias_1=q_bias_1 is not None,
+ has_k_bias_1=k_bias_1 is not None,
+ has_v_bias_1=v_bias_1 is not None,
+ rms=rms_entry,
+ rope=rope_entry,
+ rms_q_0=append_inner(saved, saved_rms_q_0),
+ rms_q_1=append_inner(saved, saved_rms_q_1),
+ rms_k_0=append_inner(saved, saved_rms_k_0),
+ rms_k_1=append_inner(saved, saved_rms_k_1),
+ rope_q=append_inner(saved, saved_rope_q),
+ rope_k=append_inner(saved, saved_rope_k),
+ )
+ return (query, key, value), SavedState(tuple(saved), meta)
+
+
+def backward(grad_output: tuple[Tensor, Tensor, Tensor], saved: SavedState) -> tuple[Tensor | None, ...]:
+ """Overlap inverse Ulysses all-to-all exchanges with routed RoPE, RMSNorm, and linear gradients."""
+ meta = saved.metadata
+ assert isinstance(meta, _BagelQKVMeta)
+ (
+ hidden_states,
+ route_0_indices,
+ route_1_indices,
+ q_weight_0,
+ k_weight_0,
+ v_weight_0,
+ q_weight_1,
+ k_weight_1,
+ v_weight_1,
+ *rest_list,
+ ) = saved.tensors
+ rest = tuple(rest_list)
+ saved_rms_q_0, rest = take_inner(rest, meta.rms_q_0)
+ saved_rms_q_1, rest = take_inner(rest, meta.rms_q_1)
+ saved_rms_k_0, rest = take_inner(rest, meta.rms_k_0)
+ saved_rms_k_1, rest = take_inner(rest, meta.rms_k_1)
+ saved_rope_q, rest = take_inner(rest, meta.rope_q)
+ saved_rope_k, rest = take_inner(rest, meta.rope_k)
+
+ grad_query, grad_key, grad_value = grad_output
+ sp_group = meta.sp_group
+ repeat_kv = meta.repeat_kv
+ num_kv_heads = meta.num_kv_heads
+ hidden_0 = hidden_states.index_select(0, route_0_indices)
+ hidden_1 = hidden_states.index_select(0, route_1_indices)
+
+ # V inverse exchange has no independent projection work before it, so
+ # collect it first. K then overlaps V projection gradients, and Q
+ # overlaps K normalization/projection gradients.
+ grad_value = padding_tensor_for_seqeunce_parallel(grad_value.contiguous(), 0, group=sp_group)
+ grad_value_wait = all_to_all_tensor(
+ grad_value,
+ scatter_dim=0,
+ gather_dim=1,
+ group=sp_group,
+ async_op=True,
+ )
+ grad_value = grad_value_wait()
+ if repeat_kv > 1:
+ grad_value = reduce_repeated_kv_gradient(
+ grad_value,
+ num_kv_heads,
+ repeat_kv,
+ head_dimension=1,
+ )
+
+ grad_key = padding_tensor_for_seqeunce_parallel(grad_key.contiguous(), 0, group=sp_group)
+ grad_key_wait = all_to_all_tensor(
+ grad_key,
+ scatter_dim=0,
+ gather_dim=1,
+ group=sp_group,
+ async_op=True,
+ )
+
+ grad_hidden_v_0, grad_v_weight_0, grad_v_bias_0 = linear_backward(
+ grad_value.index_select(0, route_0_indices),
+ hidden_0,
+ v_weight_0,
+ has_bias=meta.has_v_bias_0,
+ )
+ grad_hidden_v_1, grad_v_weight_1, grad_v_bias_1 = linear_backward(
+ grad_value.index_select(0, route_1_indices),
+ hidden_1,
+ v_weight_1,
+ has_bias=meta.has_v_bias_1,
+ )
+
+ grad_key = grad_key_wait()
+ if repeat_kv > 1:
+ grad_key = reduce_repeated_kv_gradient(
+ grad_key,
+ num_kv_heads,
+ repeat_kv,
+ head_dimension=1,
+ )
+
+ grad_query = padding_tensor_for_seqeunce_parallel(grad_query.contiguous(), 0, group=sp_group)
+ grad_query_wait = all_to_all_tensor(
+ grad_query,
+ scatter_dim=0,
+ gather_dim=1,
+ group=sp_group,
+ async_op=True,
+ )
+
+ grad_key = _rotate_single_backward(meta.rope, grad_key, saved_rope_k)
+ grad_k_0, grad_k_norm_weight_0 = meta.rms.backward(
+ grad_key.index_select(0, route_0_indices),
+ saved_rms_k_0,
+ )
+ grad_k_1, grad_k_norm_weight_1 = meta.rms.backward(
+ grad_key.index_select(0, route_1_indices),
+ saved_rms_k_1,
+ )
+ grad_hidden_k_0, grad_k_weight_0, grad_k_bias_0 = linear_backward(
+ grad_k_0, hidden_0, k_weight_0, has_bias=meta.has_k_bias_0
+ )
+ grad_hidden_k_1, grad_k_weight_1, grad_k_bias_1 = linear_backward(
+ grad_k_1, hidden_1, k_weight_1, has_bias=meta.has_k_bias_1
+ )
+
+ grad_query = grad_query_wait()
+ grad_query = _rotate_single_backward(meta.rope, grad_query, saved_rope_q)
+ grad_q_0, grad_q_norm_weight_0 = meta.rms.backward(
+ grad_query.index_select(0, route_0_indices),
+ saved_rms_q_0,
+ )
+ grad_q_1, grad_q_norm_weight_1 = meta.rms.backward(
+ grad_query.index_select(0, route_1_indices),
+ saved_rms_q_1,
+ )
+ grad_hidden_q_0, grad_q_weight_0, grad_q_bias_0 = linear_backward(
+ grad_q_0, hidden_0, q_weight_0, has_bias=meta.has_q_bias_0
+ )
+ grad_hidden_q_1, grad_q_weight_1, grad_q_bias_1 = linear_backward(
+ grad_q_1, hidden_1, q_weight_1, has_bias=meta.has_q_bias_1
+ )
+
+ grad_hidden_states = torch.zeros_like(hidden_states)
+ grad_hidden_states.index_add_(
+ 0,
+ route_0_indices,
+ grad_hidden_q_0 + grad_hidden_k_0 + grad_hidden_v_0,
+ )
+ grad_hidden_states.index_add_(
+ 0,
+ route_1_indices,
+ grad_hidden_q_1 + grad_hidden_k_1 + grad_hidden_v_1,
+ )
+
+ return (
+ grad_hidden_states,
+ None,
+ None,
+ None,
+ None,
+ grad_q_weight_0,
+ grad_q_bias_0,
+ grad_k_weight_0,
+ grad_k_bias_0,
+ grad_v_weight_0,
+ grad_v_bias_0,
+ grad_q_norm_weight_0,
+ grad_k_norm_weight_0,
+ grad_q_weight_1,
+ grad_q_bias_1,
+ grad_k_weight_1,
+ grad_k_bias_1,
+ grad_v_weight_1,
+ grad_v_bias_1,
+ grad_q_norm_weight_1,
+ grad_k_norm_weight_1,
+ )
diff --git a/veomni/trainer/omni/omni_inferencer.py b/veomni/trainer/omni/omni_inferencer.py
index 15f31d47a8..b54c2d3620 100644
--- a/veomni/trainer/omni/omni_inferencer.py
+++ b/veomni/trainer/omni/omni_inferencer.py
@@ -242,6 +242,7 @@ def _run(self, req: InferenceRequest) -> dict[str, Any]:
videos=req.videos or None,
mm_configs=req.mm_configs or None,
inference=True,
+ **req.generation_kwargs,
)
if self._distributed:
# Every module sits on this rank's device, as in training. An eager