diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml new file mode 100644 index 0000000000..f2636cab5b --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml @@ -0,0 +1,18 @@ +sources: + - /mnt/hdfs/user_dir/dataset/imagenet1k_train + - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data + - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k + - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn +names: + - imagenet1k + - tulu-3-sft-mixture + - sharegpt4v_cap_100k + - seed_edit_p23_multi_turn +schedule: + - schedule_type: const + weights: [0.4, 0.2, 0.2, 0.2] +# Heterogeneous schemas (imagenet string / tulu dict / sharegpt4v list) — keep +# them in the VeOmni weighted multisource sampler. +level: token +stopping_strategy: all_exhausted +upstream_sharded: true diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml new file mode 100644 index 0000000000..e8082463e0 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml @@ -0,0 +1,37 @@ +# BAGEL-7B-MoT inference - text+image to image edit. +# Official edit builds VAE/ViT/text context, then calls image generation directly. +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_vae.encode_context, to: bagel_flow_connector.embed_context_latents } + - { from: bagel_flow_connector.embed_context_latents, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: default }, next_state: query_denoise } + + query_denoise: + body: + - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers } + - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch } + - { from: bagel_qwen2_mot.denoise_branch, to: end } + transitions: + - { condition: { type: default }, next_state: velocity_collect } + + velocity_collect: + body: + - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity } + - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise } + - { from: bagel_flow_connector.advance_denoise, to: end } + transitions: + - { condition: { type: module_signal, key: image_complete }, next_state: image_decode } + - { condition: { type: default }, next_state: query_denoise } + + image_decode: + body: + - { from: bagel_vae.decode_generated, to: end } + transitions: + - { condition: { type: default }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml new file mode 100644 index 0000000000..74f1864075 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml @@ -0,0 +1,34 @@ +# BAGEL-7B-MoT inference - text to image. +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: default }, next_state: query_denoise } + + query_denoise: + body: + - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers } + - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch } + - { from: bagel_qwen2_mot.denoise_branch, to: end } + transitions: + - { condition: { type: default }, next_state: velocity_collect } + + velocity_collect: + body: + - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity } + - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise } + - { from: bagel_flow_connector.advance_denoise, to: end } + transitions: + - { condition: { type: module_signal, key: image_complete }, next_state: image_decode } + - { condition: { type: default }, next_state: query_denoise } + + image_decode: + body: + - { from: bagel_vae.decode_generated, to: end } + transitions: + - { condition: { type: default }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml new file mode 100644 index 0000000000..29ba541d85 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml @@ -0,0 +1,19 @@ +# BAGEL-7B-MoT inference - understanding to text (text/image -> text). +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: module_signal, key: text_done }, next_state: done } + - { condition: { type: default }, next_state: text_ar } + + text_ar: + body: + - { from: bagel_qwen2_mot, to: bagel_text_encoder } + - { from: bagel_text_encoder, to: end } + transitions: + - { condition: { type: module_signal, key: text_done }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml new file mode 100644 index 0000000000..8c062c522a --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml @@ -0,0 +1,42 @@ +# Per-module INFERENCE overrides for BAGEL-7B-MoT — all eager. +# +# Single-process load (`python tasks/omni/infer_omni.py ...`): every module +# loads via `from_pretrained(device_map='auto')`, spread across visible GPUs. + +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false + fsdp_config: + fsdp_mode: eager + +bagel_siglip_navit: + model_path: bagel_siglip_navit + accelerator: + fsdp_config: + fsdp_mode: eager + +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + fsdp_config: + fsdp_mode: eager + gradient_checkpointing: + enable: false + +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + fsdp_config: + fsdp_mode: eager + gradient_checkpointing: + enable: false + +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention + accelerator: + fsdp_config: + fsdp_mode: eager diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml new file mode 100644 index 0000000000..b6b65e8555 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml @@ -0,0 +1,33 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false + fsdp_config: + fsdp_mode: fsdp2 +bagel_siglip_navit: + model_path: bagel_siglip_navit + accelerator: + fsdp_config: + fsdp_mode: fsdp2 +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + fsdp_config: + fsdp_mode: fsdp2 + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + fsdp_config: + fsdp_mode: fsdp2 + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention + accelerator: + fsdp_config: + fsdp_mode: fsdp2 diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml new file mode 100644 index 0000000000..78b9c32181 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml @@ -0,0 +1,83 @@ +# VeOmni offline VAE posterior cache config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: false + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni + max_seq_len: 4096 + train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml + datasets_type: iterable + multisource_datasets_type: veomni_weighted_multisource + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: false + pin_memory: true + prefetch_factor: 2 + +train: + training_task: offline_embedding + offline_cache_dir: outputs/bagel_vae_cached_dataset + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_offline_cache + save_steps: 0 + save_epochs: 0 + save_hf_weights: false + hf_save_steps: 0 + hf_save_epochs: 0 + + wandb: + enable: false + project: VeOmni_omni + name: bagel_7b_mot_offline_cache diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml new file mode 100644 index 0000000000..82f52f7241 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml @@ -0,0 +1,2 @@ +# BAGEL-7B-MoT VAE posterior cache production DAG. +- { from: bagel_vae.offline_encode, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml new file mode 100644 index 0000000000..b1894fca61 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml @@ -0,0 +1,5 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml new file mode 100644 index 0000000000..41ba85608c --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml @@ -0,0 +1,83 @@ +# VeOmni training / inference config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: true + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni + max_seq_len: 4096 + train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml + # Streaming + VeOmni weighted multisource: no HF schema merge across sources. + datasets_type: iterable + multisource_datasets_type: veomni_weighted_multisource + # Iterable has no __len__; train_steps = ceil(train_sample / global_batch_size). + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: true + pin_memory: true + prefetch_factor: 2 + +train: + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_omni_sft + save_steps: 500 + save_epochs: 0 + save_hf_weights: true + hf_save_steps: 500 + hf_save_epochs: 0 + + wandb: + enable: true + project: VeOmni_omni + name: bagel_7b_mot_omni_sft diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml new file mode 100644 index 0000000000..3cae0e86bc --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml @@ -0,0 +1,9 @@ +# BAGEL-7B-MoT training DAG. +- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot } +- { from: bagel_siglip_navit, to: bagel_qwen2_mot } +- { from: bagel_vae.encode, to: bagel_flow_connector.embed_latent } +- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot } +- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode } +- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity } +- { from: bagel_text_encoder.decode, to: end } +- { from: bagel_flow_connector.decode_velocity, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml new file mode 100644 index 0000000000..c03909f5bd --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml @@ -0,0 +1,23 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false +bagel_siglip_navit: + model_path: bagel_siglip_navit +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention + accelerator: + enable_async: true diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml new file mode 100644 index 0000000000..517489071e --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml @@ -0,0 +1,81 @@ +# VeOmni cached VAE posterior training config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: true + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni_cached + max_seq_len: 4096 + train_path: outputs/bagel_vae_cached_dataset + datasets_type: iterable + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: true + pin_memory: true + prefetch_factor: 2 + +train: + training_task: offline_training + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_omni_sft_from_cache + save_steps: 500 + save_epochs: 0 + save_hf_weights: true + hf_save_steps: 500 + hf_save_epochs: 0 + + wandb: + enable: true + project: VeOmni_omni + name: bagel_7b_mot_omni_sft_from_cache diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml new file mode 100644 index 0000000000..71951d199d --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml @@ -0,0 +1,9 @@ +# BAGEL-7B-MoT training DAG from cached VAE posterior carrier items. +- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot } +- { from: bagel_siglip_navit, to: bagel_qwen2_mot } +- { from: bagel_vae.online_process, to: bagel_flow_connector.embed_latent } +- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot } +- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode } +- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity } +- { from: bagel_text_encoder.decode, to: end } +- { from: bagel_flow_connector.decode_velocity, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml new file mode 100644 index 0000000000..c03909f5bd --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml @@ -0,0 +1,23 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false +bagel_siglip_navit: + model_path: bagel_siglip_navit +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention + accelerator: + enable_async: true diff --git a/docs/index.md b/docs/index.md index 4de17b3691..b83380c0fb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -120,6 +120,7 @@ seed_omni/mixins/offline_encoding.md seed_omni/models/qwen3.md seed_omni/models/qwen3vl.md +seed_omni/models/bagel.md ``` ```{toctree} diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md new file mode 100644 index 0000000000..eb249318d0 --- /dev/null +++ b/docs/seed_omni/models/bagel.md @@ -0,0 +1,360 @@ +# BAGEL-7B-MoT (SeedOmni) + +End-to-end recipe for training and inferring **BAGEL-7B-MoT** as a SeedOmni +graph model. The upstream BAGEL checkpoint is split into five OmniModules: text +embedding / LM head, SigLIP-NaViT understanding tower, VAE codec, flow connector, +and Qwen2-MoT backbone. + +All paths below assume the upstream BAGEL checkpoint lives at +`/mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT`. Adjust to your +own storage. + +Config dir: `configs/seed_omni/Bagel/bagel_7b_mot/`. + +| Module | Holds | Role | +|--------|-------|------| +| `bagel_text_encoder` | tokenizer, token embedding, LM head | text/template markers, text logits, image-marker embeddings | +| `bagel_siglip_navit` | SigLIP-NaViT tower + connector | user image context for understanding and edit prompts | +| `bagel_vae` | VAE encoder/decoder | assistant image training latents, edit context latents, generated latent decode | +| `bagel_flow_connector` | VAE↔LLM projections, timestep embedding | latent patch embedding, velocity prediction, denoise state | +| `bagel_qwen2_mot` | Qwen2-MoT decoder backbone | text AR and flow-denoise hidden states | + +The [config layout](../usage/training_and_inference.md#1-config-layout) uses the +same `base.yaml` for training and inference. The training block references `train/modules_train.yaml` and `train/graph_train.yaml`; the +inference block maps each scenario to a separate generation graph. + +| File | Role | +|------|------| +| `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, and train blocks. | +| `train/modules_train.yaml` | Per-module training paths. `bagel_qwen2_mot` is the accelerated class with `flex_attention`. | +| `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses FlexAttention. | +| `infer/modules_infer_fsdp.yaml` | Distributed inference: every module uses FSDP2; MoT uses FlexAttention. | +| `train/graph_train.yaml` | Training DAG. | +| `infer/graph_infer_und.yaml` | Image/text understanding to text. | +| `infer/graph_infer_gen.yaml` | Text to image generation. | +| `infer/graph_infer_edit.yaml` | Text+image to image edit. | +| `data.yaml` | Weighted multisource data list. | + +The BAGEL graphs cover understanding, generation, and edit. + +--- + +## 1. Convert the checkpoint + +The converter reads upstream `llm_config.json`, `vit_config.json`, +`ema.safetensors`, and `ae.safetensors`, then writes one sub-checkpoint per +module: + +```bash +python scripts/seed_omni/convert_model.py \ + --model_path /mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT \ + --output_dir /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT +``` + +The family is picked from the source `config.json` (`model_type: bagel`). The +output root becomes `model.model_path` in `base.yaml`. It contains: + +```text +BAGEL-7B-MoT/ +├── config.json # module entries, infer_type, generation_kwargs +├── training_graph.yaml +├── generation_graph.yaml # infer_und / infer_gen / infer_edit +├── bagel_text_encoder/ +├── bagel_siglip_navit/ +├── bagel_vae/ +├── bagel_flow_connector/ +└── bagel_qwen2_mot/ +``` + +`bagel_text_encoder` also stores the tokenizer, re-saved from the upstream +checkpoint via `AutoTokenizer`; SigLIP-NaViT and VAE save their processors next +to their weights. + +--- + +## 2. Prepare data + +`data.yaml` lists a weighted multisource mixture: + +```yaml +sources: + - /mnt/hdfs/user_dir/dataset/imagenet1k_train + - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data + - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k + - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn +names: + - imagenet1k + - tulu-3-sft-mixture + - sharegpt4v_cap_100k + - seed_edit_p23_multi_turn +schedule: + - { schedule_type: const, weights: [0.4, 0.2, 0.2, 0.2] } +``` + +`ConversationItem`s carry no module ownership: each Bagel module selects its +rows by `type` / `role` / `meta` tags. Bagel defines two private meta keys in +`veomni/models/seed_omni/modules/bagel/sources.py`: + +| Key | Values | Meaning | +|---|---|---| +| `BAGEL_CONTEXT_KEY` (`"bagel_context"`) | `BAGEL_SIGLIP_CONTEXT` (`"siglip"`), `BAGEL_VAE_CONTEXT` (`"vae"`) | Which image copy a row is: SigLIP-NaViT understanding features, or VAE latent context/target. The vision start/end marker rows around each copy carry the same tag. | +| `BAGEL_PHASE_KEY` (`"bagel_phase"`) | `start_token`, `flow_query`, `flow_hidden`, `flow_velocity`, `generated_latent` | Inference-only step of a `type="output"` row (see §4). | + +The VAE CPU preprocessor (`route_image_contexts`) expands each raw training +image by its data-layer `_img_tag` into context copies: + +- `und` → one `BAGEL_SIGLIP_CONTEXT` copy for SigLIP-NaViT. +- `gen` → one `BAGEL_VAE_CONTEXT` copy, the flow-matching target. +- `edit` → a `BAGEL_VAE_CONTEXT` copy followed by a `BAGEL_SIGLIP_CONTEXT` + copy of the same image. `_img_tag` alone cannot tell these two apart, which + is why the context tag exists. + +At inference, raw user images have no `_img_tag`; they become SigLIP context, +plus a VAE context copy when `infer_type` is `infer_edit`. + +A sample with no image for SigLIP-NaViT or the VAE gets an FSDP placeholder +flagged `is_dummy=True`. It keeps the context tag (and a representative +`role` / `_img_tag`: `user`/`und` for SigLIP, `assistant`/`gen` for the VAE), +so the encoder selects it with the same filter as real rows; packing and loss +code skip it via `item.is_dummy`. + +--- + +## 3. Train + +Training uses the accelerated Qwen2-MoT class from `train/modules_train.yaml`. The +default packed attention backend is FlexAttention. + +On H20 (SM90), eligible BF16/FP16 calls automatically use Flex's FlashAttention-4 +backend from the standard `gpu` extra. The MoT training module enables async +Ulysses when sequence parallelism is active. A single-node H20 baseline is: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.accelerator.ulysses_size 4 \ + --train.micro_batch_size 1 +``` + +The online and cached-VAE training module configs both set +`bagel_qwen2_mot.accelerator.enable_async: true`. For a matched synchronous +control, override +`--model.model_config.modules.bagel_qwen2_mot.accelerator.enable_async false`. +With SP disabled, the fused synchronous QKV path is used regardless of this +flag. Both paths retain the same split-QKV external checkpoint schema. + +The async path dispatches through the `("async_ulysses_qkv", "bagel")` and +`("async_ulysses_o", "bagel")` compound ops, overlapping Q/K exchanges with +later routed projections and inverse exchanges with backward projection work. +It forwards the module's configured `rms_norm` and `rope` handles into the +compound ops and passes `skip_ulysses=True` to attention because Q/K/V have +already been redistributed. Inference continues to use its existing packed +prefill and FlashAttention-2 denoise/decode paths. + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml +``` + +The training DAG is: + +```text +bagel_text_encoder.encode -> bagel_qwen2_mot +bagel_siglip_navit -> bagel_qwen2_mot +bagel_vae.encode -> bagel_flow_connector.embed_latent +bagel_flow_connector.embed_latent -> bagel_qwen2_mot +bagel_qwen2_mot -> bagel_text_encoder.decode +bagel_qwen2_mot -> bagel_flow_connector.decode_velocity +``` + +`bagel_flow_connector.embed_latent` patchifies VAE latents, samples the training +noise/timestep, writes the velocity target, and projects noised latent patches +into LLM space. `bagel_flow_connector.decode_velocity` consumes the MoT hidden +states for those latent query positions and computes the flow velocity loss. + +Quick smoke run: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --train.max_steps 10 \ + --train.global_batch_size 8 \ + --train.micro_batch_size 1 \ + --train.wandb.enable false +``` + +### 3.1 Attention backends + +`bagel_qwen2_mot` keeps one packed visibility metadata and materializes it as +Flex, Magi, or SDPA. Visibility is the same in all three: +`same_document & (causal | same_full_span) & ~foreign_noise_key`. Packed MoT +training does not use FlashAttention-2. + +| Backend | Class | Config | Use | +|---------|-------|--------|-----| +| FlexAttention | accelerated / eager | `train/modules_train.yaml`, `infer/modules_infer_eager.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training and inference (only supported packed backend on SM90) | +| MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM100+ CUTE DSL/JIT fused training | +| SDPA | eager | CLI override onto `bagel_qwen2_mot` | Dense-mask oracle | + +Leave the YAML on `flex_attention` unless you are opting into Magi on SM100+. +Enable Magi from the CLI (the launcher rewrites it to `veomni_magi_attention`): + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.modules.bagel_qwen2_mot.ops_implementation.attn_implementation magi_attention \ + --train.micro_batch_size 1 +``` + +Magi requires physical batch size 1, `cp_size == 1`, and NVIDIA SM100 or newer +(with CUTE DSL/JIT installed via `uv sync --extra gpu --extra magi --dev`). +SM90 and older GPUs only support FlexAttention. Ulysses still works. On 8 ranks, +`--model.accelerator.ulysses_size 4` gives `dp_shard=2` and SP4. Packed +Magi/Flex is the training (and FSDP prefill) path; the denoise loop still uses +FlashAttention-2. The Magi adapter contract is in +[`docs/transformers_v5/veomni_attention_interface.md`](../../transformers_v5/veomni_attention_interface.md). + +### 3.2 Offline VAE posterior cache (two stages) + +Caching is not a special framework mode — it is a different `train_graph` plus a +different dataset type. Each stage has its own modules / graph YAML, and +`train.training_task` tells the trainer which stage it runs. `bagel_vae` can be +cached because it mixes in `OfflineEncodingMixin`; no config flag declares it (see +[Offline Encoding](../mixins/offline_encoding.md#how-a-run-builds-a-cached-module)). + +**Stage 1 — produce the cache** (`train.training_task: offline_embedding`). `offline_cache/modules_train.yaml` declares only +`bagel_vae`, and the DAG is a single edge, so nothing else is built: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml +``` + +```text +bagel_vae.offline_encode -> end +``` + +Posteriors are written to `train.offline_cache_dir` +(`outputs/bagel_vae_cached_dataset` by default), reading normal `seedomni` data. +Samples without a VAE image get the preprocessor's dummy row encoded too, so +every cached sample carries a VAE posterior. + +**Stage 2 — train from the cache** (`train.training_task: offline_training`). +`bagel_vae` mixes in `OfflineEncodingMixin`, so it is built on meta and never +loads its weights; the graph calls its `online_process`. `with_cache/modules_train.yaml` is the same file as +`train/modules_train.yaml`. `data.data_type` +becomes `seedomni_cached` and `data.train_path` points at the stage-1 output +directory. The cached items are marked as posteriors, so the VAE preprocessor +leaves them alone: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml +``` + +The DAG matches §3 except that `bagel_vae.online_process` replaces +`bagel_vae.encode` as the latent source — it rehydrates the cached posterior +instead of encoding pixels. + +--- + +## 4. Inference + +`tasks/omni/infer_omni.py` selects a generation graph with `--model.model_config.infer_type`. +Use `infer/modules_infer_eager.yaml` for a single-process run or +`infer/modules_infer_fsdp.yaml` with `bash train.sh` for a torchrun/FSDP2 run. + +### 4.1 Understanding + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_und \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.images /path/to/image.jpg \ + --infer.prompt "Describe this image." \ + --infer.output_dir bagel_out +``` + +`infer_und` runs SigLIP-NaViT for the prompt image, inserts image marker +embeddings via the text encoder, and then uses Qwen2-MoT + text encoder AR +decode until `text_done`. + +### 4.2 Text-to-image generation + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_gen \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.prompt "A watercolor painting of a small cabin beside a lake." \ + --infer.output_dir bagel_out \ + --infer.generation_kwargs.num_timesteps 50 \ + --infer.generation_kwargs.timestep_shift 3.0 +``` + +`infer_gen` runs prompt prefill, then loops: + +```text +flow_connector.prepare_denoise_query + -> text_encoder.encode_image_markers + -> qwen2_mot.denoise_branch + -> flow_connector.decode_velocity_from_hidden + -> qwen2_mot.collect_velocity + -> flow_connector.advance_denoise +``` + +Each step moves the tail `output` row through `BAGEL_PHASE_KEY` values +`flow_query` → `flow_hidden` → `flow_velocity`. When the denoise state emits +`image_complete`, the row is re-tagged `generated_latent` and +`bagel_vae.decode_generated` decodes it. + +### 4.3 Image edit + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_edit \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.images /path/to/source.jpg \ + --infer.prompt "Make it look like a snowy evening." \ + --infer.output_dir bagel_out +``` + +Edit first builds context from both image branches: VAE encodes the +`BAGEL_VAE_CONTEXT` copy of the edit image, SigLIP-NaViT encodes the +`BAGEL_SIGLIP_CONTEXT` copy as visual +context, and `flow_connector.embed_context_latents` projects the VAE context into +the denoise prompt. The downstream denoise loop is shared with `infer_gen`. + +--- + +## 5. Visualize the graphs + +```bash +python scripts/seed_omni/visualize_graph.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml +# -> graphs/bagel_7b_mot_base/{training,infer_edit,infer_gen,infer_und}.mmd +``` + +The visualized graphs are generated from the same config loader path as training +and inference, so regenerate them after changing module entrypoints or graph +YAML. + +--- + +## 6. Contract checks + +The Bagel module and graph contracts cover carrier context-tag routing, generation +state transitions, packing/cache behavior, and graph config structure. Packed +MoT also compares Flex and Magi against eager SDPA on toy CE / MSE / gradients +in `tests/seed_omni/modules/bagel/test_bagel_accel_align.py`. Magi cases skip unless an +NVIDIA SM100+ GPU with the CUTE DSL/JIT backend is present. + +```bash +.venv/bin/python -m pytest -q tests/seed_omni/modules/bagel +``` diff --git a/docs/seed_omni/usage/training_and_inference.md b/docs/seed_omni/usage/training_and_inference.md index 17a9bfc963..9d28cf60c4 100644 --- a/docs/seed_omni/usage/training_and_inference.md +++ b/docs/seed_omni/usage/training_and_inference.md @@ -206,7 +206,9 @@ flowchart TB `generate()` always lives on the native `modeling.py`, so an all-eager Inferencer run builds the same object a native user gets from `OmniModel.from_pretrained`; -the launcher only projects the YAML onto `OmniConfig` first. +the launcher only projects the YAML onto `OmniConfig` first. When the checkpoint +declares `generation_kwargs` defaults, `infer.generation_kwargs` overrides them +per key rather than replacing the bag. Inference does not use sequence or context parallelism: keep `ulysses_size` and `cp_size` at 1 on every module. The generation loop decodes one token at a time diff --git a/tasks/omni/infer_omni_native.py b/tasks/omni/infer_omni_native.py index ef43d4d1f2..776cc04b00 100644 --- a/tasks/omni/infer_omni_native.py +++ b/tasks/omni/infer_omni_native.py @@ -77,7 +77,7 @@ def main() -> None: torch_dtype="auto", **scenario, ).eval() - processor = OmniProcessor.from_pretrained(args.model_path) + processor = OmniProcessor.from_pretrained(args.model_path, **scenario) model_input = processor( text=args.prompt, images=args.image or None, diff --git a/tests/ops/async_ulysses/test_async_ulysses.py b/tests/ops/async_ulysses/test_async_ulysses.py index e3d2931caf..0e1ca4a86c 100644 --- a/tests/ops/async_ulysses/test_async_ulysses.py +++ b/tests/ops/async_ulysses/test_async_ulysses.py @@ -42,6 +42,10 @@ ("async_ulysses_o", "standard"), ("async_ulysses_o", "dit"), ) +_BAGEL_EAGER_ROWS = ( + ("async_ulysses_qkv", "bagel"), + ("async_ulysses_o", "bagel"), +) def test_linear_backward_matches_autograd_under_cpu_bf16_autocast(): @@ -134,7 +138,7 @@ def fake_pad(tensor: Tensor, dim, **kwargs): def fake_unpad(tensor: Tensor, dim, size, **kwargs): return tensor - for op, variant in _EAGER_ROWS: + for op, variant in (*_EAGER_ROWS, *_BAGEL_EAGER_ROWS): module = _eager_module(op, variant) monkeypatch.setattr(module, "all_to_all_tensor", fake_all_to_all) monkeypatch.setattr(module, "padding_tensor_for_seqeunce_parallel", fake_pad) @@ -722,3 +726,193 @@ def test_dit_parity_world_size_matches_skip_gate(): assert DIT_SP_WORLD_SIZE == 4 assert AsyncUlyssesDiTSequenceParallelTest.world_size.fget(object()) == DIT_SP_WORLD_SIZE assert callable(AsyncUlyssesDiTSequenceParallelTest._configure_repro) + + +@pytest.mark.parametrize("empty_route", [None, 0, 1]) +def test_bagel_qkv_and_o_match_sequential_and_use_nested_handles( + monkeypatch: pytest.MonkeyPatch, + empty_route: int | None, +) -> None: + """Verify two-route BAGEL async Ulysses QKV and O projections match sequential execution.""" + _mock_identity_comm(monkeypatch) + torch.manual_seed(6201) + seq, hidden, num_q_heads, num_kv_heads, head_dim = 6, 16, 4, 2, 4 + query_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + eps = 1e-6 + + if empty_route == 0: + route_0_indices = torch.empty(0, dtype=torch.long) + route_1_indices = torch.arange(seq, dtype=torch.long) + elif empty_route == 1: + route_0_indices = torch.arange(seq, dtype=torch.long) + route_1_indices = torch.empty(0, dtype=torch.long) + else: + route_0_indices = torch.tensor([0, 2, 4], dtype=torch.long) + route_1_indices = torch.tensor([1, 3, 5], dtype=torch.long) + + angles = torch.randn(seq, head_dim // 2, dtype=torch.float64) + cos = torch.cat((angles.cos(), angles.cos()), dim=-1) + sin = torch.cat((angles.sin(), angles.sin()), dim=-1) + + hidden_states = torch.randn(seq, hidden, dtype=torch.float64, requires_grad=True) + weights_0 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64) + weights_1 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64) + norm_q_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_k_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_q_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_k_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + o_weight_0 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True) + o_bias_0 = torch.randn(hidden, dtype=torch.float64, requires_grad=True) + o_weight_1 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True) + o_bias_1 = torch.randn(hidden, dtype=torch.float64, requires_grad=True) + + diff_tensors = ( + hidden_states, + *weights_0, + norm_q_0, + norm_k_0, + *weights_1, + norm_q_1, + norm_k_1, + o_weight_0, + o_bias_0, + o_weight_1, + o_bias_1, + ) + actual_leaves = _leaf(*diff_tensors) + expected_leaves = _leaf(*diff_tensors) + + ( + h_a, + qw0_a, + qb0_a, + kw0_a, + kb0_a, + vw0_a, + vb0_a, + nq0_a, + nk0_a, + qw1_a, + qb1_a, + kw1_a, + kb1_a, + vw1_a, + vb1_a, + nq1_a, + nk1_a, + ow0_a, + ob0_a, + ow1_a, + ob1_a, + ) = actual_leaves + q_a, k_a, v_a = VeomniOp("async_ulysses_qkv", "bagel")( + h_a, + route_0_indices, + route_1_indices, + cos, + sin, + qw0_a, + qb0_a, + kw0_a, + kb0_a, + vw0_a, + vb0_a, + nq0_a, + nk0_a, + qw1_a, + qb1_a, + kw1_a, + kb1_a, + vw1_a, + vb1_a, + nq1_a, + nk1_a, + head_dim=head_dim, + rms_norm_eps=eps, + unpadded_dim_size=seq, + group=object(), + rms_norm=VeomniOp("rms_norm", "standard", "eager"), + rope=VeomniOp("rope", "full", "eager"), + ) + attn_a = ( + q_a + + k_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + + v_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + ) + out_a = VeomniOp("async_ulysses_o", "bagel")( + attn_a, + route_0_indices, + route_1_indices, + ow0_a, + ob0_a, + ow1_a, + ob1_a, + group=object(), + ) + + ( + h_e, + qw0_e, + qb0_e, + kw0_e, + kb0_e, + vw0_e, + vb0_e, + nq0_e, + nk0_e, + qw1_e, + qb1_e, + kw1_e, + kb1_e, + vw1_e, + vb1_e, + nq1_e, + nk1_e, + ow0_e, + ob0_e, + ow1_e, + ob1_e, + ) = expected_leaves + rms = VeomniOp("rms_norm", "standard", "eager") + rope = VeomniOp("rope", "full", "eager") + + def routed_qkv(w0, b0, w1, b1, heads, n0=None, n1=None): + p0 = F.linear(h_e.index_select(0, route_0_indices), w0, b0).reshape(-1, heads, head_dim) + p1 = F.linear(h_e.index_select(0, route_1_indices), w1, b1).reshape(-1, heads, head_dim) + if n0 is not None and n1 is not None: + p0 = rms(p0, n0, eps=eps) + p1 = rms(p1, n1, eps=eps) + combined = p0.new_zeros((seq, heads, head_dim)) + combined = combined.index_copy(0, route_0_indices, p0) + combined = combined.index_copy(0, route_1_indices, p1) + return combined + + q_e = routed_qkv(qw0_e, qb0_e, qw1_e, qb1_e, num_q_heads, nq0_e, nq1_e) + k_e = routed_qkv(kw0_e, kb0_e, kw1_e, kb1_e, num_kv_heads, nk0_e, nk1_e) + v_e = routed_qkv(vw0_e, vb0_e, vw1_e, vb1_e, num_kv_heads) + q_e, k_e = rope( + q_e.transpose(0, 1).unsqueeze(0), + k_e.transpose(0, 1).unsqueeze(0), + cos.unsqueeze(0), + sin.unsqueeze(0), + unsqueeze_dim=1, + ) + q_e = q_e.squeeze(0).transpose(0, 1).contiguous() + k_e = k_e.squeeze(0).transpose(0, 1).contiguous() + attn_e = ( + q_e + + k_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + + v_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + ) + flat_e = attn_e.reshape(seq, query_size) + out_e = flat_e.new_zeros((seq, hidden)) + out_e = out_e.index_copy(0, route_0_indices, F.linear(flat_e.index_select(0, route_0_indices), ow0_e, ob0_e)) + out_e = out_e.index_copy(0, route_1_indices, F.linear(flat_e.index_select(0, route_1_indices), ow1_e, ob1_e)) + + torch.testing.assert_close(out_a, out_e) + grad_out = torch.randn_like(out_a) + out_a.backward(grad_out) + out_e.backward(grad_out) + for actual, expected in zip(actual_leaves, expected_leaves, strict=True): + torch.testing.assert_close(actual.grad, expected.grad) diff --git a/tests/ops/base/test_builtin_entries.py b/tests/ops/base/test_builtin_entries.py index c981ab5e4e..c849cf76f4 100644 --- a/tests/ops/base/test_builtin_entries.py +++ b/tests/ops/base/test_builtin_entries.py @@ -53,8 +53,10 @@ ("moe_unpermute", "standard", "triton", "cuda", _GPU_NVIDIA_SM70, ("triton",)), ("async_ulysses_qkv", "standard", "eager", "any", _ANY, ()), ("async_ulysses_qkv", "dit", "eager", "any", _ANY, ()), + ("async_ulysses_qkv", "bagel", "eager", "any", _ANY, ()), ("async_ulysses_o", "standard", "eager", "any", _ANY, ()), ("async_ulysses_o", "dit", "eager", "any", _ANY, ()), + ("async_ulysses_o", "bagel", "eager", "any", _ANY, ()), ("attention", "standard", "eager", "any", _ANY, ()), ("attention", "standard", "sdpa", "any", _ANY, ()), ("attention", "standard", "flash_attention_2", "cuda", _GPU_SM80, ("flash_attn",)), diff --git a/tests/seed_omni/arguments/test_module_args.py b/tests/seed_omni/arguments/test_module_args.py index 9de23dd8a4..21c9c6546a 100644 --- a/tests/seed_omni/arguments/test_module_args.py +++ b/tests/seed_omni/arguments/test_module_args.py @@ -509,6 +509,39 @@ def test_a_launcher_infer_graph_does_not_inherit_the_checkpoint_infer_type(tmp_p assert build_omni_model_runtime_args(args, for_inference=True).infer_type == "understanding" +def test_build_model_runtime_args_merges_checkpoint_generation_kwargs(tmp_path): + """Checkpoint ``generation_kwargs`` supply the defaults and ``infer.generation_kwargs`` overrides per key, + even when the launcher YAML also declares ``model.model_config.infer_graph``.""" + infer_dir = _cfg_dir() / "infer" + runtime_cfg = _model_runtime( + model_path=str(tmp_path), + infer_graph={"infer_und": str(infer_dir / "graph_infer_und.yaml")}, + infer_type="infer_und", + ) + runtime_cfg.generation_kwargs = {"num_timesteps": 50, "timestep_shift": 3.0} + export_root = tmp_path / "exported" + _with_module_configs(runtime_cfg.to_hf_config()).save_pretrained(export_root) + + args = OmniArguments( + model=OmniModelRuntimeArguments(model_path=str(export_root)), + data=OmniDataArguments(train_path=""), + infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}), + ) + resolved = build_omni_model_runtime_args(args, for_inference=True) + assert resolved.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0} + + args_with_launcher_graph = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=str(export_root), + model_config={"infer_graph": {"understanding": str(infer_dir / "graph_infer_und.yaml")}}, + ), + data=OmniDataArguments(train_path=""), + infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}), + ) + resolved_with_launcher_graph = build_omni_model_runtime_args(args_with_launcher_graph, for_inference=True) + assert resolved_with_launcher_graph.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0} + + def test_build_model_runtime_args_keeps_every_training_scenario_from_omni_checkpoint(tmp_path): """The checkpoint stores the whole ``training_graphs`` map and its ``train_type``; reading back only the active graph would rename it ``default`` and drop the rest.""" diff --git a/tests/seed_omni/model/test_processor.py b/tests/seed_omni/model/test_processor.py index d14ec82a2c..8b69afe393 100644 --- a/tests/seed_omni/model/test_processor.py +++ b/tests/seed_omni/model/test_processor.py @@ -155,6 +155,7 @@ def test_omni_processor_from_pretrained_collects_module_preprocessors( mock_from_pretrained.assert_called_once_with(tmp_path, infer_type="infer_gen") fake_mod_cls.preprocessor_class.from_pretrained.assert_called_once() assert len(processor._preprocessors) == 1 + assert processor.infer_type == "infer_gen" @patch("veomni.models.seed_omni.processing_omni.OMNI_MODEL_REGISTRY") diff --git a/tests/seed_omni/modules/bagel/__init__.py b/tests/seed_omni/modules/bagel/__init__.py new file mode 100644 index 0000000000..54ca978df6 --- /dev/null +++ b/tests/seed_omni/modules/bagel/__init__.py @@ -0,0 +1 @@ +"""BAGEL SeedOmni tests.""" diff --git a/tests/seed_omni/modules/bagel/helpers.py b/tests/seed_omni/modules/bagel/helpers.py new file mode 100644 index 0000000000..59fcb3239d --- /dev/null +++ b/tests/seed_omni/modules/bagel/helpers.py @@ -0,0 +1,382 @@ +"""Shared helpers for BAGEL tests.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Literal + +import torch +import torch.nn.functional as F + +from veomni.arguments.omni_arguments_types import ( + OmniArguments, + OmniDataArguments, + OmniInferArguments, + OmniModelRuntimeArguments, + build_omni_model_runtime_args, + build_omni_module_runtime_args, +) +from veomni.models.seed_omni import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_MODEL_REGISTRY +from veomni.models.seed_omni.configuration_omni import OmniConfig +from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY +from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import PackedConversation, preprocess_mot_inputs +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT +from veomni.models.seed_omni.modules.bagel.text_encoder.chat_template import BagelChatTemplate +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem + + +ToyCase = Literal["ce_only", "vit_ce", "mse_only", "mixed"] + +ALIGN_ATOL = 2e-2 +ALIGN_RTOL = 2e-2 +ALIGN_GRAD_ATOL = 5e-2 +ALIGN_GRAD_RTOL = 5e-2 + + +class FakeBagelTokenizer: + """Char-ordinal tokenizer with the marker tokens/ids that ``BagelChatTemplate`` resolves.""" + + bos_token = "" + eos_token = "" + boi_token = "" + eoi_token = "" + bos_token_id = 1 + eos_token_id = 2 + boi_token_id = 3 + eoi_token_id = 4 + pad_token_id = 0 + unk_token_id = -1 + special_tokens = { + "": 1, + "": 2, + "": 3, + "": 4, + "<|im_start|>": 5, + "<|im_end|>": 2, + "<|vision_start|>": 7, + "<|vision_end|>": 8, + } + + def convert_tokens_to_ids(self, token): + return self.special_tokens.get(token, self.unk_token_id) + + def decode(self, token_ids, skip_special_tokens=True): + del skip_special_tokens + special_by_id = {v: k for k, v in self.special_tokens.items()} + return "".join(special_by_id.get(int(t), chr(max(32, int(t)))) for t in token_ids) + + def __call__(self, text, add_special_tokens=False): + del add_special_tokens + input_ids = [] + index = 0 + while index < len(text): + matched = False + for token, token_id in sorted(self.special_tokens.items(), key=lambda item: len(item[0]), reverse=True): + if text.startswith(token, index): + input_ids.append(token_id) + index += len(token) + matched = True + break + if matched: + continue + input_ids.append(ord(text[index])) + index += 1 + return {"input_ids": input_ids} + + +def bagel_template() -> BagelChatTemplate: + return BagelChatTemplate(FakeBagelTokenizer()) + + +def bagel_cfg_dir() -> Path: + return Path(__file__).resolve().parents[4] / "configs" / "seed_omni" / "Bagel" / "bagel_7b_mot" + + +def config_cls(model_type: str): + return OMNI_CONFIG_REGISTRY[model_type]() + + +def model_cls(model_type: str): + """Return the VeOmni-accelerated module class (training / graph hooks).""" + return OMNI_ACCELERATED_MODEL_REGISTRY[model_type]() + + +def native_model_cls(model_type: str): + """Return the HF-native module class (eager inference weights).""" + return OMNI_MODEL_REGISTRY[model_type]() + + +def load_module_runtime_args( + *, + model_path: str = "", + modules_path: Path, + for_inference: bool = False, +) -> dict: + """Build per-module ``OmniModuleRuntimeArguments`` from ``modules_path``.""" + base = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=model_path or ".", + model_config={"modules": str(modules_path)}, + ), + data=OmniDataArguments(train_path=""), + )._to_module_global_args() + return build_omni_module_runtime_args( + global_args=base, + model_path=model_path, + modules=str(modules_path), + for_inference=for_inference, + ) + + +def load_omni_config( + *, + model_path: str = "", + modules_path: Path, + train_graph_path: Path | None = None, + infer_graph_path: Path | None = None, + generation_kwargs: dict | None = None, +) -> OmniConfig: + """Resolve and project an ``OmniConfig`` from BAGEL module and graph YAML files.""" + model_path = model_path or "." + model_config = {"modules": str(modules_path)} + if train_graph_path is not None: + model_config["train_graph"] = str(train_graph_path) + if infer_graph_path is not None: + scenario = infer_graph_path.stem.removeprefix("graph_") + model_config["infer_graph"] = {scenario: str(infer_graph_path)} + model_config["infer_type"] = scenario + args = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=model_path, + model_config=model_config, + ), + data=OmniDataArguments(train_path="."), + infer=OmniInferArguments(generation_kwargs=dict(generation_kwargs or {})), + ) + return build_omni_model_runtime_args(args).to_hf_config() + + +def tiny_bagel_qwen2_cfg() -> dict: + return dict( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "sdpa", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + ) + + +def tiny_align_qwen2_cfg() -> dict: + config = tiny_bagel_qwen2_cfg() + config["num_key_value_heads"] = 2 + return config + + +def _clone_meta(meta: dict[str, Any]) -> dict[str, Any]: + """Deep-copy a ``ConversationItem.meta`` dict while detaching and cloning tensor values.""" + cloned: dict[str, Any] = {} + for key, value in meta.items(): + cloned[key] = value.detach().clone() if torch.is_tensor(value) else copy.deepcopy(value) + return cloned + + +def clone_conversation(conversation: list[list[ConversationItem]]) -> list[list[ConversationItem]]: + """Deep-copy a packed conversation so eager and accelerated can mutate independently.""" + cloned: list[list[ConversationItem]] = [] + for sample in conversation: + cloned_sample: list[ConversationItem] = [] + for item in sample: + value = item.value + if torch.is_tensor(value): + requires_grad = value.requires_grad + value = value.detach().clone() + if requires_grad: + value = value.requires_grad_() + cloned_sample.append( + ConversationItem( + type=item.type, + value=value, + role=item.role, + is_dummy=item.is_dummy, + meta=_clone_meta(item.meta), + ) + ) + cloned.append(cloned_sample) + return cloned + + +def _embed( + generator: torch.Generator, + length: int, + hidden_size: int, + *, + device: torch.device, + dtype: torch.dtype, + requires_grad: bool = True, +) -> torch.Tensor: + """Generate a random ``(length, hidden_size)`` leaf embedding tensor.""" + value = torch.randn(length, hidden_size, generator=generator, device=device, dtype=dtype) + if requires_grad: + value = value.requires_grad_() + return value + + +def build_toy_conversation( + case: ToyCase, + *, + hidden_size: int, + vocab_size: int, + patch_latent_dim: int, + device: torch.device, + dtype: torch.dtype, + seed: int = 17, +) -> list[list[ConversationItem]]: + """Build one packed sample covering CE-only, vit-CE, MSE-only, or mixed.""" + generator = torch.Generator(device=device).manual_seed(seed) + items: list[ConversationItem] = [] + + if case in {"vit_ce", "mixed"}: + items.append( + ConversationItem( + type="image", + value=_embed(generator, 4, hidden_size, device=device, dtype=dtype), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "und"}, + ) + ) + + if case in {"ce_only", "vit_ce", "mixed"}: + labels = torch.randint(3, vocab_size, (8,), generator=generator, device=device) + items.append( + ConversationItem( + type="text", + value=_embed(generator, 8, hidden_size, device=device, dtype=dtype), + role="assistant", + meta={"labels": labels}, + ) + ) + + if case in {"mse_only", "mixed"}: + items.append( + ConversationItem( + type="image", + value=_embed(generator, 6, hidden_size, device=device, dtype=dtype), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: "gen", + "flow_velocity_target": torch.randn( + 6, + patch_latent_dim, + generator=generator, + device=device, + dtype=dtype, + ), + }, + ) + ) + + return [items] + + +def scatter_mot_hidden( + conversation: list[list[ConversationItem]], + packed: PackedConversation, + hidden_states: torch.Tensor, +) -> None: + """Write packed MoT hidden states back onto the original carrier items.""" + for span in packed.spans: + span_hidden = hidden_states[span.start : span.start + span.length] + offset = 0 + for item, length in zip(span.items, span.lengths, strict=True): + item.value = span_hidden[offset : offset + length] + offset += length + + +def run_eager_mot( + model: torch.nn.Module, + conversation: list[list[ConversationItem]], +) -> torch.Tensor: + """Pack ``conversation``, run eager ``BagelQwen2MoT``, and scatter hidden states back.""" + packed = preprocess_mot_inputs( + conversation, + device=model.device, + dtype=model.dtype, + hidden_size=int(model.config.hidden_size), + ) + if packed is None: + raise ValueError("toy conversation produced no packable tokens") + outputs = model( + packed_sequence=packed.packed_sequence, + packed_position_ids=packed.packed_position_ids, + packed_token_type_ids=packed.packed_token_type_ids, + packed_attention_metadata=packed.packed_attention_metadata, + ) + hidden_states = outputs["hidden_states"] + scatter_mot_hidden(conversation, packed, hidden_states) + return hidden_states + + +def run_accelerated_mot( + model: torch.nn.Module, + conversation: list[list[ConversationItem]], +) -> torch.Tensor: + inputs = model.forward_pre(conversation_list=conversation) + outputs = model(**inputs) + hidden_states = outputs["hidden_states"] + model.forward_post(**outputs) + return hidden_states + + +def conversation_ce_loss( + conversation: list[list[ConversationItem]], + lm_head: torch.nn.Linear, +) -> torch.Tensor | None: + """Compute shifted causal LM cross-entropy loss over text items in ``conversation``.""" + hidden_parts: list[torch.Tensor] = [] + label_parts: list[torch.Tensor] = [] + for item in (item for sample in conversation for item in sample): + labels = item.meta.get("labels") + if item.type != "text" or not torch.is_tensor(item.value) or not torch.is_tensor(labels): + continue + hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value + hidden_parts.append(hidden) + label_parts.append(labels.to(device=hidden.device)) + if not hidden_parts: + return None + hidden = torch.cat(hidden_parts, dim=0) + labels = torch.cat(label_parts, dim=0) + shift_labels = F.pad(labels[..., 1:].contiguous(), (0, 1), value=-100) + logits = lm_head(hidden.float()) + return F.cross_entropy(logits, shift_labels, ignore_index=-100) + + +def conversation_mse_loss( + conversation: list[list[ConversationItem]], + llm2vae: torch.nn.Linear, +) -> torch.Tensor | None: + """Compute flow-matching MSE velocity loss over target image items in ``conversation``.""" + velocity_parts: list[torch.Tensor] = [] + target_parts: list[torch.Tensor] = [] + for item in (item for sample in conversation for item in sample): + target = item.meta.get("flow_velocity_target") + if not torch.is_tensor(item.value) or not torch.is_tensor(target): + continue + hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value + velocity_parts.append(llm2vae(hidden.float())) + target_parts.append(target.to(device=hidden.device, dtype=torch.float32)) + if not velocity_parts: + return None + mse = (torch.cat(velocity_parts, dim=0) - torch.cat(target_parts, dim=0)).square() + return mse.mean(dim=-1).mean() diff --git a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py new file mode 100644 index 0000000000..b3fe186308 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py @@ -0,0 +1,261 @@ +"""Accelerated BAGEL Qwen2-MoT alignment: single-GPU eager parity and 4-GPU FSDP2 + Async Ulysses.""" + +from __future__ import annotations + +import copy +import importlib.util +from dataclasses import replace + +import pytest +import torch +import torch.distributed as dist +from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + ALIGN_ATOL, + ALIGN_GRAD_ATOL, + ALIGN_GRAD_RTOL, + ALIGN_RTOL, + build_toy_conversation, + clone_conversation, + config_cls, + conversation_ce_loss, + conversation_mse_loss, + native_model_cls, + run_accelerated_mot, + run_eager_mot, + tiny_align_qwen2_cfg, +) +from tests.tools.launch_utils import torchrun +from veomni.distributed.parallel_state import _init_parallel_state, use_parallel_state +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + pad_mot_attention_metadata, +) +from veomni.ops.kernels.attention.standard import flex as flex_backend +from veomni.ops.kernels.attention.standard.magi import _kernel as magi_kernel +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type, get_torch_device +from veomni.utils.helper import enable_high_precision_for_bf16 + + +_PATCH_LATENT_DIM = 4 +_MAGI_IMPLEMENTATION = "veomni_magi_attention" +_FLEX_IMPLEMENTATION = "veomni_flex_attention" + + +def _is_magi_ffa_available() -> bool: + """Return whether a runnable MagiAttention CUTE DSL/JIT backend is installed on this GPU.""" + if not IS_CUDA_AVAILABLE or importlib.util.find_spec("magi_attention") is None: + return False + + device = torch.device(get_device_type()) + return magi_kernel.get_kernel_mode(device) == magi_kernel.KERNEL_CUTE_JIT and ( + importlib.util.find_spec("flash_attn_cute") is not None + ) + + +_MAGI_FFA_AVAILABLE = _is_magi_ffa_available() +_MAGI_FFA_REASON = "MagiAttention alignment requires an NVIDIA SM100+ GPU with the CUTE DSL/JIT backend" + + +def _build_aligned_models(device: torch.device, dtype: torch.dtype, *, attn_implementation: str): + torch.manual_seed(29) + cfg = tiny_align_qwen2_cfg() + cfg.pop("ops_implementation", None) + eager_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation="sdpa") + accel_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation=attn_implementation) + eager = native_model_cls("bagel_qwen2_mot")(eager_cfg).to(device=device, dtype=dtype).train() + accel = BagelQwen2MoTAccelerated(accel_cfg).to(device=device, dtype=dtype).train() + accel.load_state_dict(copy.deepcopy(eager.state_dict())) + return eager, accel + + +def _assert_accelerated_matches_eager(attn_implementation: str) -> None: + """Verify ``BagelQwen2MoTAccelerated`` matches eager SDPA outputs, CE/MSE losses, and gradients on mixed data.""" + device = torch.device(get_device_type()) + dtype = torch.bfloat16 + eager, accel = _build_aligned_models(device, dtype, attn_implementation=attn_implementation) + hidden_size = int(eager.config.hidden_size) + vocab_size = int(eager.config.vocab_size) + conversation = build_toy_conversation( + "mixed", + hidden_size=hidden_size, + vocab_size=vocab_size, + patch_latent_dim=_PATCH_LATENT_DIM, + device=device, + dtype=dtype, + ) + eager_conversation = clone_conversation(conversation) + accel_conversation = clone_conversation(conversation) + lm_head = torch.nn.Linear(hidden_size, vocab_size, bias=False).to(device=device, dtype=torch.float32) + llm2vae = torch.nn.Linear(hidden_size, _PATCH_LATENT_DIM, bias=False).to(device=device, dtype=torch.float32) + + eager_hidden = run_eager_mot(eager, eager_conversation) + accel_hidden = run_accelerated_mot(accel, accel_conversation) + assert torch.isfinite(eager_hidden).all() and torch.isfinite(accel_hidden).all() + assert eager_hidden.shape == accel_hidden.shape + + eager_ce = conversation_ce_loss(eager_conversation, lm_head) + accel_ce = conversation_ce_loss(accel_conversation, lm_head) + eager_mse = conversation_mse_loss(eager_conversation, llm2vae) + accel_mse = conversation_mse_loss(accel_conversation, llm2vae) + assert eager_ce is not None and accel_ce is not None + assert eager_mse is not None and accel_mse is not None + torch.testing.assert_close(accel_ce, eager_ce, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + torch.testing.assert_close(accel_mse, eager_mse, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + + eager_loss = eager_hidden.float().square().mean() + eager_ce + eager_mse + accel_loss = accel_hidden.float().square().mean() + accel_ce + accel_mse + eager_loss.backward() + accel_loss.backward() + + for name in ( + "model.layers.0.self_attn.o_proj.weight", + "model.layers.0.mlp.gate_proj.weight", + "model.norm.weight", + ): + eager_grad = dict(eager.named_parameters())[name].grad + accel_grad = dict(accel.named_parameters())[name].grad + assert eager_grad is not None and accel_grad is not None + torch.testing.assert_close( + accel_grad, + eager_grad, + atol=ALIGN_GRAD_ATOL, + rtol=ALIGN_GRAD_RTOL, + msg=f"gradient mismatch for {name}", + ) + + +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1)) +@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="FlexAttention alignment requires CUDA") +def test_accelerated_ce_mse_matches_eager_on_toy_data() -> None: + _assert_accelerated_matches_eager(_FLEX_IMPLEMENTATION) + + +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1)) +@pytest.mark.skipif(not _MAGI_FFA_AVAILABLE, reason=_MAGI_FFA_REASON) +def test_magi_ce_mse_matches_eager_on_toy_data() -> None: + _assert_accelerated_matches_eager(_MAGI_IMPLEMENTATION) + + +def _bagel_fa4_async_worker() -> None: + """Compare synchronous and asynchronous Ulysses in ``BagelQwen2MoTAccelerated`` under FSDP2 + Flex-FA4 over 2 steps.""" + rank = dist.get_rank() + device = torch.device(f"{get_device_type()}:{rank}") + dtype = torch.bfloat16 + torch.manual_seed(6321) + enable_high_precision_for_bf16() + state = _init_parallel_state(dp_size=1, ulysses_size=4, name="bagel_sync_test") + async_state = replace(state, async_enabled=True) + + # SP4 leaves seven Q heads and one KV head per rank, matching BAGEL's + # non-power-of-two GQA geometry while exercising FA4 + Liger + checkpointing + FSDP2. + config = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=896, + intermediate_size=1024, + num_hidden_layers=1, + num_attention_heads=28, + num_key_value_heads=4, + max_position_embeddings=1024, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "flex_attention", + "rms_norm_implementation": "liger_kernel", + "rotary_pos_emb_implementation": "liger_kernel", + "swiglu_mlp_implementation": "eager", + }, + ) + reference = accelerated.BagelQwen2MoTAccelerated(config).to(device=device, dtype=dtype).train() + candidate = copy.deepcopy(reference) + for model in (reference, candidate): + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + model.float() + policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32) + for layer in model.model.layers: + fully_shard(layer, mesh=state.fsdp_mesh, mp_policy=policy) + fully_shard(model.model, mesh=state.fsdp_mesh, mp_policy=policy) + + selected_backends = [] + original_resolve = flex_backend.resolve_flex_attention + + def capture_backend(*args, **kwargs): + interface = original_resolve(*args, **kwargs) + selected_backends.append(interface) + return interface + + flex_backend.resolve_flex_attention = capture_backend + sequence_length, padded_length = 259, 260 + metadata = build_mot_attention_metadata( + [[65, 64, 64], [33, 33]], + [["causal", "noise", "noise"], ["full", "causal"]], + device=device, + ) + metadata = pad_mot_attention_metadata(metadata, padded_length=padded_length) + hidden = torch.randn(padded_length, config.hidden_size, device=device, dtype=dtype) + token_types = torch.zeros(padded_length, device=device, dtype=torch.long) + token_types[65:193] = 1 + token_types[sequence_length:] = -1 + hidden[sequence_length:] = 0 + output_gradient = torch.randn_like(hidden) + output_gradient[sequence_length:] = 0 + positions = torch.arange(padded_length, device=device) + start, stop = rank * 65, (rank + 1) * 65 + inputs = { + "packed_position_ids": positions[start:stop], + "packed_attention_metadata": metadata, + "packed_und_token_indexes": torch.where(token_types[start:stop] == 0)[0], + "packed_gen_token_indexes": torch.where(token_types[start:stop] == 1)[0], + } + optimizers = [torch.optim.SGD(model.parameters(), lr=1e-3) for model in (reference, candidate)] + try: + for _ in range(2): + outputs, input_gradients = [], [] + for model, model_state in ((reference, state), (candidate, async_state)): + local_input = hidden[start:stop].detach().clone().requires_grad_() + with use_parallel_state(model_state): + output = model.model(packed_sequence=local_input, **inputs).packed_query_sequence + output.backward(output_gradient[start:stop]) + outputs.append(output.detach()) + input_gradients.append(local_input.grad) + torch.testing.assert_close(outputs[1], outputs[0], atol=3e-2, rtol=3e-2) + torch.testing.assert_close(input_gradients[1], input_gradients[0], atol=3e-2, rtol=3e-2) + for (name, expected), (actual_name, actual) in zip( + reference.named_parameters(), candidate.named_parameters(), strict=True + ): + assert name == actual_name + assert expected.grad is not None and actual.grad is not None, name + expected_grad, actual_grad = expected.grad.to_local(), actual.grad.to_local() + assert torch.isfinite(actual_grad).all(), name + error = (actual_grad.float() - expected_grad.float()).norm() + scale = expected_grad.float().norm() + assert error <= 0.01 * scale + 1e-7, ( + f"{name}: relative gradient L2 error {error / scale.clamp_min(1e-12)}" + ) + + for optimizer in optimizers: + optimizer.step() + optimizer.zero_grad(set_to_none=True) + assert selected_backends and all( + selected is flex_backend._flex_attention_fa4 for selected in selected_backends + ) + finally: + flex_backend.resolve_flex_attention = original_resolve + + +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=4)) +@pytest.mark.skipif( + not IS_CUDA_AVAILABLE or get_torch_device().device_count() < 4, + reason="BAGEL FA4 + async Ulysses integration requires four SM90+ GPUs", +) +def test_bagel_fa4_async_matches_sync_forward_backward_and_updates() -> None: + """Verify BAGEL async Ulysses matches synchronous Ulysses under Flex-FA4, Liger, checkpointing, and FSDP2.""" + device = torch.device(get_device_type()) + if flex_backend.get_gpu_compute_capability(device) < 90 or not flex_backend._fa4._flash_attn_cute_available(): + pytest.skip("Flex-FA4 integration requires SM90+ and flash-attn-4") + torchrun(_bagel_fa4_async_worker, world_size=4) diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py new file mode 100644 index 0000000000..ff15caea08 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -0,0 +1,237 @@ +"""End-to-end BAGEL generation tests across ``infer_und``, ``infer_gen``, and ``infer_edit``.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch +from PIL import Image + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + FakeBagelTokenizer, + bagel_cfg_dir, + bagel_template, + config_cls, + load_omni_config, + native_model_cls, + tiny_bagel_qwen2_cfg, +) +from veomni.models.seed_omni.modeling_omni import OmniModel +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + BagelSiglipNavitPreprocessor, + BagelSiglipNavitProcessor, +) +from veomni.models.seed_omni.modules.bagel.sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_START_TOKEN, + BAGEL_VAE_CONTEXT, +) +from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor +from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEPreprocessor, BagelVAEProcessor +from veomni.models.seed_omni.processing_omni import OmniProcessor +from veomni.models.seed_omni.utils.conversation import ConversationItem + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _build_tiny_bagel_omni_model_and_processor(infer_graph_rel: str) -> tuple[OmniModel, OmniProcessor]: + """Build a tiny 5-module real BAGEL ``OmniModel`` and ``OmniProcessor`` for ``infer_graph_rel``.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / infer_graph_rel, + ) + cfg.training_graphs = {} + + hidden_size = 8 + qwen_kwargs = { + **tiny_bagel_qwen2_cfg(), + "hidden_size": hidden_size, + "intermediate_size": 16, + "num_attention_heads": 2, + "num_key_value_heads": 2, + } + text_encoder = native_model_cls("bagel_text_encoder")( + config_cls("bagel_text_encoder")(vocab_size=int(qwen_kwargs["vocab_size"]), hidden_size=hidden_size) + ).eval() + text_encoder._tokenizer = FakeBagelTokenizer() + text_encoder._chat_template = bagel_template() + + siglip = native_model_cls("bagel_siglip_navit")( + config_cls("bagel_siglip_navit")( + hidden_size=hidden_size, + output_size=hidden_size, + intermediate_size=16, + num_hidden_layers=1, + num_attention_heads=2, + image_size=4, + patch_size=2, + rope=False, + ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"}, + ) + ).eval() + + qwen_mot = native_model_cls("bagel_qwen2_mot")(config_cls("bagel_qwen2_mot")(**qwen_kwargs)).eval() + + flow = native_model_cls("bagel_flow_connector")( + config_cls("bagel_flow_connector")( + hidden_size=hidden_size, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + resolution=4, + ) + ).eval() + + vae_cfg = config_cls("bagel_vae")( + resolution=4, + ch=32, + ch_mult=[1], + num_res_blocks=1, + z_channels=2, + max_image_size=4, + min_image_size=4, + image_stride=2, + max_pixels=16, + downsample=1, + ) + vae = native_model_cls("bagel_vae")(vae_cfg).eval() + vae._image_processor = BagelVAEProcessor.from_config(vae_cfg) + + model = OmniModel( + cfg, + { + "bagel_text_encoder": text_encoder, + "bagel_siglip_navit": siglip, + "bagel_qwen2_mot": qwen_mot, + "bagel_flow_connector": flow, + "bagel_vae": vae, + }, + ).eval() + preprocessors = { + "bagel_vae": BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.float32, + ), + "bagel_siglip_navit": BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.float32, + ), + "bagel_text_encoder": BagelTextEncoderPreprocessor(bagel_template()), + } + return model, OmniProcessor(preprocessors, infer_type=model.config.infer_type) + + +def test_bagel_e2e_generation_across_und_gen_and_edit_scenarios(tmp_path: Path) -> None: + """Verify real 5-module BAGEL ``OmniProcessor`` + ``OmniModel.generate`` across ``infer_und``, ``infer_gen``, and ``infer_edit``.""" + image_path = tmp_path / "sample.png" + Image.new("RGB", (4, 4), color=(12, 34, 56)).save(str(image_path)) + + # 1. infer_und: routes image to SigLIP only, appends BAGEL_START_TOKEN, prefills + decodes text + und_model, und_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_und.yaml") + assert und_model.config.infer_type == "infer_und" + und_request = und_processor(text="hi", images=str(image_path)) + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_request["conversation_list"] if item.type == "image"] == [ + BAGEL_SIGLIP_CONTEXT + ] + assert und_request["conversation_list"][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + und_model.reset() + und_generated = und_model.generate( + und_request, + generation_kwargs={"max_new_tokens": 2, "do_sample": False}, + ) + assert any(item["type"] == "text" for item in und_generated) + assert und_model.modules_dict["bagel_qwen2_mot"]._generation_state.main.cache is not None + + # 2. infer_gen: text prompt -> CFG flow denoise loop -> decoded PIL image + gen_model, gen_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_gen.yaml") + assert gen_model.config.infer_type == "infer_gen" + gen_request = gen_processor(text="draw") + gen_model.reset() + gen_generated = gen_model.generate( + gen_request, + generation_kwargs={ + "num_timesteps": 2, + "image_height": 4, + "image_width": 4, + "latent_downsample": 2, + "cfg_text_scale": 2.0, + "cfg_img_scale": 1.0, + }, + ) + assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in gen_generated) + + # 3. infer_edit: routes user image to both VAE and SigLIP context -> 3-branch CFG denoise -> decoded PIL image + edit_model, edit_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_edit.yaml") + assert edit_model.config.infer_type == "infer_edit" + edit_request = edit_processor(text="edit", images=str(image_path)) + assert [ + item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_request["conversation_list"] if item.type == "image" + ] == [BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT] + edit_model.reset() + edit_generated = edit_model.generate( + edit_request, + generation_kwargs={ + "num_timesteps": 2, + "image_height": 4, + "image_width": 4, + "latent_downsample": 2, + "cfg_text_scale": 2.0, + "cfg_img_scale": 1.5, + }, + ) + assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in edit_generated) + + +def test_bagel_qwen2_mot_accelerated_denoise_branch_packs_cfg_branches(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify accelerated ``denoise_branch`` stacks active CFG branches into one packed ``forward_inference`` call.""" + model = BagelQwen2MoTAccelerated(config_cls("bagel_qwen2_mot")(**tiny_bagel_qwen2_cfg())) + state = model._generation_state + for branch, length, next_pos in ((state.main, 2, 3), (state.cfg_text, 0, 7), (state.cfg_img, 3, 11)): + cache = model._new_empty_cache() + if length > 0: + cache.key_cache[0] = torch.ones(length, 1, 1) + cache.value_cache[0] = torch.ones(length, 1, 1) + branch.install_cache( + cache=cache, + cache_len=length, + next_position_id=torch.tensor(next_pos), + device=model.device, + ) + + captured: dict[str, object] = {} + + def _capture_forward_inference(self, **kwargs): + del self + captured.update(kwargs) + return {"hidden_states": kwargs["packed_query_sequence"]} + + monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference) + tail = ConversationItem( + type="output", + value=torch.zeros(5, int(model.config.hidden_size)), + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, + ) + model.denoise_branch([tail], generation_kwargs={"cfg_text_scale": 2.0, "cfg_img_scale": 1.5}) + + assert captured["query_lens"].tolist() == [5, 5, 5] + assert int(captured["packed_query_sequence"].shape[0]) == 15 + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN + assert int(tail.value.shape[0]) == 15 diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py new file mode 100644 index 0000000000..ac48a1f003 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -0,0 +1,298 @@ +"""BAGEL preprocessing, MoT packing, and offline VAE cache round-trip tests.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + bagel_template, + config_cls, + model_cls, + tiny_bagel_qwen2_cfg, +) +from veomni.data.dataset import get_length_by_labels_fn, get_length_fn +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated as mot_acc +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import TrainingMixin +from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import preprocess_mot_inputs +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + _OMNI_TOKEN_LEN as BAGEL_SIGLIP_TOKEN_LEN, +) +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + BagelSiglipNavitPreprocessor, + BagelSiglipNavitProcessor, +) +from veomni.models.seed_omni.modules.bagel.sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_START_TOKEN, + BAGEL_VAE_CONTEXT, +) +from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor +from veomni.models.seed_omni.modules.bagel.vae.processing import ( + BAGEL_VAE_PIXEL_SHAPE, + BAGEL_VAE_POSTERIOR, + BagelVAEPreprocessor, + BagelVAEProcessor, +) +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _tiny_vae(meta: bool = False, **config_overrides): + """Construct a tiny ``BagelVAE`` on CPU or meta device with a bound processor.""" + BagelVAE = model_cls("bagel_vae") + BagelVAEConfig = config_cls("bagel_vae") + config_kwargs = dict( + resolution=8, + ch=32, + ch_mult=[1], + num_res_blocks=1, + z_channels=2, + max_image_size=8, + min_image_size=4, + image_stride=4, + max_pixels=64, + downsample=1, + ) + config_kwargs.update(config_overrides) + with torch.device("meta" if meta else "cpu"): + model = BagelVAE(BagelVAEConfig(**config_kwargs)) + model._image_processor = BagelVAEProcessor.from_config(model.config) + return model + + +def _build_preprocessors(): + return ( + BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ), + BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 12, dtype=torch.bfloat16), + ), + BagelTextEncoderPreprocessor(bagel_template()), + ) + + +def test_bagel_preprocessors_route_training_and_inference_contexts() -> None: + """Verify training (``edit``/``gen`` + text-only dummy injection + ``dyn_bsz`` token lens) and inference routing.""" + vae_pre, siglip_pre, text_pre = _build_preprocessors() + user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) + assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) + + # 1. Training batch: sample 0 has edit + gen images; sample 1 is text-only (receives both VAE and SigLIP dummies) + train_batch = [ + [ + ConversationItem(type="text", value="edit", role="user"), + ConversationItem(type="image", value=user_image.clone(), role="user", meta={_IMG_TAG_KEY: "edit"}), + ConversationItem( + type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} + ), + ], + [ + ConversationItem(type="text", value="hi", role="user"), + ConversationItem(type="text", value="ok", role="assistant"), + ], + ] + for pre in (vae_pre, siglip_pre, text_pre): + pre({"conversation_list": train_batch}) + + edit_sample, text_sample = train_batch + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_sample if item.type == "image"] == [ + BAGEL_VAE_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + ] + assert [item.meta.get(_IMG_TAG_KEY) for item in edit_sample if item.type == "image"] == ["edit", "edit", "gen"] + dummies = [item for item in text_sample if item.is_dummy and item.type == "image"] + assert {item.meta.get(BAGEL_CONTEXT_KEY) for item in dummies} == {BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT} + for sample in train_batch: + assert all( + isinstance(item.meta.get(_OMNI_TOKEN_LEN), int) and item.meta[_OMNI_TOKEN_LEN] > 0 for item in sample + ) + assert ( + get_length_fn({"conversation_list": edit_sample}) + > get_length_by_labels_fn({"conversation_list": edit_sample}) + > 0 + ) + + # 2. Inference routing: infer_und routes to SigLIP + appends BAGEL_START_TOKEN; infer_edit expands to VAE + SigLIP + und_batch = [[ConversationItem(type="image", value=user_image.clone(), role="user")]] + for pre in (vae_pre, siglip_pre, text_pre): + pre({"conversation_list": und_batch}, inference=True, generation_kwargs={"infer_type": "infer_und"}) + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_batch[0] if item.type == "image"] == [ + BAGEL_SIGLIP_CONTEXT + ] + assert und_batch[0][1].meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 + assert und_batch[0][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + + +def test_bagel_mot_packing_sp_padding_and_scatter(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify ``preprocess_mot_inputs`` expert routing, ``forward_pre`` SP padding, and ``forward_post`` triplet scatter.""" + BagelQwen2MoT = model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train() + hidden_size = int(model.config.hidden_size) + + monkeypatch.setattr( + mot_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=4, cp_size=1, ulysses_size=4, sp_group=object()), + ) + + def _fake_sp_pad(tensor, dim, pad_value): + pad_shape = list(tensor.shape) + pad_shape[dim] = 3 + padding = torch.full(pad_shape, pad_value, device=tensor.device, dtype=tensor.dtype) + return torch.cat((tensor, padding), dim=dim) + + monkeypatch.setattr(mot_acc, "sp_pad", _fake_sp_pad) + monkeypatch.setattr(mot_acc, "slice_input_tensor", lambda tensor, **kwargs: tensor) + + text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") + image = ConversationItem( + type="image", + value=torch.ones(3, hidden_size), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + inputs = model.forward_pre(conversation_list=[[text, image]]) + assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1, -1, -1, -1])) + assert torch.equal(inputs["packed_attention_metadata"][2, 5:], torch.tensor([-1, -1, -1], dtype=torch.int32)) + + # Triplet scatter in forward_post + def _marker(context: str) -> ConversationItem: + return ConversationItem( + type="text", + value=torch.zeros(1, 4), + role="user", + meta={"labels": torch.full((1,), -100), BAGEL_CONTEXT_KEY: context}, + ) + + latent_item = ConversationItem( + type="image", + value=torch.zeros(2, 4), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, + ) + siglip_item = ConversationItem( + type="image", + value=torch.zeros(3, 4), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "edit"}, + ) + sample = [ + _marker(BAGEL_VAE_CONTEXT), + latent_item, + _marker(BAGEL_VAE_CONTEXT), + _marker(BAGEL_SIGLIP_CONTEXT), + siglip_item, + _marker(BAGEL_SIGLIP_CONTEXT), + ] + packed = preprocess_mot_inputs([sample], device=torch.device("cpu"), dtype=torch.float32, hidden_size=4) + assert packed is not None + assert torch.equal(packed.packed_token_type_ids, torch.tensor([0, 1, 1, 0, 0, 0, 0, 0, 0])) + hidden_states = torch.arange(9 * 4, dtype=torch.float32).reshape(9, 4) + monkeypatch.setattr( + mot_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=1, cp_size=1, ulysses_size=1, sp_group=None), + ) + stub = SimpleNamespace(device=torch.device("cpu"), _conversation_carrier=[sample], _packed_training=packed) + TrainingMixin.forward_post(stub, hidden_states) + assert torch.equal(latent_item.value, hidden_states[1:3]) + assert torch.equal(siglip_item.value, hidden_states[5:8]) + + +def test_bagel_vae_encode_and_offline_cache_roundtrip(tmp_path: Path) -> None: + """Verify ragged-shape VAE ``encode``/``offline_encode`` and parquet cache replay into ``BagelFlowConnector``.""" + from datasets import load_dataset + + from veomni.data.seed_omni.seedomni_transform import process_seedomni_cached_example + from veomni.models.seed_omni.utils.offline_cache import SeedOmniOfflineCacheWriter + + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) + meta_vae = _tiny_vae(meta=True, ch_mult=[1, 1], downsample=2) + + edit_img = ConversationItem( + type="image", + value=torch.zeros(3, 4, 8), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 8])}, + ) + gen_img = ConversationItem( + type="image", + value=torch.ones(3, 8, 4), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([8, 4])}, + ) + + # 1. Ragged micro-batch offline_encode matches single-item posteriors without spatial padding + single_edit = copy.deepcopy([[edit_img]]) + vae.post_forward( + "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_edit)) + ) + single_gen = copy.deepcopy([[gen_img]]) + vae.post_forward( + "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_gen)) + ) + + batch = [[copy.deepcopy(edit_img), copy.deepcopy(gen_img)]] + pre = vae.pre_forward("offline_encode", conversation_list=batch) + encoded = vae.post_forward("offline_encode", **vae.offline_encode(**pre))["conversation_list"] + torch.testing.assert_close(encoded[0][0].value, single_edit[0][0].value) + torch.testing.assert_close(encoded[0][1].value, single_gen[0][0].value) + + # 2. Parquet round-trip -> meta-built online_process -> BagelFlowConnector.embed_latent_pre + writer = SeedOmniOfflineCacheWriter(str(tmp_path)) + writer.open() + writer.save_conversation_list(encoded) + writer.flush() + dataset = load_dataset("parquet", data_files=[str(tmp_path / "shard_000000.parquet")], split="train") + replayed = [process_seedomni_cached_example(dataset[0])[0]["conversation_list"]] + assert all(item.meta[BAGEL_VAE_POSTERIOR] for item in replayed[0]) + + assert meta_vae.encoder.conv_in.weight.is_meta + processed = meta_vae.post_forward( + "online_process", + **meta_vae.online_process(**meta_vae.pre_forward("online_process", conversation_list=replayed)), + )["conversation_list"] + assert [item.meta[_IMG_TAG_KEY] for item in processed[0]] == ["edit", "gen"] + + flow = model_cls("bagel_flow_connector")( + config_cls("bagel_flow_connector")( + hidden_size=4, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + timestep_shift=3.0, + ) + ) + edit_item, gen_item = processed[0] + inputs = flow.embed_latent_pre(conversation_list=processed) + assert flow._embed_lengths == [8, 8] + assert torch.equal(inputs["timesteps"][:8], torch.zeros(8)) + assert "flow_velocity_target" not in edit_item.meta + assert gen_item.meta["flow_velocity_target"].shape == (8, 2) + assert torch.all(gen_item.meta["timestep"] == gen_item.meta["timestep"][0]) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py new file mode 100644 index 0000000000..d0a8bf2dc6 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -0,0 +1,361 @@ +"""BAGEL module registry, YAML graph configs, MoT attention masks, QKV checkpoint conversion, and ops.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch +from safetensors import safe_open +from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict +from torch.nn.attention.flex_attention import BlockMask +from transformers.utils import SAFE_WEIGHTS_NAME + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.models.conftest import available_nvidia_ops # noqa: F401 +from tests.seed_omni.modules.bagel.helpers import ( + bagel_cfg_dir, + load_module_runtime_args, + load_omni_config, + tiny_bagel_qwen2_cfg, +) +from veomni.models.checkpoint.weights import load_model_weights +from veomni.models.seed_omni import ( + OMNI_ACCELERATED_MODEL_REGISTRY, + OMNI_CONFIG_REGISTRY, + OMNI_MODEL_REGISTRY, + OMNI_PROCESSOR_REGISTRY, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAttentionAccelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + build_mot_magi_mask, + build_mot_sdpa_mask, + pad_mot_attention_metadata, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT, BagelQwen2RotaryEmbedding +from veomni.models.seed_omni.modules.bagel.siglip_navit.accelerated.accelerated import BagelSiglipNavitAccelerated +from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig +from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from veomni.models.seed_omni.utils.conversation import ConversationItem +from veomni.ops.mask import MagiAttentionMask +from veomni.utils.save_safetensor_utils import get_model_save_state + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _flex_config() -> BagelQwen2MoTConfig: + return BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=1, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "veomni_flex_attention", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + ) + + +def _expand_magi_mask(mask: MagiAttentionMask, sequence_length: int) -> torch.Tensor: + visible = torch.zeros(sequence_length, sequence_length, dtype=torch.bool, device=mask.q_ranges.device) + attn_types = ( + mask.attn_type_map + if mask.attn_type_map is not None + else torch.zeros(mask.q_ranges.shape[0], dtype=torch.int32, device=mask.q_ranges.device) + ) + for range_index in range(mask.q_ranges.shape[0]): + q_start, q_end = int(mask.q_ranges[range_index, 0]), int(mask.q_ranges[range_index, 1]) + k_start, k_end = int(mask.k_ranges[range_index, 0]), int(mask.k_ranges[range_index, 1]) + if int(attn_types[range_index]) == 0: + visible[q_start:q_end, k_start:k_end] = True + else: + q_idx = torch.arange(q_start, q_end, device=visible.device) + k_idx = torch.arange(k_start, k_end, device=visible.device) + visible[q_start:q_end, k_start:k_end] |= q_idx[:, None] >= k_idx[None, :] + return visible + + +@pytest.mark.parametrize( + "infer_graph", + ["infer/graph_infer_und.yaml", "infer/graph_infer_gen.yaml", "infer/graph_infer_edit.yaml"], +) +def test_bagel_registry_and_yaml_graphs_load(infer_graph: str) -> None: + """Verify all five BAGEL modules are registered and train + infer YAML configs resolve cleanly.""" + expected = { + "bagel_text_encoder": ("BagelTextEncoder", "BagelTextEncoderAccelerated"), + "bagel_siglip_navit": ("BagelSiglipNavit", "BagelSiglipNavitAccelerated"), + "bagel_qwen2_mot": ("BagelQwen2MoT", "BagelQwen2MoTAccelerated"), + "bagel_flow_connector": ("BagelFlowConnector", "BagelFlowConnectorAccelerated"), + "bagel_vae": ("BagelVAE", "BagelVAEAccelerated"), + } + for model_type, (native_name, accel_name) in expected.items(): + assert model_type in OMNI_CONFIG_REGISTRY.valid_keys() + assert OMNI_MODEL_REGISTRY[model_type]().__name__ == native_name + assert OMNI_ACCELERATED_MODEL_REGISTRY[model_type]().__name__ == accel_name + assert {"bagel_siglip_navit", "bagel_vae"}.issubset(set(OMNI_PROCESSOR_REGISTRY.valid_keys())) + + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / infer_graph, + ) + assert set(cfg._module_entries) == set(expected) + assert cfg.generation_graph["initial"] == "prompt_encode" + assert any( + t.get("next_state") == "done" + for state in cfg.generation_graph["states"].values() + for t in state.get("transitions", []) + ) + runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") + assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == "veomni_flex_attention" + assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True + + +def test_mot_attention_masks_and_sp_padding() -> None: + """Verify SDPA, Magi, and Flex masks agree on multi-sample causal/full/noise visibility and SP padding.""" + metadata = build_mot_attention_metadata( + [[2, 3, 2], [4, 2]], + [["full", "causal", "noise"], ["causal", "noise"]], + device=torch.device("cpu"), + ) + padded = pad_mot_attention_metadata(metadata, padded_length=16) + sdpa = build_mot_sdpa_mask(padded) + magi = build_mot_magi_mask(padded) + visible = _expand_magi_mask(magi, 16) + torch.testing.assert_close(visible[None, None], sdpa) + assert not bool(visible[:13, 13:].any()) + assert bool(visible[13:, 13:].all()) + assert isinstance( + BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_flex_attention"), + BlockMask, + ) + assert isinstance( + BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_magi_attention"), + MagiAttentionMask, + ) + + +def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path: Path) -> None: + """Verify fused runtime QKV parameters keep fused keys in ``state_dict`` / DCP and split on HF safetensors export.""" + torch.manual_seed(3014) + eager_cfg = BagelQwen2MoTConfig(**{**tiny_bagel_qwen2_cfg(), "num_key_value_heads": 1}) + eager_ref = BagelQwen2MoT(eager_cfg) + eager_split_state = eager_ref.state_dict() + + reference = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + reference.load_state_dict(eager_split_state) + runtime_state = reference.state_dict() + dcp_state = get_model_state_dict(reference) + assert "model.layers.0.self_attn.qkv_proj_und.weight" in runtime_state + assert "model.layers.0.self_attn.qkv_proj_gen.weight" in runtime_state + assert "model.layers.0.self_attn.q_proj.weight" not in runtime_state + assert set(dcp_state) == set(runtime_state) + + dcp_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + set_model_state_dict(dcp_loaded, dcp_state) + for name in ("qkv_proj_und", "qkv_proj_gen"): + expected = getattr(reference.model.layers[0].self_attn, name) + actual = getattr(dcp_loaded.model.layers[0].self_attn, name) + torch.testing.assert_close(actual.weight, expected.weight) + torch.testing.assert_close(actual.bias, expected.bias) + + reverted_hf_state = get_model_save_state(reference, None) + for key in ( + "model.layers.0.self_attn.q_proj.weight", + "model.layers.0.self_attn.k_proj.weight", + "model.layers.0.self_attn.v_proj.weight", + "model.layers.0.self_attn.q_proj_moe_gen.weight", + ): + torch.testing.assert_close(reverted_hf_state[key], eager_split_state[key].to(torch.bfloat16)) + + reference.save_pretrained(tmp_path) + with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint: + checkpoint_keys = set(checkpoint.keys()) + assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_keys + assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_keys + + hf_loaded = accelerated.BagelQwen2MoTAccelerated.from_pretrained(tmp_path, config=_flex_config()) + veomni_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + load_model_weights(veomni_loaded, str(tmp_path), init_device="cpu") + for name in ("qkv_proj_und", "qkv_proj_gen"): + expected = getattr(reference.model.layers[0].self_attn, name) + torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).weight, expected.weight) + torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).weight, expected.weight) + + +def test_bagel_modules_ops_dispatch_meta_rotary_and_flops(available_nvidia_ops) -> None: # noqa: F811 + """Verify module-level ``VeomniOp`` binding, ``meta -> to_empty`` FP32 rotary table, and multi-image SigLIP FLOPs.""" + default_cfg = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=1, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ) + assert default_cfg._attn_implementation == "veomni_flex_attention" + + with torch.device("meta"): + meta_model = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + materialized = meta_model.to_empty(device="cpu").to(dtype=torch.bfloat16) + layer = materialized.model.layers[0] + assert layer.self_attn.veomni_async_ulysses_qkv.variant == "bagel" + assert layer.self_attn.veomni_async_ulysses_o.variant == "bagel" + assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention" + assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2" + + ref_rotary = BagelQwen2RotaryEmbedding(_flex_config()) + rotary = materialized.model.rotary_emb + assert rotary.inv_freq.dtype == torch.float32 + assert rotary.original_inv_freq.dtype == torch.float32 + torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) + + # BagelFlowConnector meta -> to_empty -> init_weights preserves zero-initialized llm2vae velocity head + flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()( + hidden_size=16, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + ) + eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + with torch.device("meta"): + meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + materialized_flow = meta_flow.to_empty(device="cpu") + materialized_flow.init_weights() + assert torch.count_nonzero(materialized_flow.llm2vae.weight) == 0 + assert torch.count_nonzero(materialized_flow.llm2vae.bias) == 0 + assert torch.count_nonzero(materialized_flow.vae2llm.weight) > 0 + torch.testing.assert_close(materialized_flow.latent_pos_embed.pos_embed, eager_flow.latent_pos_embed.pos_embed) + + siglip_cfg = BagelSiglipNavitConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=4, + image_size=28, + patch_size=14, + ops_implementation={"attn_implementation": "sdpa", "rotary_pos_emb_vision_implementation": "eager"}, + ) + siglip = BagelSiglipNavit(siglip_cfg).eval() + assert siglip.vision_model.encoder.layers[0].self_attn.veomni_attn.impl == "veomni_sdpa" + + # Multi-image SigLIP FLOPs meter uses per-image lengths while keeping per-sample token counts + acc_siglip = BagelSiglipNavitAccelerated(siglip_cfg) + acc_siglip.config.num_hidden_layers = 26 + patch_dim = siglip_cfg.num_channels * siglip_cfg.patch_size * siglip_cfg.patch_size + items = [ + ConversationItem( + type="image", + role="user", + value=torch.zeros(1024, patch_dim, dtype=torch.bfloat16), + meta={ + BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, + _OMNI_TOKEN_LEN: 1024, + _OMNI_POSITION_IDS: torch.zeros(1024, dtype=torch.long), + }, + ) + for _ in range(2) + ] + acc_siglip.forward_pre(conversation_list=[items]) + acc_siglip.metric_meter_add("forward") + flops, seqlens = acc_siglip.metric_meter_collect() + assert seqlens == [2048] + assert flops == pytest.approx(acc_siglip.estimate_flops([1024, 1024])) + assert flops < acc_siglip.estimate_flops([2048]) + + +def _two_rank_fsdp2_pos_embed_materialize_worker(rank: int, rendezvous: str) -> None: + """Verify two-rank FSDP2 ``meta -> fully_shard -> to_empty -> init_weights`` materializes sharded ``PositionEmbedding``.""" + import torch.distributed as dist + from torch.distributed.fsdp import fully_shard + from torch.distributed.tensor import DTensor + + from veomni.distributed.parallel_state import _init_parallel_state, clear_parallel_state + + dist.init_process_group("gloo", init_method=f"file://{rendezvous}", world_size=2, rank=rank) + try: + ps = _init_parallel_state(dp_size=2, dp_shard_size=2, ulysses_size=1, dp_mode="fsdp2", device_type="cpu") + + flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()( + hidden_size=16, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + ) + eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + with torch.device("meta"): + meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + fully_shard(meta_flow, mesh=ps.fsdp_mesh) + meta_flow.to_empty(device="cpu") + meta_flow.init_weights() + + assert isinstance(meta_flow.latent_pos_embed.pos_embed.data, DTensor) + assert meta_flow.latent_pos_embed.pos_embed.to_local().shape == (8, 16) + torch.testing.assert_close( + meta_flow.latent_pos_embed.pos_embed.full_tensor(), + eager_flow.latent_pos_embed.pos_embed, + ) + assert torch.count_nonzero(meta_flow.llm2vae.weight.full_tensor()) == 0 + assert torch.count_nonzero(meta_flow.llm2vae.bias.full_tensor()) == 0 + assert torch.count_nonzero(meta_flow.vae2llm.weight.full_tensor()) > 0 + + siglip_cfg = BagelSiglipNavitConfig( + hidden_size=16, + output_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=2, + image_size=8, + patch_size=4, + vit_max_num_patch_per_side=4, + ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"}, + ) + eager_siglip = BagelSiglipNavit(siglip_cfg) + with torch.device("meta"): + meta_siglip = BagelSiglipNavit(siglip_cfg) + fully_shard(meta_siglip, mesh=ps.fsdp_mesh) + meta_siglip.to_empty(device="cpu") + meta_siglip.init_weights() + + assert isinstance(meta_siglip.vit_pos_embed.pos_embed.data, DTensor) + assert meta_siglip.vit_pos_embed.pos_embed.to_local().shape == (8, 16) + torch.testing.assert_close( + meta_siglip.vit_pos_embed.pos_embed.full_tensor(), + eager_siglip.vit_pos_embed.pos_embed, + ) + finally: + clear_parallel_state() + dist.destroy_process_group() + + +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=2, device="cpu")) +def test_bagel_pos_embed_two_rank_fsdp2_meta_materialization(tmp_path: Path) -> None: + """Verify two-rank FSDP2 config-only initialization distributes ``PositionEmbedding.pos_embed`` across shards.""" + import torch.multiprocessing as mp + + mp.spawn( + _two_rank_fsdp2_pos_embed_materialize_worker, + args=(str(tmp_path / "rendezvous"),), + nprocs=2, + join=True, + ) diff --git a/tests/seed_omni/trainer/test_omni_inferencer.py b/tests/seed_omni/trainer/test_omni_inferencer.py index 258ced4713..d9a3b10f96 100644 --- a/tests/seed_omni/trainer/test_omni_inferencer.py +++ b/tests/seed_omni/trainer/test_omni_inferencer.py @@ -88,15 +88,20 @@ def generate(request, generation_kwargs=None): seen["input_ids"] = request["input_ids"] return [] + def processor(**kwargs): + seen["processor_kwargs"] = kwargs + return {"input_ids": torch.ones(2, dtype=torch.long)} + inferencer = _inferencer_with("infer_gen", {}) inferencer._distributed = distributed inferencer.device = torch.device("meta") - inferencer.processor = lambda **kwargs: {"input_ids": torch.ones(2, dtype=torch.long)} + inferencer.processor = processor inferencer.model = SimpleNamespace(reset=lambda: None, generate=generate) - inferencer._run(InferenceRequest(prompt="hi")) + inferencer._run(InferenceRequest(prompt="hi", generation_kwargs={"infer_type": "infer_edit"})) assert seen["input_ids"].device.type == expected_device + assert seen["processor_kwargs"]["infer_type"] == "infer_edit" def test_extract_generated_text_keeps_only_filled_text_items(): diff --git a/veomni/arguments/omni_arguments_types.py b/veomni/arguments/omni_arguments_types.py index f9eb8595d3..10b414e146 100644 --- a/veomni/arguments/omni_arguments_types.py +++ b/veomni/arguments/omni_arguments_types.py @@ -290,7 +290,10 @@ def build_omni_model_runtime_args(args: "OmniArguments", *, for_inference: bool generation_graphs=generation_graphs, train_type=train_type, infer_type=infer_type, - generation_kwargs=dict(args.infer.generation_kwargs), + generation_kwargs={ + **(omni_cfg.generation_kwargs or {} if omni_cfg is not None else {}), + **args.infer.generation_kwargs, + }, ) diff --git a/veomni/models/checkpoint/convert.py b/veomni/models/checkpoint/convert.py index d4b4149478..f5bbffd78f 100644 --- a/veomni/models/checkpoint/convert.py +++ b/veomni/models/checkpoint/convert.py @@ -223,6 +223,8 @@ def get_checkpoint_tensor_converter( model: Union["nn.Module", "PreTrainedModel"], ) -> Optional["CheckpointTensorConverter"]: """Return the checkpoint tensor converter registered on *model*, or ``None``.""" + while isinstance(getattr(model, "module", None), nn.Module): + model = model.module factory = getattr(model, "_create_checkpoint_tensor_converter", None) if factory is None: return None diff --git a/veomni/models/seed_omni/configuration_omni.py b/veomni/models/seed_omni/configuration_omni.py index 323b8a30ca..a02d2f45b0 100644 --- a/veomni/models/seed_omni/configuration_omni.py +++ b/veomni/models/seed_omni/configuration_omni.py @@ -68,11 +68,10 @@ def select_graph( empty_hint: str = "", unknown_hint: str = "scenario", ) -> Any: - """Pick the active graph out of a scenario map; unset ``scenario`` takes the first.""" + """Pick the active graph out of a scenario map.""" if not graphs: raise ValueError(f"No graph scenarios are declared. {empty_hint}".strip()) - if scenario is None: - return next(iter(graphs.values())) + assert scenario is not None, f"`{unknown_hint}` must be normalized before calling `select_graph`." if scenario not in graphs: known = ", ".join(graphs) raise KeyError(f"Unknown {unknown_hint} {scenario!r}; expected one of: {known}.") @@ -220,6 +219,7 @@ def __init__( check_infer_type_names(generation_graphs or {}) self.train_type = train_type self.infer_type = infer_type + self._resolve_graph_types() self.generation_kwargs = generation_kwargs # Model-level kernels: this composed model's defaults for every module # it holds. Sparse — only what was explicitly selected. A loaded module @@ -454,6 +454,17 @@ def load_checkpoint_sidecars(self, checkpoint_root: Union[str, os.PathLike]) -> base_ops_implementation=self.ops_implementation, ) + def _resolve_graph_types(self) -> None: + """Bind the active scenario in each graph map once the map is final.""" + for graphs, attr in ((self.training_graphs, "train_type"), (self.generation_graphs, "infer_type")): + if not graphs: + continue + selected = getattr(self, attr) + if selected is None: + setattr(self, attr, next(iter(graphs))) + elif selected not in graphs: + raise KeyError(f"Unknown {attr} {selected!r}; expected one of: {', '.join(graphs)}.") + def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike]) -> None: """Load ``training_graphs`` and ``generation_graphs`` from YAML sidecars when present. @@ -469,6 +480,7 @@ def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike]) if not self.generation_graphs: self.generation_graphs = self._read_graph_sidecar(root, DEFAULT_GENERATION_GRAPH_FILE, dict) check_infer_type_names(self.generation_graphs) + self._resolve_graph_types() @classmethod def _read_graph_sidecar(cls, root: str, filename: str, payload_type: type) -> Dict: diff --git a/veomni/models/seed_omni/modules/__init__.py b/veomni/models/seed_omni/modules/__init__.py index 0aa5efc852..9d98923cad 100644 --- a/veomni/models/seed_omni/modules/__init__.py +++ b/veomni/models/seed_omni/modules/__init__.py @@ -110,8 +110,8 @@ def read_model_type(model_path: str) -> str: return model_type -# Side-effect only: attach @register factories under base/, fake_model/, qwen3/, qwen3vl/, text_encoder/. -from . import base, fake_model, qwen3, qwen3vl, text_encoder # noqa: F401 E402 +# Side-effect only: attach @register factories under bagel/, base/, fake_model/, qwen3/, qwen3vl/, text_encoder/. +from . import bagel, base, fake_model, qwen3, qwen3vl, text_encoder # noqa: F401 E402 __all__ = [ diff --git a/veomni/models/seed_omni/modules/bagel/__init__.py b/veomni/models/seed_omni/modules/bagel/__init__.py new file mode 100644 index 0000000000..07280c870b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/__init__.py @@ -0,0 +1,25 @@ +"""BAGEL SeedOmni modules. + +The SeedOmni graph keeps BAGEL modules split by producer/consumer boundaries: + +- training: text/SigLIP/VAE/flow hooks write embedded carrier items, then + ``bagel_qwen2_mot.forward`` packs them into the MoT backbone and scatters + hidden states back to the same items. +- prompt inference: ``bagel_text_encoder`` and optional VAE/SigLIP context + feed ``bagel_qwen2_mot.generate`` for prompt prefill or text decoding. +- image denoise: ``flow_connector.prepare_denoise_query`` writes + ``bagel_flow_query``; text marker wrapping feeds ``qwen2_mot.denoise_branch``; + ``flow_connector.decode_velocity_from_hidden`` writes ``bagel_flow_velocity``; + ``qwen2_mot.collect_velocity`` strips branch markers and handles CFG merge; finally + ``flow_connector.advance_denoise`` either loops or emits generated latents + for VAE decode. +""" + +from . import ( + convert_model, # noqa: F401 + flow_connector, # noqa: F401 + qwen2_mot, # noqa: F401 + siglip_navit, # noqa: F401 + text_encoder, # noqa: F401 + vae, # noqa: F401 +) diff --git a/veomni/models/seed_omni/modules/bagel/convert_model.py b/veomni/models/seed_omni/modules/bagel/convert_model.py new file mode 100644 index 0000000000..cb5923f115 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/convert_model.py @@ -0,0 +1,299 @@ +"""Split a BAGEL checkpoint into SeedOmni modules. + +Registered under ``OMNI_CONVERT_REGISTRY["bagel"]``; :func:`convert_checkpoint` +writes the omni checkpoint. Graphs come from ``configs/seed_omni/Bagel/bagel_7b_mot/``. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable + +import torch +from safetensors import safe_open +from transformers import AutoTokenizer +from transformers.initialization import no_init_weights + +from veomni.models.checkpoint.weights import init_empty_weights +from veomni.models.seed_omni.utils.convert_registry import ( + OMNI_CONVERT_REGISTRY, + attach_module_assets, + load_family_graphs, +) + + +BAGEL_GENERATION_KWARGS = { + "max_new_tokens": 2048, + "temperature": 0.0, + "top_p": 1.0, + "do_sample": False, + "image_height": 1024, + "image_width": 1024, + "latent_downsample": 16, + "num_timesteps": 50, + "timestep_shift": 3.0, + "cfg_text_scale": 1.0, + "cfg_img_scale": 1.0, + "cfg_interval": [0.0, 1.0], + "cfg_renorm_min": 0.0, + "cfg_renorm_type": "global", + "enable_taylorseer": False, +} + + +def _read_json(path: Path) -> dict[str, Any]: + """Read and parse a UTF-8 JSON file.""" + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _load_prefixed_safetensors( + path: Path, + remap: Callable[[str], str | None], + consumed_keys: set[str] | None = None, +) -> dict[str, Any]: + """Load safetensors keys selected by ``remap`` onto CPU.""" + state_dict: dict[str, Any] = {} + with safe_open(path, framework="pt", device="cpu") as f: + for key in f.keys(): + target_key = remap(key) + if target_key is None: + continue + if consumed_keys is not None: + consumed_keys.add(key) + state_dict[target_key] = f.get_tensor(key) + return state_dict + + +def _safetensor_keys(path: Path) -> set[str]: + """Return the set of tensor keys stored in ``path``.""" + with safe_open(path, framework="pt", device="cpu") as f: + return set(f.keys()) + + +def _assert_no_unhandled_keys(path: Path, consumed_keys: set[str], ignored_keys: set[str]) -> None: + """Raise if any safetensors key in ``path`` is neither consumed nor ignored.""" + unhandled = sorted(_safetensor_keys(path) - consumed_keys - ignored_keys) + if unhandled: + preview = ", ".join(unhandled[:20]) + raise RuntimeError(f"Unhandled source keys in {path}: {preview}") + + +def _instantiate(model_type: str, config: Any): + """Instantiate an empty-weight SeedOmni module for ``model_type``.""" + from veomni.models.seed_omni.modules import OMNI_MODEL_REGISTRY + + model_cls = OMNI_MODEL_REGISTRY[model_type]() + with no_init_weights(), init_empty_weights(): + return model_cls._from_config(config) + + +def _materialize_allowed_missing_parameters(model: Any, missing: list[str]) -> None: + """Materialize and initialize deterministic missing position-embedding parameters.""" + for key in missing: + if not key.endswith(".pos_embed"): + continue + module_name, _, param_name = key.rpartition(".") + module = model.get_submodule(module_name) + parameter = getattr(module, param_name) + setattr( + module, + param_name, + torch.nn.Parameter(torch.empty(tuple(parameter.shape)), requires_grad=parameter.requires_grad), + ) + if not hasattr(module, "_init_weights"): + raise RuntimeError(f"Cannot initialize deterministic missing parameter: {key}") + module._init_weights() + + +def _build_module( + model_type: str, + config: Any, + state_dict: dict[str, Any], + *, + allowed_missing: set[str] | None = None, +) -> Any: + """Build ``model_type`` from ``config`` and assign ``state_dict`` tensors.""" + model = _instantiate(model_type, config) + allowed_missing = set() if allowed_missing is None else allowed_missing + missing, unexpected = model.load_state_dict(state_dict, strict=False, assign=True) + unexpected = list(unexpected) + missing = list(missing) + unexpected_missing = [key for key in missing if key not in allowed_missing] + if unexpected_missing or unexpected: + raise RuntimeError(f"{model_type} load mismatch: missing={unexpected_missing}, unexpected={unexpected}") + _materialize_allowed_missing_parameters(model, [key for key in missing if key in allowed_missing]) + print(f"[bagel] extracted {model_type}") + return model + + +def convert_bagel_checkpoint( + model_path: str, + *, + max_latent_size: int = 64, + **kwargs: Any, +) -> dict[str, Any]: + """Split an upstream BAGEL checkpoint into five SeedOmni modules.""" + training_graphs, generation_graphs = load_family_graphs( + "configs/seed_omni/Bagel/bagel_7b_mot", + training="train/graph_train.yaml", + generation={ + "infer_edit": "infer/graph_infer_edit.yaml", + "infer_gen": "infer/graph_infer_gen.yaml", + "infer_und": "infer/graph_infer_und.yaml", + }, + training_graph=kwargs.pop("training_graph", None), + generation_graph=kwargs.pop("generation_graph", None), + ) + del kwargs + max_latent_size = int(max_latent_size) + + model_root = Path(model_path) + + ema_path = model_root / "ema.safetensors" + ae_path = model_root / "ae.safetensors" + if not ema_path.exists(): + raise FileNotFoundError(f"Missing BAGEL EMA weights: {ema_path}") + if not ae_path.exists(): + raise FileNotFoundError(f"Missing BAGEL VAE weights: {ae_path}") + + consumed_ema_keys: set[str] = set() + consumed_ae_keys: set[str] = set() + + llm_dict = _read_json(model_root / "llm_config.json") + vit_dict = _read_json(model_root / "vit_config.json") + hidden_size = int(llm_dict["hidden_size"]) + vit_hidden_size = int(vit_dict["hidden_size"]) + + from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY + + text_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_text_encoder"]() + text_cfg = text_cfg_cls( + vocab_size=int(llm_dict["vocab_size"]), + hidden_size=hidden_size, + tie_word_embeddings=False, + lm_head_bias=False, + ) + text_state = _load_prefixed_safetensors( + ema_path, + lambda key: { + "language_model.model.embed_tokens.weight": "embed_tokens.weight", + "language_model.lm_head.weight": "lm_head.weight", + }.get(key), + consumed_ema_keys, + ) + text_encoder = _build_module("bagel_text_encoder", text_cfg, text_state) + attach_module_assets(text_encoder, tokenizer=AutoTokenizer.from_pretrained(model_path)) + + qwen_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_qwen2_mot"]() + qwen_cfg = qwen_cfg_cls( + **llm_dict, + qk_norm=True, + layer_module="Qwen2MoTDecoderLayer", + freeze_und=False, + ) + qwen_state = _load_prefixed_safetensors( + ema_path, + lambda key: ( + key.removeprefix("language_model.") + if key.startswith("language_model.") + and not key.startswith("language_model.model.embed_tokens.") + and not key.startswith("language_model.lm_head.") + else None + ), + consumed_ema_keys, + ) + qwen2_mot = _build_module("bagel_qwen2_mot", qwen_cfg, qwen_state) + + siglip_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_siglip_navit"]() + siglip_cfg = siglip_cfg_cls( + hidden_size=vit_hidden_size, + output_size=hidden_size, + image_size=int(vit_dict["image_size"]), + intermediate_size=int(vit_dict["intermediate_size"]), + num_attention_heads=int(vit_dict["num_attention_heads"]), + # Official inference/training uses one fewer active layer than vit_config.json. + num_hidden_layers=int(vit_dict["num_hidden_layers"]) - 1, + patch_size=int(vit_dict["patch_size"]), + connector_act="gelu_pytorch_tanh", + vit_max_num_patch_per_side=int(vit_dict["image_size"]) // int(vit_dict["patch_size"]), + rope=False, + ) + siglip_state = _load_prefixed_safetensors( + ema_path, + lambda key: ( + key.removeprefix("vit_model.") + if key.startswith("vit_model.") + else key + if key.startswith("connector.") + else None + ), + consumed_ema_keys, + ) + siglip = _build_module( + "bagel_siglip_navit", + siglip_cfg, + siglip_state, + allowed_missing={"vit_pos_embed.pos_embed"}, + ) + from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import BagelSiglipNavitProcessor + + attach_module_assets(siglip, image_processor=BagelSiglipNavitProcessor.from_config(siglip_cfg)) + + flow_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]() + patch_latent_dim = 2 * 2 * 16 + flow_cfg = flow_cfg_cls( + hidden_size=hidden_size, + patch_latent_dim=patch_latent_dim, + max_latent_size=max_latent_size, + timestep_shift=1.0, + ) + flow_prefixes = ("vae2llm.", "llm2vae.", "time_embedder.") + flow_state = _load_prefixed_safetensors( + ema_path, + lambda key: key if key.startswith(flow_prefixes) else None, + consumed_ema_keys, + ) + flow_connector = _build_module( + "bagel_flow_connector", + flow_cfg, + flow_state, + allowed_missing={"latent_pos_embed.pos_embed"}, + ) + + vae_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_vae"]() + vae_cfg = vae_cfg_cls() + vae_state = _load_prefixed_safetensors(ae_path, lambda key: key, consumed_ae_keys) + vae = _build_module("bagel_vae", vae_cfg, vae_state) + from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEProcessor + + attach_module_assets(vae, image_processor=BagelVAEProcessor.from_config(vae_cfg)) + + # Position embeddings are deterministic sin-cos buffers regenerated from config. + ignored_ema_keys = {"latent_pos_embed.pos_embed", "vit_pos_embed.pos_embed"} + _assert_no_unhandled_keys(ema_path, consumed_ema_keys, ignored_ema_keys) + _assert_no_unhandled_keys(ae_path, consumed_ae_keys, set()) + + return { + "modules": { + "bagel_vae": vae, + "bagel_siglip_navit": siglip, + "bagel_text_encoder": text_encoder, + "bagel_flow_connector": flow_connector, + "bagel_qwen2_mot": qwen2_mot, + }, + "training_graphs": training_graphs, + "generation_graphs": generation_graphs, + "infer_type": "infer_und", + "generation_kwargs": deepcopy(BAGEL_GENERATION_KWARGS), + } + + +@OMNI_CONVERT_REGISTRY.register("bagel") +def _register_bagel_convert(): + return convert_bagel_checkpoint + + +__all__ = ["convert_bagel_checkpoint"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py new file mode 100644 index 0000000000..a84b9c7f23 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL VAE/LLM flow connector module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_config(): + from .configuration import BagelFlowConnectorConfig + + return BagelFlowConnectorConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_model(): + from .modeling import BagelFlowConnector + + return BagelFlowConnector + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_accelerated_model(): + from .accelerated import BagelFlowConnectorAccelerated + + return BagelFlowConnectorAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py new file mode 100644 index 0000000000..d1c9c0a21c --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/flow_connector OmniModule (conversation graph).""" + +from .accelerated import ( + BagelFlowConnectorAccelerated, +) + + +__all__ = [ + "BagelFlowConnectorAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py new file mode 100644 index 0000000000..25c9037283 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py @@ -0,0 +1,381 @@ +"""SeedOmni carrier hooks for BAGEL's flow connector — training-graph hooks only. + +``embed_context_latents()`` / ``prepare_denoise_query()`` / ``decode_velocity_from_hidden()`` +/ ``advance_denoise()`` and the shared :class:`~.generation_state.FlowGenerationState` +live on the native :class:`~.modeling.BagelFlowConnector` — this file only carries +the training pre/forward/post hooks (carrier selection, SP token slicing, dummy +alignment). +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import ( + gather_outputs, + reduce_sequence_parallel_loss, + slice_input_tensor, + sp_pad, +) +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ...sources import BAGEL_FLOW_HIDDEN, BAGEL_FLOW_VELOCITY, BAGEL_PHASE_KEY, bagel_phase +from ..configuration import BagelFlowConnectorConfig +from ..modeling import BagelFlowConnector, scatter_flow_latent_embeds, select_vae_context_latent_items +from ..processing import preprocess_context_latent_embed, preprocess_decode_velocity, preprocess_latent_embed + + +def slice_sp_token_inputs(method: str, **inputs: torch.Tensor) -> tuple[int, dict[str, torch.Tensor]]: + """Pad and slice token-level ``inputs`` across the active Ulysses SP group.""" + ps = get_parallel_state() + if ps.cp_size != 1: + raise ValueError(f"BAGEL flow connector {method} supports Ulysses groups only; got cp_size={ps.cp_size}.") + + lengths = {name: int(tensor.shape[0]) for name, tensor in inputs.items()} + if not lengths or next(iter(lengths.values())) == 0: + raise ValueError(f"BAGEL flow connector {method} SP requires at least one token.") + if len(set(lengths.values())) != 1: + raise ValueError(f"BAGEL flow connector {method} SP inputs must have matching token lengths: {lengths}.") + + full_length = next(iter(lengths.values())) + local_inputs = { + name: slice_input_tensor( + sp_pad(tensor, dim=0, pad_value=0), + dim=0, + padding=False, + group=ps.sp_group, + ) + for name, tensor in inputs.items() + } + return full_length, local_inputs + + +def gather_sp_token_output(method: str, output: torch.Tensor, full_length: int | None) -> torch.Tensor: + """All-gather token-level ``output`` across the SP group and trim SP padding.""" + if full_length is None: + raise RuntimeError(f"BAGEL flow connector {method} SP token length was not initialized.") + output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group) + return output.narrow(0, 0, full_length) + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph carrier hooks — depends on :class:`BagelFlowConnector` modeling APIs.""" + + config: BagelFlowConnectorConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._embed_items: list[ConversationItem] = [] + self._embed_lengths: list[int] = [] + self._sp_embed_length: int | None = None + self._decode_target_groups: list[list[ConversationItem]] = [] + self._decode_lengths: list[int] = [] + self._decode_target: torch.Tensor | None = None + self._sp_decode_length: int | None = None + + @pre_forward("embed_latent") + def embed_latent_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Prepare patchified latent tokens, position IDs, and timesteps for ``embed_latent``.""" + del kwargs + self._conversation_carrier = conversation_list + self._embed_lengths = [] + self._sp_embed_length = None + + self._embed_items = select_vae_context_latent_items(conversation_list) + if not self._embed_items: + raise ValueError("BAGEL flow connector requires per-sample VAE context carriers before embed_latent.") + + parts: list[dict[str, torch.Tensor]] = [] + meter_lengths: list[int] = [] + for item in self._embed_items: + if item.is_dummy: + inputs, lengths = preprocess_context_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + ) + anchor = item.value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + inputs["latents"] = inputs["latents"] + anchor + parts.append(inputs) + self._embed_lengths.extend(lengths) + meter_lengths.extend(int(v) for v in lengths) + continue + + tag = item.meta.get(_IMG_TAG_KEY) + if tag == "gen": + inputs, lengths = preprocess_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + timestep_shift=float(self.config.timestep_shift), + ) + elif tag == "edit": + inputs, lengths = preprocess_context_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + ) + else: + raise ValueError( + f"BAGEL flow connector training expects VAE image {_IMG_TAG_KEY} to be 'edit' or 'gen', got {tag!r}." + ) + + parts.append(inputs) + self._embed_lengths.extend(lengths) + meter_lengths.extend(int(v) for v in lengths) + + self.metric_meter_set_seqlens("embed_latent", meter_lengths) + + inputs = { + "latents": torch.cat([part["latents"] for part in parts], dim=0), + "position_ids": torch.cat([part["position_ids"] for part in parts], dim=0), + "timesteps": torch.cat([part["timesteps"] for part in parts], dim=0), + } + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_embed_length = None + return inputs + + self._sp_embed_length, inputs = slice_sp_token_inputs("embed_latent", **inputs) + return inputs + + @post_forward("embed_latent") + def embed_latent_post(self, latent_embeds: torch.Tensor) -> dict[str, Any]: + """Gather SP shards if needed and scatter ``latent_embeds`` back onto conversation items.""" + if get_parallel_state().sp_size > 1: + latent_embeds = gather_sp_token_output("embed_latent", latent_embeds, self._sp_embed_length) + + conversation = self._conversation_carrier + embed_items = self._embed_items + embed_lengths = self._embed_lengths + self._conversation_carrier = None + self._embed_items = [] + self._embed_lengths = [] + self._sp_embed_length = None + + scatter_flow_latent_embeds( + embed_items, + embed_lengths, + latent_embeds, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation} + + @pre_forward("decode_velocity") + def decode_velocity_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Collect latent-token hidden states and velocity targets for ``decode_velocity``.""" + del kwargs + self._conversation_carrier = conversation_list + self._decode_lengths = [] + self._decode_target = None + self._sp_decode_length = None + + self._decode_target_groups = self._select_velocity_target_groups(conversation_list) + + inputs_parts: list[dict[str, torch.Tensor]] = [] + target_parts: list[torch.Tensor] = [] + for sample, group in zip(conversation_list, self._decode_target_groups, strict=True): + if not group: + # No velocity target for this sample; run an anchored dummy decode + # so the decode head stays in the graph with zero loss. + dummy = self._anchor_dummy_decode_velocity_inputs([sample]) + inputs_parts.append(dummy) + self._decode_lengths.append(1) + continue + + inputs, lengths, target = preprocess_decode_velocity( + group, + config=self.config, + device=self.device, + dtype=self.dtype, + ) + inputs_parts.append(inputs) + self._decode_lengths.extend(lengths) + target_parts.append(target) + if target_parts: + self._decode_target = torch.cat(target_parts, dim=0) + + inputs = {"hidden_states": torch.cat([part["hidden_states"] for part in inputs_parts], dim=0)} + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_decode_length = None + return inputs + + self._sp_decode_length, inputs = slice_sp_token_inputs("decode_velocity", **inputs) + return inputs + + @post_forward("decode_velocity") + def decode_velocity_post(self, velocity: torch.Tensor) -> dict[str, Any]: + """Gather predicted velocity tokens, scatter onto items, and compute MSE flow loss.""" + if get_parallel_state().sp_size > 1: + velocity = gather_sp_token_output("decode_velocity", velocity, self._sp_decode_length) + + conversation = self._conversation_carrier + decode_target_groups = self._decode_target_groups + decode_lengths = self._decode_lengths + target = self._decode_target + self._conversation_carrier = None + self._decode_target_groups = [] + self._decode_lengths = [] + self._decode_target = None + self._sp_decode_length = None + + real_velocity_parts = self._scatter_velocity(decode_target_groups, decode_lengths, velocity) + loss = velocity.float().sum() * 0.0 + if target is not None and real_velocity_parts: + real_velocity = torch.cat(real_velocity_parts, dim=0) + mse = (real_velocity - target.to(device=velocity.device, dtype=velocity.dtype)).float().square() + num_valid_tokens = torch.tensor(mse.shape[0], device=velocity.device, dtype=torch.long) + loss = loss + mse.mean(dim=-1).sum() / num_valid_tokens.to(dtype=loss.dtype) + else: + num_valid_tokens = torch.zeros((), device=velocity.device, dtype=torch.long) + + ps = get_parallel_state() + loss_reduction_group = ( + ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None) + ) + if loss_reduction_group is not None: + loss = reduce_sequence_parallel_loss(loss, num_valid_tokens, group=loss_reduction_group) + return {"conversation_list": conversation, "_loss": loss} + + def _anchor_dummy_decode_velocity_inputs( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> dict[str, torch.Tensor]: + """Tie dummy flow loss to MoT hidden states without changing its value.""" + dummy = { + "hidden_states": torch.zeros( + 1, + int(self.config.hidden_size), + device=self.device, + dtype=self.dtype, + ) + } + + if conversation_list is None: + return dummy + + anchor = None + for item in iter_desired_items( + conversation_list, + types=["text", "image", "output"], + roles=["user", "assistant"], + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size): + anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + break + if anchor is None: + return dummy + + return {"hidden_states": dummy["hidden_states"] + anchor} + + def _select_velocity_target_groups( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> list[list[ConversationItem]]: + """Group non-dummy image items carrying ``flow_velocity_target`` per sample.""" + if conversation_list is None: + raise ValueError("BAGEL flow connector requires conversation_list to select velocity targets.") + + target_groups: list[list[ConversationItem]] = [] + for sample in conversation_list: + # Keep one target group per sample. An empty group means the hook + # should run a sample-level dummy decode anchored to this sample's + # MoT hidden states. + sample_target_items: list[ConversationItem] = [] + for item in iter_desired_items([sample], types=["image"]): + if not item.is_dummy and torch.is_tensor(item.meta.get("flow_velocity_target")): + sample_target_items.append(item) + target_groups.append(sample_target_items) + return target_groups + + def _scatter_velocity( + self, + target_groups: list[list[ConversationItem]], + decode_lengths: list[int], + velocity: torch.Tensor, + ) -> list[torch.Tensor]: + """Write predicted velocity spans onto target items and return non-dummy spans.""" + real_velocity_parts: list[torch.Tensor] = [] + offset = 0 + length_iter = iter(decode_lengths) + for group in target_groups: + if not group: + length = next(length_iter) + offset += length + continue + for item in group: + length = next(length_iter) + span = velocity[offset : offset + length] + offset += length + item.value = span.to(device=self.device, dtype=self.dtype) + if bagel_phase(item) == BAGEL_FLOW_HIDDEN: + item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY + real_velocity_parts.append(span) + if offset != int(velocity.shape[0]): + raise RuntimeError("BAGEL flow connector token count mismatch during velocity scatter.") + return real_velocity_parts + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's flow connector.""" + + config: BagelFlowConnectorConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + # Only ``embed_latent`` stashes lengths. Per latent token it runs vae2llm + # and the per-token timestep MLP (freq -> hidden -> hidden); the hidden x + # hidden layer dominates. ``decode_velocity``'s llm2vae is folded in per + # embedded token, which slightly over-counts edit-context tokens that are + # never decoded (llm2vae is ~2% of the per-token cost). + cfg = self.config + hidden = int(cfg.hidden_size) + latent = int(cfg.patch_latent_dim) + freq = int(cfg.timestep_frequency_embedding_size) + per_token_n = latent * hidden + freq * hidden + hidden * hidden + hidden * latent + return 6 * per_token_n * sum(seqlens) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks for latent embedding and velocity projection. + + ``embed_context_latents()`` / ``prepare_denoise_query()`` / + ``decode_velocity_from_hidden()`` / ``advance_denoise()`` and the shared + flow-matching generation-FSM state already live on the native + :class:`~.modeling.BagelFlowConnector` (via its own :class:`~.modeling.InferenceMixin`), + so no ``InferenceMixin`` is needed here. + """ + + +class BagelFlowConnectorAccelerated(VeOmniMixin, BagelFlowConnector): + pass + + +__all__ = ["BagelFlowConnectorAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py new file mode 100644 index 0000000000..5d6eb4000f --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py @@ -0,0 +1,34 @@ +"""Configuration for BAGEL's VAE/LLM flow connector.""" + +from ...module_configuration_base import OmniModuleConfig + + +class BagelFlowConnectorConfig(OmniModuleConfig): + """BAGEL latent-token connector config.""" + + model_type = "bagel_flow_connector" + + def __init__( + self, + hidden_size: int = 3584, + z_channels: int = 16, + latent_patch_size: int = 2, + patch_latent_dim: int | None = None, + max_latent_size: int = 64, + timestep_frequency_embedding_size: int = 256, + timestep_shift: float = 1.0, + **kwargs, + ) -> None: + self.hidden_size = hidden_size + self.z_channels = z_channels + self.latent_patch_size = latent_patch_size + self.patch_latent_dim = ( + latent_patch_size * latent_patch_size * z_channels if patch_latent_dim is None else patch_latent_dim + ) + self.max_latent_size = max_latent_size + self.timestep_frequency_embedding_size = timestep_frequency_embedding_size + self.timestep_shift = timestep_shift + super().__init__(**kwargs) + + +__all__ = ["BagelFlowConnectorConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py new file mode 100644 index 0000000000..fb483a7408 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py @@ -0,0 +1,156 @@ +"""Generation-time denoise state for BAGEL flow connector.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + + +@dataclass +class FlowGenerationState: + """Flow-side latent denoise state. + + The flow connector owns the numerical denoise state: noisy latent tokens, + timesteps, dt values, and step progress. Branch/cache/controller state + belongs to the MoT module and is intentionally not represented here. + """ + + _latents: torch.Tensor | None = None + _timestep_values: torch.Tensor | None = None + _dt_values: torch.Tensor | None = None + _step_index: int = 0 + _image_shape: tuple[int, int] | None = None + _grid_shape: tuple[int, int] | None = None + _token_count: int = 0 + + def reset(self) -> None: + self._latents = None + self._timestep_values = None + self._dt_values = None + self._step_index = 0 + self._image_shape = None + self._grid_shape = None + self._token_count = 0 + + def initialize( + self, + generation_kwargs: dict[str, Any], + *, + resolution: int, + patch_latent_dim: int, + device: torch.device, + ) -> None: + """Initialize initial Gaussian latent tokens and shifted Euler timestep schedule.""" + if bool(generation_kwargs.get("enable_taylorseer", False)): + raise NotImplementedError("BAGEL infer_gen TaylorSeer is not implemented in the SeedOmni graph path.") + + image_height = int(generation_kwargs.get("image_height", resolution)) + image_width = int(generation_kwargs.get("image_width", image_height)) + latent_downsample = int(generation_kwargs.get("latent_downsample", 16)) + if image_height % latent_downsample != 0 or image_width % latent_downsample != 0: + raise ValueError("BAGEL infer_gen image size must be divisible by latent_downsample.") + + grid_shape = (image_height // latent_downsample, image_width // latent_downsample) + token_count = int(grid_shape[0] * grid_shape[1]) + num_timesteps = int(generation_kwargs.get("num_timesteps", 50)) + if num_timesteps < 2: + raise ValueError("BAGEL infer_gen num_timesteps must be at least 2.") + + timestep_shift = float(generation_kwargs.get("timestep_shift", 1.0)) + timesteps = torch.linspace(1.0, 0.0, num_timesteps, device=device, dtype=torch.float32) + timesteps = timestep_shift * timesteps / (1.0 + (timestep_shift - 1.0) * timesteps) + + self._timestep_values = timesteps[:-1] + self._dt_values = timesteps[:-1] - timesteps[1:] + self._latents = torch.randn( + token_count, + int(patch_latent_dim), + device=device, + dtype=torch.float32, + ) + self._step_index = 0 + self._image_shape = (image_height, image_width) + self._grid_shape = grid_shape + self._token_count = token_count + + @property + def initialized(self) -> bool: + return self._latents is not None + + @property + def step_index(self) -> int: + return self._step_index + + @property + def image_shape(self) -> tuple[int, int] | None: + return self._image_shape + + @property + def token_count(self) -> int: + return self._token_count + + @property + def latents(self) -> torch.Tensor: + """Return the current noisy latent tokens ``x_t``.""" + if self._latents is None: + raise RuntimeError("BAGEL flow denoise latent state is not initialized.") + return self._latents + + @property + def timestep_values(self) -> torch.Tensor: + """Return the shifted timestep schedule tensor.""" + if self._timestep_values is None: + raise RuntimeError("BAGEL flow denoise timesteps are not initialized.") + return self._timestep_values + + @property + def dt_values(self) -> torch.Tensor: + """Return the step size deltas ``dt`` between consecutive timesteps.""" + if self._dt_values is None: + raise RuntimeError("BAGEL flow denoise dt values are not initialized.") + return self._dt_values + + @property + def grid_shape(self) -> tuple[int, int]: + """Return the ``(height, width)`` patch grid dimensions.""" + if self._grid_shape is None: + raise RuntimeError("BAGEL flow denoise latent grid shape is not initialized.") + return self._grid_shape + + def current_timestep(self) -> torch.Tensor: + """Return the scalar timestep tensor for the current Euler step.""" + timesteps = self.timestep_values + if self._step_index >= int(timesteps.numel()): + raise RuntimeError("BAGEL flow denoise timestep index is past the final step.") + return timesteps[self._step_index] + + def current_timestep_tokens(self) -> torch.Tensor: + x_t = self.latents + return self.current_timestep().reshape(1).expand(int(x_t.shape[0])) + + def is_complete(self) -> bool: + timesteps = self.timestep_values + return self._step_index >= int(timesteps.numel()) + + def advance(self, velocity: torch.Tensor) -> bool: + """Apply one Euler step ``x_t <- x_t - v_t * dt`` and return whether denoising finished.""" + x_t = self.latents + dts = self.dt_values + # Official generate_image does `x_t - v_t.to(x_t.device) * dts[i]` without + # pre-casting velocity to fp32. bf16 * fp32 is not always the same as + # fp32(v) * fp32(dt); keep the mixed-dtype multiply. + velocity = velocity.to(device=x_t.device) + if velocity.shape != x_t.shape: + raise ValueError( + f"BAGEL flow velocity shape mismatch: got {tuple(velocity.shape)}, expected {tuple(x_t.shape)}." + ) + + dt = dts[self._step_index].to(device=x_t.device) + self._latents = x_t - velocity * dt + self._step_index += 1 + return self.step_index >= int(dts.numel()) + + +__all__ = ["FlowGenerationState"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py new file mode 100644 index 0000000000..4d052c526f --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py @@ -0,0 +1,403 @@ +"""BAGEL flow connector. + +``embed_latent`` projects patchified VAE latent tokens, timestep embeddings, +and latent position embeddings to MoT hidden width. ``decode_velocity`` projects +MoT hidden states back to patch-latent velocity tokens. Carrier selection, +dummy alignment, and loss computation live in the SeedOmni module mixin. + +The flow-matching FSM ``generate``-family endpoints (``embed_context_latents`` +/ ``prepare_denoise_query`` / ``decode_velocity_from_hidden`` / ``advance_denoise``) +and their shared :class:`~.generation_state.FlowGenerationState` live here too — +pure inference over an already-loaded ``BagelFlowConnector`` needs no VeOmni +training-graph machinery. ``accelerated.py`` only carries the training-graph +pre/forward/post hooks (carrier selection, SP token slicing, dummy alignment). +""" + +from __future__ import annotations + +import math +from typing import Any + +import torch +import torch.nn as nn + +from ....graphs.generation_graph import FSM_SIGNAL_KEY +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..position_embedding import PositionEmbedding +from ..sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_FLOW_VELOCITY, + BAGEL_GENERATED_LATENT, + BAGEL_PHASE_KEY, + BAGEL_VAE_CONTEXT, + get_tail_phase_item, +) +from .configuration import BagelFlowConnectorConfig +from .generation_state import FlowGenerationState +from .processing import flattened_position_ids, preprocess_context_latent_embed, unpatchify_latent_tokens + + +SIGNAL_IMAGE_COMPLETE = "image_complete" + + +def select_vae_context_latent_items( + conversation_list: list[list[ConversationItem]] | None, +) -> list[ConversationItem]: + """Select per-sample VAE context latents shared by training and inference hooks.""" + if conversation_list is None: + raise ValueError("BAGEL flow connector requires conversation_list to select VAE context latents.") + + # VAE preprocessing already makes BAGEL_VAE_CONTEXT per-sample by + # appending a dummy carrier only for samples without real VAE context. + # Reuse those existing carriers here; do not append another dummy. + return list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]})) + + +def scatter_flow_latent_embeds( + embed_items: list[ConversationItem], + embed_lengths: list[int], + latent_embeds: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Write flow latent embeds back onto carrier items (training + inference).""" + offset = 0 + for item, length in zip(embed_items, embed_lengths, strict=True): + item.value = latent_embeds[offset : offset + length].to(device=device, dtype=dtype) + offset += length + + # Hand the upstream VAE dummy carrier off as a flow dummy anchor; + # downstream MoT should not treat it as another VAE image span. Only + # the context tag survives: it is how MoT recognizes the flow anchor. + if item.is_dummy: + item.type = "output" + item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} + + if offset != int(latent_embeds.shape[0]): + raise RuntimeError("BAGEL flow connector latent count mismatch during embed scatter.") + + +class InferenceMixin: + """Flow-matching denoise FSM (``embed_context_latents`` / ``prepare_denoise_query`` + / ``decode_velocity_from_hidden`` / ``advance_denoise``) — HF ``GenerationMixin`` + analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelFlowConnector`'s bases: ``PretrainedOmniModule`` ships a no-op + ``reset_local_inference_state`` default (kept as a safety net for modules + that don't need real inference state), and MRO resolves left-to-right — + put second, that no-op would shadow the real one below. + """ + + def reset_local_inference_state(self) -> None: + self._generation_state.reset() + + def embed_context_latents( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Embed clean VAE context latents into LLM hidden space during prompt encoding.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + embed_items = [item for item in select_vae_context_latent_items(batched) if not item.is_dummy] + if not embed_items: + return {"conversation_list": conversation_list} + + inputs, embed_lengths = preprocess_context_latent_embed( + embed_items, + config=self.config, + device=self.device, + dtype=self.dtype, + ) + outputs = self.embed_latent(**inputs) + scatter_flow_latent_embeds( + embed_items, + embed_lengths, + outputs["latent_embeds"], + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation_list} + + def prepare_denoise_query( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Embed the current noisy latent tokens and write a ``flow_query`` output item.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + state = self._generation_state + if not state.initialized: + self._generation_state.initialize( + generation_kwargs or {}, + resolution=int(getattr(self.config, "resolution", 1024)), + patch_latent_dim=int(self.config.patch_latent_dim), + device=self._vae2llm_device, + ) + + x_t = state.latents + timestep = state.current_timestep() + timestep_tokens = state.current_timestep_tokens() + position_ids = flattened_position_ids( + state.grid_shape, + max_latent_size=int(self.config.max_latent_size), + device=x_t.device, + ) + outputs = self.embed_latent( + latents=x_t, + position_ids=position_ids, + timesteps=timestep_tokens, + ) + query = outputs["latent_embeds"].to(device=self.device, dtype=self.dtype) + timestep_meta = timestep.detach().to(device=query.device, dtype=torch.float32) + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if item is None: + conversation_list.append( + ConversationItem( + type="output", + value=query, + role="assistant", + meta={"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY}, + ) + ) + else: + item.type = "output" + item.role = "assistant" + item.value = query + item.meta = {"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY} + return {"conversation_list": conversation_list} + + def decode_velocity_from_hidden( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Project MoT denoise hidden states to patch-latent velocity tokens.""" + del generation_kwargs, kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_HIDDEN) + if item is None or not torch.is_tensor(item.value): + raise ValueError(f"BAGEL flow decode_velocity requires an output in phase {BAGEL_FLOW_HIDDEN!r}.") + + hidden = item.value + if hidden.dim() == 3 and hidden.shape[0] == 1: + hidden = hidden.squeeze(0) + if hidden.dim() != 2: + raise ValueError(f"BAGEL flow decode_velocity expected rank-2 hidden states, got {tuple(hidden.shape)}.") + if int(hidden.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + "BAGEL flow decode_velocity hidden-size mismatch: " + f"got {hidden.shape[-1]}, expected {self.config.hidden_size}." + ) + + # Official llm2vae is one GEMM per CFG branch. A stacked GEMM over + # concatenated branches is not bitwise even when each branch hidden + # matches, so split back to the per-branch query length. + query_len = int(self._generation_state.token_count) + 2 + if ( + self._generation_state.initialized + and query_len >= 3 + and int(hidden.shape[0]) >= query_len + and int(hidden.shape[0]) % query_len == 0 + ): + velocity = torch.cat( + [self.decode_velocity(hidden_states=chunk)["velocity"] for chunk in hidden.split(query_len, dim=0)], + dim=0, + ) + else: + velocity = self.decode_velocity(hidden_states=hidden)["velocity"] + item.type = "output" + item.role = "assistant" + item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY + item.value = velocity.to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + def advance_denoise( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Advance the Euler denoise state by one step or emit the final unpatchified latent.""" + del generation_kwargs, kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if item is None or not torch.is_tensor(item.value): + raise ValueError(f"BAGEL flow advance requires an output in phase {BAGEL_FLOW_VELOCITY!r}.") + + velocity = item.value + if velocity.dim() == 3 and velocity.shape[0] == 1: + velocity = velocity.squeeze(0) + complete = self._generation_state.advance(velocity) + + if complete: + return self._emit_final_latent(conversation_list) + item.meta.pop("timestep", None) + return {"conversation_list": conversation_list} + + def _emit_final_latent( + self, + conversation_list: list[ConversationItem], + ) -> dict[str, Any]: + """Unpatchify the completed latent grid and emit ``image_complete``.""" + x_t = self._generation_state.latents + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + latent = unpatchify_latent_tokens( + x_t, + self._generation_state.grid_shape, + z_channels=int(self.config.z_channels), + latent_patch_size=int(self.config.latent_patch_size), + ) + if item is None: + conversation_list.append( + ConversationItem( + type="output", + value=latent, + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT}, + ) + ) + else: + item.type = "output" + item.role = "assistant" + item.value = latent.to(device=self.device, dtype=self.dtype) + item.meta = {BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT} + + self._generation_state.reset() + return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: SIGNAL_IMAGE_COMPLETE} + + +def _module_weight(module: nn.Module) -> torch.Tensor: + """Return the underlying weight parameter for plain or LoRA-wrapped linear modules.""" + return getattr(module, "base_layer", module).weight + + +class BagelFlowConnector(InferenceMixin, PretrainedOmniModule): + config_class = BagelFlowConnectorConfig + base_model_prefix = "bagel_flow_connector" + main_input_name = "hidden_states" + _no_split_modules: list[str] = [] + + def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): + # The flow connector projects latent tokens and timesteps; it does not execute attention. + return attn_implementation + + def __init__(self, config: BagelFlowConnectorConfig) -> None: + super().__init__(config) + self.time_embedder = TimestepEmbedder(config.hidden_size, config.timestep_frequency_embedding_size) + self.vae2llm = nn.Linear(config.patch_latent_dim, config.hidden_size) + self.llm2vae = nn.Linear(config.hidden_size, config.patch_latent_dim) + self.latent_pos_embed = PositionEmbedding(config.max_latent_size, config.hidden_size) + self._generation_state = FlowGenerationState() + self.post_init() + + def _init_weights(self, module: nn.Module) -> None: + if isinstance(module, PositionEmbedding): + module.reset_parameters() + return + if module is self.llm2vae: + nn.init.constant_(module.weight, 0) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + return + super()._init_weights(module) + + @property + def _vae2llm_device(self) -> torch.device: + return _module_weight(self.vae2llm).device + + @property + def _llm2vae_device(self) -> torch.device: + return _module_weight(self.llm2vae).device + + @property + def _pos_embed_device(self) -> torch.device: + return self.latent_pos_embed.pos_embed.device + + @property + def _time_embedder_device(self) -> torch.device: + return _module_weight(self.time_embedder.mlp[0]).device + + def embed_latent( + self, + latents: torch.Tensor, + position_ids: torch.LongTensor, + timesteps: torch.Tensor, + ) -> dict[str, torch.Tensor]: + """Project patchified latents and add timestep and 2D sinusoidal position embeddings.""" + vae2llm_weight = _module_weight(self.vae2llm) + latents = latents.to(device=vae2llm_weight.device, dtype=vae2llm_weight.dtype) + position_ids = position_ids.to(device=self._pos_embed_device, dtype=torch.long).reshape(-1) + timesteps = timesteps.to(device=self._time_embedder_device, dtype=torch.float32).reshape(-1) + if position_ids.numel() != latents.shape[0]: + raise ValueError("position_ids must have one value per latent token.") + + latent_embeds = self.vae2llm(latents) + time_embeds = self.time_embedder(timesteps) + pos_embeds = self.latent_pos_embed(position_ids) + return { + "latent_embeds": latent_embeds + + time_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype) + + pos_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype) + } + + def decode_velocity(self, hidden_states: torch.Tensor) -> dict[str, torch.Tensor]: + llm2vae_weight = _module_weight(self.llm2vae) + hidden_states = hidden_states.to(device=llm2vae_weight.device, dtype=llm2vae_weight.dtype) + return {"velocity": self.llm2vae(hidden_states)} + + +class TimestepEmbedder(nn.Module): + def __init__(self, hidden_size: int, frequency_embedding_size: int = 256) -> None: + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + """Create sinusoidal timestep embeddings of width ``dim``.""" + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(half, dtype=torch.float32, device=t.device) / half) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + first_weight = _module_weight(self.mlp[0]) + return self.mlp(t_freq.to(device=first_weight.device, dtype=first_weight.dtype)) + + +__all__ = [ + "BagelFlowConnector", + "BagelFlowConnectorConfig", + "InferenceMixin", + "SIGNAL_IMAGE_COMPLETE", + "select_vae_context_latent_items", + "scatter_flow_latent_embeds", +] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py new file mode 100644 index 0000000000..dc9c0d0deb --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py @@ -0,0 +1,216 @@ +"""Latent patch and timestep helpers for BAGEL flow connector.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from ....utils.conversation import ConversationItem + + +def preprocess_latent_embed( + embed_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, + timestep_shift: float, +) -> tuple[dict[str, torch.Tensor], list[int]]: + """Build noised latent embed inputs and training velocity targets.""" + latents: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + timesteps: list[torch.Tensor] = [] + embed_lengths: list[int] = [] + for item in embed_items: + clean, grid_shape = _patchify_latent_grid( + item.value, + z_channels=int(config.z_channels), + latent_patch_size=int(config.latent_patch_size), + ) + clean = clean.to(device=device, dtype=dtype) + token_count = int(clean.shape[0]) + + noise = torch.randn_like(clean) + timestep = torch.sigmoid(torch.randn(1, device=device, dtype=torch.float32)).expand(token_count) + timestep = float(timestep_shift) * timestep / (1.0 + (float(timestep_shift) - 1.0) * timestep) + timestep_values = timestep.reshape(-1, 1).to(device=clean.device, dtype=torch.float32) + noised = (1.0 - timestep_values) * clean + timestep_values * noise + noised = noised.to(dtype=dtype) + + item.meta["timestep"] = timestep.detach() + item.meta["noise"] = noise.detach() + item.meta["flow_velocity_target"] = (noise - clean).detach() + latents.append(noised) + position_ids.append( + flattened_position_ids( + grid_shape, + max_latent_size=int(config.max_latent_size), + device=device, + ) + ) + timesteps.append(timestep) + embed_lengths.append(token_count) + + return ( + { + "latents": torch.cat(latents, dim=0), + "position_ids": torch.cat(position_ids, dim=0), + "timesteps": torch.cat(timesteps, dim=0), + }, + embed_lengths, + ) + + +def preprocess_context_latent_embed( + embed_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, +) -> tuple[dict[str, torch.Tensor], list[int]]: + """Build clean latent embed inputs for inference context latents.""" + latents: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + embed_lengths: list[int] = [] + for item in embed_items: + clean, grid_shape = _patchify_latent_grid( + item.value, + z_channels=int(config.z_channels), + latent_patch_size=int(config.latent_patch_size), + ) + clean = clean.to(device=device) + token_count = int(clean.shape[0]) + + latents.append(clean) + position_ids.append( + flattened_position_ids( + grid_shape, + max_latent_size=int(config.max_latent_size), + device=device, + ) + ) + embed_lengths.append(token_count) + + return ( + { + "latents": torch.cat(latents, dim=0), + "position_ids": torch.cat(position_ids, dim=0), + "timesteps": torch.zeros(sum(embed_lengths), device=device, dtype=torch.float32), + }, + embed_lengths, + ) + + +def preprocess_decode_velocity( + decode_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, +) -> tuple[dict[str, torch.Tensor], list[int], torch.Tensor]: + """Build packed hidden states and velocity targets for training decode.""" + hidden_states: list[torch.Tensor] = [] + targets: list[torch.Tensor] = [] + decode_lengths: list[int] = [] + for item in decode_items: + hidden = item.value + if not torch.is_tensor(hidden): + raise ValueError("BAGEL flow decode_velocity expects tensor hidden states.") + if hidden.dim() == 3 and hidden.shape[0] == 1: + hidden = hidden.squeeze(0) + if hidden.dim() != 2: + raise ValueError( + f"BAGEL flow decode_velocity expects rank-2 hidden states, got {tuple(item.value.shape)}." + ) + if int(hidden.shape[-1]) != int(config.hidden_size): + raise ValueError( + "BAGEL flow decode_velocity hidden-size mismatch: " + f"got {hidden.shape[-1]}, expected {config.hidden_size}." + ) + hidden = hidden.to(device=device) + hidden_states.append(hidden) + decode_lengths.append(int(hidden.shape[0])) + + target = item.meta.get("flow_velocity_target").to(device=device, dtype=dtype) + expected_shape = (hidden.shape[0], int(config.patch_latent_dim)) + if target.shape != expected_shape: + raise ValueError( + f"BAGEL flow_velocity_target shape mismatch: got {tuple(target.shape)}, expected {expected_shape}." + ) + targets.append(target) + + return {"hidden_states": torch.cat(hidden_states, dim=0)}, decode_lengths, torch.cat(targets, dim=0) + + +def _patchify_latent_grid( + value: torch.Tensor, + *, + z_channels: int, + latent_patch_size: int, +) -> tuple[torch.Tensor, tuple[int, int]]: + """Patchify a latent grid into a list of tokens.""" + latent = value.detach() + if latent.dim() == 4 and latent.shape[0] == 1: + latent = latent.squeeze(0) + if latent.dim() != 3: + raise ValueError(f"BAGEL flow connector expects latent grid tensors, got shape {tuple(value.shape)}.") + if int(latent.shape[0]) != int(z_channels): + raise ValueError( + f"BAGEL flow connector latent channel mismatch: got {int(latent.shape[0])}, expected {int(z_channels)}." + ) + channels, height, width = latent.shape + + patch_size = int(latent_patch_size) + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError( + "BAGEL flow connector latent grid is not divisible by latent_patch_size: " + f"shape={tuple(latent.shape)}, latent_patch_size={patch_size}." + ) + + grid_h, grid_w = height // patch_size, width // patch_size + tokens = latent.reshape(channels, grid_h, patch_size, grid_w, patch_size) + tokens = torch.einsum("chpwq->hwpqc", tokens) + return tokens.reshape(-1, patch_size * patch_size * channels), (grid_h, grid_w) + + +def unpatchify_latent_tokens( + tokens: torch.Tensor, + grid_shape: tuple[int, int], + *, + z_channels: int, + latent_patch_size: int, +) -> torch.Tensor: + grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1]) + patch = int(latent_patch_size) + channels = int(z_channels) + latent = tokens.reshape(grid_h, grid_w, patch, patch, channels) + latent = torch.einsum("hwpqc->chpwq", latent) + return latent.reshape(channels, grid_h * patch, grid_w * patch) + + +def flattened_position_ids( + grid_shape: tuple[int, int], + *, + max_latent_size: int, + device: torch.device | None = None, +) -> torch.Tensor: + """Compute flattened 2D grid position IDs bounded by ``max_latent_size``.""" + grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1]) + if grid_h > max_latent_size or grid_w > max_latent_size: + raise ValueError( + "BAGEL flow connector latent grid exceeds max_latent_size: " + f"grid={grid_shape}, max_latent_size={max_latent_size}." + ) + coords_h = torch.arange(grid_h, device=device, dtype=torch.long) + coords_w = torch.arange(grid_w, device=device, dtype=torch.long) + return (coords_h[:, None] * int(max_latent_size) + coords_w).flatten() + + +__all__ = [ + "flattened_position_ids", + "preprocess_context_latent_embed", + "preprocess_decode_velocity", + "preprocess_latent_embed", + "unpatchify_latent_tokens", +] diff --git a/veomni/models/seed_omni/modules/bagel/position_embedding.py b/veomni/models/seed_omni/modules/bagel/position_embedding.py new file mode 100644 index 0000000000..39e179a417 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/position_embedding.py @@ -0,0 +1,70 @@ +"""Shared 2D sin-cos position embedding for BAGEL vision and latent patch grids.""" + +from __future__ import annotations + +import numpy as np +import torch +import torch.nn as nn +from torch.distributed.tensor import DTensor, distribute_tensor + + +class PositionEmbedding(nn.Module): + """Fixed 2D sin-cos position embedding table indexed by flattened patch IDs.""" + + def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: + super().__init__() + self.max_num_patch_per_side = max_num_patch_per_side + self.hidden_size = hidden_size + self.pos_embed = nn.Parameter( + torch.zeros(max_num_patch_per_side**2, hidden_size), + requires_grad=False, + ) + self.reset_parameters() + + def reset_parameters(self) -> None: + """Populate ``pos_embed`` with deterministic 2D sin-cos values.""" + if self.pos_embed.is_meta: + return + new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) + if isinstance(self.pos_embed.data, DTensor): + local_target = self.pos_embed.data.to_local() + sharded = distribute_tensor( + new_data.to(device=local_target.device, dtype=local_target.dtype), + self.pos_embed.data.device_mesh, + self.pos_embed.data.placements, + src_data_rank=None, + ) + local_target.copy_(sharded.to_local().to(device=local_target.device, dtype=local_target.dtype)) + elif hasattr(self.pos_embed.data, "to_local"): + self.pos_embed.data.to_local().copy_(new_data) + else: + self.pos_embed.data.copy_(new_data) + + def _init_weights(self) -> None: + self.reset_parameters() + + def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: + return self.pos_embed[position_ids.to(device=self.pos_embed.device)] + + +def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: + """Build a ``[grid_size**2, embed_dim]`` 2D sin-cos position embedding tensor.""" + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0).reshape(2, 1, grid_size, grid_size) + emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + return torch.from_numpy(np.concatenate([emb_h, emb_w], axis=1)).float() + + +def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: + """Build a 1D sin-cos position embedding array from flattened grid coordinates.""" + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + out = np.einsum("m,d->md", pos.reshape(-1), omega) + return np.concatenate([np.sin(out), np.cos(out)], axis=1) + + +__all__ = ["PositionEmbedding"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py new file mode 100644 index 0000000000..8c774dee1a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL Qwen2 MoT backbone module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_config(): + from .configuration import BagelQwen2MoTConfig + + return BagelQwen2MoTConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_model(): + from .modeling import BagelQwen2MoT + + return BagelQwen2MoT + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_accelerated_model(): + from .accelerated import BagelQwen2MoTAccelerated + + return BagelQwen2MoTAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py new file mode 100644 index 0000000000..30fc075956 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py @@ -0,0 +1,12 @@ +"""Accelerated runtime for the bagel/qwen2_mot OmniModule (conversation graph).""" + +from .accelerated import ( + BagelQwen2MoTAccelerated, + InferenceMixinAccelerated, +) + + +__all__ = [ + "BagelQwen2MoTAccelerated", + "InferenceMixinAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py new file mode 100644 index 0000000000..68671c8848 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -0,0 +1,722 @@ +"""VeOmni-accelerated BAGEL Qwen2-MoT — instance-local VeomniOp ops plus training-graph hooks. + +Native weights and packed ``forward`` live on :class:`BagelQwen2MoTCore`. +This module adds packed CFG denoise, async Ulysses, fused RMSNorm / SwiGLU / RoPE, +attention ``VeomniOp`` dispatch, and the training pre/forward/post hooks. +""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.distributed as dist +from torch.nn.attention.flex_attention import BlockMask +from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import ( + gather_outputs, + slice_input_tensor, + sp_pad, +) +from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import ( + Qwen2MLP as PatchedQwen2MLP, +) +from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import ( + Qwen2RMSNorm as PatchedQwen2RMSNorm, +) +from .......ops import VeomniOp +from .......ops.mask import MagiAttentionMask +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ...sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + bagel_context, +) +from ..checkpoint_conversion import ( + BagelQwen2MoTCheckpointTensorConverter, + combine_qkv_state_dict_pre_hook, + register_accelerated_qkv_conversion_mapping, +) +from ..configuration import BagelQwen2MoTConfig +from ..masking import build_mot_block_mask, build_mot_magi_mask, pad_mot_attention_metadata +from ..modeling import ( + BagelQwen2MoTAttention, + BagelQwen2MoTCore, + InferenceMixin, +) +from ..processing import PackedConversation, preprocess_mot_inputs + + +_FLASH_ATTENTION_2 = "veomni_flash_attention_2" + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelQwen2MoTCore` modeling APIs.""" + + config: BagelQwen2MoTConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._packed_training: PackedConversation | None = None + self._sp_full_sequence_length: int | None = None + self._validated_ulysses_size: int | None = None + + @pre_forward("forward") + def forward_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Pack multimodal conversation items into MoT training tensors and slice for Ulysses SP.""" + del kwargs + self._conversation_carrier = conversation_list + self._packed_training = preprocess_mot_inputs( + conversation_list, + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if self._packed_training is None: + raise ValueError( + "BAGEL Qwen2-MoT forward requires a non-empty conversation_list with real tokens; " + "got no packable tokens across the whole batch." + ) + + # Meter the full replicated batch before SP slices its packed sequence. + self.metric_meter_set_seqlens( + "forward", + [int(sum(splits)) for splits in self._packed_training.sample_splits], + ) + + packed_sequence = self._fold_dummy_anchors(self._packed_training.packed_sequence, conversation_list) + sequence_length = int(packed_sequence.shape[0]) + attention_metadata = self._packed_training.packed_attention_metadata + expected_metadata_shape = (3, sequence_length) + if tuple(attention_metadata.shape) != expected_metadata_shape: + raise ValueError( + "BAGEL Qwen2-MoT attention metadata must match the full packed sample: " + f"expected {expected_metadata_shape}, got {tuple(attention_metadata.shape)}." + ) + + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_full_sequence_length = None + return { + "packed_sequence": packed_sequence, + "packed_position_ids": self._packed_training.packed_position_ids, + "packed_token_type_ids": self._packed_training.packed_token_type_ids, + "packed_attention_metadata": attention_metadata, + } + + if ps.cp_size != 1: + raise ValueError( + f"BAGEL Qwen2-MoT training supports Ulysses sequence parallelism only; got cp_size={ps.cp_size}." + ) + if self._validated_ulysses_size != ps.ulysses_size: + num_heads = int(self.config.num_attention_heads) + num_key_value_heads = int(self.config.num_key_value_heads) + if num_heads % ps.ulysses_size != 0: + raise ValueError( + "BAGEL Qwen2-MoT attention heads must be divisible by " + f"ulysses_size: num_heads={num_heads}, ulysses_size={ps.ulysses_size}." + ) + if num_key_value_heads % ps.ulysses_size != 0: + raise ValueError( + "BAGEL Qwen2-MoT KV heads must be divisible by " + f"ulysses_size for native GQA: num_key_value_heads={num_key_value_heads}, " + f"ulysses_size={ps.ulysses_size}." + ) + self._validated_ulysses_size = ps.ulysses_size + + # Tensor inputs are padded then sliced for Ulysses. Attention metadata + # stays full because the Flex adapter reconstructs the full sequence + # before building/using its BlockMask. + self._sp_full_sequence_length = sequence_length + packed_sequence = sp_pad(packed_sequence, dim=0, pad_value=0) + packed_position_ids = sp_pad(self._packed_training.packed_position_ids, dim=0, pad_value=0) + packed_token_type_ids = sp_pad(self._packed_training.packed_token_type_ids, dim=0, pad_value=-1) + attention_metadata = pad_mot_attention_metadata( + attention_metadata, + padded_length=int(packed_sequence.shape[0]), + ) + packed_sequence = slice_input_tensor(packed_sequence, dim=0, padding=False, group=ps.sp_group) + packed_position_ids = slice_input_tensor( + packed_position_ids, + dim=0, + padding=False, + group=ps.sp_group, + ) + packed_token_type_ids = slice_input_tensor( + packed_token_type_ids, + dim=0, + padding=False, + group=ps.sp_group, + ) + + return { + "packed_sequence": packed_sequence, + "packed_position_ids": packed_position_ids, + "packed_token_type_ids": packed_token_type_ids, + # FlexAttention's generic Ulysses adapter reconstructs the complete + # padded sequence, so every rank keeps the same full metadata. + "packed_attention_metadata": attention_metadata, + } + + @post_forward("forward") + def forward_post(self, hidden_states: torch.Tensor, **kwargs: Any) -> dict[str, Any]: + """Gather SP sequence shards if active and scatter MoT hidden states onto carrier items.""" + del kwargs + ps = get_parallel_state() + if ps.sp_size > 1: + if self._sp_full_sequence_length is None: + raise RuntimeError("BAGEL Qwen2-MoT SP sequence length was not initialized.") + hidden_states = gather_outputs(hidden_states, gather_dim=0, group=ps.sp_group) + hidden_states = hidden_states.narrow(0, 0, self._sp_full_sequence_length) + + conversation = self._conversation_carrier + packed = self._packed_training + self._conversation_carrier = None + self._packed_training = None + self._sp_full_sequence_length = None + + for span in packed.spans: + span_hidden = hidden_states[span.start : span.start + span.length].to(device=self.device) + offset = 0 + for item, length in zip(span.items, span.lengths, strict=True): + item.value = span_hidden[offset : offset + length] + offset += length + return {"conversation_list": conversation} + + def _fold_dummy_anchors( + self, + packed_sequence: torch.Tensor, + conversation_list: list[list[ConversationItem]] | None, + ) -> torch.Tensor: + """Fold zero-valued FSDP dummy anchors from upstream SigLIP and flow encoders.""" + # Dummy encoder outputs must still touch MoT on ranks where another rank + # has the real branch, otherwise FSDP sees different gradient buckets. + anchor = packed_sequence.new_zeros(()) + has_anchor = False + include_siglip_dummy, include_flow_dummy = self._has_valid_upstream_embeddings(conversation_list) + + for item in iter_desired_items(conversation_list or []): + if not item.is_dummy or not torch.is_tensor(item.value): + continue + + # The flow connector hands its embedded VAE placeholder on as an + # ``output`` row; a VAE placeholder still of type ``image`` never + # reached the flow connector and holds no MoT-width embedding. + context = bagel_context(item) + is_flow_dummy = item.type == "output" and context == BAGEL_VAE_CONTEXT + if not is_flow_dummy and context != BAGEL_SIGLIP_CONTEXT: + continue + if is_flow_dummy and not include_flow_dummy: + continue + if context == BAGEL_SIGLIP_CONTEXT and not include_siglip_dummy: + continue + + has_anchor = True + anchor = ( + anchor + + item.value.to( + device=packed_sequence.device, + dtype=packed_sequence.dtype, + ).sum() + * 0.0 + ) + + if not has_anchor: + return packed_sequence + return packed_sequence + anchor + + def _has_valid_upstream_embeddings( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> tuple[bool, bool]: + """Return whether any rank in the distributed group has real SigLIP or flow embeddings.""" + has_siglip = int( + self._has_valid_upstream_embedding( + conversation_list, + label="SigLIP", + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}, + ) + ) + has_flow = int( + self._has_valid_upstream_embedding( + conversation_list, + label="flow", + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}, + meta_keys=["flow_velocity_target"], + ) + ) + + if not dist.is_available() or not dist.is_initialized(): + return bool(has_siglip), bool(has_flow) + flags = torch.tensor([has_siglip, has_flow], device=self.device, dtype=torch.int32) + dist.all_reduce(flags, op=dist.ReduceOp.MAX) + return bool(flags[0].item()), bool(flags[1].item()) + + def _has_valid_upstream_embedding( + self, + conversation_list: list[list[ConversationItem]] | None, + *, + label: str, + types: list[str], + meta: dict[str, list[Any]] | None = None, + meta_keys: list[str] | None = None, + ) -> bool: + """Check whether ``conversation_list`` contains a non-dummy rank-2 embedding matching the filter.""" + for item in iter_desired_items( + conversation_list or [], + types=types, + roles=["user", "assistant"], + meta_keys=meta_keys, + meta=meta, + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT {label} alignment expects rank-2 embeddings, got {tuple(value.shape)}." + ) + if int(value.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + f"BAGEL Qwen2-MoT {label} alignment hidden-size mismatch: " + f"got {value.shape[-1]}, expected {self.config.hidden_size}." + ) + return True + return False + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's Qwen2-MoT backbone (transformer layers only).""" + + config: BagelQwen2MoTConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + """Estimate TFLOPs for the Qwen2-MoT transformer layers over ``seqlens``.""" + cfg = self.config + hidden = cfg.hidden_size + num_layers = cfg.num_hidden_layers + num_heads = cfg.num_attention_heads + num_kv_heads = cfg.num_key_value_heads + # Attention uses head_dim = hidden // num_heads (see BagelQwen2MoTAttention). + head_dim = hidden // num_heads + + # SwiGLU MLP (gate/up/down) + attention projections (q, o over num_heads; + # k, v over num_kv_heads). Biases are O(hidden) → negligible, ignored. + mlp_n = hidden * cfg.intermediate_size * 3 + attn_linear_n = hidden * head_dim * (num_heads * 2 + num_kv_heads * 2) + dense_n = (mlp_n + attn_linear_n) * num_layers + + tokens = sum(seqlens) + seqlen_sq = sum(s * s for s in seqlens) + dense_flops = 6 * dense_n * tokens + attn_flops = 12 * seqlen_sq * head_dim * num_heads * num_layers + return (dense_flops + attn_flops) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks and graph entrypoints for BAGEL's packed MoT backbone. + + Packed CFG denoise is mixed in separately via :class:`InferenceMixinAccelerated` + so this mixin stays runtime-only. + """ + + +def _apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + *, + rope_op: VeomniOp | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Packed RoPE dispatched through the instance-local ``VeomniOp``.""" + if rope_op is None: + rope_op = VeomniOp("rope", "full", "eager") + if unsqueeze_dim != 1: + raise NotImplementedError("BAGEL packed fused RoPE requires unsqueeze_dim=1.") + if q.ndim != 3 or k.ndim != 3 or cos.ndim != 2 or sin.ndim != 2: + raise NotImplementedError( + "BAGEL fused RoPE requires packed q/k tensors shaped [tokens, heads, head_dim] " + "and cos/sin tensors shaped [tokens, head_dim]." + ) + if not (q.shape[0] == k.shape[0] == cos.shape[0] == sin.shape[0]): + raise ValueError("BAGEL packed q/k/cos/sin tensors must share the token dimension.") + if not (q.shape[-1] == k.shape[-1] == cos.shape[-1] == sin.shape[-1]): + raise NotImplementedError("Liger full RoPE does not support partial rotary dimensions.") + + # The shared full-RoPE kernels consume [batch, heads, sequence, head_dim] + # for LLM-dtype arithmetic (whereas 3-D [S, H, D] triggers vision FP32 promotion). + # BAGEL stores packed Q/K as [tokens, heads, head_dim], so adapt through + # a synthetic batch dimension and restore the packed layout afterwards. + q_embed, k_embed = rope_op( + q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), + cos.unsqueeze(0), + sin.unsqueeze(0), + unsqueeze_dim=1, + ) + return ( + q_embed.squeeze(0).transpose(0, 1).contiguous(), + k_embed.squeeze(0).transpose(0, 1).contiguous(), + ) + + +_PACKED_FUSED_ATTN_IMPLEMENTATIONS = ( + "veomni_flex_attention", + "veomni_magi_attention", +) +_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"}) +_MAGI_ATTN_IMPLEMENTATIONS = frozenset({"magi_attention", "veomni_magi_attention"}) + + +def _normalize_packed_attn_impl(attn_implementation: str | None) -> str | None: + """Normalize Flex/Magi attention aliases to their canonical ``veomni_*`` names.""" + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS: + return "veomni_flex_attention" + if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS: + return "veomni_magi_attention" + return attn_implementation + + +class BagelQwen2MoTAttentionAccelerated(BagelQwen2MoTAttention): + """MoT attention with instance-local RoPE and attention VeomniOps.""" + + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__(config, layer_idx, rms_norm_cls=rms_norm_cls) + self.veomni_rms_norm = VeomniOp("rms_norm", "standard", config.ops_config.rms_norm_implementation) + self.veomni_rope = VeomniOp("rope", "full", config.ops_config.rotary_pos_emb_implementation) + self.veomni_async_ulysses_qkv = VeomniOp("async_ulysses_qkv", "bagel") + self.veomni_async_ulysses_o = VeomniOp("async_ulysses_o", "bagel") + attn_impl = _normalize_packed_attn_impl( + config.ops_config.attn_implementation + if "attn_implementation" in config.ops_implementation + else "veomni_flex_attention" + ) + self.veomni_attn_impl = attn_impl + if attn_impl in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: + try: + self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl) + except RuntimeError: + self.veomni_attn = None + else: + self.veomni_attn = None + try: + self.veomni_denoise_attn: VeomniOp | None = VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + except RuntimeError: + self.veomni_denoise_attn = None + self.register_load_state_dict_pre_hook(combine_qkv_state_dict_pre_hook) + + def _build_qkv_proj(self) -> None: + qkv_size = sum(self.qkv_split_sizes) + self.qkv_proj_und = torch.nn.Linear(self.hidden_size, qkv_size, bias=True) + self.qkv_proj_gen = torch.nn.Linear(self.hidden_size, qkv_size, bias=True) + + def _project_qkv( + self, + hidden: torch.Tensor, + *, + is_gen: bool, + split: bool = False, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project ``hidden`` through the fused und or gen QKV linear layer.""" + projected = (self.qkv_proj_gen if is_gen else self.qkv_proj_und)(hidden) + if split: + return projected.split(self.qkv_split_sizes, dim=-1) + return projected + + @staticmethod + def build_attention_mask( + packed_attention_metadata: torch.Tensor, + *, + attn_implementation: str | None = None, + ) -> BlockMask | MagiAttentionMask: + """Materialize a Flex ``BlockMask`` or ``MagiAttentionMask`` from ``packed_attention_metadata``.""" + if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS: + return build_mot_magi_mask(packed_attention_metadata) + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS or attn_implementation is None: + return build_mot_block_mask(packed_attention_metadata) + raise ValueError( + "BAGEL Qwen2-MoT packed fused attention requires " + f"{', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}, got {attn_implementation!r}." + ) + + def apply_rotary_pos_emb( + self, + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + ) -> tuple[torch.Tensor, torch.Tensor]: + return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim, rope_op=self.veomni_rope) + + def _resolve_packed_attn_op(self, *, context: str) -> VeomniOp: + """Resolve the packed fused attention ``VeomniOp`` bound at initialization for ``context``.""" + if self.veomni_attn is not None: + return self.veomni_attn + if self.veomni_attn_impl not in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: + raise ValueError( + f"BAGEL Qwen2-MoT {context} requires packed fused attention " + f"({', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}), got " + f"{self.veomni_attn_impl!r}." + ) + return VeomniOp("attention", "standard", self.veomni_attn_impl) + + def attend( + self, + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + attention_mask: BlockMask | MagiAttentionMask, + ) -> torch.Tensor: + attn_op = self._resolve_packed_attn_op(context="training") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + ) + packed_attn_output = packed_attn_output.squeeze(0) + return packed_attn_output.reshape(-1, self.num_heads * self.head_dim) + + def attend_inference( + self, + packed_query_states: torch.Tensor, + merged_key_states: torch.Tensor, + merged_value_states: torch.Tensor, + *, + attention_mask: BlockMask | MagiAttentionMask | None, + is_causal: bool, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + ) -> torch.Tensor: + """Run packed inference attention using Flex/Magi for prefill and FlashAttention-2 for decode/denoise.""" + if attention_mask is not None: + attn_op = self._resolve_packed_attn_op(context="packed prefill") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + # Inference packs complete logical documents locally and does + # not enter the training-only Ulysses redistribution. + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + + denoise_op = self.veomni_denoise_attn or VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + packed_attn_output, _ = denoise_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask=None, + dropout=0.0, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + # Inference owns its packed KV-cache layout and does not enter the + # training-only module Ulysses redistribution. + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + + def _has_plain_linear_projections(self) -> bool: + """Whether async Ulysses may read this layer's projection weights directly. + + A wrapped projection such as ``LoraLinear`` has no ``weight`` and must run + its own forward, so attention falls back to the synchronous projection path. + """ + return all( + isinstance(getattr(self, name), torch.nn.Linear) + for name in ("qkv_proj_und", "qkv_proj_gen", "o_proj", "o_proj_moe_gen") + ) + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: BlockMask | MagiAttentionMask, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + """Run packed training attention, using async Ulysses when enabled on the active parallel state.""" + parallel_state = get_parallel_state() + use_async_ulysses = ( + parallel_state.ulysses_enabled + and bool(parallel_state.async_enabled) + and self._has_plain_linear_projections() + ) + if not use_async_ulysses: + return super().forward_packed_train( + packed_sequence, + attention_mask, + packed_position_cos, + packed_position_sin, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + attn_op = self._resolve_packed_attn_op(context="training") + + q_weight_und, k_weight_und, v_weight_und = self.qkv_proj_und.weight.split(self.qkv_split_sizes, dim=0) + q_bias_und, k_bias_und, v_bias_und = ( + (None, None, None) + if self.qkv_proj_und.bias is None + else self.qkv_proj_und.bias.split(self.qkv_split_sizes, dim=0) + ) + q_weight_gen, k_weight_gen, v_weight_gen = self.qkv_proj_gen.weight.split(self.qkv_split_sizes, dim=0) + q_bias_gen, k_bias_gen, v_bias_gen = ( + (None, None, None) + if self.qkv_proj_gen.bias is None + else self.qkv_proj_gen.bias.split(self.qkv_split_sizes, dim=0) + ) + rms_norm_op = getattr(self.q_norm, "veomni_rms_norm", self.veomni_rms_norm) + packed_query_states, packed_key_states, packed_value_states = self.veomni_async_ulysses_qkv( + hidden_states=packed_sequence, + route_0_indices=packed_und_token_indexes, + route_1_indices=packed_gen_token_indexes, + position_cos=packed_position_cos, + position_sin=packed_position_sin, + q_weight_0=q_weight_und, + q_bias_0=q_bias_und, + k_weight_0=k_weight_und, + k_bias_0=k_bias_und, + v_weight_0=v_weight_und, + v_bias_0=v_bias_und, + q_norm_weight_0=self.q_norm.weight, + k_norm_weight_0=self.k_norm.weight, + q_weight_1=q_weight_gen, + q_bias_1=q_bias_gen, + k_weight_1=k_weight_gen, + k_bias_1=k_bias_gen, + v_weight_1=v_weight_gen, + v_bias_1=v_bias_gen, + q_norm_weight_1=self.q_norm_moe_gen.weight, + k_norm_weight_1=self.k_norm_moe_gen.weight, + head_dim=self.head_dim, + rms_norm_eps=self.config.rms_norm_eps, + # The SP pre-hook already pads the packed sequence and builds a + # BlockMask for that full padded length. Keep those padding rows + # through attention; the module post-hook removes them after the + # local output shards have been gathered. + unpadded_dim_size=packed_sequence.shape[0] * parallel_state.ulysses_size, + group=parallel_state.ulysses_group, + rms_norm=rms_norm_op, + rope=self.veomni_rope, + ) + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return self.veomni_async_ulysses_o( + hidden_states=packed_attn_output.squeeze(0), + route_0_indices=packed_und_token_indexes, + route_1_indices=packed_gen_token_indexes, + proj_weight_0=self.o_proj.weight, + proj_bias_0=self.o_proj.bias, + proj_weight_1=self.o_proj_moe_gen.weight, + proj_bias_1=self.o_proj_moe_gen.bias, + group=parallel_state.ulysses_group, + ) + + +class InferenceMixinAccelerated(InferenceMixin): + """Packed CFG denoise for the FSDP/runtime MoT class. + + Stacks active CFG branches into one ``forward_inference`` call so decoder + layers unshard once per denoise step. ``generate`` and ``collect_velocity`` + stay on :class:`~.modeling.InferenceMixin`. + """ + + def denoise_branch( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Run one stacked CFG denoise forward across all active branches.""" + del kwargs + tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs) + inputs = self._generation_state.preprocess_parallel_denoise_inputs( + query, + generation_kwargs or {}, + timestep=tail.meta.get("timestep"), + empty_cache_factory=self._new_empty_cache, + device=self.device, + dtype=self.dtype, + ) + outputs = self.forward_inference( + **inputs, + update_past_key_values=False, + is_causal=False, + mode="gen", + ) + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = outputs["hidden_states"].to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + +class BagelQwen2MoTAccelerated(VeOmniMixin, InferenceMixinAccelerated, BagelQwen2MoTCore): + _supports_flex_attn = True + _supports_magi_attn = True + attention_cls = BagelQwen2MoTAttentionAccelerated + mlp_cls = PatchedQwen2MLP + rms_norm_cls = PatchedQwen2RMSNorm + + @staticmethod + def _create_checkpoint_tensor_converter( + model: BagelQwen2MoTCore, + ) -> BagelQwen2MoTCheckpointTensorConverter: + del model + return BagelQwen2MoTCheckpointTensorConverter() + + +register_accelerated_qkv_conversion_mapping() + + +__all__ = ["BagelQwen2MoTAccelerated", "InferenceMixinAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py new file mode 100644 index 0000000000..f11535eba3 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py @@ -0,0 +1,233 @@ +"""Checkpoint compatibility for accelerated BAGEL fused QKV projections. + +Eager attention owns official split ``q_proj`` / ``k_proj`` / ``v_proj`` +parameters. Accelerated attention stores one merged ``qkv_proj_und`` or +``qkv_proj_gen`` parameter per MoT branch. Existing checkpoints keep their +separate Q/K/V keys, so only the accelerated runtime needs an adapter: +PyTorch state-dict hooks, Transformers weight converters, or VeOmni's streaming +checkpoint converter. +""" + +from __future__ import annotations + +import re +from collections.abc import Generator +from typing import Any + +import torch +from transformers.conversion_mapping import register_checkpoint_conversion_mapping +from transformers.core_model_loading import ConversionOps, WeightConverter + +from ......models.checkpoint.convert import ConvertedCheckpointTensor, export_weights + + +_QKV_COMPONENTS = ("q", "k", "v") +_QKV_BRANCHES = ( + ("qkv_proj_und", ("q_proj", "k_proj", "v_proj")), + ("qkv_proj_gen", ("q_proj_moe_gen", "k_proj_moe_gen", "v_proj_moe_gen")), +) +_QKV_BRANCH_TARGETS = dict(_QKV_BRANCHES) +_QKV_CHECKPOINT_RE = re.compile( + r"^(?P.*\.self_attn\.)" + r"(?P[qkv]_proj(?:_moe_gen)?)\." + r"(?Pweight|bias)$" +) +_QKV_FUSED_RE = re.compile( + r"^(?P.*\.self_attn\.)" + r"(?Pqkv_proj_(?:und|gen))\." + r"(?Pweight|bias)$" +) + + +def combine_qkv_state_dict_pre_hook( + module: torch.nn.Module, + state_dict: dict[str, torch.Tensor], + prefix: str, + local_metadata: dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], +) -> None: + """Accept the legacy split Q/K/V schema in direct PyTorch and DCP loads.""" + del module, local_metadata, strict, missing_keys, unexpected_keys + for combined_name, checkpoint_names in _QKV_BRANCHES: + for kind in ("weight", "bias"): + combined_key = f"{prefix}{combined_name}.{kind}" + checkpoint_keys = [f"{prefix}{name}.{kind}" for name in checkpoint_names] + if combined_key in state_dict or not any(key in state_dict for key in checkpoint_keys): + continue + if not all(key in state_dict for key in checkpoint_keys): + present = [key for key in checkpoint_keys if key in state_dict] + error_msgs.append( + f"Incomplete BAGEL QKV checkpoint group for {combined_key}: found {present}, " + f"expected {checkpoint_keys}." + ) + continue + + state_dict[combined_key] = torch.cat([state_dict.pop(key) for key in checkpoint_keys], dim=0) + + +class _ConcatenateQKV(ConversionOps): + """Concatenate checkpoint Q/K/V tensors along the projection dimension.""" + + @torch.no_grad() + def convert( + self, + input_dict: dict[str, list[torch.Tensor]], + source_patterns: list[str], + target_patterns: list[str], + **kwargs: Any, + ) -> dict[str, torch.Tensor]: + """Concatenate ``source_patterns`` tensors into a single fused target tensor.""" + del kwargs + if len(target_patterns) != 1: + raise ValueError("Combined BAGEL QKV conversion requires one target pattern.") + + tensors: list[torch.Tensor] = [] + for source_pattern in source_patterns: + values = input_dict.pop(source_pattern) + tensors.extend(values if isinstance(values, list) else [values]) + return {target_patterns[0]: torch.cat(tensors, dim=0)} + + @property + def reverse_op(self) -> ConversionOps: + return _SplitQKV() + + +class _SplitQKV(ConversionOps): + """Restore combined QKV tensors to BAGEL's external checkpoint schema.""" + + @torch.no_grad() + def convert( + self, + input_dict: dict[str, list[torch.Tensor]], + source_patterns: list[str], + target_patterns: list[str], + **kwargs: Any, + ) -> dict[str, torch.Tensor]: + """Split a fused QKV tensor into separate Q, K, and V checkpoint tensors.""" + del source_patterns + if len(input_dict) != 1 or len(target_patterns) != 3: + raise ValueError("BAGEL QKV export requires one combined source and three target patterns.") + + values = next(iter(input_dict.values())) + tensor = values[0] if isinstance(values, list) else values + config = kwargs["config"] + head_dim = config.hidden_size // config.num_attention_heads + split_sizes = ( + config.num_attention_heads * head_dim, + config.num_key_value_heads * head_dim, + config.num_key_value_heads * head_dim, + ) + return { + target: chunk.clone() + for target, chunk in zip(target_patterns, tensor.split(split_sizes, dim=0), strict=True) + } + + @property + def reverse_op(self) -> ConversionOps: + return _ConcatenateQKV() + + +def _qkv_weight_converters() -> list[WeightConverter]: + """Build Transformers ``WeightConverter`` rules for both MoT QKV branches.""" + converters: list[WeightConverter] = [] + for branch, source_suffix in (("und", ""), ("gen", "_moe_gen")): + for kind in ("weight", "bias"): + converters.append( + WeightConverter( + source_patterns=[ + f"self_attn.{component}_proj{source_suffix}.{kind}" for component in _QKV_COMPONENTS + ], + target_patterns=f"self_attn.qkv_proj_{branch}.{kind}", + operations=[_ConcatenateQKV()], + ) + ) + return converters + + +def register_accelerated_qkv_conversion_mapping() -> None: + """Register fused-QKV converters for the accelerated class only. + + Lookups prefer class name over ``model_type``. Do not register + ``bagel_qwen2_mot`` here: eager ``BagelQwen2MoT`` uses native split keys. + """ + register_checkpoint_conversion_mapping( + "BagelQwen2MoTAccelerated", + _qkv_weight_converters(), + overwrite=True, + ) + + +class BagelQwen2MoTCheckpointTensorConverter: + """Stream legacy Q/K/V tensors into accelerated combined runtime parameters. + + VeOmni's ``load_model_weights`` assigns tensors directly instead of calling + ``load_state_dict``, so the PyTorch pre-hook cannot perform this merge. + """ + + def __init__(self) -> None: + self._pending: dict[tuple[str, str, str], dict[str, torch.Tensor]] = {} + + def can_handle(self, name: str) -> bool: + return _QKV_CHECKPOINT_RE.fullmatch(name) is not None + + def convert(self, name: str, tensor: torch.Tensor) -> ConvertedCheckpointTensor | None: + """Buffer split Q/K/V tensors and emit the fused ``qkv_proj_{und,gen}`` tensor once complete.""" + match = _QKV_CHECKPOINT_RE.fullmatch(name) + if match is None: + return ConvertedCheckpointTensor(name=name, tensor=tensor) + + projection = match.group("projection") + component = projection[0] + branch = "gen" if projection.endswith("_moe_gen") else "und" + group_key = (match.group("prefix"), branch, match.group("kind")) + group = self._pending.setdefault(group_key, {}) + if component in group: + raise ValueError(f"Duplicate BAGEL QKV checkpoint tensor: {name}.") + group[component] = tensor + if group.keys() != set(_QKV_COMPONENTS): + return None + + self._pending.pop(group_key) + combined = torch.cat([group[component] for component in _QKV_COMPONENTS], dim=0) + prefix, branch, kind = group_key + return ConvertedCheckpointTensor( + name=f"{prefix}qkv_proj_{branch}.{kind}", + tensor=combined, + ) + + def finalize(self) -> list[ConvertedCheckpointTensor]: + """Verify no partial QKV groups remain buffered at the end of checkpoint loading.""" + if self._pending: + incomplete = ", ".join( + f"{prefix}qkv_proj_{branch}.{kind}" for prefix, branch, kind in sorted(self._pending) + ) + raise ValueError(f"Incomplete BAGEL QKV checkpoint groups: {incomplete}.") + return [] + + def export_weights(self, model: torch.nn.Module) -> Generator[tuple[str, torch.Tensor]]: + """Export model parameters in HF checkpoint format with fused ``qkv_proj_{und,gen}`` split into ``{q,k,v}_proj``.""" + base_model = model + while isinstance(getattr(base_model, "module", None), torch.nn.Module): + base_model = base_model.module + config = base_model.config + head_dim = config.hidden_size // config.num_attention_heads + split_sizes = ( + config.num_attention_heads * head_dim, + config.num_key_value_heads * head_dim, + config.num_key_value_heads * head_dim, + ) + for name, tensor in export_weights(base_model): + if tensor.dtype == torch.float32: + tensor = tensor.to(torch.bfloat16) + match = _QKV_FUSED_RE.fullmatch(name) + if match is None: + yield name, tensor + continue + prefix = match.group("prefix") + fused = match.group("fused") + kind = match.group("kind") + for target_proj, chunk in zip(_QKV_BRANCH_TARGETS[fused], tensor.split(split_sizes, dim=0), strict=True): + yield f"{prefix}{target_proj}.{kind}", chunk.clone() diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py new file mode 100644 index 0000000000..5a330813fc --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py @@ -0,0 +1,65 @@ +"""Configuration for BAGEL's Qwen2 MoT backbone module.""" + +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config + +from ...module_configuration_base import OmniModuleConfig + + +class BagelQwen2MoTConfig(OmniModuleConfig, Qwen2Config): + """Qwen2 config with BAGEL's runtime MoT defaults. + + The upstream ``llm_config.json`` is a plain Qwen2 config. BAGEL's loader + mutates it at runtime with ``qk_norm=True`` and + ``layer_module="Qwen2MoTDecoderLayer"``; this config bakes those defaults + into the split module so a SeedOmni checkpoint is self-describing. + """ + + model_type = "bagel_qwen2_mot" + + def __init__( + self, + vocab_size: int = 152064, + hidden_size: int = 3584, + intermediate_size: int = 18944, + num_hidden_layers: int = 28, + num_attention_heads: int = 28, + num_key_value_heads: int = 4, + max_position_embeddings: int = 32768, + rms_norm_eps: float = 1e-6, + rope_theta: float = 1000000.0, + bos_token_id: int = 151643, + eos_token_id: int = 151645, + tie_word_embeddings: bool = False, + qk_norm: bool = True, + layer_module: str = "Qwen2MoTDecoderLayer", + freeze_und: bool = False, + **kwargs, + ): + ops_impl = kwargs.get("ops_implementation") + if "attn_implementation" not in kwargs and not ( + isinstance(ops_impl, dict) and ops_impl.get("attn_implementation") is not None + ): + kwargs["attn_implementation"] = "flex_attention" + super().__init__( + vocab_size=vocab_size, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + max_position_embeddings=max_position_embeddings, + rms_norm_eps=rms_norm_eps, + rope_theta=rope_theta, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + if self.num_attention_heads % self.num_key_value_heads != 0: + raise ValueError( + "BAGEL Qwen2-MoT query heads must be divisible by KV heads, " + f"got query={self.num_attention_heads}, key_value={self.num_key_value_heads}." + ) + self.qk_norm = qk_norm + self.layer_module = layer_module + self.freeze_und = freeze_und diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py new file mode 100644 index 0000000000..530cbffc54 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py @@ -0,0 +1,516 @@ +"""Internal generation state for BAGEL Qwen2-MoT inference.""" + +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Any, Callable, Iterator + +import torch + +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + + +def _build_denoise_index_layout( + key_values_lens: torch.Tensor, + query_len: int, + *, + device: torch.device, +) -> dict[str, torch.Tensor]: + """Build attention-space and query-space indexes for stacked CFG branches.""" + query_indexes: list[torch.Tensor] = [] + key_value_indexes: list[torch.Tensor] = [] + text_indexes: list[torch.Tensor] = [] + vae_token_indexes: list[torch.Tensor] = [] + # Attention indexes address the merged [branch cache, branch query] + # layout. Expert indexes address the query-only tensor, so they use a + # separate cursor that does not include cached rows. + attention_offset = 0 + query_offset = 0 + for key_values_len in key_values_lens.tolist(): + key_values_len = int(key_values_len) + key_value_indexes.append( + torch.arange(attention_offset, attention_offset + key_values_len, device=device, dtype=torch.long) + ) + query_indexes.append( + torch.arange( + attention_offset + key_values_len, + attention_offset + key_values_len + query_len, + device=device, + dtype=torch.long, + ) + ) + text_indexes.append( + torch.tensor([query_offset, query_offset + query_len - 1], device=device, dtype=torch.long) + ) + vae_token_indexes.append( + torch.arange(query_offset + 1, query_offset + query_len - 1, device=device, dtype=torch.long) + ) + attention_offset += key_values_len + query_len + query_offset += query_len + + return { + "packed_query_indexes": torch.cat(query_indexes, dim=0), + "packed_key_value_indexes": ( + torch.cat(key_value_indexes, dim=0) + if key_value_indexes + else torch.empty(0, device=device, dtype=torch.long) + ), + "packed_text_indexes": torch.cat(text_indexes, dim=0), + "packed_vae_token_indexes": torch.cat(vae_token_indexes, dim=0), + } + + +@dataclass +class MotCacheContext: + """KV-cache state for one logical inference branch. + + FA receives all active branches in one packed call. Each branch therefore + keeps its own cache plus the indexes needed to place that cache in the + temporary packed ``[past, query]`` attention layout. + """ + + name: str + _cache: Any | None = None + _key_values_lens: torch.Tensor | None = None + _packed_key_value_indexes: torch.Tensor | None = None + _next_position_ids: torch.Tensor | None = None + + # ── Cache state ───────────────────────────────────────────── + + def reset(self) -> None: + self._cache = None + self._key_values_lens = None + self._packed_key_value_indexes = None + self._next_position_ids = None + + @property + def cache(self) -> Any | None: + return self._cache + + @property + def key_values_lens(self) -> torch.Tensor | None: + return self._key_values_lens + + @property + def packed_key_value_indexes(self) -> torch.Tensor | None: + return self._packed_key_value_indexes + + @property + def next_position_ids(self) -> torch.Tensor | None: + return self._next_position_ids + + # ── Cache mutation ────────────────────────────────── + + def _set_cache_state( + self, + *, + cache: Any, + key_values_lens: torch.Tensor | None, + packed_key_value_indexes: torch.Tensor | None, + next_position_ids: torch.Tensor | None, + ) -> None: + self._cache = cache + self._key_values_lens = key_values_lens + self._packed_key_value_indexes = packed_key_value_indexes + self._next_position_ids = next_position_ids + + def install_cache( + self, + *, + cache: Any, + cache_len: int, + next_position_id: torch.Tensor, + device: torch.device, + ) -> None: + """Install one contiguous cache extracted from the packed Flex prefill.""" + if cache_len < 0: + raise ValueError(f"BAGEL {self.name} branch cache length must be non-negative.") + self._set_cache_state( + cache=cache, + key_values_lens=torch.tensor([cache_len], device=device, dtype=torch.int32), + packed_key_value_indexes=torch.arange(cache_len, device=device, dtype=torch.long), + next_position_ids=next_position_id.detach().reshape(1).to(device=device, dtype=torch.long), + ) + + def append_packed_query( + self, + *, + cache: Any, + query_lens: torch.Tensor, + device: torch.device, + ) -> None: + """Advance the main AR cache after a successful one-branch decode.""" + self.require_ready() + next_key_values_lens = self._key_values_lens + query_lens + + self._set_cache_state( + cache=cache, + key_values_lens=next_key_values_lens, + packed_key_value_indexes=torch.arange( + int(next_key_values_lens.sum().item()), + device=device, + dtype=torch.long, + ), + next_position_ids=self._next_position_ids + query_lens, + ) + + # ── Cache accessors ────────────────────────────────── + + def require_ready(self) -> None: + """Raise if this branch's KV-cache state has not been initialized.""" + if ( + self._cache is None + or self._key_values_lens is None + or self._packed_key_value_indexes is None + or self._next_position_ids is None + ): + raise RuntimeError(f"BAGEL {self.name} branch context is not initialized.") + + def cache_len(self) -> int: + """Return the total cached sequence length for this branch.""" + if self._key_values_lens is None: + raise RuntimeError(f"BAGEL {self.name} branch cache lengths are not initialized.") + return int(self._key_values_lens.sum().item()) + + def repeated_position_ids(self, query_len: int, *, device: torch.device) -> torch.Tensor: + """Use one logical position for every token in a denoise image span.""" + if self._next_position_ids is None: + raise RuntimeError(f"BAGEL {self.name} branch position ids are not initialized.") + return self._next_position_ids.reshape(1).expand(query_len).to(device=device) + + def packed_query_args( + self, + query_len: int, + *, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build sequential positions and cache-relative indexes for AR decode.""" + self.require_ready() + cache_len = self.cache_len() + query_lens = torch.tensor([query_len], device=device, dtype=torch.int32) + packed_query_indexes = torch.arange(cache_len, cache_len + query_len, device=device, dtype=torch.long) + packed_position_ids = self._next_position_ids.reshape(1) + torch.arange( + query_len, + device=device, + dtype=torch.long, + ) + + return query_lens, packed_query_indexes, packed_position_ids + + +@dataclass +class MotGenerationState: + """Scheduling state shared by Qwen2-MoT inference graph invocations. + + ``main`` is the conditional prompt, ``cfg_text`` is a prefix of ``main``, + and ``cfg_img`` is the text-only document used for image CFG. The state + also remembers the branch layout between denoise and velocity collection. + """ + + _velocity_buffer: dict[str, torch.Tensor] = field(default_factory=dict) + _pending_denoise_branches: tuple[str, ...] | None = None + _pending_denoise_query_len: int | None = None + _main: MotCacheContext = field(default_factory=lambda: MotCacheContext("main")) + _cfg_text: MotCacheContext = field(default_factory=lambda: MotCacheContext("text CFG")) + _cfg_img: MotCacheContext = field(default_factory=lambda: MotCacheContext("image CFG")) + + def reset(self) -> None: + self._velocity_buffer.clear() + self._pending_denoise_branches = None + self._pending_denoise_query_len = None + self._main.reset() + self._cfg_text.reset() + self._cfg_img.reset() + + # ── CFG cache contexts ────────────────────────────────── + + @property + def main(self) -> MotCacheContext: + return self._main + + @property + def cfg_text(self) -> MotCacheContext: + return self._cfg_text + + @property + def cfg_img(self) -> MotCacheContext: + return self._cfg_img + + def _cache_for_branch(self, branch: str) -> MotCacheContext: + """Return the ``MotCacheContext`` for ``branch`` (``'main'``, ``'cfg_text'``, or ``'cfg_img'``).""" + if branch == "main": + return self._main + if branch == "cfg_text": + return self._cfg_text + if branch == "cfg_img": + return self._cfg_img + raise RuntimeError(f"Unsupported BAGEL denoise branch {branch!r}.") + + # ── CFG request policy ────────────────────────────────── + + def validate_cfg_request(self, generation_kwargs: dict[str, object]) -> None: + """Validate that image CFG is only enabled when text CFG is also active.""" + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0)) + if cfg_img_scale > 1.0 and cfg_text_scale <= 1.0: + raise ValueError("cfg_img_scale > 1.0 requires cfg_text_scale > 1.0") + + def cfg_img_requested(self, generation_kwargs: dict[str, object]) -> bool: + return float(generation_kwargs.get("cfg_img_scale", 1.0)) > 1.0 + + # ── Denoise velocity round ────────────────────────────────── + + def iter_serial_denoise_inputs( + self, + query: torch.Tensor, + generation_kwargs: dict[str, object], + *, + timestep: object, + device: torch.device, + dtype: torch.dtype, + ) -> Iterator[dict[str, Any]]: + """Yield one ``forward_inference`` input dict per active CFG branch.""" + query = query.to(device=device, dtype=dtype) + query_len = int(query.shape[0]) + if query_len < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + + branches = self._denoise_branches_for_timestep(generation_kwargs, timestep) + self._pending_denoise_branches = branches + self._pending_denoise_query_len = query_len + for branch in branches: + context = self._cache_for_branch(branch) + context.require_ready() + cache_len = context.cache_len() + packed_key_value_indexes = context.packed_key_value_indexes + if packed_key_value_indexes is None: + packed_key_value_indexes = torch.empty(0, device=device, dtype=torch.long) + yield { + "packed_query_sequence": query, + "query_lens": torch.tensor([query_len], device=device, dtype=torch.int32), + "packed_query_position_ids": context.repeated_position_ids(query_len, device=device), + "packed_query_indexes": torch.arange( + cache_len, cache_len + query_len, device=device, dtype=torch.long + ), + "past_key_values": context.cache, + "key_values_lens": context.key_values_lens, + "packed_key_value_indexes": packed_key_value_indexes, + "packed_text_indexes": torch.tensor([0, query_len - 1], device=device, dtype=torch.long), + "packed_vae_token_indexes": torch.arange(1, query_len - 1, device=device, dtype=torch.long), + } + + def preprocess_parallel_denoise_inputs( + self, + query: torch.Tensor, + generation_kwargs: dict[str, object], + *, + timestep: object, + empty_cache_factory: Callable[[], Any], + device: torch.device, + dtype: torch.dtype, + ) -> dict[str, Any]: + """Stack active CFG branches into one packed ``forward_inference`` call.""" + query = query.to(device=device, dtype=dtype) + query_len = int(query.shape[0]) + if query_len < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + + branches = self._denoise_branches_for_timestep(generation_kwargs, timestep) + contexts = [self._cache_for_branch(branch) for branch in branches] + for context in contexts: + context.require_ready() + + query_lens = torch.full((len(branches),), query_len, device=device, dtype=torch.int32) + key_values_lens = torch.tensor([context.cache_len() for context in contexts], device=device, dtype=torch.int32) + packed_query_sequence = torch.cat([query] * len(branches), dim=0) + packed_query_position_ids = torch.cat( + [context.repeated_position_ids(query_len, device=device) for context in contexts], + dim=0, + ) + index_layout = _build_denoise_index_layout(key_values_lens, query_len, device=device) + + # The flow connector preserves row order but does not carry branch + # labels, so collect_velocity must replay this exact layout. + self._pending_denoise_branches = branches + self._pending_denoise_query_len = query_len + return { + "packed_query_sequence": packed_query_sequence, + "query_lens": query_lens, + "key_values_lens": key_values_lens, + "packed_query_position_ids": packed_query_position_ids, + "past_key_values": self._merged_branch_cache(contexts, key_values_lens, empty_cache_factory), + **index_layout, + } + + def collect_velocity( + self, + velocity: torch.Tensor, + generation_kwargs: dict[str, object], + *, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + """Split stacked velocities by branch and apply BAGEL CFG equations.""" + branches = self._pending_denoise_branches + query_len = self._pending_denoise_query_len + if branches is None or query_len is None: + raise RuntimeError("BAGEL MoT velocity collection requires a pending denoise branch layout.") + expected_rows = len(branches) * query_len + if int(velocity.shape[0]) != expected_rows: + raise ValueError( + f"BAGEL MoT velocity collection expected {expected_rows} stacked rows, got {velocity.shape[0]}." + ) + + self._velocity_buffer.clear() + offset = 0 + for branch in branches: + branch_velocity = velocity[offset : offset + query_len].to(device=device, dtype=dtype) + self._velocity_buffer[branch] = self._strip_denoise_query_markers(branch_velocity) + offset += query_len + + merged_velocity = ( + self._merged_velocity_without_cfg(device=device) + if "cfg_text" not in self._velocity_buffer + else self._merged_velocity_with_cfg(generation_kwargs, device=device) + ) + self._velocity_buffer.clear() + self._pending_denoise_branches = None + self._pending_denoise_query_len = None + return merged_velocity + + def _denoise_branches_for_timestep( + self, generation_kwargs: dict[str, object], timestep: object + ) -> tuple[str, ...]: + """Return the ordered tuple of active CFG branch names for ``timestep``.""" + self.validate_cfg_request(generation_kwargs) + cfg_text_active = self._cfg_text_active(generation_kwargs, timestep) + branches = ["main"] + if cfg_text_active: + branches.append("cfg_text") + if cfg_text_active and self.cfg_img_requested(generation_kwargs): + branches.append("cfg_img") + return tuple(branches) + + def _cfg_text_active(self, generation_kwargs: dict[str, object], timestep: object) -> bool: + """Return whether text CFG guidance is active at ``timestep``.""" + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + if cfg_text_scale <= 1.0: + return False + + # CFG applies only inside the configured open-left, closed-right + # diffusion-time interval. + interval = generation_kwargs.get("cfg_interval", (0.0, 1.0)) + if interval is None: + lower, upper = 0.0, 1.0 + elif isinstance(interval, (list, tuple)) and len(interval) == 2: + lower, upper = float(interval[0]), float(interval[1]) + else: + lower, upper = float(interval), 1.0 + + if torch.is_tensor(timestep): + if timestep.numel() == 0: + raise ValueError("BAGEL CFG timestep tensor must not be empty.") + t = float(timestep.detach().reshape(-1)[0].item()) + elif timestep is None: + raise ValueError("BAGEL CFG requires current-round meta['timestep'].") + else: + t = float(timestep) + return t > lower and t <= upper + + def _strip_denoise_query_markers(self, velocity: torch.Tensor) -> torch.Tensor: + """Strip the leading and trailing vision marker rows from a branch velocity tensor.""" + if int(velocity.shape[0]) < 3: + raise ValueError("BAGEL MoT denoise velocity must include start/end marker rows.") + return velocity[1:-1] + + def _merged_velocity_without_cfg(self, *, device: torch.device) -> torch.Tensor: + """Return the unguided ``main`` branch velocity.""" + main_velocity = self._velocity_buffer.get("main") + if main_velocity is None: + raise RuntimeError("BAGEL CFG merge requires a collected main branch velocity.") + return main_velocity.to(device=device) + + def _merged_velocity_with_cfg(self, generation_kwargs: dict[str, object], *, device: torch.device) -> torch.Tensor: + """Combine ``main``, ``cfg_text``, and optional ``cfg_img`` velocities with CFG renormalization.""" + main_velocity = self._merged_velocity_without_cfg(device=device) + + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + cfg_text_velocity = self._velocity_buffer.get("cfg_text") + if cfg_text_velocity is None: + raise RuntimeError("BAGEL CFG merge requires a collected cfg_text branch velocity.") + cfg_text_velocity = cfg_text_velocity.to(device=device, dtype=main_velocity.dtype) + + cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0)) + cfg_img_velocity = self._velocity_buffer.get("cfg_img") + if cfg_img_scale > 1.0 and cfg_img_velocity is None: + raise ValueError("BAGEL image CFG merge requires a collected cfg_img velocity.") + if cfg_img_velocity is not None: + cfg_img_velocity = cfg_img_velocity.to(device=device, dtype=main_velocity.dtype) + + cfg_renorm_min = float(generation_kwargs.get("cfg_renorm_min", 0.0)) + cfg_renorm_type = str(generation_kwargs.get("cfg_renorm_type", "global")) + + # Official InterleaveInferencer wraps generate_image in bf16 autocast, so + # the CFG scale/norm/renorm math is not the same as eager bf16 * python float. + autocast = ( + torch.amp.autocast(device.type, dtype=torch.bfloat16) + if IS_CUDA_AVAILABLE and device.type == get_device_type() + else nullcontext() + ) + with autocast: + guided = cfg_text_velocity + cfg_text_scale * (main_velocity - cfg_text_velocity) + if cfg_renorm_type == "text_channel": + norm_main = torch.norm(main_velocity, dim=-1, keepdim=True) + norm_text = torch.norm(guided, dim=-1, keepdim=True) + scale = (norm_main / (norm_text + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) + merged = guided * scale + if cfg_img_scale > 1.0: + assert cfg_img_velocity is not None + merged = cfg_img_velocity + cfg_img_scale * (merged - cfg_img_velocity) + return merged.to(device=device) + + if cfg_img_scale > 1.0: + assert cfg_img_velocity is not None + guided = cfg_img_velocity + cfg_img_scale * (guided - cfg_img_velocity) + + if cfg_renorm_type == "global": + norm_main = torch.norm(main_velocity) + norm_guided = torch.norm(guided) + elif cfg_renorm_type == "channel": + norm_main = torch.norm(main_velocity, dim=-1, keepdim=True) + norm_guided = torch.norm(guided, dim=-1, keepdim=True) + else: + raise NotImplementedError(f"BAGEL infer_gen CFG renorm type {cfg_renorm_type!r} is not implemented.") + scale = (norm_main / (norm_guided + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) + return (guided * scale).to(device=device) + + def _merged_branch_cache( + self, + contexts: list[MotCacheContext], + key_values_lens: torch.Tensor, + empty_cache_factory: Callable[[], Any], + ) -> Any: + """Concatenate per-branch KV caches into one packed ``NaiveCache``.""" + # Concatenation order must match preprocess_parallel_denoise_inputs: + # main, then cfg_text, then cfg_img when those branches are active. + merged_cache = empty_cache_factory() + for layer_idx in merged_cache.key_cache: + key_parts = [] + value_parts = [] + for context, key_values_len in zip(contexts, key_values_lens.tolist(), strict=True): + key_values_len = int(key_values_len) + if key_values_len == 0: + continue + cache = context.cache + if cache is None or cache.key_cache[layer_idx] is None or cache.value_cache[layer_idx] is None: + raise RuntimeError(f"BAGEL {context.name} branch cache is missing layer {layer_idx}.") + key_parts.append(cache.key_cache[layer_idx]) + value_parts.append(cache.value_cache[layer_idx]) + if key_parts: + merged_cache.key_cache[layer_idx] = torch.cat(key_parts, dim=0) + merged_cache.value_cache[layer_idx] = torch.cat(value_parts, dim=0) + return merged_cache + + +__all__ = ["MotCacheContext", "MotGenerationState"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py new file mode 100644 index 0000000000..52270435de --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py @@ -0,0 +1,346 @@ +"""Attention visibility metadata and mask materializers for BAGEL Qwen2-MoT. + +`packed_attention_metadata` is the source of truth. `build_mot_block_mask` +materializes a Flex `BlockMask` for accelerated training. `build_mot_magi_mask` +materializes MagiAttention ranges. `build_mot_sdpa_mask` materializes a dense +boolean mask for eager SDPA (``True`` means attend). +""" + +from __future__ import annotations + +import torch +from torch.nn.attention.flex_attention import BlockMask + +from ......ops.mask import MagiAttentionMask + + +_MOT_BLOCK_SIZE = 128 +_MAGI_ATTN_FULL = 0 +_MAGI_ATTN_CAUSAL = 1 + + +def build_mot_attention_metadata( + sample_splits: list[list[int]], + sample_attn_modes: list[list[str]], + *, + device: torch.device, +) -> torch.Tensor: + """Encode BAGEL packed visibility in three O(sequence) int32 rows. + + Row 0 identifies independent packed documents. Row 1 identifies spans + whose tokens can attend bidirectionally within the span (``full`` and + ``noise``). Row 2 identifies noise spans so their keys remain invisible + outside that same noise span. ``-1`` means that a row does not apply. + + Callers materialize this compact representation as a Flex ``BlockMask``, + MagiAttention ranges, or a dense SDPA mask, without changing the metadata. + """ + total_length = sum(sum(split_lens) for split_lens in sample_splits) + metadata = torch.full((3, total_length), -1, device=device, dtype=torch.int32) + cursor = 0 + span_id = 0 + for document_id, (split_lens, attn_modes) in enumerate(zip(sample_splits, sample_attn_modes, strict=True)): + for length, mode in zip(split_lens, attn_modes, strict=True): + if mode not in {"causal", "full", "noise"}: + raise ValueError(f"Unsupported BAGEL attention mode: {mode!r}.") + span_end = cursor + length + metadata[0, cursor:span_end] = document_id + if mode != "causal": + metadata[1, cursor:span_end] = span_id + if mode == "noise": + metadata[2, cursor:span_end] = span_id + cursor = span_end + span_id += 1 + return metadata.contiguous() + + +def _validate_mot_attention_metadata(packed_attention_metadata: torch.Tensor) -> None: + """Validate that ``packed_attention_metadata`` is a ``[3, S]`` int32 tensor.""" + if packed_attention_metadata.ndim != 2 or packed_attention_metadata.shape[0] != 3: + raise ValueError( + "BAGEL Qwen2-MoT attention metadata must have shape [3, sequence], " + f"got {tuple(packed_attention_metadata.shape)}." + ) + if packed_attention_metadata.dtype != torch.int32: + raise ValueError( + f"BAGEL Qwen2-MoT attention metadata must use torch.int32, got {packed_attention_metadata.dtype}." + ) + + +def pad_mot_attention_metadata( + packed_attention_metadata: torch.Tensor, + padded_length: int, +) -> torch.Tensor: + """Pad metadata with one isolated full-attention span for SP alignment. + + Giving padding its own document prevents real tokens from attending to it; + leaving its noise ID at ``-1`` makes the padding block safe to classify as + a regular clean block. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + + sequence_length = int(packed_attention_metadata.shape[1]) + if padded_length < sequence_length: + raise ValueError( + "BAGEL Qwen2-MoT padded attention length cannot shrink the sequence: " + f"sequence_length={sequence_length}, padded_length={padded_length}." + ) + if padded_length == sequence_length: + return packed_attention_metadata + + padded = torch.full( + (3, padded_length), + -1, + device=packed_attention_metadata.device, + dtype=packed_attention_metadata.dtype, + ) + padded[:, :sequence_length] = packed_attention_metadata + padding_id = sequence_length + padded[0, sequence_length:] = padding_id + padded[1, sequence_length:] = padding_id + return padded.contiguous() + + +def _metadata_block_ranges( + token_ids: torch.Tensor, + *, + block_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Summarize monotonic token IDs for conservative block classification.""" + padding = (-int(token_ids.numel())) % block_size + if padding: + token_ids = torch.nn.functional.pad(token_ids, (0, padding), value=-1) + token_ids = token_ids.view(-1, block_size) + valid = token_ids >= 0 + minimum = torch.where(valid, token_ids, torch.iinfo(token_ids.dtype).max).amin(dim=-1) + maximum = token_ids.amax(dim=-1) + return minimum, maximum, valid.any(dim=-1), valid.all(dim=-1) + + +def _ordered_block_indices(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Convert a dense block-level predicate to FlexAttention ordered rows.""" + block_count = int(blocks.shape[-1]) + block_ids = torch.arange(block_count, device=blocks.device, dtype=torch.int32).expand(block_count, -1) + sentinel = torch.full_like(block_ids, block_count) + indices = torch.where(blocks, block_ids, sentinel).sort(dim=-1).values + num_blocks = blocks.sum(dim=-1, dtype=torch.int32) + return num_blocks.unsqueeze(0).unsqueeze(0), indices.unsqueeze(0).unsqueeze(0) + + +def build_mot_block_mask(packed_attention_metadata: torch.Tensor) -> BlockMask: + """Build BAGEL's head-independent BlockMask from compact metadata. + + Token visibility is: + ``same_document & (causal | same_full_span) & ~foreign_noise_key``. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + + def mask_mod(batch_idx, head_idx, query_idx, key_idx): + # MoT visibility is identical across batch and attention heads, but + # FlexAttention requires the four-argument callback signature. + del batch_idx, head_idx + same_document = document_ids[query_idx] == document_ids[key_idx] + causal = query_idx >= key_idx + same_full_or_noise_span = (full_span_ids[query_idx] >= 0) & ( + full_span_ids[query_idx] == full_span_ids[key_idx] + ) + foreign_noise_key = (noise_span_ids[key_idx] >= 0) & (noise_span_ids[query_idx] != noise_span_ids[key_idx]) + return same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key + + doc_min, doc_max, doc_valid, doc_complete = _metadata_block_ranges( + document_ids, + block_size=_MOT_BLOCK_SIZE, + ) + full_min, full_max, full_valid, full_complete = _metadata_block_ranges( + full_span_ids, + block_size=_MOT_BLOCK_SIZE, + ) + _, noise_max, _, _ = _metadata_block_ranges( + noise_span_ids, + block_size=_MOT_BLOCK_SIZE, + ) + + # Range overlap produces a conservative candidate set: mixed boundary + # blocks may be included here and are filtered token-by-token by mask_mod. + same_document_block = ( + doc_valid[:, None] + & doc_valid[None, :] + & (doc_min[:, None] <= doc_max[None, :]) + & (doc_min[None, :] <= doc_max[:, None]) + ) + shared_full_span_block = ( + full_valid[:, None] + & full_valid[None, :] + & (full_min[:, None] <= full_max[None, :]) + & (full_min[None, :] <= full_max[:, None]) + ) + block_count = int(doc_min.numel()) + block_ids = torch.arange(block_count, device=packed_attention_metadata.device) + causal_block = block_ids[:, None] >= block_ids[None, :] + candidate_blocks = same_document_block & (causal_block | shared_full_span_block) + + # Full blocks bypass mask_mod. Mark a block pair full only when every token + # is provably in one document and visibility is unconditional: either the + # key block is strictly earlier and clean, or both blocks share one + # bidirectional span. + single_document_block = doc_complete & (doc_min == doc_max) + same_single_document = ( + single_document_block[:, None] & single_document_block[None, :] & (doc_min[:, None] == doc_min[None, :]) + ) + single_full_span_block = full_complete & full_valid & (full_min == full_max) + same_single_full_span = ( + single_full_span_block[:, None] & single_full_span_block[None, :] & (full_min[:, None] == full_min[None, :]) + ) + strictly_earlier_clean_key_block = (block_ids[:, None] > block_ids[None, :]) & (noise_max[None, :] < 0) + full_blocks = candidate_blocks & same_single_document & (strictly_earlier_clean_key_block | same_single_full_span) + partial_blocks = candidate_blocks & ~full_blocks + + partial_num_blocks, partial_indices = _ordered_block_indices(partial_blocks) + full_num_blocks, full_indices = _ordered_block_indices(full_blocks) + return BlockMask.from_kv_blocks( + partial_num_blocks, + partial_indices, + full_num_blocks, + full_indices, + BLOCK_SIZE=_MOT_BLOCK_SIZE, + mask_mod=mask_mod, + seq_lengths=(sequence_length, sequence_length), + ) + + +def _mot_span_mode(full_span_id: int, noise_span_id: int) -> str: + """Classify a token span as ``'noise'``, ``'full'``, or ``'causal'`` from its span IDs.""" + if noise_span_id >= 0: + return "noise" + if full_span_id >= 0: + return "full" + return "causal" + + +def _iter_mot_spans(packed_attention_metadata: torch.Tensor) -> list[tuple[int, int, int, str]]: + """Recover contiguous (start, end, document_id, mode) spans from metadata.""" + sequence_length = int(packed_attention_metadata.shape[1]) + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + changed = document_ids.new_ones(sequence_length, dtype=torch.bool) + changed[1:] = ( + (document_ids[1:] != document_ids[:-1]) + | (full_span_ids[1:] != full_span_ids[:-1]) + | (noise_span_ids[1:] != noise_span_ids[:-1]) + ) + starts = torch.nonzero(changed, as_tuple=False).flatten() + ends = torch.cat((starts[1:], starts.new_tensor([sequence_length]))) + spans: list[tuple[int, int, int, str]] = [] + for start, end, document_id, full_span_id, noise_span_id in zip( + starts.tolist(), + ends.tolist(), + document_ids[starts].tolist(), + full_span_ids[starts].tolist(), + noise_span_ids[starts].tolist(), + strict=True, + ): + spans.append((start, end, int(document_id), _mot_span_mode(int(full_span_id), int(noise_span_id)))) + return spans + + +def _merge_adjacent_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]: + """Coalesce contiguous ``(start, end)`` intervals into maximal ranges.""" + if not ranges: + return [] + merged = [ranges[0]] + for start, end in ranges[1:]: + last_start, last_end = merged[-1] + if start == last_end: + merged[-1] = (last_start, end) + else: + merged.append((start, end)) + return merged + + +def build_mot_magi_mask(packed_attention_metadata: torch.Tensor) -> MagiAttentionMask: + """Build BAGEL's MagiAttention ranges from compact metadata. + + Token visibility is the same as Flex/SDPA: + ``same_document & (causal | same_full_span) & ~foreign_noise_key``. + + Each document is encoded as: + - a self range per span (causal inside ``causal`` spans, full inside ``full``/``noise``) + - full ranges from each query span onto earlier non-noise spans in that document + + Noise keys are omitted from every other span's key ranges. Metadata is + already the full packed sequence, including Ulysses padding, so this + builder does not rescale ranges by ``ulysses_size``. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + + q_ranges: list[tuple[int, int]] = [] + k_ranges: list[tuple[int, int]] = [] + attn_types: list[int] = [] + document_spans: list[tuple[int, int, str]] = [] + current_document_id: int | None = None + + def flush_document(spans: list[tuple[int, int, str]]) -> None: + """Append Magi Q/K range pairs and attention types for one document's spans.""" + for query_index, (query_start, query_end, query_mode) in enumerate(spans): + earlier_clean = [ + (key_start, key_end) for key_start, key_end, key_mode in spans[:query_index] if key_mode != "noise" + ] + for key_start, key_end in _merge_adjacent_ranges(earlier_clean): + q_ranges.append((query_start, query_end)) + k_ranges.append((key_start, key_end)) + attn_types.append(_MAGI_ATTN_FULL) + q_ranges.append((query_start, query_end)) + k_ranges.append((query_start, query_end)) + attn_types.append(_MAGI_ATTN_FULL if query_mode != "causal" else _MAGI_ATTN_CAUSAL) + + for start, end, document_id, mode in _iter_mot_spans(packed_attention_metadata): + if current_document_id is not None and document_id != current_document_id: + flush_document(document_spans) + document_spans = [] + current_document_id = document_id + document_spans.append((start, end, mode)) + flush_document(document_spans) + + device = packed_attention_metadata.device + return MagiAttentionMask( + q_ranges=torch.tensor(q_ranges, device=device, dtype=torch.int32), + k_ranges=torch.tensor(k_ranges, device=device, dtype=torch.int32), + attn_type_map=torch.tensor(attn_types, device=device, dtype=torch.int32), + ) + + +def build_mot_sdpa_mask(packed_attention_metadata: torch.Tensor) -> torch.Tensor: + """Materialize a dense ``[1, 1, S, S]`` boolean SDPA mask. + + ``True`` means the query may attend that key. Do not store this as float32 + ``0 / -inf``; PyTorch SDPA already treats boolean ``False`` as masked out. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + query_idx = torch.arange(sequence_length, device=packed_attention_metadata.device) + same_document = document_ids[:, None] == document_ids[None, :] + causal = query_idx[:, None] >= query_idx[None, :] + same_full_or_noise_span = (full_span_ids[:, None] >= 0) & (full_span_ids[:, None] == full_span_ids[None, :]) + foreign_noise_key = (noise_span_ids[None, :] >= 0) & (noise_span_ids[:, None] != noise_span_ids[None, :]) + visible = same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key + return visible.to(dtype=torch.bool)[None, None].contiguous() + + +__all__ = [ + "build_mot_attention_metadata", + "build_mot_block_mask", + "build_mot_magi_mask", + "build_mot_sdpa_mask", + "pad_mot_attention_metadata", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py new file mode 100644 index 0000000000..e2b2975a5a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -0,0 +1,1568 @@ +"""BAGEL Qwen2 MoT backbone.""" + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +import torch.nn as nn +from torch.nn.attention import SDPBackend, sdpa_kernel +from torch.nn.attention.flex_attention import BlockMask +from torch.nn.functional import scaled_dot_product_attention +from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm, Qwen2RotaryEmbedding +from transformers.utils import ModelOutput + +from veomni.ops import VeomniOp +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + +from ....utils.conversation import ConversationItem +from ...module_modeling_base import PretrainedOmniModule +from ..sources import ( + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_FLOW_VELOCITY, + BAGEL_PHASE_KEY, + BAGEL_START_TOKEN, + bagel_phase, + get_tail_phase_item, +) +from .configuration import BagelQwen2MoTConfig +from .generation_state import MotGenerationState +from .masking import build_mot_block_mask, build_mot_sdpa_mask +from .processing import PackedConversation, preprocess_mot_inputs + + +_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"}) + + +class InferenceMixin: + """FSM ``generate`` / serial denoise / velocity-collection — HF ``GenerationMixin`` analog. + + Listed *before* :class:`BagelQwen2MoTCore` (and therefore before + :class:`~...module_modeling_base.PretrainedOmniModule`): the pretrained + base ships a no-op ``reset_local_inference_state``, and MRO resolves + left-to-right — put second, that no-op would shadow the real one below. + """ + + def forward_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_ids: torch.Tensor, + packed_query_indexes: torch.Tensor, + past_key_values: Optional["NaiveCache"] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + packed_attention_metadata: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + """Run one packed inference step through the MoT backbone and return hidden states and KV cache.""" + call_kwargs: Dict[str, Any] = { + "packed_query_sequence": packed_query_sequence, + "query_lens": query_lens, + "packed_query_position_ids": packed_query_position_ids, + "packed_query_indexes": packed_query_indexes, + "past_key_values": past_key_values, + "key_values_lens": key_values_lens, + "packed_key_value_indexes": packed_key_value_indexes, + "update_past_key_values": update_past_key_values, + "is_causal": is_causal, + "mode": mode, + "packed_attention_metadata": packed_attention_metadata, + } + + is_gen = _check_packed_inference_mode(mode) + if is_gen: + call_kwargs["packed_vae_token_indexes"] = packed_vae_token_indexes + call_kwargs["packed_text_indexes"] = packed_text_indexes + + output = self.model.forward_packed_inference(**call_kwargs) + + return { + "hidden_states": output.packed_query_sequence, + "past_key_values": output.past_key_values, + } + + def reset_local_inference_state(self) -> None: + self._generation_state.reset() + + def generate( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Prefill prompt caches or decode the next understanding token in the generation FSM.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT generate requires conversation_list.") + + generation_kwargs = generation_kwargs or {} + tail = conversation_list[-1] + is_start_token = bagel_phase(tail) == BAGEL_START_TOKEN + # Prefill the full conversation_list (including the assistant start token + # on understanding requests). The first text_ar pass returns the prefilled + # start-token hidden state; subsequent AR steps decode the tail output item. + if self._generation_state.main.cache is None or tail.type != "output": + hidden_states = self._prefill_prompt(conversation_list, generation_kwargs) + if is_start_token: + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + if hidden_states.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}." + ) + tail.type = "output" + tail.value = hidden_states[-1:].contiguous() + return {"conversation_list": conversation_list} + + if is_start_token: + tail.meta.pop(BAGEL_PHASE_KEY, None) + return {"conversation_list": conversation_list} + + hidden_states = self._decode_next_token(conversation_list) + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + if hidden_states.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}.") + tail.value = hidden_states[-1:].contiguous() + return {"conversation_list": conversation_list} + + def _prepare_denoise_query( + self, + conversation_list: list[ConversationItem] | None, + generation_kwargs: dict[str, Any] | None, + ) -> tuple[ConversationItem, torch.Tensor]: + """Validate CFG parameters and extract the tail ``flow_query`` tensor.""" + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT denoise_branch requires conversation_list.") + + self._generation_state.validate_cfg_request(generation_kwargs or {}) + self._generation_state.main.require_ready() + tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_QUERY) + if tail is None or not torch.is_tensor(tail.value): + raise ValueError(f"BAGEL Qwen2-MoT denoise branch requires an output in phase {BAGEL_FLOW_QUERY!r}.") + + query = tail.value + if query.dim() == 3 and query.shape[0] == 1: + query = query.squeeze(0) + if query.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT denoise branch expects rank-2 query tensor, got {tuple(query.shape)}.") + if int(query.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + "BAGEL Qwen2-MoT denoise branch hidden-size mismatch: " + f"got {query.shape[-1]}, expected {self.config.hidden_size}." + ) + if int(query.shape[0]) < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + return tail, query + + def denoise_branch( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Run serial CFG denoise forwards over active branches and write ``flow_hidden``.""" + del kwargs + tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs) + + hiddens: list[torch.Tensor] = [] + for inputs in self._generation_state.iter_serial_denoise_inputs( + query, + generation_kwargs or {}, + timestep=tail.meta.get("timestep"), + device=self.device, + dtype=self.dtype, + ): + outputs = self.forward_inference( + **inputs, + update_past_key_values=False, + is_causal=False, + mode="gen", + ) + hiddens.append(outputs["hidden_states"]) + if not hiddens: + raise RuntimeError("BAGEL Qwen2-MoT denoise branch produced no branch hidden states.") + + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = torch.cat(hiddens, dim=0).to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + def collect_velocity( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Strip marker rows from ``flow_velocity`` and apply CFG combination.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT collect_velocity requires conversation_list.") + + self._generation_state.validate_cfg_request(generation_kwargs or {}) + tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if tail is None or not torch.is_tensor(tail.value): + raise ValueError( + f"BAGEL Qwen2-MoT velocity collection requires an output in phase {BAGEL_FLOW_VELOCITY!r}." + ) + + velocity = tail.value + if velocity.dim() == 3 and velocity.shape[0] == 1: + velocity = velocity.squeeze(0) + if velocity.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT velocity collection expects rank-2 velocity, got {tuple(velocity.shape)}." + ) + + # Restore branch identity from the layout recorded by denoise_branch, + # strip marker rows, and replace the stacked tensor with guided velocity. + tail.value = self._generation_state.collect_velocity( + velocity, + generation_kwargs or {}, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation_list} + + def _prefill_prompt( + self, + conversation_list: list[ConversationItem], + generation_kwargs: dict[str, Any], + ) -> torch.Tensor: + """Pack and prefill the main prompt and optional CFG branch caches.""" + main_packed = preprocess_mot_inputs( + [conversation_list], + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if main_packed is None: + raise ValueError("BAGEL Qwen2-MoT generate requires at least one embedded text/image item.") + + state = self._generation_state + state.main.reset() + state.cfg_text.reset() + state.cfg_img.reset() + + # Text CFG reuses the latest valid prefix of the main prompt: its cache + # boundary is before a text span and after an image span. + cfg_text_slice, cfg_text_next_position_id = self._cfg_text_prefix_slice(main_packed) + use_cfg_img = state.cfg_img_requested(generation_kwargs) + + # Image CFG keeps logical text spans only. Inspect the grouped spans so + # marker tokens belonging to image triplets are not treated as text. + cfg_img_items = [] + if use_cfg_img: + cfg_img_items = [item for span in main_packed.spans if span.item.type == "text" for item in span.items] + + # Pack main and cfg_img as independent logical documents. The attention + # metadata prevents cross-document attention inside the Flex kernel. + packed = main_packed + if cfg_img_items: + packed = preprocess_mot_inputs( + [conversation_list, cfg_img_items], + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if packed is None: + raise RuntimeError("BAGEL Qwen2-MoT CFG prompt packing produced no tokens.") + + main_length = sum(main_packed.sample_splits[0]) + total_length = int(packed.packed_sequence.shape[0]) + main_slice = slice(0, main_length) + cfg_img_slice = slice(main_length, total_length) + + # This is a fresh-cache prefill: forward_inference allocates the cache, + # while packed attention metadata selects FlexAttention. + outputs = self.forward_packed_prefill(packed) + packed_cache = outputs["past_key_values"] + + # Detach main and cfg_img from the temporary packed cache allocation. + # cfg_text is a read-only prefix view of main and can share its storage. + main_cache = self._slice_prefill_cache(packed_cache, main_slice, clone=True) + cfg_text_cache = self._slice_prefill_cache(main_cache, cfg_text_slice, clone=False) + + state.main.install_cache( + cache=main_cache, + cache_len=main_length, + next_position_id=self._next_position_id(packed.packed_position_ids[main_slice]), + device=self.device, + ) + state.cfg_text.install_cache( + cache=cfg_text_cache, + cache_len=cfg_text_slice.stop, + next_position_id=cfg_text_next_position_id, + device=self.device, + ) + if use_cfg_img: + cfg_img_cache = self._slice_prefill_cache(packed_cache, cfg_img_slice, clone=True) + state.cfg_img.install_cache( + cache=cfg_img_cache, + cache_len=cfg_img_slice.stop - cfg_img_slice.start, + next_position_id=self._next_position_id(packed.packed_position_ids[cfg_img_slice]), + device=self.device, + ) + + # cfg_img is an auxiliary packed document; graph consumers should only + # observe hidden states from the main prompt. + hidden_states = outputs["hidden_states"] + if int(hidden_states.shape[0]) != total_length: + raise RuntimeError( + "BAGEL Qwen2-MoT FlexAttention prefill returned an unexpected sequence length: " + f"expected {total_length}, got {hidden_states.shape[0]}." + ) + return hidden_states[main_slice] + + def forward_packed_prefill(self, packed: PackedConversation) -> dict[str, Any]: + """Run one fresh-cache Flex prefill over all logical documents.""" + total_length = int(packed.packed_sequence.shape[0]) + document_lens = [sum(split_lens) for split_lens in packed.sample_splits] + query_lens = torch.tensor(document_lens, device=self.device, dtype=torch.int32) + + # A packed prefill may contain both understanding and generation expert + # tokens even though it executes as one model forward. + packed_gen_token_indexes = torch.nonzero( + packed.packed_token_type_ids == 1, + as_tuple=False, + ).flatten() + mode = "gen" if int(packed_gen_token_indexes.numel()) > 0 else "und" + call_kwargs: dict[str, Any] = { + "packed_query_sequence": packed.packed_sequence, + "query_lens": query_lens, + "packed_query_position_ids": packed.packed_position_ids, + "packed_query_indexes": torch.arange(total_length, device=self.device, dtype=torch.long), + "update_past_key_values": True, + "is_causal": False, + "mode": mode, + "packed_attention_metadata": packed.packed_attention_metadata, + } + if mode == "gen": + call_kwargs["packed_text_indexes"] = torch.nonzero( + packed.packed_token_type_ids == 0, + as_tuple=False, + ).flatten() + call_kwargs["packed_vae_token_indexes"] = packed_gen_token_indexes + return self.forward_inference(**call_kwargs) + + def _decode_next_token(self, conversation_list: list[ConversationItem]) -> torch.Tensor: + """Run one AR decode step against the cached ``main`` prompt context.""" + # AR has one query token and an existing contiguous cache, so the + # dedicated FlashAttention path is preferable to rebuilding a packed mask. + main_context = self._generation_state.main + main_context.require_ready() + tail = conversation_list[-1] + if tail.type != "output": + raise ValueError(f"BAGEL Qwen2-MoT decode expects tail output item, got {tail.type!r}.") + + packed_query_sequence = tail.value + if not torch.is_tensor(packed_query_sequence): + raise ValueError("BAGEL Qwen2-MoT decode expects tail output.value to be an embedding tensor.") + if packed_query_sequence.dim() == 3 and packed_query_sequence.shape[0] == 1: + packed_query_sequence = packed_query_sequence.squeeze(0) + if packed_query_sequence.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT expected tail output embedding rank 2, got {tuple(packed_query_sequence.shape)}." + ) + packed_query_sequence = packed_query_sequence[-1:].contiguous().to(device=self.device, dtype=self.dtype) + + query_lens, packed_query_indexes, packed_position_ids = main_context.packed_query_args( + 1, + device=self.device, + ) + outputs = self.forward_inference( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_ids=packed_position_ids, + packed_query_indexes=packed_query_indexes, + past_key_values=main_context.cache, + key_values_lens=main_context.key_values_lens, + packed_key_value_indexes=main_context.packed_key_value_indexes, + update_past_key_values=True, + is_causal=True, + mode="und", + ) + main_context.append_packed_query( + cache=outputs["past_key_values"], + query_lens=query_lens, + device=self.device, + ) + + return outputs["hidden_states"] + + def _cfg_text_prefix_slice(self, packed: PackedConversation) -> tuple[slice, torch.Tensor]: + """Compute the ``main`` cache prefix slice and next position ID for text CFG.""" + # Moving across an image keeps the image in text-CFG conditioning; + # encountering text moves the boundary to just before that text span. + prefix_end = 0 + next_position_id = torch.zeros(1, device=self.device, dtype=torch.long) + for span in packed.spans: + span_end = span.start + span.length + if span.item.type == "text": + prefix_end = span.start + next_position_id = packed.packed_position_ids[span.start].reshape(1) + elif span.item.type == "image": + prefix_end = span_end + next_position_id = self._next_position_id(packed.packed_position_ids[:span_end]) + return slice(0, prefix_end), next_position_id + + def _slice_prefill_cache(self, packed_cache: Any, rows: slice, *, clone: bool) -> Any: + """Extract a row slice from ``packed_cache`` into a standalone ``NaiveCache``.""" + # main/cfg_img outlive the temporary packed cache and own their storage. + # cfg_text is an immutable prefix of main, so it may use cheap views. + cache = self._new_empty_cache() + if rows.stop <= rows.start: + return cache + for layer_idx in cache.key_cache: + key = packed_cache.key_cache[layer_idx][rows] + value = packed_cache.value_cache[layer_idx][rows] + cache.key_cache[layer_idx] = key.clone() if clone else key + cache.value_cache[layer_idx] = value.clone() if clone else value + return cache + + def _next_position_id(self, position_ids: torch.Tensor) -> torch.Tensor: + """Return ``max(position_ids) + 1`` or ``0`` when ``position_ids`` is empty.""" + if int(position_ids.numel()) == 0: + return torch.zeros(1, device=self.device, dtype=torch.long) + return position_ids.max().reshape(1).to(device=self.device, dtype=torch.long) + 1 + + def _new_empty_cache(self) -> Any: + return NaiveCache(len(self.model.layers)) + + +class BagelQwen2MoTCore(PretrainedOmniModule): + """Shared weights, packed train/infer forward, and generation-state shell. + + Public eager and accelerated classes mix their inference mixins onto this + core. It is not registered on ``OMNI_*_REGISTRY``. + """ + + config_class = BagelQwen2MoTConfig + base_model_prefix = "bagel_qwen2_mot" + main_input_name = "inputs_embeds" + _no_split_modules = ["BagelQwen2MoTDecoderLayer"] + supports_gradient_checkpointing = True + _supports_sdpa = True + _supports_flex_attn = True + + def __init__(self, config: BagelQwen2MoTConfig): + super().__init__(config) + self.model = BagelQwen2MoTBackbone( + config, + attention_cls=type(self).attention_cls, + mlp_cls=type(self).mlp_cls, + rms_norm_cls=type(self).rms_norm_cls, + ) + self._generation_state = MotGenerationState() + self.post_init() + + def forward( # type: ignore[override] + self, + packed_sequence: torch.Tensor, + packed_position_ids: torch.Tensor, + packed_token_type_ids: torch.Tensor, + packed_attention_metadata: torch.Tensor, + ) -> Dict[str, Any]: + packed_und_token_indexes = torch.nonzero(packed_token_type_ids == 0, as_tuple=False).flatten() + packed_gen_token_indexes = torch.nonzero(packed_token_type_ids == 1, as_tuple=False).flatten() + output = self.model( + packed_sequence=packed_sequence, + packed_position_ids=packed_position_ids, + packed_attention_metadata=packed_attention_metadata, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + return {"hidden_states": output.packed_query_sequence} + + +class BagelQwen2MoT(InferenceMixin, BagelQwen2MoTCore): + """HF-native eager Qwen2-MoT with serial CFG denoise.""" + + +class NaiveCache: + """Official BAGEL packed KV cache.""" + + def __init__(self, num_layers: int): + self.key_cache = dict.fromkeys(range(num_layers)) + self.value_cache = dict.fromkeys(range(num_layers)) + + +@dataclass +class BaseNavitOutputWithPast(ModelOutput): + packed_query_sequence: torch.FloatTensor | None = None + past_key_values: Optional[NaiveCache] = None + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, +) -> tuple[torch.Tensor, torch.Tensor]: + """Eager packed RoPE. Fused dispatch lives on the accelerated attention class.""" + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (_rotate_half(q) * sin) + k_embed = (k * cos) + (_rotate_half(k) * sin) + return q_embed, k_embed + + +def _sdpa_context(device: torch.device): + """Pin mem-efficient SDPA on CUDA, matching official dense-train Bagel. + + Official 2D-mask training forces ``SDPBackend.EFFICIENT_ATTENTION`` so the + dispatcher cannot pick MATH (OOM) or a newer CUDNN kernel. CPU/NPU keep the + default dispatcher because Efficient is CUDA-only. + """ + if IS_CUDA_AVAILABLE and device.type == get_device_type(): + return sdpa_kernel(backends=[SDPBackend.EFFICIENT_ATTENTION]) + return nullcontext() + + +def _repeat_kv_heads(packed_states: torch.Tensor, num_query_heads: int) -> torch.Tensor: + """Expand packed KV from ``[tokens, kv_heads, dim]`` to query heads. + + Official dense-train Bagel repeats KV before Efficient SDPA. The CUDA + Efficient kernel does not accept ``enable_gqa`` with a 2D mask. + """ + num_kv_heads = int(packed_states.shape[1]) + if num_kv_heads == num_query_heads: + return packed_states + if num_query_heads % num_kv_heads != 0: + raise ValueError( + "BAGEL Qwen2-MoT GQA requires query heads divisible by KV heads: " + f"num_query_heads={num_query_heads}, num_kv_heads={num_kv_heads}." + ) + groups = num_query_heads // num_kv_heads + packed_states = packed_states[:, :, None, :].repeat(1, 1, groups, 1) + return packed_states.reshape(-1, num_query_heads, packed_states.shape[-1]) + + +def _sdpa_packed_attention( + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + *, + attention_mask: Optional[torch.Tensor] = None, + is_causal: bool = False, + cu_seq_lens_q: Optional[torch.Tensor] = None, + cu_seq_lens_k: Optional[torch.Tensor] = None, + scale: float, + enable_gqa: bool, +) -> torch.Tensor: + """Packed SDPA returning ``[tokens, heads, head_dim]``. + + The 2D-mask path pins Efficient SDPA on CUDA. The varlen / ``is_causal`` + path is left unpinned: official inference uses ``flash_attn_varlen``, not + Efficient. + """ + if packed_query_states.shape[0] == 0: + return packed_query_states.new_zeros(packed_query_states.shape) + + if attention_mask is not None: + if enable_gqa: + num_query_heads = int(packed_query_states.shape[1]) + packed_key_states = _repeat_kv_heads(packed_key_states, num_query_heads) + packed_value_states = _repeat_kv_heads(packed_value_states, num_query_heads) + query = packed_query_states.transpose(0, 1).unsqueeze(0) + key = packed_key_states.transpose(0, 1).unsqueeze(0) + value = packed_value_states.transpose(0, 1).unsqueeze(0) + with _sdpa_context(packed_query_states.device): + output = scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=False, + scale=scale, + ) + return output.squeeze(0).transpose(0, 1).contiguous() + + if cu_seq_lens_q is None or cu_seq_lens_k is None: + raise ValueError("BAGEL Qwen2-MoT SDPA without an attention mask requires cu_seq_lens_q/k.") + + outputs: list[torch.Tensor] = [] + for seq_idx in range(int(cu_seq_lens_q.shape[0]) - 1): + query_start = int(cu_seq_lens_q[seq_idx].item()) + query_end = int(cu_seq_lens_q[seq_idx + 1].item()) + key_start = int(cu_seq_lens_k[seq_idx].item()) + key_end = int(cu_seq_lens_k[seq_idx + 1].item()) + if query_end <= query_start: + continue + + query = packed_query_states[query_start:query_end].transpose(0, 1).unsqueeze(0) + key = packed_key_states[key_start:key_end].transpose(0, 1).unsqueeze(0) + value = packed_value_states[key_start:key_end].transpose(0, 1).unsqueeze(0) + + query_length = query_end - query_start + key_length = key_end - key_start + causal_mask = None + sdpa_is_causal = is_causal + if is_causal and query_length != key_length: + # FlashAttention varlen uses bottom-right causal alignment. PyTorch + # SDPA's ``is_causal=True`` is upper-left aligned when Q and K have + # different lengths, which makes a one-token cached decode see only + # the first key. The common Q=1 case needs no mask because the final + # logical query can see the entire cache; use an explicit lower-right + # mask only for multi-token cached decode. + sdpa_is_causal = False + if query_length != 1: + causal_mask = torch.ones( + (query_length, key_length), + dtype=torch.bool, + device=query.device, + ).tril(diagonal=key_length - query_length) + + output = scaled_dot_product_attention( + query, + key, + value, + attn_mask=causal_mask, + dropout_p=0.0, + is_causal=sdpa_is_causal, + scale=scale, + enable_gqa=enable_gqa, + ) + outputs.append(output.squeeze(0).transpose(0, 1)) + if not outputs: + return packed_query_states.new_zeros(packed_query_states.shape) + return torch.cat(outputs, dim=0) + + +def _fold_zero_anchors(target: torch.Tensor, *anchors: torch.Tensor) -> torch.Tensor: + """Add ``0 * sum(anchor)`` to ``target`` so empty-route parameters remain in the autograd graph.""" + # When a packed batch has no generation tokens, the MoT "gen" expert weights still + # run on an empty slice and produce zero-sized outputs. Folding ``sum() * 0.0`` of + # those outputs into ``target`` keeps the gen-expert parameters in the autograd graph + # (with zero gradient) so FSDP/DP gradient reduction sees the same parameter set on + # every rank regardless of which modalities a micro-batch contains. + anchor = target.new_zeros(()) + has_anchor = False + for value in anchors: + if torch.is_tensor(value): + anchor = anchor + value.sum() * 0.0 + has_anchor = True + if not has_anchor: + return target + return target + anchor + + +def _scatter_moe( + packed: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + und_value: torch.Tensor, + gen_value: torch.Tensor, + *, + extra_und_anchors: tuple[torch.Tensor, ...] = (), + extra_gen_anchors: tuple[torch.Tensor, ...] = (), +) -> torch.Tensor: + """Write MoT und/gen expert outputs into a packed buffer and keep empty-route FSDP anchors.""" + packed[packed_und_token_indexes] = und_value + packed[packed_gen_token_indexes] = gen_value + if int(packed_und_token_indexes.numel()) == 0: + packed = _fold_zero_anchors(packed, und_value, *extra_und_anchors) + if int(packed_gen_token_indexes.numel()) == 0: + packed = _fold_zero_anchors(packed, gen_value, *extra_gen_anchors) + return packed + + +def _check_packed_inference_mode( + mode: str, +) -> bool: + """Validate ``mode`` (``'und'`` or ``'gen'``) and return ``True`` for ``'gen'``.""" + if mode == "und": + return False + if mode == "gen": + return True + raise ValueError(f"Unsupported BAGEL Qwen2 MoT inference mode: {mode!r}") + + +class BagelQwen2RotaryEmbedding(Qwen2RotaryEmbedding): + """Qwen2 RoPE preserving fp32 inverse frequencies across dtype casts and meta materialization.""" + + def _apply(self, fn: Any, recurse: bool = True) -> nn.Module: + # Rebuild the true fp32 table after ``fn`` so ``.to(bfloat16)`` does not + # quantize rotary buffers and ``to_empty(device=...)`` from ``meta`` + # materializes valid fp32 inverse frequencies on the target device. + module = super()._apply(fn, recurse=recurse) + device = self.inv_freq.device + if device.type == "meta": + return module + inv_freq, _ = self.compute_default_rope_parameters(self.config, device) + self.inv_freq = inv_freq + self.original_inv_freq = inv_freq.clone() + return module + + +def _build_rms_norm( + rms_norm_cls: type[Qwen2RMSNorm], + hidden_size: int, + config: BagelQwen2MoTConfig, +) -> Qwen2RMSNorm: + """Instantiate ``rms_norm_cls``, forwarding ``config`` when the subclass accepts it.""" + try: + return rms_norm_cls(hidden_size, eps=config.rms_norm_eps, config=config) + except TypeError: + return rms_norm_cls(hidden_size, eps=config.rms_norm_eps) + + +class BagelQwen2MoTAttention(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.num_key_value_heads = config.num_key_value_heads + self.head_dim = self.hidden_size // self.num_heads + + self.query_size = self.num_heads * self.head_dim + self.key_value_size = self.num_key_value_heads * self.head_dim + self.qkv_split_sizes = (self.query_size, self.key_value_size, self.key_value_size) + + self._build_qkv_proj() + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.q_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.k_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.o_proj_moe_gen = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.q_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.k_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) + attn_impl = ( + config.ops_config.attn_implementation if "attn_implementation" in config.ops_implementation else "sdpa" + ) + self.veomni_attn: VeomniOp | None = None + if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: + try: + self.veomni_attn = VeomniOp("attention", "standard", "veomni_flex_attention") + except RuntimeError: + self.veomni_attn = None + attn_impl = "sdpa" + self.veomni_attn_impl = attn_impl + + def _build_qkv_proj(self) -> None: + self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.q_proj_moe_gen = nn.Linear(self.hidden_size, self.query_size, bias=True) + self.k_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.v_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + + def _project_qkv( + self, + hidden: torch.Tensor, + *, + is_gen: bool, + split: bool = False, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project ``hidden`` through the und or gen Q/K/V linear layers.""" + query_proj, key_proj, value_proj = ( + (self.q_proj_moe_gen, self.k_proj_moe_gen, self.v_proj_moe_gen) + if is_gen + else (self.q_proj, self.k_proj, self.v_proj) + ) + query_states = query_proj(hidden) + key_states = key_proj(hidden) + value_states = value_proj(hidden) + if split: + return query_states, key_states, value_states + return torch.cat((query_states, key_states, value_states), dim=-1) + + @staticmethod + def build_attention_mask( + packed_attention_metadata: torch.Tensor, + *, + attn_implementation: str | None = None, + ) -> torch.Tensor | BlockMask: + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS: + return build_mot_block_mask(packed_attention_metadata) + return build_mot_sdpa_mask(packed_attention_metadata) + + def apply_rotary_pos_emb( + self, + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + ) -> tuple[torch.Tensor, torch.Tensor]: + return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim) + + def project_qkv( + self, + packed_sequence: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + packed_qkv_states = _scatter_moe( + packed_sequence.new_zeros((packed_sequence.shape[0], sum(self.qkv_split_sizes))), + packed_und_token_indexes, + packed_gen_token_indexes, + self._project_qkv(packed_sequence[packed_und_token_indexes], is_gen=False), + self._project_qkv(packed_sequence[packed_gen_token_indexes], is_gen=True), + ) + packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split( + self.qkv_split_sizes, dim=-1 + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + + packed_query_states = _scatter_moe( + packed_query_states.new_zeros(packed_query_states.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.q_norm(packed_query_states[packed_und_token_indexes]), + self.q_norm_moe_gen(packed_query_states[packed_gen_token_indexes]), + ) + packed_key_states = _scatter_moe( + packed_key_states.new_zeros(packed_key_states.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.k_norm(packed_key_states[packed_und_token_indexes]), + self.k_norm_moe_gen(packed_key_states[packed_gen_token_indexes]), + ) + packed_query_states, packed_key_states = self.apply_rotary_pos_emb( + packed_query_states, + packed_key_states, + packed_position_cos, + packed_position_sin, + unsqueeze_dim=1, + ) + return packed_query_states, packed_key_states, packed_value_states + + def attend( + self, + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + attention_mask: torch.Tensor | BlockMask, + ) -> torch.Tensor: + if isinstance(attention_mask, BlockMask): + attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0).reshape(-1, self.num_heads * self.head_dim) + + packed_attn_output = _sdpa_packed_attention( + packed_query_states, + packed_key_states, + packed_value_states, + attention_mask=attention_mask, + scale=self.head_dim**-0.5, + enable_gqa=self.num_heads != self.num_key_value_heads, + ) + return packed_attn_output.reshape(-1, self.num_heads * self.head_dim) + + def project_o( + self, + packed_attn_output: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + return _scatter_moe( + packed_attn_output.new_zeros(packed_attn_output.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.o_proj(packed_attn_output[packed_und_token_indexes]), + self.o_proj_moe_gen(packed_attn_output[packed_gen_token_indexes]), + ) + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + packed_query_states, packed_key_states, packed_value_states = self.project_qkv( + packed_sequence, + packed_und_token_indexes, + packed_gen_token_indexes, + packed_position_cos, + packed_position_sin, + ) + packed_attn_output = self.attend( + packed_query_states, + packed_key_states, + packed_value_states, + attention_mask, + ) + return self.project_o( + packed_attn_output, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + + def project_qkv_inference( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project packed inference tokens through the matching MoT attention expert.""" + if not is_gen: + packed_query_states, packed_key_states, packed_value_states = self._project_qkv( + packed_query_sequence, is_gen=False, split=True + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_query_states = self.q_norm(packed_query_states) + packed_key_states = self.k_norm(packed_key_states) + return packed_query_states, packed_key_states, packed_value_states + + packed_qkv_states = packed_query_sequence.new_zeros( + (packed_query_sequence.shape[0], sum(self.qkv_split_sizes)) + ) + packed_qkv_states[packed_text_indexes] = self._project_qkv( + packed_query_sequence[packed_text_indexes], is_gen=False + ) + packed_qkv_states[packed_vae_token_indexes] = self._project_qkv( + packed_query_sequence[packed_vae_token_indexes], is_gen=True + ) + packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split( + self.qkv_split_sizes, dim=-1 + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + normed_query_states = packed_query_states.new_zeros(packed_query_states.shape) + normed_query_states[packed_text_indexes] = self.q_norm(packed_query_states[packed_text_indexes]) + normed_query_states[packed_vae_token_indexes] = self.q_norm_moe_gen( + packed_query_states[packed_vae_token_indexes] + ) + normed_key_states = packed_key_states.new_zeros(packed_key_states.shape) + normed_key_states[packed_text_indexes] = self.k_norm(packed_key_states[packed_text_indexes]) + normed_key_states[packed_vae_token_indexes] = self.k_norm_moe_gen(packed_key_states[packed_vae_token_indexes]) + return normed_query_states, normed_key_states, packed_value_states + + def merge_kv_cache( + self, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + *, + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache], + key_values_lens: Optional[torch.Tensor], + packed_key_value_indexes: Optional[torch.Tensor], + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Merge the current query span with this layer's packed KV cache.""" + if past_key_values is None or past_key_values.key_cache[self.layer_idx] is None: + return packed_key_states, packed_value_states + if key_values_lens is None or packed_key_value_indexes is None: + raise ValueError("key_values_lens and packed_key_value_indexes are required when cache is non-empty.") + + past_key_states = past_key_values.key_cache[self.layer_idx] + past_value_states = past_key_values.value_cache[self.layer_idx] + merged_key_states = past_key_states.new_zeros( + (total_key_value_tokens, self.num_key_value_heads, self.head_dim) + ) + merged_value_states = past_key_states.new_zeros( + (total_key_value_tokens, self.num_key_value_heads, self.head_dim) + ) + merged_key_states[packed_query_indexes] = packed_key_states + merged_key_states[packed_key_value_indexes] = past_key_states + merged_value_states[packed_query_indexes] = packed_value_states + merged_value_states[packed_key_value_indexes] = past_value_states + return merged_key_states, merged_value_states + + def attend_inference( + self, + packed_query_states: torch.Tensor, + merged_key_states: torch.Tensor, + merged_value_states: torch.Tensor, + *, + attention_mask: Optional[torch.Tensor | BlockMask], + is_causal: bool, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + ) -> torch.Tensor: + del max_length_q, max_length_k + + if isinstance(attention_mask, BlockMask): + attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + + return _sdpa_packed_attention( + packed_query_states, + merged_key_states, + merged_value_states, + attention_mask=attention_mask, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + scale=self.head_dim**-0.5, + enable_gqa=self.num_heads != self.num_key_value_heads, + ) + + def project_o_inference( + self, + packed_attn_output: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Project packed attention outputs through the matching MoT output expert.""" + packed_attn_output = packed_attn_output.reshape(-1, self.hidden_size) + if not is_gen: + return self.o_proj(packed_attn_output) + + output = packed_attn_output.new_zeros(packed_attn_output.shape) + output[packed_text_indexes] = self.o_proj(packed_attn_output[packed_text_indexes]) + output[packed_vae_token_indexes] = self.o_proj_moe_gen(packed_attn_output[packed_vae_token_indexes]) + return output + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor], + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + attention_mask: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + *, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, Optional[NaiveCache]]: + is_gen = _check_packed_inference_mode(mode) + compute_dtype = packed_query_sequence.dtype + packed_query_states, packed_key_states, packed_value_states = self.project_qkv_inference( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + + packed_cos, packed_sin = packed_query_position_embeddings + packed_query_states, packed_key_states = self.apply_rotary_pos_emb( + packed_query_states, + packed_key_states, + packed_cos, + packed_sin, + unsqueeze_dim=1, + ) + + packed_query_states = packed_query_states.to(compute_dtype) + packed_key_states = packed_key_states.to(compute_dtype) + packed_value_states = packed_value_states.to(compute_dtype) + + merged_key_states, merged_value_states = self.merge_kv_cache( + packed_key_states, + packed_value_states, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + total_key_value_tokens=total_key_value_tokens, + ) + packed_attn_output = self.attend_inference( + packed_query_states, + merged_key_states, + merged_value_states, + attention_mask=attention_mask, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + ) + packed_attn_output = self.project_o_inference( + packed_attn_output.to(compute_dtype), + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + + if update_past_key_values: + if past_key_values is None: + raise ValueError("past_key_values is required when update_past_key_values=True.") + past_key_values.key_cache[self.layer_idx] = merged_key_states + past_key_values.value_cache[self.layer_idx] = merged_value_states + + return packed_attn_output, past_key_values + + def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]: + if self.training: + return self.forward_packed_train(*args, **kwargs), None + return self.forward_packed_inference(*args, **kwargs) + + +class BagelQwen2MoTDecoderLayer(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention, + mlp_cls: type[Qwen2MLP] = Qwen2MLP, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.self_attn = attention_cls(config, layer_idx, rms_norm_cls=rms_norm_cls) + self.mlp = mlp_cls(config) + self.mlp_moe_gen = mlp_cls(config) + self.input_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.input_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.post_attention_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.post_attention_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + + def _apply_moe_norm( + self, + packed_sequence: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + und_norm: nn.Module, + gen_norm: nn.Module, + ) -> torch.Tensor: + return _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + und_norm(packed_sequence[packed_und_token_indexes]), + gen_norm(packed_sequence[packed_gen_token_indexes]), + ) + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + residual = packed_sequence + packed_sequence = self._apply_moe_norm( + packed_sequence, + packed_und_token_indexes, + packed_gen_token_indexes, + self.input_layernorm, + self.input_layernorm_moe_gen, + ) + packed_sequence, _ = self.self_attn( + packed_sequence=packed_sequence, + attention_mask=attention_mask, + packed_position_cos=packed_position_cos, + packed_position_sin=packed_position_sin, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + packed_sequence = residual + packed_sequence + + residual = packed_sequence + post_attn_und = self.post_attention_layernorm(packed_sequence[packed_und_token_indexes]) + post_attn_gen = self.post_attention_layernorm_moe_gen(packed_sequence[packed_gen_token_indexes]) + packed_sequence = _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.mlp(post_attn_und), + self.mlp_moe_gen(post_attn_gen), + extra_und_anchors=(post_attn_und,), + extra_gen_anchors=(post_attn_gen,), + ) + return residual + packed_sequence + + def _apply_inference_input_norm( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Normalize inference tokens with the matching MoT expert.""" + if not is_gen: + return self.input_layernorm(packed_query_sequence) + + normed_sequence = torch.zeros_like(packed_query_sequence) + normed_sequence[packed_text_indexes] = self.input_layernorm(packed_query_sequence[packed_text_indexes]) + normed_sequence[packed_vae_token_indexes] = self.input_layernorm_moe_gen( + packed_query_sequence[packed_vae_token_indexes] + ) + return normed_sequence + + def _apply_inference_mlp( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Apply the inference MLP with text and VAE tokens routed independently.""" + if not is_gen: + return self.mlp(self.post_attention_layernorm(packed_query_sequence)) + + packed_text_query_sequence = self.post_attention_layernorm(packed_query_sequence[packed_text_indexes]) + packed_vae_query_sequence = self.post_attention_layernorm_moe_gen( + packed_query_sequence[packed_vae_token_indexes] + ) + mlp_output = torch.zeros_like(packed_query_sequence) + mlp_output[packed_text_indexes] = self.mlp(packed_text_query_sequence) + mlp_output[packed_vae_token_indexes] = self.mlp_moe_gen(packed_vae_query_sequence) + return mlp_output + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor], + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + attention_mask: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + *, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, Optional[NaiveCache]]: + is_gen = _check_packed_inference_mode(mode) + residual = packed_query_sequence + packed_query_sequence = self._apply_inference_input_norm( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + packed_query_sequence, past_key_values = self.self_attn( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_embeddings=packed_query_position_embeddings, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + update_past_key_values=update_past_key_values, + is_causal=is_causal, + mode=mode, + attention_mask=attention_mask, + packed_vae_token_indexes=packed_vae_token_indexes, + packed_text_indexes=packed_text_indexes, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + total_key_value_tokens=total_key_value_tokens, + ) + packed_query_sequence = residual + packed_query_sequence + + residual = packed_query_sequence + packed_query_sequence = self._apply_inference_mlp( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + packed_query_sequence = residual + packed_query_sequence + + return packed_query_sequence, past_key_values + + def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]: + if self.training: + return self.forward_packed_train(*args, **kwargs), None + return self.forward_packed_inference(*args, **kwargs) + + +class BagelQwen2MoTBackbone(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + *, + attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention, + mlp_cls: type[Qwen2MLP] = Qwen2MLP, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.config = config + self.gradient_checkpointing = False + self.layers = nn.ModuleList( + [ + BagelQwen2MoTDecoderLayer( + config, + layer_idx, + attention_cls=attention_cls, + mlp_cls=mlp_cls, + rms_norm_cls=rms_norm_cls, + ) + for layer_idx in range(config.num_hidden_layers) + ] + ) + self.norm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.norm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.rotary_emb = BagelQwen2RotaryEmbedding(config=config) + self.use_moe = "Mo" in config.layer_module + self.attention_cls = attention_cls + + def _bound_attn_implementation(self) -> str | None: + """Return the attention implementation bound onto the decoder layers at initialization.""" + if len(self.layers) > 0: + bound = getattr(self.layers[0].self_attn, "veomni_attn_impl", None) + if bound is not None: + return bound + return self.config._attn_implementation + + def _build_inference_attention_mask( + self, + packed_query_sequence: torch.Tensor, + packed_attention_metadata: Optional[torch.Tensor], + *, + cache_has_values: bool, + ) -> Any: + """Validate packed-prefill invariants and materialize its attention mask.""" + if packed_attention_metadata is None: + return None + if cache_has_values: + raise ValueError("BAGEL packed prefill requires an empty KV cache.") + + expected_metadata_shape = (3, int(packed_query_sequence.shape[0])) + if tuple(packed_attention_metadata.shape) != expected_metadata_shape: + raise ValueError( + "BAGEL packed prefill metadata must match the packed query sequence: " + f"expected {expected_metadata_shape}, got {tuple(packed_attention_metadata.shape)}." + ) + return self.attention_cls.build_attention_mask( + packed_attention_metadata.to(device=packed_query_sequence.device), + attn_implementation=self._bound_attn_implementation(), + ) + + def _apply_inference_final_norm( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Apply the final backbone norm with the matching MoT expert.""" + if not is_gen: + return self.norm(packed_query_sequence) + + normed_sequence = torch.zeros_like(packed_query_sequence) + normed_sequence[packed_text_indexes] = self.norm(packed_query_sequence[packed_text_indexes]) + normed_sequence[packed_vae_token_indexes] = self.norm_moe_gen(packed_query_sequence[packed_vae_token_indexes]) + return normed_sequence + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + packed_position_ids: torch.Tensor, + packed_attention_metadata: torch.Tensor, + packed_und_token_indexes: Optional[torch.Tensor] = None, + packed_gen_token_indexes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run packed training forward across all MoT decoder layers and final expert norms.""" + attention_mask = self.attention_cls.build_attention_mask( + packed_attention_metadata, + attn_implementation=self._bound_attn_implementation(), + ) + cos, sin = self.rotary_emb(packed_sequence, packed_position_ids.unsqueeze(0)) + packed_position_cos = cos.squeeze(0) + packed_position_sin = sin.squeeze(0) + + if self.use_moe: + if packed_und_token_indexes is None: + raise ValueError("packed_und_token_indexes is required for BAGEL MoT training.") + if packed_gen_token_indexes is None: + packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0]) + else: + packed_und_token_indexes = torch.arange(packed_sequence.shape[0], device=packed_sequence.device) + packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0]) + + for decoder_layer in self.layers: + if self.gradient_checkpointing and self.training: + packed_sequence, _ = self._gradient_checkpointing_func( + decoder_layer.__call__, + packed_sequence, + attention_mask, + packed_position_cos, + packed_position_sin, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + else: + packed_sequence, _ = decoder_layer( + packed_sequence=packed_sequence, + attention_mask=attention_mask, + packed_position_cos=packed_position_cos, + packed_position_sin=packed_position_sin, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + + if self.use_moe: + return _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.norm(packed_sequence[packed_und_token_indexes]), + self.norm_moe_gen(packed_sequence[packed_gen_token_indexes]), + ) + return self.norm(packed_sequence) + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_ids: torch.Tensor, + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + packed_attention_metadata: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + ) -> BaseNavitOutputWithPast: + """Execute all MoT decoder layers for packed inference prefill, decode, or denoise.""" + is_gen = _check_packed_inference_mode(mode) + query_device = packed_query_sequence.device + packed_query_indexes = packed_query_indexes.to(device=query_device) + packed_query_position_ids = packed_query_position_ids.to(device=query_device) + if packed_key_value_indexes is not None: + packed_key_value_indexes = packed_key_value_indexes.to(device=query_device) + if packed_vae_token_indexes is not None: + packed_vae_token_indexes = packed_vae_token_indexes.to(device=query_device) + if packed_text_indexes is not None: + packed_text_indexes = packed_text_indexes.to(device=query_device) + if past_key_values is None: + past_key_values = NaiveCache(len(self.layers)) + + cos, sin = self.rotary_emb(packed_query_sequence, packed_query_position_ids.unsqueeze(0)) + packed_query_position_embeddings = (cos.squeeze(0), sin.squeeze(0)) + + cache_has_values = past_key_values.key_cache[0] is not None + if cache_has_values and key_values_lens is None: + raise ValueError("key_values_lens is required when cache is non-empty.") + attention_mask = self._build_inference_attention_mask( + packed_query_sequence, + packed_attention_metadata, + cache_has_values=cache_has_values, + ) + + effective_key_values_lens = key_values_lens + query_lens if cache_has_values else query_lens + cu_seq_lens_q = torch.nn.functional.pad(torch.cumsum(query_lens, dim=0), (1, 0)).to(torch.int32) + cu_seq_lens_k = torch.nn.functional.pad(torch.cumsum(effective_key_values_lens, dim=0), (1, 0)).to(torch.int32) + max_length_q = int(query_lens.max().item()) + max_length_k = int(effective_key_values_lens.max().item()) + total_key_value_tokens = int(effective_key_values_lens.sum().item()) + + for decoder_layer in self.layers: + packed_query_sequence, past_key_values = decoder_layer( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_embeddings=packed_query_position_embeddings, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + update_past_key_values=update_past_key_values, + is_causal=is_causal, + mode=mode, + attention_mask=attention_mask, + packed_vae_token_indexes=packed_vae_token_indexes, + packed_text_indexes=packed_text_indexes, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + total_key_value_tokens=total_key_value_tokens, + ) + + packed_query_sequence = self._apply_inference_final_norm( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + return BaseNavitOutputWithPast( + packed_query_sequence=packed_query_sequence, + past_key_values=past_key_values, + ) + + def forward(self, *args: Any, **kwargs: Any) -> BaseNavitOutputWithPast: + if self.training: + return BaseNavitOutputWithPast(packed_query_sequence=self.forward_packed_train(*args, **kwargs)) + return self.forward_packed_inference(*args, **kwargs) + + +BagelQwen2MoTCore.attention_cls = BagelQwen2MoTAttention +BagelQwen2MoTCore.mlp_cls = Qwen2MLP +BagelQwen2MoTCore.rms_norm_cls = Qwen2RMSNorm + + +__all__ = [ + "BaseNavitOutputWithPast", + "BagelQwen2MoT", + "BagelQwen2MoTCore", + "InferenceMixin", + "NaiveCache", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py new file mode 100644 index 0000000000..5795f8656b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py @@ -0,0 +1,336 @@ +"""Conversation packing helpers for BAGEL Qwen2-MoT.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context +from .masking import build_mot_attention_metadata + + +_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"}) + + +@dataclass(frozen=True) +class PackedSpan: + """Describe one logical span in the packed sequence. + + Most spans contain one carrier item. A VAE image span can group its leading + marker, image, and trailing marker; ``primary_index`` identifies the image, + while ``lengths`` preserves each item's subrange for routing and writeback. + """ + + start: int + items: tuple[ConversationItem, ...] + lengths: tuple[int, ...] + primary_index: int = 0 + + @property + def item(self) -> ConversationItem: + return self.items[self.primary_index] + + @property + def length(self) -> int: + return sum(self.lengths) + + @property + def primary_start(self) -> int: + return sum(self.lengths[: self.primary_index]) + + @property + def primary_length(self) -> int: + return self.lengths[self.primary_index] + + @property + def is_image_triplet(self) -> bool: + return ( + len(self.items) == 3 + and self.primary_index == 1 + and self.items[0].type == "text" + and self.items[1].type == "image" + and self.items[2].type == "text" + ) + + +@dataclass +class PackedConversation: + """Result of packing an embedded conversation batch for MoT. + + Packed tensors concatenate all samples. ``sample_splits`` preserves their + document/span boundaries for attention metadata. ``spans`` uses global + packed offsets to map model outputs back to the original carrier items. + Token type 0 selects the understanding expert and type 1 the generation + expert. + """ + + packed_sequence: torch.Tensor + sample_splits: list[list[int]] + packed_attention_metadata: torch.Tensor + packed_position_ids: torch.Tensor + packed_token_type_ids: torch.Tensor + spans: list[PackedSpan] + + +def preprocess_mot_inputs( + conversation_list: list[list[ConversationItem]] | None, + *, + device: torch.device, + dtype: torch.dtype, + hidden_size: int, +) -> PackedConversation | None: + """Preprocess already-embedded carrier items into MoT packed inputs. + + Text and SigLIP spans route through the understanding expert. VAE latent + spans and output spans route through the generation expert. Attention mode + and expert routing are deliberately computed separately: a clean VAE + conditioning image still uses the generation expert. Packed layout stays + local to MoT. + """ + + if not conversation_list: + raise ValueError("BAGEL Qwen2-MoT preprocessing requires a non-empty conversation_list.") + + sequence_parts: list[torch.Tensor] = [] + sample_splits_by_sample: list[list[int]] = [] + sample_attn_modes_by_sample: list[list[str]] = [] + position_parts: list[torch.Tensor] = [] + token_type_parts: list[torch.Tensor] = [] + spans: list[PackedSpan] = [] + sequence_cursor = 0 + + for sample in conversation_list: + sample_start = sequence_cursor + sample_splits: list[int] = [] + sample_attn_modes: list[str] = [] + sample_position_cursor = 0 + + # Dummy carrier items are handled separately as zero-gradient anchors; + # only real user/assistant embeddings participate in packed attention. + items = [ + item + for item in iter_desired_items( + [sample], + types=["text", "image", "output"], + roles=["user", "assistant"], + ) + if not item.is_dummy + ] + item_index = 0 + while item_index < len(items): + if _is_vision_marker_triplet_at(items, item_index): + # Treat , image tokens, and as one + # logical attention span while preserving three writeback + # ranges and routing only the image tokens to the gen expert. + span_items = (items[item_index], items[item_index + 1], items[item_index + 2]) + span_values = [ + _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size) + for item in span_items + ] + span_lengths = tuple(int(value.shape[0]) for value in span_values) + item = span_items[1] + value = torch.cat(span_values, dim=0) + length = int(value.shape[0]) + primary_index = 1 + item_index += 3 + else: + item = items[item_index] + value = _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size) + length = int(value.shape[0]) + span_items = (item,) + span_lengths = (length,) + primary_index = 0 + item_index += 1 + + if length == 0: + continue + + # sequence_cursor is global across the packed batch; positions + # restart independently for each logical document. + indexes = torch.arange(sequence_cursor, sequence_cursor + length, device=device, dtype=torch.long) + position_ids, sample_position_cursor = _mot_position_ids_for_span( + item, + start=sample_position_cursor, + length=length, + device=device, + ) + mode = _mot_attn_mode_for_item(item) + + sequence_parts.append(value) + position_parts.append(position_ids) + sample_splits.append(length) + sample_attn_modes.append(mode) + span = PackedSpan( + start=sequence_cursor, + items=span_items, + lengths=span_lengths, + primary_index=primary_index, + ) + spans.append(span) + sequence_cursor += length + + gen_token_indexes = _mot_gen_token_indexes_for_span(span, indexes) + token_type_ids = torch.zeros(length, device=device, dtype=torch.long) + if gen_token_indexes.numel() > 0: + token_type_ids[(gen_token_indexes - indexes[0]).to(device=device, dtype=torch.long)] = 1 + token_type_parts.append(token_type_ids) + + sample_len = sequence_cursor - sample_start + if sample_len > 0: + sample_splits_by_sample.append(list(sample_splits)) + sample_attn_modes_by_sample.append(list(sample_attn_modes)) + + if not sequence_parts: + return None + + packed_sequence = torch.cat(sequence_parts, dim=0).to(device=device, dtype=dtype) + return PackedConversation( + packed_sequence=packed_sequence, + sample_splits=sample_splits_by_sample, + packed_attention_metadata=build_mot_attention_metadata( + sample_splits_by_sample, + sample_attn_modes_by_sample, + device=device, + ), + packed_position_ids=torch.cat(position_parts, dim=0).to(device=device, dtype=torch.long), + packed_token_type_ids=torch.cat(token_type_parts, dim=0).to(device=device, dtype=torch.long), + spans=spans, + ) + + +def _mot_value_for_item( + item: ConversationItem, + *, + device: torch.device, + dtype: torch.dtype, + hidden_size: int, +) -> torch.Tensor: + """Validate and cast an item's rank-2 embedding tensor to ``(device, dtype)``.""" + value = item.value + if not torch.is_tensor(value): + raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got {type(value).__name__}.") + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got shape {tuple(item.value.shape)}.") + if int(value.shape[-1]) != int(hidden_size): + raise ValueError(f"BAGEL Qwen2-MoT item hidden-size mismatch: got {value.shape[-1]}, expected {hidden_size}.") + return value.to(device=device, dtype=dtype) + + +def _mot_position_ids_for_span( + item: ConversationItem, + *, + start: int, + length: int, + device: torch.device, +) -> tuple[torch.Tensor, int]: + """Assign BAGEL positions and return the next document-local position.""" + meta_position_ids = item.meta.get("position_ids") + if torch.is_tensor(meta_position_ids): + position_ids = meta_position_ids.detach().to(device=device, dtype=torch.long).reshape(-1) + if int(position_ids.numel()) != length: + raise ValueError("BAGEL Qwen2-MoT position_ids length must match the item span length.") + next_start = max(start + 1, int(position_ids.max().item()) + 1) if length else start + return position_ids, next_start + + # All latent/image tokens in one span share a position; text remains AR + # sequential. Thus a whole image advances the logical position only once. + if item.type in {"image", "output"}: + return torch.full((length,), start, device=device, dtype=torch.long), start + 1 + return torch.arange(start, start + length, device=device, dtype=torch.long), start + length + + +def _mot_attn_mode_for_item(item: ConversationItem) -> str: + """Classify visibility independently from MoT expert routing.""" + if item.type == "output": + return "noise" + if item.type == "image": + if bagel_context(item) != BAGEL_VAE_CONTEXT: + # SigLIP context (and any non-VAE image) is clean/full-attention + # regardless of ``_img_tag``; the tag only disambiguates VAE roles. + return "full" + tag = item.meta.get(_IMG_TAG_KEY) + if tag is None: + # Untagged VAE images come from inference prompt routing. Training + # targets must carry ``_img_tag="gen"``. + return "full" + if tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.") + if tag == "gen": + return "noise" + if tag == "edit": + # Edit VAE context is a clean conditioning image, not a noised target. + return "full" + raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.") + return "causal" + + +def _mot_gen_token_indexes_for_span(span: PackedSpan, indexes: torch.Tensor) -> torch.Tensor: + """Return packed rows routed through the generation expert.""" + item = span.item + if item.type == "output": + return indexes + if item.type != "image" or bagel_context(item) != BAGEL_VAE_CONTEXT: + return indexes.new_empty(0) + + tag = item.meta.get(_IMG_TAG_KEY) + if tag is not None and tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.") + if tag == "und": + raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.") + + # Expert routing follows token modality, independently of whether the VAE + # latent is clean conditioning (untagged/edit) or a noised generation + # target. Marker tokens remain on the understanding path. + if span.is_image_triplet: + primary_start = span.primary_start + primary_end = primary_start + span.primary_length + return indexes[primary_start:primary_end] + return indexes + + +def _is_vision_marker_triplet_at(items: list[ConversationItem], index: int) -> bool: + """Return whether ``items[index : index + 3]`` forms a ``(start, image, end)`` triplet.""" + # Context-tag equality prevents unrelated one-token text items around an + # image from being mistaken for BAGEL's encoder-specific marker pair. + if index + 2 >= len(items): + return False + start, image, end = items[index], items[index + 1], items[index + 2] + context = bagel_context(image) + if image.type != "image" or context not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + return False + return _is_bagel_vision_marker(start, context=context) and _is_bagel_vision_marker(end, context=context) + + +def _is_bagel_vision_marker(item: ConversationItem, *, context: str) -> bool: + """Return whether ``item`` is a single-token vision marker for ``context``.""" + return item.type == "text" and bagel_context(item) == context and _text_item_length(item) == 1 + + +def _text_item_length(item: ConversationItem) -> int | None: + """Return the token length of a text ``ConversationItem``, or ``None`` if unavailable.""" + value = item.value + if torch.is_tensor(value): + if value.dim() == 0: + return 1 + if value.dim() == 1: + return int(value.shape[0]) + if value.dim() == 2: + return int(value.shape[0]) + if value.dim() == 3 and int(value.shape[0]) == 1: + return int(value.shape[1]) + return None + input_ids = item.meta.get("input_ids") + if torch.is_tensor(input_ids): + return int(input_ids.reshape(-1).shape[0]) + return None + + +__all__ = [ + "PackedConversation", + "PackedSpan", + "preprocess_mot_inputs", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py new file mode 100644 index 0000000000..23fabe4009 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py @@ -0,0 +1,31 @@ +"""BAGEL SigLIP NaViT vision encoder module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_config(): + from .configuration import BagelSiglipNavitConfig + + return BagelSiglipNavitConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_model(): + from .modeling import BagelSiglipNavit + + return BagelSiglipNavit + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_accelerated_model(): + from .accelerated import BagelSiglipNavitAccelerated + + return BagelSiglipNavitAccelerated + + +@OMNI_PROCESSOR_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_processor(): + from .processing import BagelSiglipNavitProcessor + + return BagelSiglipNavitProcessor diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py new file mode 100644 index 0000000000..f4ac46ddca --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/siglip_navit OmniModule (conversation graph).""" + +from .accelerated import ( + BagelSiglipNavitAccelerated, +) + + +__all__ = [ + "BagelSiglipNavitAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py new file mode 100644 index 0000000000..607c4481b0 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py @@ -0,0 +1,267 @@ +"""SeedOmni training-graph hooks for BAGEL's SigLIP NaViT vision encoder. + +``generate()`` lives natively on :class:`~.modeling.BagelSiglipNavit` — this +file only carries the SP-aware training pre/forward/post hooks. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import gather_outputs +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ...sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from ..configuration import BagelSiglipNavitConfig +from ..modeling import ( + BagelSiglipNavit, + scatter_siglip_image_embeds, + select_siglip_image_items, + siglip_inputs_from_preprocessed_items, +) +from ..processing import _OMNI_TOKEN_LEN + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelSiglipNavit` modeling APIs.""" + + config: BagelSiglipNavitConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._image_items: list[ConversationItem] = [] + self._sp_image_count: int | None = None + self._sp_token_count: int | None = None + + @pre_forward("forward") + def forward_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Select preprocessed SigLIP image items and slice across Ulysses SP ranks if enabled.""" + del kwargs + self._conversation_carrier = conversation_list + + self._image_items = select_siglip_image_items(conversation_list) + out = siglip_inputs_from_preprocessed_items(self._image_items, device=self.device, dtype=self.dtype) + self.metric_meter_set_seqlens( + "forward", + self._metric_sample_token_lens(conversation_list), + image_seqlens=[int(item.meta[_OMNI_TOKEN_LEN]) for item in self._image_items], + ) + + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_image_count = None + self._sp_token_count = None + return out + + if ps.cp_size != 1: + raise ValueError(f"BAGEL SigLIP NaViT training supports Ulysses groups only; got cp_size={ps.cp_size}.") + + token_lens = out["token_lens"] + image_count = int(token_lens.numel()) + if image_count == 0: + raise ValueError("BAGEL SigLIP NaViT SP requires at least one real or dummy image.") + self._sp_image_count = image_count + self._sp_token_count = int(token_lens.sum().item()) + + pad_images = (-image_count) % ps.sp_size + if pad_images: + token_lens = torch.cat((token_lens, token_lens.new_ones(pad_images)), dim=0) + out["patchified_pixel_values"] = torch.cat( + ( + out["patchified_pixel_values"], + out["patchified_pixel_values"].new_zeros((pad_images, *out["patchified_pixel_values"].shape[1:])), + ), + dim=0, + ) + out["patchified_position_ids"] = torch.cat( + ( + out["patchified_position_ids"], + out["patchified_position_ids"].new_zeros((pad_images, *out["patchified_position_ids"].shape[1:])), + ), + dim=0, + ) + + images_per_rank = int(token_lens.numel()) // ps.sp_size + image_start = ps.sp_rank * images_per_rank + image_end = image_start + images_per_rank + full_cu_seqlens = torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0)) + token_start = int(full_cu_seqlens[image_start].item()) + token_end = int(full_cu_seqlens[image_end].item()) + local_token_lens = token_lens[image_start:image_end].contiguous() + return { + "patchified_pixel_values": out["patchified_pixel_values"][token_start:token_end].contiguous(), + "patchified_position_ids": out["patchified_position_ids"][token_start:token_end].contiguous(), + "cu_seqlens": torch.nn.functional.pad(torch.cumsum(local_token_lens, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(local_token_lens.max().item()), + "token_lens": local_token_lens, + } + + def forward(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """Isolate SP-sharded attention dropout RNG from downstream modules when enabled.""" + if ( + get_parallel_state().sp_size > 1 + and self.training + and float(getattr(self.config, "attention_dropout", 0.0)) > 0.0 + ): + fork_kwargs: dict[str, Any] = ( + {"devices": []} + if self.device.type == "cpu" + else {"devices": [self.device], "device_type": self.device.type} + ) + with torch.random.fork_rng(**fork_kwargs): + return super().forward(*args, **kwargs) + return super().forward(*args, **kwargs) + + @post_forward("forward") + def forward_post( + self, + image_embeds: torch.Tensor, + token_lens: torch.Tensor | None = None, + ) -> dict[str, Any]: + """Gather SP image feature shards and scatter ``image_embeds`` onto conversation items.""" + if token_lens is None: + raise ValueError("BagelSiglipNavit.forward_post requires token_lens for non-dummy outputs.") + + ps = get_parallel_state() + if ps.sp_size > 1: + if self._sp_image_count is None or self._sp_token_count is None: + raise RuntimeError("BAGEL SigLIP NaViT SP image shape was not initialized.") + image_embeds = gather_outputs(image_embeds, gather_dim=0, group=ps.sp_group) + token_lens = gather_outputs(token_lens, gather_dim=0, group=ps.sp_group) + image_embeds = image_embeds.narrow(0, 0, self._sp_token_count) + token_lens = token_lens.narrow(0, 0, self._sp_image_count) + + conversation = self._conversation_carrier + image_items = self._image_items + self._conversation_carrier = None + self._image_items = [] + self._sp_image_count = None + self._sp_token_count = None + + scatter_siglip_image_embeds( + image_items, + image_embeds, + token_lens, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation} + + def _metric_sample_token_lens( + self, + conversation_list: list[list[ConversationItem]], + ) -> list[int]: + """Sum SigLIP patch token lengths per sample for FLOPs metering.""" + sample_lens: list[int] = [] + for sample in conversation_list: + sample_len = 0 + # SigLIP encodes per image, including dummy carriers, but + # multi-source metering needs one aggregated length per sample. + for item in iter_desired_items( + [sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]} + ): + sample_len += int(item.meta[_OMNI_TOKEN_LEN]) + sample_lens.append(sample_len) + return sample_lens + + +class MeterMixin(MetricMeterMixin): + """Per-module training trace for BAGEL SigLIP NaViT.""" + + config: BagelSiglipNavitConfig + + def metric_meter_set_seqlens( + self, + method: str, + seqlens: list[int], + *, + image_seqlens: list[int] | None = None, + ) -> None: + """Stash per-sample ``seqlens`` plus per-image ``image_seqlens`` for attention FLOPs.""" + super().metric_meter_set_seqlens(method, seqlens) + if not hasattr(self, "_metric_full_image_seqlens"): + self._metric_full_image_seqlens: dict[str, list[int]] = {} + resolved_image_seqlens = seqlens if image_seqlens is None else image_seqlens + self._metric_full_image_seqlens[method] = [int(s) for s in resolved_image_seqlens] + + def _metric_meter_image_seqlen_buffer(self) -> list[int]: + if not hasattr(self, "_metric_meter_image_seqlens"): + self._metric_meter_image_seqlens: list[int] = [] + return self._metric_meter_image_seqlens + + def metric_meter_add(self, method: str) -> None: + """Move stashed per-sample and per-image token lengths into the step buffers.""" + super().metric_meter_add(method) + image_stash = getattr(self, "_metric_full_image_seqlens", None) + if image_stash: + self._metric_meter_image_seqlen_buffer().extend(image_stash.get(method, [])) + image_stash.clear() + + def metric_meter_collect(self) -> tuple[float, list[int]]: + """Return ``(theoretical_flops, sample_seqlens)`` using per-image lengths for attention FLOPs.""" + seqlens = self._metric_meter_seqlen_buffer() + self._metric_meter_seqlens = [] + image_seqlens = self._metric_meter_image_seqlen_buffer() + self._metric_meter_image_seqlens = [] + return self.estimate_flops(seqlens, image_seqlens=image_seqlens), seqlens + + def estimate_flops( + self, + seqlens: list[int], + *, + image_seqlens: list[int] | None = None, + ) -> float: + """Estimate TFLOPs for the SigLIP-NaViT encoder and MLP connector over ``seqlens``.""" + cfg = self.config + dim = cfg.hidden_size + heads = cfg.num_attention_heads + head_dim = dim // heads + patch_embed_n = dim * cfg.num_channels * cfg.patch_size * cfg.patch_size + attn_linear_n = dim * 4 * dim + mlp_n = dim * cfg.intermediate_size * 2 + connector_n = dim * cfg.output_size + cfg.output_size * cfg.output_size + dense_n = patch_embed_n + (attn_linear_n + mlp_n) * cfg.num_hidden_layers + connector_n + if image_seqlens is None: + buffered = getattr(self, "_metric_meter_image_seqlens", None) + stashed = getattr(self, "_metric_full_image_seqlens", {}).get("forward") + if buffered and sum(buffered) == sum(seqlens): + attn_seqlens = buffered + elif stashed and sum(stashed) == sum(seqlens): + attn_seqlens = stashed + else: + attn_seqlens = seqlens + else: + attn_seqlens = image_seqlens + tokens = sum(seqlens) + seqlen_sq = sum(length * length for length in attn_seqlens) + dense_flops = 6 * dense_n * tokens + attn_flops = 12 * seqlen_sq * head_dim * heads * cfg.num_hidden_layers + return (dense_flops + attn_flops) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks for BAGEL visual-understanding image features. + + ``generate()`` already lives on the native :class:`~.modeling.BagelSiglipNavit` + (via its own :class:`~.modeling.InferenceMixin`), so no ``InferenceMixin`` is + needed here. + """ + + +class BagelSiglipNavitAccelerated(VeOmniMixin, BagelSiglipNavit): + pass + + +__all__ = ["BagelSiglipNavitAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py new file mode 100644 index 0000000000..f59232e2f6 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py @@ -0,0 +1,56 @@ +"""Configuration for :class:`BagelSiglipNavit`.""" + +from typing import List, Optional + +from ...module_configuration_base import OmniModuleConfig + + +class BagelSiglipNavitConfig(OmniModuleConfig): + """BAGEL SigLIP NaViT vision encoder config.""" + + model_type = "bagel_siglip_navit" + + def __init__( + self, + hidden_size: int = 1152, + output_size: int = 3584, + image_size: int = 980, + min_image_size: int = 378, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: Optional[List[float]] = None, + image_std: Optional[List[float]] = None, + intermediate_size: int = 4304, + num_attention_heads: int = 16, + num_hidden_layers: int = 26, + num_channels: int = 3, + patch_size: int = 14, + hidden_act: str = "gelu_pytorch_tanh", + connector_act: str = "gelu_pytorch_tanh", + vit_max_num_patch_per_side: int = 70, + layer_norm_eps: float = 1e-6, + attention_dropout: float = 0.0, + rope: bool = False, + **kwargs, + ) -> None: + self.hidden_size = hidden_size + self.output_size = output_size + self.image_size = image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.intermediate_size = intermediate_size + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.num_channels = num_channels + self.patch_size = patch_size + self.hidden_act = hidden_act + self.connector_act = connector_act + self.vit_max_num_patch_per_side = vit_max_num_patch_per_side + self.layer_norm_eps = layer_norm_eps + self.attention_dropout = attention_dropout + self.rope = rope + super().__init__(**kwargs) + + +__all__ = ["BagelSiglipNavitConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py new file mode 100644 index 0000000000..4147254847 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -0,0 +1,514 @@ +"""BAGEL SigLIP NaViT vision encoder. + +The training node is ``bagel_siglip_navit.forward``. It receives local +patchified image tensors from ``accelerated.py``, runs SigLIP/NaViT attention, +projects features to the Bagel MoT hidden size, adds visual position embeddings, +and returns ``{"image_embeds": ..., "token_lens": ...}``. The post-hook writes +those feature spans back to the image ``ConversationItem.value`` fields. +""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn as nn +from transformers.activations import ACT2FN + +from veomni.ops import VeomniOp + +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..position_embedding import PositionEmbedding +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from .configuration import BagelSiglipNavitConfig +from .processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN, BagelSiglipNavitPreprocessor, BagelSiglipNavitProcessor + + +def select_siglip_image_items( + conversation_list: list[list[ConversationItem]] | None, +) -> list[ConversationItem]: + """Select BAGEL SigLIP image carriers (training + inference).""" + if conversation_list is None: + raise ValueError("BagelSiglipNavit requires conversation_list to select image items.") + + return list( + iter_desired_items( + conversation_list, + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}, + ) + ) + + +def siglip_inputs_from_preprocessed_items( + image_items: list[ConversationItem], + *, + device: torch.device, + dtype: torch.dtype, +) -> dict[str, Any]: + """Build NaViT forward inputs from CPU-preprocessed image items.""" + token_lens = torch.tensor( + [int(item.meta[_OMNI_TOKEN_LEN]) for item in image_items], + dtype=torch.int32, + device=device, + ) + return { + "patchified_pixel_values": torch.cat([item.value for item in image_items], dim=0).to( + device=device, dtype=dtype, non_blocking=True + ), + "patchified_position_ids": torch.cat( + [item.meta[_OMNI_POSITION_IDS] for item in image_items], + dim=0, + ).to(device=device, dtype=torch.long, non_blocking=True), + "cu_seqlens": torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(token_lens.max().item()), + "token_lens": token_lens, + } + + +def scatter_siglip_image_embeds( + image_items: list[ConversationItem], + image_embeds: torch.Tensor, + token_lens: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Scatter NaViT outputs back onto conversation image items.""" + offset = 0 + lengths = token_lens.detach().cpu().reshape(-1).tolist() + for item, length in zip(image_items, lengths, strict=True): + item.value = image_embeds[offset : offset + int(length)].to(device=device, dtype=dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + offset += int(length) + + if offset != int(image_embeds.shape[0]): + raise RuntimeError("BAGEL SigLIP token count mismatch during feature scatter.") + + +class InferenceMixin: + """FSM ``generate`` — HF ``GenerationMixin`` analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelSiglipNavit`'s bases for consistency with every other + module's native / accelerated split (see ``janus/llama/modeling.py`` for + the full MRO rationale where a competing no-op default exists). + """ + + def generate( + self, + conversation_list: list[ConversationItem] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Encode SigLIP context images and scatter visual embeddings onto ``conversation_list``.""" + del kwargs + batched = [conversation_list] + image_items = select_siglip_image_items(batched) + if not image_items: + return {"conversation_list": conversation_list} + + inputs = siglip_inputs_from_preprocessed_items(image_items, device=self.device, dtype=self.dtype) + outputs = self.forward(**inputs) + token_lens = outputs.get("token_lens", inputs["token_lens"]) + scatter_siglip_image_embeds( + image_items, + outputs["image_embeds"], + token_lens, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": batched[0]} + + +def _module_weight(module: nn.Module) -> torch.Tensor: + """Return the underlying weight parameter for plain or LoRA-wrapped modules.""" + return getattr(module, "base_layer", module).weight + + +class BagelSiglipNavit(InferenceMixin, PretrainedOmniModule): + config_class = BagelSiglipNavitConfig + image_processor_class = BagelSiglipNavitProcessor + preprocessor_class = BagelSiglipNavitPreprocessor + base_model_prefix = "bagel_siglip_navit" + main_input_name = "patchified_pixel_values" + _no_split_modules = ["BagelSiglipEncoderLayer"] + supports_gradient_checkpointing = True + _supports_sdpa = True + _supports_flash_attn = True + + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__(config) + self.vision_model = BagelSiglipVisionTransformer(config) + self.connector = MLPConnector(config.hidden_size, config.output_size, config.connector_act) + self.vit_pos_embed = PositionEmbedding(config.vit_max_num_patch_per_side, config.output_size) + self._image_processor: BagelSiglipNavitProcessor = None + self.post_init() + + def _init_weights(self, module: nn.Module) -> None: + if isinstance(module, PositionEmbedding): + module.reset_parameters() + return + super()._init_weights(module) + + @property + def _vit_device(self) -> torch.device: + return _module_weight(self.vision_model.embeddings.patch_embedding).device + + @property + def _connector_device(self) -> torch.device: + return _module_weight(self.connector.fc1).device + + @property + def _pos_embed_device(self) -> torch.device: + return self.vit_pos_embed.pos_embed.device + + def forward( # type: ignore[override] + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + token_lens: torch.IntTensor, + **kwargs: Any, + ) -> dict[str, Any]: + del kwargs + vit_hidden = self._encode_patches( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + image_embeds = self._project_with_position( + vit_hidden=vit_hidden, + patchified_position_ids=patchified_position_ids, + ) + return {"image_embeds": image_embeds, "token_lens": token_lens} + + def _encode_patches( + self, + *, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + ) -> torch.Tensor: + patchified_pixel_values = patchified_pixel_values.to(device=self._vit_device, dtype=self.dtype) + patchified_position_ids = patchified_position_ids.to(device=self._vit_device, dtype=torch.long) + cu_seqlens = cu_seqlens.to(device=self._vit_device, dtype=torch.int32) + return self.vision_model( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + + def _project_with_position( + self, + *, + vit_hidden: torch.Tensor, + patchified_position_ids: torch.LongTensor, + ) -> torch.Tensor: + fc1_weight = _module_weight(self.connector.fc1) + vit_hidden = vit_hidden.to(device=fc1_weight.device, dtype=fc1_weight.dtype) + patchified_position_ids = patchified_position_ids.to(device=self._pos_embed_device) + image_embeds = self.connector(vit_hidden) + return image_embeds + self.vit_pos_embed(patchified_position_ids).to( + device=image_embeds.device, + dtype=image_embeds.dtype, + ) + + +class RotaryEmbedding2D(nn.Module): + def __init__(self, dim: int, max_h: int, max_w: int, base: int = 10000) -> None: + super().__init__() + freq = torch.arange(0, dim, 2, dtype=torch.int64).float() / dim + inv_freq = 1.0 / (base**freq) + + grid_h = torch.arange(0, max_h).to(inv_freq.dtype) + grid_h = grid_h[:, None].repeat(1, max_w) + + grid_w = torch.arange(0, max_w).to(inv_freq.dtype) + grid_w = grid_w[None, :].repeat(max_h, 1) + + cos_h, sin_h = self._forward_one_side(grid_h, inv_freq) + cos_w, sin_w = self._forward_one_side(grid_w, inv_freq) + + self.register_buffer("cos_h", cos_h, persistent=False) + self.register_buffer("sin_h", sin_h, persistent=False) + self.register_buffer("cos_w", cos_w, persistent=False) + self.register_buffer("sin_w", sin_w, persistent=False) + + @staticmethod + def _forward_one_side(grid: torch.Tensor, inv_freq: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + freqs = grid[..., None] * inv_freq[None, None, :] + emb = torch.cat((freqs, freqs), dim=-1).flatten(0, 1) + return emb.cos(), emb.sin() + + +class MLPConnector(nn.Module): + def __init__(self, in_dim: int, out_dim: int, hidden_act: str) -> None: + super().__init__() + self.activation_fn = ACT2FN[hidden_act] + self.fc1 = nn.Linear(in_dim, out_dim) + self.fc2 = nn.Linear(out_dim, out_dim) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.fc2(self.activation_fn(self.fc1(hidden_states))) + + +class BagelSiglipVisionEmbeddings(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + patch_dim = config.num_channels * config.patch_size * config.patch_size + self.patch_embedding = nn.Linear(patch_dim, config.hidden_size, bias=True) + self.position_embedding = ( + None if config.rope else nn.Embedding((config.image_size // config.patch_size) ** 2, config.hidden_size) + ) + + def forward( + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + ) -> torch.Tensor: + patch_weight = _module_weight(self.patch_embedding) + patchified_pixel_values = patchified_pixel_values.to( + device=patch_weight.device, + dtype=patch_weight.dtype, + ) + patch_embeds = self.patch_embedding(patchified_pixel_values) + if self.position_embedding is None: + return patch_embeds + position_weight = _module_weight(self.position_embedding) + position_ids = patchified_position_ids.to(device=position_weight.device, dtype=torch.long) + position_embeds = self.position_embedding(position_ids) + return patch_embeds + position_embeds.to(device=patch_embeds.device, dtype=patch_embeds.dtype) + + +class BagelSiglipAttention(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + """Initialize NaViT self-attention projections and instance-local ``VeomniOp`` handles.""" + super().__init__() + self.config = config + self.num_heads = config.num_attention_heads + self.head_dim = config.hidden_size // config.num_attention_heads + self.num_key_value_groups = 1 + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = False + self.q_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.k_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.v_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + + rope_impl = ( + config.ops_config.rotary_pos_emb_vision_implementation + if "rotary_pos_emb_vision_implementation" in config.ops_implementation + or "rotary_pos_emb_implementation" not in config.ops_implementation + else config.ops_config.rotary_pos_emb_implementation + ) + self.veomni_rope = VeomniOp("rope", "full", rope_impl) + + attn_impl = config.ops_config.attn_implementation + if "attn_implementation" not in config.ops_implementation: + try: + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + except RuntimeError: + self.veomni_attn = VeomniOp("attention", "standard", "veomni_sdpa") + else: + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + """Run varlen non-causal self-attention over packed NaViT patch tokens.""" + total_q_len, _ = hidden_states.size() + query_states = self.q_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + key_states = self.k_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + value_states = self.v_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + + if self.config.rope: + if cos_h is None or sin_h is None or cos_w is None or sin_w is None: + raise ValueError("2D RoPE tensors are required when BagelSiglipNavitConfig.rope=True.") + qh, qw = query_states[:, :, : self.head_dim // 2], query_states[:, :, self.head_dim // 2 :] + kh, kw = key_states[:, :, : self.head_dim // 2], key_states[:, :, self.head_dim // 2 :] + cos_h = cos_h.to(device=query_states.device, dtype=query_states.dtype) + sin_h = sin_h.to(device=query_states.device, dtype=query_states.dtype) + cos_w = cos_w.to(device=query_states.device, dtype=query_states.dtype) + sin_w = sin_w.to(device=query_states.device, dtype=query_states.dtype) + qh, kh = self.veomni_rope(qh, kh, cos_h, sin_h) + qw, kw = self.veomni_rope(qw, kw, cos_w, sin_w) + query_states = torch.cat([qh, qw], dim=-1) + key_states = torch.cat([kh, kw], dim=-1) + + attn_dtype = ( + torch.bfloat16 + if value_states.dtype == torch.float32 and "flash_attention" in self.veomni_attn.impl + else value_states.dtype + ) + query_states = query_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) + key_states = key_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) + value_states = value_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) + dropout = 0.0 if not self.training else self.attention_dropout + + attn_output, _ = self.veomni_attn( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + skip_ulysses=True, + ) + return self.out_proj(attn_output.reshape(total_q_len, -1).to(hidden_states.dtype)) + + +class BagelSiglipMLP(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=True) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=True) + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.fc2(self.activation_fn(self.fc1(hidden_states))) + + +class BagelSiglipEncoderLayer(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.self_attn = BagelSiglipAttention(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = BagelSiglipMLP(config) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn( + hidden_states=hidden_states, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + cos_h=cos_h, + sin_h=sin_h, + cos_w=cos_w, + sin_w=sin_w, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + return residual + hidden_states + + +class BagelSiglipEncoder(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.gradient_checkpointing = False + self.layers = nn.ModuleList([BagelSiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + inputs_embeds: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + hidden_states = inputs_embeds + for encoder_layer in self.layers: + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func( + encoder_layer.__call__, + hidden_states, + cu_seqlens, + max_seqlen, + cos_h, + sin_h, + cos_w, + sin_w, + ) + else: + hidden_states = encoder_layer( + hidden_states, + cu_seqlens, + max_seqlen, + cos_h=cos_h, + sin_h=sin_h, + cos_w=cos_w, + sin_w=sin_w, + ) + return hidden_states + + +class BagelSiglipVisionTransformer(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.config = config + self.embeddings = BagelSiglipVisionEmbeddings(config) + if config.rope: + max_size = config.image_size // config.patch_size + dim_head = config.hidden_size // config.num_attention_heads + self.rope = RotaryEmbedding2D(dim_head // 2, max_size, max_size) + self.encoder = BagelSiglipEncoder(config) + self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + ) -> torch.Tensor: + hidden_states = self.embeddings( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + ) + extra_inputs: dict[str, torch.Tensor] = {} + if self.config.rope: + rope_position_ids = patchified_position_ids.to(device=self.rope.cos_h.device, dtype=torch.long) + extra_inputs.update( + cos_h=self.rope.cos_h[rope_position_ids], + sin_h=self.rope.sin_h[rope_position_ids], + cos_w=self.rope.cos_w[rope_position_ids], + sin_w=self.rope.sin_w[rope_position_ids], + ) + last_hidden_state = self.encoder( + inputs_embeds=hidden_states, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + **extra_inputs, + ) + return self.post_layernorm(last_hidden_state) + + +__all__ = [ + "BagelSiglipNavit", + "BagelSiglipNavitConfig", + "InferenceMixin", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py new file mode 100644 index 0000000000..ba7ac3fe72 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py @@ -0,0 +1,438 @@ +"""Image processor + worker-side preprocessor for BAGEL SigLIP NaViT. + +:class:`BagelSiglipNavitPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) — +built straight off the checkpoint dir, with no model instance involved. Unlike +Janus, BAGEL ships no separate ``preprocessor_config.json``: the image +processor is fully derived from the module's own ``config.json`` via +:meth:`BagelSiglipNavitProcessor.from_config`. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TVF +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature + +from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ....utils.conversation import _OMNI_TOKEN_LEN as _CONVERSATION_TOKEN_LEN +from ...module_processing_base import ModulePreprocessorBase +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from .configuration import BagelSiglipNavitConfig + + +class BagelSiglipNavitProcessor(BaseImageProcessor): + """BAGEL SigLIP NaViT image processor. + + Converts raw images into flattened patch rows plus the varlen metadata that + the NaViT tower consumes. Carrier selection and embed scatter stay in the + module mixin because they are SeedOmni conversation semantics. + """ + + model_input_names = [ + "patchified_pixel_values", + "patchified_position_ids", + "cu_seqlens", + "max_seqlen", + "token_lens", + ] + + def __init__( + self, + patch_size: int = 14, + image_size: int = 980, + min_image_size: int = 378, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + vit_max_num_patch_per_side: int = 70, + num_channels: int = 3, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.patch_size = patch_size + self.image_size = image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.vit_max_num_patch_per_side = vit_max_num_patch_per_side + self.num_channels = num_channels + + @classmethod + def from_config(cls, config: Any) -> BagelSiglipNavitProcessor: + return cls( + patch_size=int(config.patch_size), + image_size=int(config.image_size), + min_image_size=int(config.min_image_size), + max_pixels=int(config.max_pixels), + image_mean=list(config.image_mean), + image_std=list(config.image_std), + vit_max_num_patch_per_side=int(config.vit_max_num_patch_per_side), + num_channels=int(config.num_channels), + ) + + def preprocess( + self, + images: Any, + *, + return_tensors: str | None = None, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + **kwargs: Any, + ) -> BatchFeature: + """Resize, normalize, and patchify ``images`` into NaViT varlen batch tensors.""" + del return_tensors, kwargs + image_list = images if isinstance(images, list) else [images] + + pixel_values: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + token_lens: list[int] = [] + for image in image_list: + image_tensor = _preprocess_image( + image, + patch_size=self.patch_size, + image_size=self.image_size, + min_image_size=self.min_image_size, + max_pixels=self.max_pixels, + image_mean=self.image_mean, + image_std=self.image_std, + ) + patches = _patchify_image(image_tensor, self.patch_size) + positions = _flattened_position_ids( + image_tensor.shape[-2], + image_tensor.shape[-1], + patch_size=self.patch_size, + max_num_patches_per_side=self.vit_max_num_patch_per_side, + ) + pixel_values.append(patches) + position_ids.append(positions) + token_lens.append(int(patches.shape[0])) + + tensor_device = torch.device("cpu") if device is None else device + token_lens_tensor = torch.tensor(token_lens, dtype=torch.int32, device=tensor_device) + pixel_tensor = torch.cat(pixel_values, dim=0) + if dtype is not None: + pixel_tensor = pixel_tensor.to(dtype=dtype) + return BatchFeature( + data={ + "patchified_pixel_values": pixel_tensor.to(device=tensor_device), + "patchified_position_ids": torch.cat(position_ids, dim=0).to(device=tensor_device, dtype=torch.long), + "cu_seqlens": F.pad(torch.cumsum(token_lens_tensor, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(token_lens_tensor.max().item()), + "token_lens": token_lens_tensor, + } + ) + + +def _preprocess_image( + image: Any, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + image_mean: list[float], + image_std: list[float], +) -> torch.Tensor: + """Convert ``image`` to a normalized ``[3, H, W]`` float32 tensor aligned to ``patch_size``.""" + if isinstance(image, (Image.Image, np.ndarray)): + pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image)) + pil_image = _resize_pil( + pil_image, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + tensor = _pil_to_rgb_tensor(pil_image) + else: + tensor = _to_rgb_tensor(image) + tensor = _resize_tensor( + tensor, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + return tensor.sub(mean).div(std) + + +def _to_rgb_tensor(image: Any) -> torch.Tensor: + """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``.""" + if isinstance(image, Image.Image): + pil_image = _to_rgb_pil(image) + array = np.array(pil_image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + if isinstance(image, np.ndarray): + return _to_rgb_tensor(Image.fromarray(image)) + if torch.is_tensor(image): + tensor = image.detach().to(dtype=torch.float32) + if tensor.dim() != 3: + raise TypeError(f"BAGEL image tensor must be 3-D, got shape {tuple(tensor.shape)}.") + if tensor.shape[0] in (1, 3, 4): + pass + elif tensor.shape[-1] in (1, 3, 4): + tensor = tensor.permute(2, 0, 1).contiguous() + else: + raise TypeError(f"Unable to infer channel dimension for BAGEL image tensor {tuple(tensor.shape)}.") + if tensor.shape[0] == 1: + tensor = tensor.repeat(3, 1, 1) + elif tensor.shape[0] == 4: + alpha = tensor[3:4] + if alpha.max() > 1: + alpha = alpha / 255.0 + rgb = tensor[:3] + if rgb.max() > 1: + rgb = rgb / 255.0 + tensor = rgb * alpha + (1.0 - alpha) + if tensor.max() > 1: + tensor = tensor / 255.0 + return tensor[:3].clamp(0.0, 1.0) + raise TypeError(f"BAGEL image item value must be PIL, numpy, or tensor, got {type(image).__name__}.") + + +def _resize_tensor( + tensor: torch.Tensor, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> torch.Tensor: + """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to the target patch grid.""" + height, width = tensor.shape[-2:] + new_width, new_height = _target_size( + width, + height, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + img_num=img_num, + ) + if new_width == width and new_height == height: + return tensor + resized = F.interpolate( + tensor.unsqueeze(0), + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + antialias=True, + ) + return resized.squeeze(0).clamp(0.0, 1.0) + + +def _resize_pil( + image: Image.Image, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> Image.Image: + """Resize a PIL image with bicubic interpolation to the target patch grid.""" + width, height = image.size + new_width, new_height = _target_size( + width, + height, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + img_num=img_num, + ) + if new_width == width and new_height == height: + return image + return TVF.resize( + image, + (new_height, new_width), + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ) + + +def _target_size( + width: int, + height: int, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int, +) -> tuple[int, int]: + """Compute patch-stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds.""" + scale = min(image_size / max(width, height), 1.0) + scale = max(scale, min_image_size / min(width, height)) + new_width, new_height = _apply_scale(width, height, scale, patch_size) + if new_width * new_height > max_pixels / img_num: + scale = max_pixels / img_num / (new_width * new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size) + if max(new_width, new_height) > image_size: + scale = image_size / max(new_width, new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size) + return new_width, new_height + + +def _to_rgb_pil(image: Image.Image) -> Image.Image: + """Convert ``image`` to RGB, compositing transparency over a white background.""" + if image.mode == "RGBA" or image.info.get("transparency", None) is not None: + rgba = image.convert("RGBA") + white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255)) + white.paste(rgba, mask=rgba.split()[3]) + return white + return image.convert("RGB") + + +def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor: + array = np.array(image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + + +def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]: + new_width = round(width * scale) + new_height = round(height * scale) + return ( + max(stride, int(round(new_width / stride) * stride)), + max(stride, int(round(new_height / stride) * stride)), + ) + + +def _patchify_image(image: torch.Tensor, patch_size: int) -> torch.Tensor: + """Flatten a ``[C, H, W]`` image tensor into ``[(H/P)*(W/P), P*P*C]`` patch rows.""" + channels, height, width = image.shape + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError("BAGEL preprocessed image height and width must be divisible by patch_size.") + image = image.reshape(channels, height // patch_size, patch_size, width // patch_size, patch_size) + image = torch.einsum("chpwq->hwpqc", image) + return image.reshape(-1, patch_size**2 * channels) + + +def _flattened_position_ids( + height: int, + width: int, + *, + patch_size: int, + max_num_patches_per_side: int, +) -> torch.Tensor: + num_patches_h = height // patch_size + num_patches_w = width // patch_size + coords_h = torch.arange(0, num_patches_h, dtype=torch.long) + coords_w = torch.arange(0, num_patches_w, dtype=torch.long) + return (coords_h[:, None] * max_num_patches_per_side + coords_w).flatten() + + +_OMNI_POSITION_IDS = "bagel_siglip_navit_position_ids" +_OMNI_TOKEN_LEN = "bagel_siglip_navit_token_len" + + +class BagelSiglipNavitPreprocessor(ModulePreprocessorBase): + """Worker-side image patchify for BAGEL SigLIP NaViT context images.""" + + def __init__( + self, + image_processor: Any, + dtype: torch.dtype | None = None, + dummy_pixel_values: torch.Tensor | None = None, + ) -> None: + self._image_processor = image_processor + self._dtype = dtype + self._dummy_pixel_values = dummy_pixel_values + + @classmethod + def from_pretrained( + cls, module_path: str, *, config_overrides: dict[str, Any] | None = None, **kwargs: Any + ) -> BagelSiglipNavitPreprocessor: + """Build straight from the checkpoint dir — no model instance needed. + + BAGEL ships no standalone ``preprocessor_config.json``; the image + processor is derived from the module's own ``config.json``. + + ``config_overrides`` (the module's YAML ``model_config:`` block) is + applied on top of the on-disk default before deriving the image + processor's geometry, so it agrees with the live model's own config. + """ + del kwargs + # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare. + config = BagelSiglipNavitConfig.from_pretrained(module_path, model_config=config_overrides) + return cls(BagelSiglipNavitProcessor.from_config(config)) + + def bind_dummy_inputs(self, config: BagelSiglipNavitConfig, dtype: torch.dtype | None = None) -> None: + """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model.""" + patch_dim = int(config.num_channels) * int(config.patch_size) * int(config.patch_size) + self._dtype = dtype + self._dummy_pixel_values = torch.zeros(1, patch_dim, dtype=dtype) + + def preprocess_conversations( + self, + conversation_list: list[list[ConversationItem]], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> None: + """Patchify SigLIP context images in place and append per-sample dummy anchors in training.""" + del generation_kwargs + + image_items: list[ConversationItem] = [] + for sample in conversation_list: + sample_image_items = list( + iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}) + ) + if sample_image_items: + image_items.extend(item for item in sample_image_items if not item.is_dummy) + elif not inference: + if self._dummy_pixel_values is None: + raise RuntimeError( + f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() " + "before training use (pure inference never reaches this branch)." + ) + sample.append( + ConversationItem( + type="image", + value=self._dummy_pixel_values.to(dtype=self._dtype).clone(), + role="user", + is_dummy=True, + meta={ + _IMG_TAG_KEY: "und", + BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, + _OMNI_POSITION_IDS: torch.zeros(1, dtype=torch.long), + _OMNI_TOKEN_LEN: 1, + _CONVERSATION_TOKEN_LEN: 1, + }, + ) + ) + + if not image_items: + return + + inputs = self._image_processor( + images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype + ) + lengths = inputs["token_lens"].detach().cpu().reshape(-1).tolist() + pixel_chunks = torch.split(inputs["patchified_pixel_values"], lengths, dim=0) + position_chunks = torch.split(inputs["patchified_position_ids"], lengths, dim=0) + for item, pixels, position_ids, length in zip( + image_items, pixel_chunks, position_chunks, lengths, strict=True + ): + item.value = pixels.to(dtype=self._dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + item.meta[_OMNI_POSITION_IDS] = position_ids.to(dtype=torch.long) + item.meta[_OMNI_TOKEN_LEN] = int(length) + item.meta[_CONVERSATION_TOKEN_LEN] = int(length) + + +__all__ = [ + "BagelSiglipNavitProcessor", + "BagelSiglipNavitPreprocessor", +] diff --git a/veomni/models/seed_omni/modules/bagel/sources.py b/veomni/models/seed_omni/modules/bagel/sources.py new file mode 100644 index 0000000000..e58bdd1f9a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/sources.py @@ -0,0 +1,69 @@ +"""BAGEL-local ``ConversationItem.meta`` routing tags. + +Items carry no module ownership (see :mod:`...utils.conversation`), so BAGEL +routes its rows with two BAGEL-private meta keys: + +* ``meta[BAGEL_CONTEXT_KEY]`` — which image context copy a row belongs to: + ``BAGEL_SIGLIP_CONTEXT`` (understanding features, read by SigLIP-NaViT) or + ``BAGEL_VAE_CONTEXT`` (latent context/target, read by the VAE and flow + connector). The VAE preprocessor expands each raw image by its data-layer + ``_IMG_TAG_KEY`` into one or both copies — ``_img_tag`` alone cannot tell the + two copies of one ``edit`` image apart. The chat template stamps the same tag + on the vision start/end marker rows around each copy, and MoT reads it to + pick attention mode and expert routing. +* ``meta[BAGEL_PHASE_KEY]`` — the inference/flow step an ``output`` row is in: + the understanding start token, then per denoise step flow query -> hidden -> + velocity, and finally the generated latent handed to VAE decode. + +An FSDP placeholder is flagged ``is_dummy=True`` and keeps the context tag of +the rows it stands in for, so each encoder selects real and dummy rows with +one filter. +""" + +from __future__ import annotations + +from ...utils.conversation import ConversationItem, iter_desired_items + + +BAGEL_CONTEXT_KEY = "bagel_context" +BAGEL_SIGLIP_CONTEXT = "siglip" +BAGEL_VAE_CONTEXT = "vae" + +BAGEL_PHASE_KEY = "bagel_phase" +BAGEL_START_TOKEN = "start_token" +BAGEL_FLOW_QUERY = "flow_query" +BAGEL_FLOW_HIDDEN = "flow_hidden" +BAGEL_FLOW_VELOCITY = "flow_velocity" +BAGEL_GENERATED_LATENT = "generated_latent" + + +def bagel_context(item: ConversationItem) -> str | None: + return item.meta.get(BAGEL_CONTEXT_KEY) + + +def bagel_phase(item: ConversationItem) -> str | None: + return item.meta.get(BAGEL_PHASE_KEY) + + +def get_tail_phase_item(conversation: list[ConversationItem], phase: str) -> ConversationItem | None: + """Return the latest ``output`` row in ``phase`` of a single conversation.""" + return next( + iter_desired_items([conversation], types=["output"], reverse_item=True, meta={BAGEL_PHASE_KEY: [phase]}), + None, + ) + + +__all__ = [ + "BAGEL_CONTEXT_KEY", + "BAGEL_FLOW_HIDDEN", + "BAGEL_FLOW_QUERY", + "BAGEL_FLOW_VELOCITY", + "BAGEL_GENERATED_LATENT", + "BAGEL_PHASE_KEY", + "BAGEL_SIGLIP_CONTEXT", + "BAGEL_START_TOKEN", + "BAGEL_VAE_CONTEXT", + "bagel_context", + "bagel_phase", + "get_tail_phase_item", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py new file mode 100644 index 0000000000..f780d497c9 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL token embedding and LM head module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_config(): + from .configuration import BagelTextEncoderConfig + + return BagelTextEncoderConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_model(): + from .modeling import BagelTextEncoder + + return BagelTextEncoder + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_accelerated_model(): + from .accelerated import BagelTextEncoderAccelerated + + return BagelTextEncoderAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py new file mode 100644 index 0000000000..1a106ff116 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/text_encoder OmniModule (conversation graph).""" + +from .accelerated import ( + BagelTextEncoderAccelerated, +) + + +__all__ = [ + "BagelTextEncoderAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py new file mode 100644 index 0000000000..48ec08ae65 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py @@ -0,0 +1,176 @@ +"""VeOmni-accelerated BagelTextEncoder — training graph hooks. + +FSM ``generate`` (and ``encode_image_markers``) now live natively on +:class:`~.modeling.BagelTextEncoder`; this file only owns the training +pre/forward/post hooks (packed-token embedding + MoT CE loss). +""" + +from typing import Any, Dict, List, Optional, Tuple + +import torch +from transformers import PreTrainedTokenizerBase + +from .....mixins.training_module_mixin import post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ....text_encoder.accelerated import ( + TrainingMixin as BaseTrainingMixin, +) +from ....text_encoder.accelerated import ( + VeOmniMixin as BaseVeOmniMixin, +) +from ..chat_template import BagelChatTemplate +from ..configuration import BagelTextEncoderConfig +from ..modeling import BagelTextEncoder, prepare_bagel_encode_input_ids + + +class TrainingMixin(BaseTrainingMixin): + config: BagelTextEncoderConfig + device: torch.device + dtype: torch.dtype + _chat_template: BagelChatTemplate | None + _encode_batch_shape: torch.LongTensor | None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._chat_template: Optional[BagelChatTemplate] = None + + def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]: + """Delegate to native impl — overrides ``TrainingModuleMixin.dummy_inputs``'s + unrelated ``(*, batch_size, device, dtype)`` signature, which otherwise wins + this name collision by sitting ahead of :class:`BagelTextEncoder` in MRO. + """ + return BagelTextEncoder.dummy_inputs(self, kind=kind) + + @pre_forward("encode") + def encode_pre( + self, + conversation_list: Optional[List[List[ConversationItem]]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + return super().encode_pre(conversation_list, **kwargs) + + @post_forward("encode") + def encode_post(self, **outputs: Any) -> Dict[str, Any]: + return super().encode_post(**outputs) + + @pre_forward("decode") + def decode_pre( + self, + conversation_list: Optional[List[List[ConversationItem]]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + return super().decode_pre(conversation_list, **kwargs) + + @post_forward("decode") + def decode_post(self, **outputs: Any) -> Dict[str, Any]: + return super().decode_post(**outputs) + + def _anchor_dummy_decode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + dummy: Dict[str, torch.Tensor], + ) -> Dict[str, torch.Tensor]: + """Tie dummy CE loss to MoT hidden states without changing its value.""" + if conversation_list is None: + return dummy + + anchor = None + for item in iter_desired_items( + conversation_list, types=["text", "image", "output"], roles=["user", "assistant"] + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size): + anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + break + if anchor is None: + return dummy + + return { + "hidden_states": dummy["hidden_states"] + anchor, + "labels": dummy["labels"], + } + + def _prepare_encode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + ) -> torch.Tensor: + """Pack non-dummy text token IDs into a flat tensor and record segment lengths.""" + if conversation_list is None: + raise ValueError("BagelTextEncoder._prepare_encode_inputs requires conversation_list.") + + input_ids, self._encode_batch_shape = prepare_bagel_encode_input_ids( + conversation_list, + device=self.device, + fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"], + ) + return input_ids + + def _prepare_decode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Collect text hidden states and pre-shifted causal LM labels across ``conversation_list``.""" + if conversation_list is None: + raise ValueError("BagelTextEncoder._prepare_decode_inputs requires conversation_list.") + + hidden_parts: List[torch.Tensor] = [] + shift_label_parts: List[torch.Tensor] = [] + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + + hidden_states = item.value + labels = item.meta["labels"] + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + labels = labels.reshape(-1) + if hidden_states.shape[0] != labels.shape[0]: + raise ValueError( + "BAGEL text decode requires hidden-state and label lengths to match: " + f"got {hidden_states.shape[0]} and {labels.shape[0]}." + ) + + shift_labels = torch.full_like(labels, -100, dtype=torch.long) + shift_labels[:-1] = labels[1:] + hidden_parts.append(hidden_states.to(device=self.device, dtype=self.dtype)) + shift_label_parts.append(shift_labels) + + if hidden_parts: + hidden_states = torch.cat(hidden_parts, dim=0) + shift_labels = torch.cat(shift_label_parts, dim=0).to(device=hidden_states.device, non_blocking=True) + if torch.any(shift_labels != -100): + return hidden_states, shift_labels + + dummy = self._anchor_dummy_decode_inputs(conversation_list, self.dummy_inputs(kind="decode")) + return dummy["hidden_states"], dummy["labels"] + + +class VeOmniMixin(TrainingMixin, BaseVeOmniMixin): + """BAGEL text-embedding accelerated wrapper — chat-template binding + training hooks. + + The T2I generation FSM (``generate`` / ``encode_image_markers``) lives on + the native :class:`~.modeling.BagelTextEncoder`; only the chat template + binding and CE-loss training hooks are accelerated-specific. + """ + + @property + def tokenizer(self) -> PreTrainedTokenizerBase: + return self._tokenizer + + @tokenizer.setter + def tokenizer(self, tokenizer: PreTrainedTokenizerBase) -> None: + self._tokenizer = tokenizer + self._chat_template = BagelChatTemplate(tokenizer) + + +class BagelTextEncoderAccelerated(VeOmniMixin, BagelTextEncoder): + pass + + +__all__ = ["BagelTextEncoderAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py new file mode 100644 index 0000000000..8c2aba26a3 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py @@ -0,0 +1,215 @@ +"""BAGEL chat template for text/materialized image prompt rows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +from transformers import PreTrainedTokenizerBase + +from ....utils.conversation import _OMNI_TOKEN_LEN, ConversationItem +from ...text_encoder.chat_template import ChatMarkers, TextEncoderChatTemplate +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_PHASE_KEY, BAGEL_START_TOKEN, bagel_context +from .processing import is_bagel_vision_marker + + +_OMNI_TOKENIZED = "_omni_tokenized" + + +@dataclass(frozen=True) +class BagelChatMarkers(ChatMarkers): + vision_start_token: str + vision_end_token: str + + +class BagelChatTemplate(TextEncoderChatTemplate): + """BAGEL-specific template over SeedOmni conversation rows. + + The template is intentionally row-preserving: text rows are tokenized, and + image rows are bracketed by marker text rows, but marker-image-marker packing + remains a MoT responsibility. + """ + + chat_markers: BagelChatMarkers + + def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None: + super().__init__(tokenizer) + + bos_token = "<|im_start|>" + eos_token = "<|im_end|>" + vision_start_token = "<|vision_start|>" + vision_end_token = "<|vision_end|>" + + self.bos_token_id = self._resolve_token_id_from_token(tokenizer, bos_token) + self.eos_token_id = self._resolve_token_id_from_token(tokenizer, eos_token) + self.vision_start_token_id = self._resolve_token_id_from_token(tokenizer, vision_start_token) + self.vision_end_token_id = self._resolve_token_id_from_token(tokenizer, vision_end_token) + + self.chat_markers = BagelChatMarkers( + bos_token=bos_token, + eos_token=eos_token, + vision_start_token=vision_start_token, + vision_end_token=vision_end_token, + ) + + @staticmethod + def _resolve_token_id_from_token(tokenizer: PreTrainedTokenizerBase, token: str) -> int: + """Look up ``token`` in ``tokenizer`` and raise if it is missing or maps to ``unk_token_id``.""" + token_id = tokenizer.convert_tokens_to_ids(token) + if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None): + raise ValueError(f"BAGEL tokenizer is missing required token: {token!r}.") + return int(token_id) + + def tokenize_conversation( + self, + sample: list[ConversationItem], + *, + add_generation_prompt: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Apply the BAGEL chat template, wrap image rows with vision markers, and tokenize text rows.""" + parts = self.apply_chat_template( + sample, + inference=add_generation_prompt, + generation_kwargs=generation_kwargs, + ) + if add_generation_prompt: + parts = self.apply_generation_prompt(parts, generation_kwargs=generation_kwargs) + self.tokenize(parts) + return self.merge_text_embeds(parts) + + def apply_chat_template( + self, + sample: list[ConversationItem], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Wrap untokenized text items with ``<|im_start|>...<|im_end|>`` and insert image marker items.""" + del inference, generation_kwargs + + for item in sample: + if item.type == "text" and not is_bagel_vision_marker(item): + if not item.meta.get(_OMNI_TOKENIZED) and not torch.is_tensor(item.value): + item.value = f"{self.chat_markers.bos_token}{item.value}{self.chat_markers.eos_token}" + item.meta["loss_mask"] = int(item.role == "assistant") + + return self._insert_prompt_context_image_markers(sample) + + def apply_generation_prompt( + self, + sample: list[ConversationItem], + *, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Append the assistant ``<|im_start|>`` start token for understanding inference.""" + infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type") + if infer_type != "infer_und": + return sample + out = list(sample) + out.append( + ConversationItem( + type="text", + value=self.chat_markers.bos_token, + role="assistant", + meta={"loss_mask": 0, BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, + ) + ) + return out + + def tokenize(self, parts: list[ConversationItem]) -> None: + """Tokenize untokenized text items in place and populate ``input_ids``, ``labels``, and ``attention_mask``.""" + for item in parts: + if item.type != "text" or item.is_dummy or item.meta.get(_OMNI_TOKENIZED): + continue + + raw_value = item.value + token_ids = self._token_ids_for_text_item(item) + loss_mask = int(item.meta.pop("loss_mask", int(item.role == "assistant"))) + item.value = token_ids + item.meta["input_ids"] = token_ids.detach() + item.meta["attention_mask"] = torch.ones_like(token_ids, dtype=torch.long) + item.meta["labels"] = ( + token_ids.detach().clone() if loss_mask else torch.full_like(token_ids, -100, dtype=torch.long) + ) + item.meta[_OMNI_TOKEN_LEN] = int(token_ids.numel()) + item.meta[_OMNI_TOKENIZED] = True + + if ( + isinstance(raw_value, str) + and raw_value in {self.chat_markers.bos_token, self.chat_markers.eos_token} + and int(token_ids.numel()) != 1 + ): + raise ValueError("BAGEL start/end tokens must tokenize to exactly one token.") + + @staticmethod + def merge_text_embeds(parts: list[ConversationItem]) -> list[ConversationItem]: + return parts + + def _token_ids_for_text_item(self, item: ConversationItem) -> torch.Tensor: + """Convert ``item.value`` (tensor or string) into a 1-D int64 token-id tensor.""" + value = item.value + if torch.is_tensor(value): + token_ids = value.detach().reshape(-1).to(dtype=torch.long) + else: + token_ids = torch.tensor( + self.tokenizer(str(value), add_special_tokens=False)["input_ids"], + dtype=torch.long, + ) + return token_ids + + def _insert_prompt_context_image_markers(self, sample: list[ConversationItem]) -> list[ConversationItem]: + """Bracket each non-dummy image item with ``<|vision_start|>`` and ``<|vision_end|>`` marker items.""" + out: list[ConversationItem] = [] + + index = 0 + while index < len(sample): + item = sample[index] + + if item.type != "image" or item.is_dummy: + out.append(item) + index += 1 + continue + + # Add vision marker for image items: + image + . + # Markers carry the image's context tag so MoT can regroup the triplet. + context = bagel_context(item) + marker_meta = {"loss_mask": 0} if context is None else {"loss_mask": 0, BAGEL_CONTEXT_KEY: context} + if not (out and bagel_context(out[-1]) == context and is_bagel_vision_marker(out[-1], context=context)): + out.append( + ConversationItem( + type="text", + value=self.chat_markers.vision_start_token, + role=item.role, + meta=dict(marker_meta), + ) + ) + + out.append(item) + + if ( + index + 1 < len(sample) + and bagel_context(sample[index + 1]) == context + and is_bagel_vision_marker(sample[index + 1], context=context) + ): + out.append(sample[index + 1]) + index += 2 + else: + out.append( + ConversationItem( + type="text", + value=self.chat_markers.vision_end_token, + role=item.role, + meta=dict(marker_meta), + ) + ) + index += 1 + + return out + + +__all__ = [ + "BagelChatMarkers", + "BagelChatTemplate", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py new file mode 100644 index 0000000000..da13aff996 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py @@ -0,0 +1,12 @@ +"""Configuration for :class:`BagelTextEncoder`.""" + +from ...text_encoder.configuration import TextEncoderConfig + + +class BagelTextEncoderConfig(TextEncoderConfig): + """BAGEL text encoder config.""" + + model_type = "bagel_text_encoder" + + def __init__(self, *args, tie_word_embeddings: bool = True, lm_head_bias: bool = False, **kwargs): + super().__init__(*args, tie_word_embeddings=tie_word_embeddings, lm_head_bias=lm_head_bias, **kwargs) diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py new file mode 100644 index 0000000000..469caebc28 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py @@ -0,0 +1,174 @@ +"""BAGEL token embedding and LM head module.""" + +from typing import Any, Dict, List, Optional + +import torch + +from veomni.utils.tensor_utils import naflatten, unflatten + +from ....graphs.generation_graph import FSM_SIGNAL_KEY +from ....utils.conversation import ConversationItem, iter_desired_items, maybe_merge_outputs +from ...text_encoder.modeling import TextEncoder +from ..sources import BAGEL_FLOW_QUERY, BAGEL_PHASE_KEY +from .configuration import BagelTextEncoderConfig +from .processing import BagelTextEncoderPreprocessor, apply_image_marker + + +SIGNAL_START_IMAGE_GEN = "start_image_gen" +SIGNAL_TEXT_DONE = "text_done" + +# Sentinel written by BagelTextEncoderPreprocessor onto every text item so +# encode_pre can skip tokenizer work already completed by a DataLoader worker. +_OMNI_TOKENIZED = "_omni_tokenized" + + +def prepare_bagel_encode_input_ids( + conversation_list: Optional[List[List[ConversationItem]]], + *, + device: torch.device, + fallback_input_ids: torch.Tensor, +) -> tuple[torch.Tensor, torch.LongTensor | None]: + """Pack CPU-preprocessed BAGEL text token ids (training + inference).""" + if conversation_list is None: + raise ValueError("BagelTextEncoder.prepare_bagel_encode_input_ids requires conversation_list.") + + input_ids: List[torch.Tensor] = [] + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + if not item.meta.get(_OMNI_TOKENIZED): + raise ValueError("BAGEL text encoder expects CPU-preprocessed text items.") + token_ids = item.meta.get("input_ids") + input_ids.append(token_ids.reshape(-1)) + + if not input_ids: + return fallback_input_ids, None + + flat_ids, batch_shape = naflatten(input_ids) + return flat_ids.to(device, non_blocking=True), batch_shape + + +def scatter_bagel_text_embeds( + conversation_list: List[List[ConversationItem]], + segment_embeds: List[torch.Tensor], + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Write BAGEL text embeds back onto conversation text items.""" + segment_embeds_iterator = iter(segment_embeds) + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + item.value = next(segment_embeds_iterator).to(device=device, dtype=dtype) + if next(segment_embeds_iterator, None) is not None: + raise RuntimeError("BAGEL text segment count mismatch during embed scatter.") + + +class BagelTextEncoder(TextEncoder): + """BAGEL token embedding + LM head, plus the T2I generation FSM. + + Only the T2I generation FSM (image-gen signal, marker embedding) is + genuinely BAGEL-specific; accelerated-only behavior lives in + ``accelerated.py``. + """ + + config_class = BagelTextEncoderConfig + preprocessor_class = BagelTextEncoderPreprocessor + + def generate( + self, + conversation_list: Optional[List[ConversationItem]] = None, + generation_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Embed prompt text tokens or sample the next AR decode token.""" + tail = conversation_list[-1] + batched = [conversation_list] + + if tail.type == "text": + input_ids, batch_shape = prepare_bagel_encode_input_ids( + batched, + device=self.device, + fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"], + ) + inputs_embeds = self.encode(input_ids)["inputs_embeds"] + scatter_bagel_text_embeds( + batched, + unflatten(inputs_embeds, batch_shape), + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": batched[0]} + + if tail.type == "output": + outputs: Dict[str, Any] = {"conversation_list": batched[0]} + hidden_states = tail.value + if not torch.is_tensor(hidden_states): + raise TypeError("BAGEL text generate expects the tail output value to be a hidden-state tensor.") + if hidden_states.dim() == 2: + hidden_states = hidden_states.unsqueeze(0) + sampling = self._extract_sampling_kwargs(generation_kwargs, 1.0, 1.0, kwargs) + + output_token_id = self._sample_token(hidden_states, **sampling) + self._text_token_cache.append(output_token_id) + + input_ids = self._token_id_tensor(output_token_id) + inputs_embeds = self.encode(input_ids)["inputs_embeds"] + tail.value = inputs_embeds.to(device=self.device, dtype=self.dtype) + tail.meta["input_ids"] = input_ids.reshape(-1).detach() + maybe_merge_outputs(batched[0]) + if output_token_id == self._chat_template.eos_token_id: + outputs[FSM_SIGNAL_KEY] = SIGNAL_TEXT_DONE + outputs["generated"] = self._flush_text_generated(batched[0]) + return outputs + + raise ValueError(f"Invalid type: {tail.type}") + + def encode_image_markers( + self, + conversation_list: Optional[List[ConversationItem]] = None, + generation_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Bracket ``flow_query`` output items with embedded ``<|vision_start|>`` and ``<|vision_end|>`` tokens.""" + del generation_kwargs, kwargs + marker_embeds: Optional[torch.Tensor] = None + for item in iter_desired_items( + [conversation_list], + types=["output"], + meta={BAGEL_PHASE_KEY: [BAGEL_FLOW_QUERY]}, + ): + if item.is_dummy: + continue + + if marker_embeds is None: + marker_ids = torch.tensor( + [[self._chat_template.vision_start_token_id, self._chat_template.vision_end_token_id]], + dtype=torch.long, + device=self.device, + ) + marker_embeds = self.encode(marker_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype) + marker_embeds = marker_embeds.squeeze(0) + apply_image_marker(item, marker_embeds, device=self.device, dtype=self.dtype) + + return {"conversation_list": conversation_list} + + def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]: + """Return minimal dummy ``encode`` or ``decode`` inputs for FSDP anchor execution.""" + if kind == "encode": + return {"input_ids": torch.zeros(1, device=self.device, dtype=torch.long)} + return { + "hidden_states": torch.zeros(1, int(self.config.hidden_size), device=self.device, dtype=self.dtype), + "labels": torch.full((1,), -100, device=self.device, dtype=torch.long), + } + + +__all__ = [ + "BagelTextEncoder", + "BagelTextEncoderConfig", + "SIGNAL_START_IMAGE_GEN", + "SIGNAL_TEXT_DONE", + "prepare_bagel_encode_input_ids", + "scatter_bagel_text_embeds", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py new file mode 100644 index 0000000000..9fbda4b240 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py @@ -0,0 +1,110 @@ +"""Stateless text carrier helpers + worker-side preprocessor for BAGEL text encoder. + +:class:`BagelTextEncoderPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`). +Its :meth:`from_pretrained` loads the tokenizer and builds the +:class:`~veomni.models.seed_omni.modules.bagel.text_encoder.chat_template.BagelChatTemplate` +straight off the checkpoint dir — no model instance involved. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import torch + +from .....auto import build_tokenizer +from ....utils.conversation import ConversationItem +from ...text_encoder.processing import TextEncoderPreprocessor +from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context + + +if TYPE_CHECKING: + # chat_template.py imports is_bagel_vision_marker from this module, so the + # reverse import must stay deferred (see from_pretrained) to avoid a cycle. + from .chat_template import BagelChatTemplate + + +def is_bagel_vision_marker(item: ConversationItem, *, context: str | None = None) -> bool: + """Return whether ``item`` is a single-token text vision marker tagged with a BAGEL context.""" + if item.type != "text": + return False + if context is not None and bagel_context(item) != context: + return False + if bagel_context(item) not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + return False + return _text_item_length(item) == 1 + + +def _text_item_length(item: ConversationItem) -> int | None: + """Return the token count of a text ``ConversationItem``, or ``None`` if untokenized.""" + value = item.value + if torch.is_tensor(value): + if value.dim() == 0: + return 1 + if value.dim() == 1: + return int(value.shape[0]) + if value.dim() == 2: + return int(value.shape[0]) + if value.dim() == 3 and int(value.shape[0]) == 1: + return int(value.shape[1]) + return None + input_ids = item.meta.get("input_ids") + if torch.is_tensor(input_ids): + return int(input_ids.reshape(-1).shape[0]) + return None + + +def apply_image_marker( + item: ConversationItem, + marker_embeds: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Prepend and append ``marker_embeds`` to ``item.value`` if not already bracketed.""" + image_embeds = item.value + if not torch.is_tensor(image_embeds): + return + if image_embeds.dim() == 3 and image_embeds.shape[0] == 1: + image_embeds = image_embeds.squeeze(0) + if image_embeds.dim() != 2: + return + + image_embeds = image_embeds.to(device=device, dtype=dtype) + if ( + image_embeds.shape[0] >= 2 + and torch.equal(image_embeds[:1], marker_embeds[:1]) + and torch.equal(image_embeds[-1:], marker_embeds[1:]) + ): + return + item.value = torch.cat([marker_embeds[:1], image_embeds, marker_embeds[1:]], dim=0) + + +class BagelTextEncoderPreprocessor(TextEncoderPreprocessor): + """Worker-side chat-template + tokenize for BAGEL text encoder inputs.""" + + @classmethod + def build_chat_template( + cls, + module_path: str, + *, + config_overrides: dict[str, Any] | None = None, + ) -> BagelChatTemplate: + del config_overrides + from .chat_template import BagelChatTemplate # deferred: see module-level note + + return BagelChatTemplate(build_tokenizer(module_path)) + + def _tokenize_conversation_kwargs(self, inference: bool, **kwargs: Any) -> dict[str, Any]: + return { + "add_generation_prompt": inference, + "generation_kwargs": kwargs.get("generation_kwargs"), + } + + +__all__ = [ + "apply_image_marker", + "is_bagel_vision_marker", + "BagelTextEncoderPreprocessor", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/__init__.py new file mode 100644 index 0000000000..6ac7773a52 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/__init__.py @@ -0,0 +1,31 @@ +"""BAGEL latent autoencoder module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_vae") +def register_bagel_vae_config(): + from .configuration import BagelVAEConfig + + return BagelVAEConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_vae") +def register_bagel_vae_model(): + from .modeling import BagelVAE + + return BagelVAE + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_vae") +def register_bagel_vae_accelerated_model(): + from .accelerated import BagelVAEAccelerated + + return BagelVAEAccelerated + + +@OMNI_PROCESSOR_REGISTRY.register("bagel_vae") +def register_bagel_vae_processor(): + from .processing import BagelVAEProcessor + + return BagelVAEProcessor diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py new file mode 100644 index 0000000000..907701452f --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py @@ -0,0 +1,14 @@ +"""Accelerated runtime for the bagel/vae OmniModule (conversation graph).""" + +from .accelerated import ( + BAGEL_VAE_PIXEL_SHAPE, + BagelVAEAccelerated, + BagelVAEOfflineMixin, +) + + +__all__ = [ + "BAGEL_VAE_PIXEL_SHAPE", + "BagelVAEAccelerated", + "BagelVAEOfflineMixin", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py new file mode 100644 index 0000000000..cb0542b568 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -0,0 +1,457 @@ +"""SeedOmni carrier hooks for BAGEL's latent VAE module — training-graph hooks only. + +``encode_context()`` and ``decode_generated()`` (the FSM generation nodes) live +natively on :class:`~.modeling.BagelVAE` — this file only carries the training +pre/forward/post hooks and offline-encoding-cache endpoints. +""" + +from __future__ import annotations + +import math +from typing import Any + +import torch +import torch.distributed as dist + +from veomni.utils.device import get_device_id, get_device_type + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import gather_outputs +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.offline_encoding_mixin import OfflineEncodingMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem +from ...sources import BAGEL_CONTEXT_KEY, BAGEL_VAE_CONTEXT +from ..configuration import BagelVAEConfig +from ..modeling import BagelVAE, select_bagel_vae_context_items +from ..processing import ( + BAGEL_VAE_PIXEL_SHAPE, + BAGEL_VAE_POSTERIOR, + crop_latent_to_image_shape, + unpad_vae_item_pixel_values, +) + + +def _posterior_from_cache( + cache: torch.Tensor, + *, + z_channels: int, +) -> tuple[tuple[torch.Tensor, torch.Tensor], bool]: + """Unpack ``(mean, logvar)`` from a cached ``(2, C, H, W)`` or ``(1, 2, C, H, W)`` posterior tensor.""" + if cache.dim() == 5 and int(cache.shape[0]) == 1: + cache = cache.squeeze(0) + if cache.dim() == 4 and int(cache.shape[0]) == 2 and int(cache.shape[1]) == z_channels: + return (cache[0].unsqueeze(0), cache[1].unsqueeze(0)), True + raise ValueError( + "BAGEL VAE posterior cache tensor must be shaped (2, C, H, W) " + f"or singleton-batched (1, 2, C, H, W); got {tuple(cache.shape)}." + ) + + +class BagelVAEOfflineMixin(OfflineEncodingMixin): + """Offline-cache tensor endpoints and training-graph hooks for VAE encode / process.""" + + config: BagelVAEConfig + device: torch.device + dtype: torch.dtype + _conversation_carrier: list[list[ConversationItem]] | None + _encode_items: list[ConversationItem] + _sp_encode_count: int | None + + def offline_encode( + self, + pixel_values: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values`` into ``(mean, logvar)`` posterior tensor(s) for offline caching.""" + del kwargs + if isinstance(pixel_values, list): + posteriors = self._encode_posterior_list(pixel_values) + return {"encoded_cache": [torch.cat(posterior, dim=0).to(dtype=self.dtype) for posterior in posteriors]} + + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) + with self._runtime_context(pixel_values): + posterior = self._encode_posterior(pixel_values) + return {"encoded_cache": torch.cat(posterior, dim=1).to(dtype=self.dtype)} + + def online_process( + self, + encoded_cache: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Sample scaled latent grids from cached ``(mean, logvar)`` posteriors without running the encoder.""" + del kwargs + if not isinstance(encoded_cache, list): + encoded_cache = [encoded_cache] + + latents = [] + for cache in encoded_cache: + posterior, is_item_cache = _posterior_from_cache(cache, z_channels=int(self.config.z_channels)) + latent = self._sample_scaled_latents(posterior) + if is_item_cache: + latent = latent.squeeze(0) + latents.append(latent.to(dtype=cache.dtype)) + return {"latents": latents} + + @post_forward("offline_encode") + def offline_encode_post(self, encoded_cache: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: + """Gather SP shards, crop cached posteriors to each image's shape, and store on carrier items.""" + if get_parallel_state().sp_size > 1: + if isinstance(encoded_cache, list): + encoded_cache = self._gather_encode_output_list(encoded_cache) + else: + encoded_cache = self._gather_encode_output(encoded_cache) + + conversation = self._conversation_carrier + encode_items = self._encode_items + self._conversation_carrier = None + self._encode_items = [] + self._sp_encode_count = None + self._sp_latent_shapes = None + + z_channels = int(self.config.z_channels) + for item, cache_tensor in zip(encode_items, encoded_cache, strict=True): + item.type = "image" + if not isinstance(encoded_cache, list): + cache_tensor = crop_latent_to_image_shape( + cache_tensor, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels: + cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:]) + item.value = cache_tensor.detach().to(device=self.device, dtype=self.dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_VAE_POSTERIOR] = True + return {"conversation_list": conversation} + + @pre_forward("online_process") + def online_process_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Collect cached posterior tensors from VAE context items for ``online_process``.""" + del kwargs + self._conversation_carrier = conversation_list + self._encode_items = [] + self._sp_encode_count = None + + self._encode_items = select_bagel_vae_context_items(conversation_list) + encoded_cache: list[torch.Tensor] = [] + for item in self._encode_items: + cache = item.value + if not torch.is_tensor(cache): + raise ValueError("BAGEL VAE online_process requires tensor posterior cache.") + encoded_cache.append(cache.detach().to(device=self._online_process_device())) + return {"encoded_cache": encoded_cache} + + def _online_process_dtype(self) -> torch.dtype: + """Resolve the target dtype for ``online_process`` when the VAE is built on meta.""" + config_dtype = getattr(self.config, "dtype", None) or getattr(self.config, "torch_dtype", None) + if isinstance(config_dtype, torch.dtype): + return config_dtype + if isinstance(config_dtype, str) and hasattr(torch, config_dtype): + dtype = getattr(torch, config_dtype) + if isinstance(dtype, torch.dtype): + return dtype + return torch.get_default_dtype() + + def _online_process_device(self) -> torch.device: + """Resolve the target device for ``online_process`` when the VAE is built on meta.""" + device_type = get_device_type() + if device_type == "cpu": + return torch.device("cpu") + return torch.device(device_type, get_device_id()) + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelVAE` modeling APIs.""" + + config: BagelVAEConfig + device: torch.device + dtype: torch.dtype + _conversation_carrier: list[list[ConversationItem]] | None + _encode_items: list[ConversationItem] + _sp_encode_count: int | None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._encode_items: list[ConversationItem] = [] + self._sp_encode_count: int | None = None + self._sp_latent_shapes: list[tuple[int, int]] | None = None + + @pre_forward("encode", "offline_encode") + def encode_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Collect unpadded VAE context images and slice across Ulysses SP ranks if enabled.""" + del kwargs + self._conversation_carrier = conversation_list + + self._encode_items = select_bagel_vae_context_items(conversation_list) + + self._metric_meter_stash_latent_tokens(self._vae_latent_token_lengths()) + + pixel_values = [ + unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True) + for item in self._encode_items + ] + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_encode_count = None + self._sp_latent_shapes = None + return {"pixel_values": pixel_values} + + if ps.cp_size != 1: + raise ValueError(f"BAGEL VAE training supports Ulysses groups only; got cp_size={ps.cp_size}.") + + self._sp_encode_count = len(pixel_values) + if self._sp_encode_count == 0: + raise ValueError("BAGEL VAE SP requires at least one real or dummy image.") + + pad_count = (-self._sp_encode_count) % ps.sp_size + padded_pixels = pixel_values + [pixel_values[-1]] * pad_count + ds = max(int(self.config.downsample), 1) + self._sp_latent_shapes = [(int(image.shape[-2]) // ds, int(image.shape[-1]) // ds) for image in padded_pixels] + per_rank = len(padded_pixels) // ps.sp_size + start = ps.sp_rank * per_rank + return {"pixel_values": padded_pixels[start : start + per_rank]} + + def encode( + self, + pixel_values: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values``; under SP, defer posterior sampling until ``encode_post`` gathers all shards.""" + if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None: + return {"latents": self.offline_encode(pixel_values=pixel_values, **kwargs)["encoded_cache"]} + return super().encode(pixel_values=pixel_values, **kwargs) + + @post_forward("encode", "online_process") + def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: + """Gather SP posterior shards, sample scaled latents after gather, and write latents onto conversation items.""" + if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None: + if isinstance(latents, list): + gathered_cache = self._gather_encode_output_list(latents) + latents = self._sample_posteriors_list(gathered_cache) + else: + gathered_cache = self._gather_encode_output(latents) + latents = self._sample_posterior_tensor(gathered_cache, self._encode_items) + + conversation = self._conversation_carrier + encode_items = self._encode_items + self._conversation_carrier = None + self._encode_items = [] + self._sp_encode_count = None + self._sp_latent_shapes = None + + if isinstance(latents, list): + for item, latent in zip(encode_items, latents, strict=True): + item.type = "image" + item.value = latent + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + else: + for item, latent in zip(encode_items, latents, strict=True): + item.type = "image" + latent = crop_latent_to_image_shape( + latent, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + item.value = latent + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + return {"conversation_list": conversation} + + def _sample_posteriors_list(self, encoded_cache: list[torch.Tensor]) -> list[torch.Tensor]: + """Sample scaled latents in batch order from gathered ``(mean, logvar)`` posteriors.""" + z_channels = int(self.config.z_channels) + latents: list[torch.Tensor] = [] + for cache in encoded_cache: + if cache.dim() == 3 and int(cache.shape[0]) == 2 * z_channels: + cache = cache.reshape(2, z_channels, *cache.shape[-2:]) + posterior, is_item_cache = _posterior_from_cache(cache, z_channels=z_channels) + with self._runtime_context(posterior[0]): + latent = self._sample_scaled_latents(posterior) + if is_item_cache: + latent = latent.squeeze(0) + latents.append(latent.to(dtype=cache.dtype)) + return latents + + def _sample_posterior_tensor( + self, + encoded_cache: torch.Tensor, + encode_items: list[ConversationItem], + ) -> list[torch.Tensor]: + """Crop and sample scaled latents in batch order from a gathered 4-D posterior tensor.""" + z_channels = int(self.config.z_channels) + cropped_caches: list[torch.Tensor] = [] + for item, cache_tensor in zip(encode_items, encoded_cache, strict=True): + cache_tensor = crop_latent_to_image_shape( + cache_tensor, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels: + cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:]) + cropped_caches.append(cache_tensor) + return self._sample_posteriors_list(cropped_caches) + + def _encode_posterior_list( + self, + pixel_values: list[torch.Tensor], + *, + min_passes: int = 0, + ) -> list[tuple[torch.Tensor, torch.Tensor]]: + """Align the number of encoder passes across ``fsdp_group`` / ``sp_group`` so FSDP unshards stay in lockstep.""" + if pixel_values and dist.is_initialized(): + ps = get_parallel_state() + shard_group = ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None) + if shard_group is not None: + local_shapes = {(int(image.shape[-2]), int(image.shape[-1])) for image in pixel_values} + passes_tensor = torch.tensor( + max(len(local_shapes), min_passes), + device=self._encoder_device, + dtype=torch.int64, + ) + dist.all_reduce(passes_tensor, op=dist.ReduceOp.MAX, group=shard_group) + min_passes = int(passes_tensor.item()) + return super()._encode_posterior_list(pixel_values, min_passes=min_passes) + + def _gather_encode_output(self, output: torch.Tensor) -> torch.Tensor: + """All-gather batch-sharded VAE outputs across the SP group and trim SP batch padding.""" + if self._sp_encode_count is None: + raise RuntimeError("BAGEL VAE SP image count was not initialized.") + output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group) + return output.narrow(0, 0, self._sp_encode_count) + + def _gather_encode_output_list(self, outputs: list[torch.Tensor]) -> list[torch.Tensor]: + """All-gather variable-shape per-image VAE outputs across the SP group and trim SP padding.""" + if self._sp_encode_count is None or self._sp_latent_shapes is None: + raise RuntimeError("BAGEL VAE SP image shapes were not initialized.") + ps = get_parallel_state() + per_rank = len(self._sp_latent_shapes) // ps.sp_size + if len(outputs) != per_rank: + raise RuntimeError(f"BAGEL VAE SP local output count mismatch: expected {per_rank}, got {len(outputs)}.") + leading_shape = tuple(int(dim) for dim in outputs[0].shape[:-2]) + leading_numel = math.prod(leading_shape) + rank_numels = [ + sum( + leading_numel * height * width + for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank] + ) + for rank in range(ps.sp_size) + ] + max_rank_numel = max(rank_numels) + flat_local = torch.cat([tensor.reshape(-1) for tensor in outputs], dim=0) + if int(flat_local.numel()) < max_rank_numel: + flat_local = torch.nn.functional.pad(flat_local, (0, max_rank_numel - int(flat_local.numel()))) + gathered = gather_outputs(flat_local.unsqueeze(0), gather_dim=0, group=ps.sp_group) + + restored: list[torch.Tensor] = [] + for rank in range(ps.sp_size): + offset = 0 + rank_flat = gathered[rank] + for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank]: + numel = leading_numel * height * width + restored.append(rank_flat[offset : offset + numel].reshape(*leading_shape, height, width)) + offset += numel + if len(restored) == self._sp_encode_count: + return restored + return restored + + def _vae_latent_token_lengths(self) -> list[int]: + """Compute the unpadded latent grid token count per encoded image for FLOPs metering.""" + # Latent tokens per image = (H // ds) * (W // ds), using the real pixel shape + ds = max(int(self.config.downsample), 1) + lengths: list[int] = [] + for item in self._encode_items: + shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + if torch.is_tensor(shape): + dims = shape.detach().reshape(-1).tolist() + height, width = int(dims[0]), int(dims[1]) + else: + height, width = int(item.value.shape[-2]), int(item.value.shape[-1]) + lengths.append((height // ds) * (width // ds)) + return lengths + + def _metric_meter_stash_latent_tokens(self, lengths: list[int]) -> None: + # currently use the same counts for encode and offline_encode + self.metric_meter_set_seqlens("encode", lengths) + self.metric_meter_set_seqlens("offline_encode", lengths) + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's latent VAE codec (FLUX-style conv AE).""" + + config: BagelVAEConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + """Estimate TFLOPs for the FLUX-style convolutional VAE encoder over ``seqlens``.""" + cfg = self.config + + if getattr(cfg, "freeze", True): + return 0.0 + + ch = int(cfg.ch) + ch_mult = [int(m) for m in cfg.ch_mult] + in_ch_mult = [1, *ch_mult] + num_res = len(ch_mult) + num_res_blocks = int(cfg.num_res_blocks) + in_channels = int(cfg.in_channels) + z_channels = int(cfg.z_channels) + + # Conv MACs per latent token (linear term): area(level i) = N · 4**(num_res-1-i). + def area_factor(level: int) -> int: + return 4 ** (num_res - 1 - level) + + lin_macs_per_token = in_channels * ch * 9 * area_factor(0) # conv_in (3×3) + + for i in range(num_res): + af = area_factor(i) + block_in = ch * in_ch_mult[i] + block_out = ch * ch_mult[i] + for j in range(num_res_blocks): + cin = block_in if j == 0 else block_out + lin_macs_per_token += cin * block_out * 9 * af # ResnetBlock.conv1 + lin_macs_per_token += block_out * block_out * 9 * af # ResnetBlock.conv2 + if j == 0 and block_in != block_out: + lin_macs_per_token += block_in * block_out * af # 1×1 nin_shortcut + if i != num_res - 1: + # Strided Downsample (3×3) emits the next level's spatial area. + lin_macs_per_token += block_out * block_out * 9 * area_factor(i + 1) + + # Mid block at the latent resolution (area factor 1): two ResnetBlocks + AttnBlock. + bc = ch * ch_mult[-1] + lin_macs_per_token += 4 * (bc * bc * 9) # block_1 + block_2 (conv1 + conv2 each) + lin_macs_per_token += 4 * (bc * bc) # attn q/k/v/proj_out are 1×1 convs + lin_macs_per_token += bc * (2 * z_channels) * 9 # conv_out (3×3) + + tokens = sum(seqlens) + # Mid self-attention (single head, dim = bc): QK^T + attn·V ⇒ 2·N**2·bc MACs. + attn_macs = sum(2 * (n * n) * bc for n in seqlens) + + total_macs = lin_macs_per_token * tokens + attn_macs + return 6 * total_macs / 1e12 + + +class VeOmniMixin(BaseMixin, BagelVAEOfflineMixin, TrainingMixin, MeterMixin): + """Carrier hooks for raw-image VAE encode and latent decode. + + ``encode_context()`` / ``decode_generated()`` already live on the native + :class:`~.modeling.BagelVAE` (via its own :class:`~.modeling.InferenceMixin`), + so no ``InferenceMixin`` is needed here. + """ + + config: BagelVAEConfig + + +class BagelVAEAccelerated(VeOmniMixin, BagelVAE): + _no_split_modules: list[str] = [] + + +__all__ = ["BAGEL_VAE_PIXEL_SHAPE", "BagelVAEAccelerated", "BagelVAEOfflineMixin"] diff --git a/veomni/models/seed_omni/modules/bagel/vae/configuration.py b/veomni/models/seed_omni/modules/bagel/vae/configuration.py new file mode 100644 index 0000000000..c7cc266ec2 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/configuration.py @@ -0,0 +1,52 @@ +"""Configuration for BAGEL's latent VAE module.""" + +from ...module_configuration_base import OmniModuleConfig + + +class BagelVAEConfig(OmniModuleConfig): + """BAGEL FLUX-style latent autoencoder config.""" + + model_type = "bagel_vae" + + def __init__( + self, + resolution: int = 256, + in_channels: int = 3, + downsample: int = 8, + ch: int = 128, + out_ch: int = 3, + ch_mult: list[int] | None = None, + num_res_blocks: int = 2, + z_channels: int = 16, + scale_factor: float = 0.3611, + shift_factor: float = 0.1159, + max_image_size: int = 1024, + min_image_size: int = 512, + image_stride: int = 16, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + freeze: bool = True, + **kwargs, + ) -> None: + self.resolution = resolution + self.in_channels = in_channels + self.downsample = downsample + self.ch = ch + self.out_ch = out_ch + self.ch_mult = [1, 2, 4, 4] if ch_mult is None else ch_mult + self.num_res_blocks = num_res_blocks + self.z_channels = z_channels + self.scale_factor = scale_factor + self.shift_factor = shift_factor + self.max_image_size = max_image_size + self.min_image_size = min_image_size + self.image_stride = image_stride + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.freeze = freeze + super().__init__(**kwargs) + + +__all__ = ["BagelVAEConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py new file mode 100644 index 0000000000..fa518bd977 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -0,0 +1,552 @@ +"""BAGEL latent VAE module. + +Tensor call-sites: +* ``bagel_vae.encode`` maps normalized image tensors to scaled latent grids. +* ``bagel_vae.decode`` maps generated latent grids to decoded image tensors. + +The VAE module is a codec boundary only. Flow timestep/noise sampling, latent +patchification, packed MoT indexes, and conversation-carrier mutation belong to +the Bagel VAE module mixin and downstream Bagel nodes. + +``encode_context()`` (image-understanding VAE encode) and ``decode_generated()`` +(final generated-latent decode) are the FSM generation-node methods, native here +so pure HF inference (no accelerated mixins) can run BAGEL's generation graphs. +""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from typing import Any + +import torch +import torch.nn as nn +from einops import rearrange +from torch import Tensor + +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_VAE_CONTEXT +from .configuration import BagelVAEConfig +from .processing import ( + BagelVAEPreprocessor, + BagelVAEProcessor, + unpad_vae_item_pixel_values, +) + + +def select_bagel_vae_context_items( + conversation_list: list[list[ConversationItem]] | None, + *, + exclude_dummy: bool = False, +) -> list[ConversationItem]: + """Select VAE context image items (training includes dummies; inference skips them).""" + if conversation_list is None: + raise ValueError("BagelVAE requires conversation_list to select VAE context items.") + + items = list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]})) + if exclude_dummy: + return [item for item in items if not item.is_dummy] + return items + + +class InferenceMixin: + """FSM ``encode_context`` / ``decode_generated`` — HF ``GenerationMixin`` analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelVAE`'s bases for consistency with every other module's + native / accelerated split (see ``janus/llama/modeling.py`` for the full + MRO rationale where a competing no-op default exists — this module + doesn't override ``reset_local_inference_state`` / ``finalize`` so there + is nothing to shadow here). + """ + + def encode_context( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """FSM node: encode image-understanding context images into VAE latents.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + image_items = select_bagel_vae_context_items(batched, exclude_dummy=True) + if not image_items: + return {"conversation_list": conversation_list} + + pixel_values = [ + unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True) + for item in image_items + ] + outputs = self.encode(pixel_values=pixel_values) + for image_item, latent in zip(image_items, outputs["latents"], strict=True): + image_item.value = latent.to(device=self.device, dtype=self.dtype) + image_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + return {"conversation_list": conversation_list} + + def decode_generated( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """FSM node: decode the flow connector's completed latent into a PIL image.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + decode_items = self._select_vae_decode_items(batched) + if not decode_items: + return {"conversation_list": conversation_list} + + latents = [] + for item in decode_items: + latents.append( + item.value.detach().squeeze(0) + if item.value.dim() == 4 and item.value.shape[0] == 1 + else item.value.detach() + ) + latents = torch.stack(latents, dim=0).to(device=self.device, dtype=self.dtype) + + outputs = self.decode(latents=latents) + pixel_values = outputs["pixel_values"] + for item, image in zip(decode_items, pixel_values, strict=True): + item.type = "image" + item.value = image.to(device=self.device, dtype=self.dtype) + if self._image_processor is None: + raise RuntimeError( + "BagelVAE: cannot postprocess decoded image — no image processor was " + "loaded. Ensure `preprocessor_config.json` ships next to the weights." + ) + return { + "conversation_list": conversation_list, + "generated": {"type": "image", "value": self._image_processor.postprocess(pixel_values[-1])[0]}, + } + + def _select_vae_decode_items( + self, conversation_list: list[list[ConversationItem]] | None + ) -> list[ConversationItem]: + """Select non-dummy ``generated_latent`` output items for VAE image decoding.""" + if conversation_list is None: + raise ValueError("BagelVAE decode requires conversation_list to select latent items.") + + # Final image decode consumes the completed latent emitted by the flow connector. + decode_items: list[ConversationItem] = [] + for item in iter_desired_items( + conversation_list, types=["output"], meta={BAGEL_PHASE_KEY: [BAGEL_GENERATED_LATENT]} + ): + if not item.is_dummy: + decode_items.append(item) + return decode_items + + +class BagelVAE(InferenceMixin, PretrainedOmniModule): + config_class = BagelVAEConfig + image_processor_class = BagelVAEProcessor + preprocessor_class = BagelVAEPreprocessor + base_model_prefix = "bagel_vae" + main_input_name = "pixel_values" + _no_split_modules: list[str] = ["ResnetBlock", "AttnBlock"] + supports_gradient_checkpointing = False + + def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): + # The VAE uses fixed local torch SDPA in its mid-block, with no HF backend dispatch. + return attn_implementation + + def __init__(self, config: BagelVAEConfig, **kwargs: Any) -> None: + super().__init__(config, **kwargs) + self.encoder = Encoder( + resolution=config.resolution, + in_channels=config.in_channels, + ch=config.ch, + ch_mult=config.ch_mult, + num_res_blocks=config.num_res_blocks, + z_channels=config.z_channels, + ) + self.decoder = Decoder( + resolution=config.resolution, + in_channels=config.in_channels, + ch=config.ch, + out_ch=config.out_ch, + ch_mult=config.ch_mult, + num_res_blocks=config.num_res_blocks, + z_channels=config.z_channels, + ) + self._image_processor: BagelVAEProcessor | None = None + self.post_init() + + def freeze_model(self) -> None: + """Set the VAE to eval mode and disable gradients when ``config.freeze`` is true.""" + if self.config.freeze: + self.eval() + self.requires_grad_(False) + + @property + def _encoder_device(self) -> torch.device: + return self.encoder.conv_in.weight.device + + @property + def _decoder_device(self) -> torch.device: + return self.decoder.conv_in.weight.device + + @contextmanager + def _runtime_context(self, tensor: torch.Tensor): + """Enter ``no_grad`` (when frozen) and autocast for VAE encoder/decoder execution.""" + grad_context = torch.enable_grad() + if self.config.freeze: + grad_context = torch.no_grad() + + autocast_context = nullcontext() + if self.dtype != torch.float32: + autocast_context = torch.amp.autocast(tensor.device.type, enabled=True, dtype=self.dtype) + + with grad_context, autocast_context: + yield + + def encode( + self, + pixel_values: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values`` into scaled VAE latent grids.""" + del kwargs + if isinstance(pixel_values, list): + posteriors = self._encode_posterior_list(pixel_values) + latents: list[torch.Tensor] = [] + for posterior in posteriors: + with self._runtime_context(posterior[0]): + latents.append(self._sample_scaled_latents(posterior).squeeze(0).to(dtype=self.dtype)) + return {"latents": latents} + + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) + with self._runtime_context(pixel_values): + posterior = self._encode_posterior(pixel_values) + latents = self._sample_scaled_latents(posterior) + return {"latents": latents.to(dtype=self.dtype)} + + def decode( + self, + latents: torch.Tensor, + **kwargs: object, + ) -> dict[str, Any]: + """Unscale ``latents`` and decode them back to normalized pixel tensors.""" + del kwargs + latents = latents.to(device=self._decoder_device, dtype=self.dtype) + latents = latents / self.config.scale_factor + self.config.shift_factor + with self._runtime_context(latents): + pixel_values = self.decoder(latents) + return {"pixel_values": pixel_values.to(dtype=self.dtype)} + + def _encode_posterior(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return torch.chunk(self.encoder(pixel_values), 2, dim=1) + + def _encode_posterior_list( + self, + pixel_values: list[torch.Tensor], + *, + min_passes: int = 0, + ) -> list[tuple[torch.Tensor, torch.Tensor]]: + """Encode a list of ``[C, H, W]`` images at their native shapes, batching equal shapes.""" + if not pixel_values: + return [] + + shape_groups: dict[tuple[int, int], list[int]] = {} + normalized: list[torch.Tensor] = [] + for idx, image in enumerate(pixel_values): + if image.dim() == 4 and int(image.shape[0]) == 1: + image = image.squeeze(0) + if image.dim() != 3: + raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(image.shape)}.") + normalized.append(image) + shape_key = (int(image.shape[-2]), int(image.shape[-1])) + shape_groups.setdefault(shape_key, []).append(idx) + + group_indices = list(shape_groups.values()) + posteriors: list[tuple[torch.Tensor, torch.Tensor] | None] = [None] * len(normalized) + for indices in group_indices: + batch = torch.stack([normalized[i] for i in indices], dim=0).to( + device=self._encoder_device, dtype=self.dtype + ) + with self._runtime_context(batch): + mean_batch, logvar_batch = self._encode_posterior(batch) + for idx, mean_i, logvar_i in zip(indices, mean_batch, logvar_batch, strict=True): + posteriors[idx] = (mean_i.unsqueeze(0), logvar_i.unsqueeze(0)) + + if min_passes > len(group_indices): + dummy_batch = normalized[0].unsqueeze(0).to(device=self._encoder_device, dtype=self.dtype) + for _ in range(min_passes - len(group_indices)): + with self._runtime_context(dummy_batch): + mean_dummy, logvar_dummy = self._encode_posterior(dummy_batch) + assert posteriors[0] is not None + anchor = (mean_dummy.sum() + logvar_dummy.sum()) * 0.0 + posteriors[0] = (posteriors[0][0] + anchor, posteriors[0][1]) + + return posteriors # type: ignore[return-value] + + def _sample_scaled_latents(self, posterior: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + mean, logvar = posterior + latents = mean + torch.exp(0.5 * logvar) * torch.randn_like(mean) + return self.config.scale_factor * (latents - self.config.shift_factor) + + +def swish(x: Tensor) -> Tensor: + return x * torch.sigmoid(x) + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.in_channels = in_channels + self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1) + + def attention(self, hidden_states: Tensor) -> Tensor: + hidden_states = self.norm(hidden_states) + query = self.q(hidden_states) + key = self.k(hidden_states) + value = self.v(hidden_states) + + batch, channels, height, width = query.shape + query = rearrange(query, "b c h w -> b 1 (h w) c").contiguous() + key = rearrange(key, "b c h w -> b 1 (h w) c").contiguous() + value = rearrange(value, "b c h w -> b 1 (h w) c").contiguous() + hidden_states = nn.functional.scaled_dot_product_attention(query, key, value) + return rearrange(hidden_states, "b 1 (h w) c -> b c h w", h=height, w=width, c=channels, b=batch) + + def forward(self, x: Tensor) -> Tensor: + return x + self.proj_out(self.attention(x)) + + +class ResnetBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: Tensor) -> Tensor: + hidden_states = self.norm1(x) + hidden_states = swish(hidden_states) + hidden_states = self.conv1(hidden_states) + hidden_states = self.norm2(hidden_states) + hidden_states = swish(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return x + hidden_states + + +class Downsample(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x: Tensor) -> Tensor: + x = nn.functional.pad(x, (0, 1, 0, 1), mode="constant", value=0) + return self.conv(x) + + +class Upsample(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor) -> Tensor: + x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + return self.conv(x) + + +class Encoder(nn.Module): + def __init__( + self, + resolution: int, + in_channels: int, + ch: int, + ch_mult: list[int], + num_res_blocks: int, + z_channels: int, + ) -> None: + super().__init__() + self.gradient_checkpointing = False + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.conv_in = nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + in_ch_mult = (1,) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + block_in = self.ch + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in) + self.down.append(down) + + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor) -> Tensor: + """Encode input image tensor ``x`` into ``2 * z_channels`` mean/logvar parameters.""" + hidden_stack = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + block = self.down[i_level].block[i_block] + hidden_states = ( + self._gradient_checkpointing_func(block.__call__, hidden_stack[-1]) + if self.gradient_checkpointing and self.training + else block(hidden_stack[-1]) + ) + if len(self.down[i_level].attn) > 0: + attn = self.down[i_level].attn[i_block] + hidden_states = ( + self._gradient_checkpointing_func(attn.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else attn(hidden_states) + ) + hidden_stack.append(hidden_states) + if i_level != self.num_resolutions - 1: + downsample = self.down[i_level].downsample + hidden_stack.append( + self._gradient_checkpointing_func(downsample.__call__, hidden_stack[-1]) + if self.gradient_checkpointing and self.training + else downsample(hidden_stack[-1]) + ) + + hidden_states = hidden_stack[-1] + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states) + else: + hidden_states = self.mid.block_1(hidden_states) + hidden_states = self.mid.attn_1(hidden_states) + hidden_states = self.mid.block_2(hidden_states) + hidden_states = self.norm_out(hidden_states) + hidden_states = swish(hidden_states) + return self.conv_out(hidden_states) + + +class Decoder(nn.Module): + def __init__( + self, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + in_channels: int, + resolution: int, + z_channels: int, + ) -> None: + super().__init__() + self.gradient_checkpointing = False + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.ffactor = 2 ** (self.num_resolutions - 1) + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + + self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in) + self.up.insert(0, up) + + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z: Tensor) -> Tensor: + """Decode latent tensor ``z`` into reconstructed RGB image space.""" + hidden_states = self.conv_in(z) + + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states) + else: + hidden_states = self.mid.block_1(hidden_states) + hidden_states = self.mid.attn_1(hidden_states) + hidden_states = self.mid.block_2(hidden_states) + + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + block = self.up[i_level].block[i_block] + hidden_states = ( + self._gradient_checkpointing_func(block.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else block(hidden_states) + ) + if len(self.up[i_level].attn) > 0: + attn = self.up[i_level].attn[i_block] + hidden_states = ( + self._gradient_checkpointing_func(attn.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else attn(hidden_states) + ) + if i_level != 0: + upsample = self.up[i_level].upsample + hidden_states = ( + self._gradient_checkpointing_func(upsample.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else upsample(hidden_states) + ) + + hidden_states = self.norm_out(hidden_states) + hidden_states = swish(hidden_states) + return self.conv_out(hidden_states) + + +__all__ = [ + "BagelVAE", + "BagelVAEConfig", + "InferenceMixin", + "select_bagel_vae_context_items", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/processing.py b/veomni/models/seed_omni/modules/bagel/vae/processing.py new file mode 100644 index 0000000000..879b019a94 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/processing.py @@ -0,0 +1,584 @@ +"""Image processor + worker-side preprocessor for BAGEL VAE. + +:class:`BagelVAEPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) — +built straight off the checkpoint dir, with no model instance involved. Like +:mod:`..siglip_navit.processing`, BAGEL ships no separate +``preprocessor_config.json``: the image processor is derived from the +module's own ``config.json`` via :meth:`BagelVAEProcessor.from_config`. +""" + +from __future__ import annotations + +import copy +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TVF +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature + +from ....utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem, iter_desired_items +from ...module_processing_base import ModulePreprocessorBase +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context +from .configuration import BagelVAEConfig + + +_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"}) + + +class BagelVAEProcessor(BaseImageProcessor): + """BAGEL VAE image processor. + + Owns raw-image resize and normalization for VAE encode. Carrier selection, + context tagging, and latent scatter stay in the module mixin. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + image_stride: int = 16, + max_image_size: int = 1024, + min_image_size: int = 512, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.image_stride = image_stride + self.max_image_size = max_image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + + @classmethod + def from_config(cls, config: Any) -> BagelVAEProcessor: + return cls( + image_stride=int(config.image_stride), + max_image_size=int(config.max_image_size), + min_image_size=int(config.min_image_size), + max_pixels=int(config.max_pixels), + image_mean=list(config.image_mean), + image_std=list(config.image_std), + ) + + def preprocess( + self, + images: Any, + *, + return_tensors: str | None = None, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + **kwargs: Any, + ) -> BatchFeature: + """Resize, normalize, and batch-pad ``images`` into ``pixel_values`` and ``pixel_shapes``.""" + del return_tensors, kwargs + image_list = images if isinstance(images, list) else [images] + pixel_values = [ + _preprocess_image( + image, + image_stride=self.image_stride, + max_image_size=self.max_image_size, + min_image_size=self.min_image_size, + max_pixels=self.max_pixels, + image_mean=self.image_mean, + image_std=self.image_std, + ) + for image in image_list + ] + pixel_shapes = torch.tensor( + [[int(pixel.shape[-2]), int(pixel.shape[-1])] for pixel in pixel_values], + dtype=torch.long, + ) + pixel_values = torch.stack(_pad_to_batch_size(pixel_values), dim=0) + + if dtype is not None: + pixel_values = pixel_values.to(dtype=dtype) + if device is not None: + pixel_values = pixel_values.to(device=device) + pixel_shapes = pixel_shapes.to(device=device) + return BatchFeature(data={"pixel_values": pixel_values, "pixel_shapes": pixel_shapes}) + + def postprocess(self, images: torch.Tensor | list[torch.Tensor], **kwargs: Any) -> list[Image.Image]: + """Convert decoded VAE image tensors in ``[-1, 1]`` back to RGB PIL images.""" + del kwargs + image_list = images if isinstance(images, list) else [images] + out: list[Image.Image] = [] + for image in image_list: + if image.dim() == 4: + out.extend(_decoded_tensor_to_pil(item) for item in image) + else: + out.append(_decoded_tensor_to_pil(image)) + return out + + +def _preprocess_image( + image: Any, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, + image_mean: list[float], + image_std: list[float], +) -> torch.Tensor: + """Resize and normalize ``image`` into a stride-aligned ``[3, H, W]`` float32 tensor.""" + if isinstance(image, (Image.Image, np.ndarray)): + pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image)) + pil_image = _resize_pil( + pil_image, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + tensor = _pil_to_rgb_tensor(pil_image) + else: + tensor = _to_rgb_tensor(image) + tensor = _resize_tensor( + tensor, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + return tensor.sub(mean).div(std) + + +def _to_rgb_tensor(image: Any) -> torch.Tensor: + """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``.""" + if isinstance(image, Image.Image): + pil_image = _to_rgb_pil(image) + array = np.array(pil_image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + if isinstance(image, np.ndarray): + return _to_rgb_tensor(Image.fromarray(image)) + if torch.is_tensor(image): + tensor = image.detach().to(dtype=torch.float32) + if tensor.dim() != 3: + raise TypeError(f"BAGEL VAE image tensor must be 3-D, got shape {tuple(tensor.shape)}.") + if tensor.shape[0] in (1, 3, 4): + pass + elif tensor.shape[-1] in (1, 3, 4): + tensor = tensor.permute(2, 0, 1).contiguous() + else: + raise TypeError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(tensor.shape)}.") + if tensor.shape[0] == 1: + tensor = tensor.repeat(3, 1, 1) + elif tensor.shape[0] == 4: + alpha = tensor[3:4] + if alpha.max() > 1: + alpha = alpha / 255.0 + rgb = tensor[:3] + if rgb.max() > 1: + rgb = rgb / 255.0 + tensor = rgb * alpha + (1.0 - alpha) + if tensor.max() > 1: + tensor = tensor / 255.0 + return tensor[:3].clamp(0.0, 1.0) + raise TypeError(f"BAGEL VAE image item value must be PIL, numpy, or tensor, got {type(image).__name__}.") + + +def _resize_tensor( + tensor: torch.Tensor, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, +) -> torch.Tensor: + """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to stride-aligned bounds.""" + height, width = tensor.shape[-2:] + new_width, new_height = _target_size( + width, + height, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + if new_width == width and new_height == height: + return tensor + resized = F.interpolate( + tensor.unsqueeze(0), + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + antialias=True, + ) + return resized.squeeze(0).clamp(0.0, 1.0) + + +def _resize_pil( + image: Image.Image, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> Image.Image: + """Resize a PIL image with bicubic interpolation to stride-aligned bounds.""" + width, height = image.size + new_width, new_height = _target_size( + width, + height, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + if new_width == width and new_height == height: + return image + return TVF.resize( + image, + (new_height, new_width), + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ) + + +def _pad_to_batch_size(images: list[torch.Tensor]) -> list[torch.Tensor]: + """Zero-pad variable-size ``[C, H, W]`` image tensors on the bottom/right to a common canvas.""" + if not images: + return images + max_height = max(int(image.shape[-2]) for image in images) + max_width = max(int(image.shape[-1]) for image in images) + padded = [] + for image in images: + pad_height = max_height - int(image.shape[-2]) + pad_width = max_width - int(image.shape[-1]) + if pad_height or pad_width: + image = F.pad(image, (0, pad_width, 0, pad_height), value=0.0) + padded.append(image) + return padded + + +def unpad_vae_item_pixel_values(item: ConversationItem) -> torch.Tensor: + """Return ``item.value`` cropped to its ``BAGEL_VAE_PIXEL_SHAPE`` if present.""" + pixel_values = item.value + if not torch.is_tensor(pixel_values): + raise TypeError("BAGEL VAE item value must be a tensor.") + if pixel_values.dim() == 4 and int(pixel_values.shape[0]) == 1: + pixel_values = pixel_values.squeeze(0) + if pixel_values.dim() != 3: + raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(pixel_values.shape)}.") + pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + if torch.is_tensor(pixel_shape): + dims = pixel_shape.detach().reshape(-1).tolist() + if len(dims) != 2: + raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].") + height, width = int(dims[0]), int(dims[1]) + pixel_values = pixel_values[..., :height, :width] + return pixel_values + + +def crop_latent_to_image_shape( + latent: torch.Tensor, + pixel_shape: torch.Tensor | None, + *, + downsample: int, +) -> torch.Tensor: + """Crop spatial batch padding off ``latent`` using the pre-pad ``pixel_shape`` metadata.""" + if not torch.is_tensor(pixel_shape): + return latent + + shape = pixel_shape.detach().reshape(-1).to(device="cpu", dtype=torch.long) + if int(shape.numel()) != 2: + raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].") + + downsample = max(int(downsample), 1) + latent_height = int(shape[0].item()) // downsample + latent_width = int(shape[1].item()) // downsample + if latent_height <= 0 or latent_width <= 0: + raise ValueError( + "BAGEL VAE pixel shape metadata produces an empty latent crop: " + f"pixel_shape={tuple(int(v.item()) for v in shape)}, downsample={downsample}." + ) + if latent_height > int(latent.shape[-2]) or latent_width > int(latent.shape[-1]): + raise ValueError( + "BAGEL VAE latent crop exceeds encoded latent shape: " + f"crop=({latent_height}, {latent_width}), latent_shape={tuple(latent.shape)}." + ) + return latent[..., :latent_height, :latent_width] + + +def _target_size( + width: int, + height: int, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, +) -> tuple[int, int]: + """Compute stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds.""" + scale = min(max_image_size / max(width, height), 1.0) + scale = max(scale, min_image_size / min(width, height)) + new_width, new_height = _apply_scale(width, height, scale, image_stride) + if new_width * new_height > max_pixels: + scale = max_pixels / (new_width * new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride) + if max(new_width, new_height) > max_image_size: + scale = max_image_size / max(new_width, new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride) + return new_width, new_height + + +def _to_rgb_pil(image: Image.Image) -> Image.Image: + """Convert ``image`` to RGB, compositing transparency over a white background.""" + if image.mode == "RGBA" or image.info.get("transparency", None) is not None: + rgba = image.convert("RGBA") + white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255)) + white.paste(rgba, mask=rgba.split()[3]) + return white + return image.convert("RGB") + + +def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor: + array = np.array(image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + + +def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]: + new_width = round(width * scale) + new_height = round(height * scale) + return ( + max(stride, int(round(new_width / stride) * stride)), + max(stride, int(round(new_height / stride) * stride)), + ) + + +def _decoded_tensor_to_pil(image: torch.Tensor) -> Image.Image: + """Convert a single decoded ``[-1, 1]`` tensor into an RGB ``PIL.Image``.""" + if image.dim() == 4 and image.shape[0] == 1: + image = image.squeeze(0) + if image.dim() != 3: + raise ValueError(f"BAGEL VAE generated image must be rank 3, got shape {tuple(image.shape)}.") + image = (image.detach().to(dtype=torch.float32).clamp(-1.0, 1.0) * 0.5 + 0.5).clamp(0.0, 1.0) + if image.shape[0] in (1, 3, 4): + image = image[:3].permute(1, 2, 0) + elif image.shape[-1] not in (1, 3, 4): + raise ValueError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(image.shape)}.") + image = (image[..., :3] * 255.0).round().to(dtype=torch.uint8).cpu().numpy() + return Image.fromarray(image) + + +def copy_image_item(item: ConversationItem) -> ConversationItem: + """Deep-copy an image ``ConversationItem`` so VAE and SigLIP context copies do not alias.""" + value = item.value + if torch.is_tensor(value): + value = value.clone() + elif isinstance(value, Image.Image): + value = value.copy() + else: + value = copy.deepcopy(value) + return ConversationItem( + type=item.type, + value=value, + role=item.role, + is_dummy=item.is_dummy, + meta=copy.deepcopy(item.meta), + ) + + +def route_image_contexts( + conversation_list: list[list[ConversationItem]], + *, + inference: bool, + infer_type: object, +) -> None: + """Expand raw images into their SigLIP and/or VAE context copies (``BAGEL_CONTEXT_KEY``).""" + for sample in conversation_list: + routed: list[ConversationItem] = [] + for item in sample: + if item.type != "image" or item.is_dummy: + routed.append(item) + continue + + if bagel_context(item) in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + routed.append(item) + continue + + # Interactive inference has no dataset preprocessor, so raw user + # request images use ``infer_type`` in generation_kwargs for routing. + if inference and item.role == "user": + if infer_type == "infer_edit": + vae_item = copy_image_item(item) + vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.extend([vae_item, item]) + else: + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.append(item) + continue + + # Training data declares each raw image's role via ``_img_tag``. + tag = item.meta.get(_IMG_TAG_KEY) + if tag is None or tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL route_image_contexts received image with invalid {_IMG_TAG_KEY}: {tag!r}.") + if tag == "und": + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.append(item) + elif tag == "gen": + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + routed.append(item) + else: + # tag == "edit": the same physical image is both VAE context and + # SigLIP context. Preserve the infer_edit ordering (VAE before + # SigLIP) so downstream marker wrapping stays stable. + vae_item = copy_image_item(item) + vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.extend([vae_item, item]) + + sample[:] = routed + + +BAGEL_VAE_PIXEL_SHAPE = "bagel_vae_pixel_shape" +# Set by ``offline_encode``: the item already holds its cached posterior, not pixels. +BAGEL_VAE_POSTERIOR = "bagel_vae_posterior" + + +class BagelVAEPreprocessor(ModulePreprocessorBase): + """Worker-side image normalize for BAGEL VAE context/target images.""" + + def __init__( + self, + image_processor: Any, + dtype: torch.dtype | None = None, + dummy_pixel_values: torch.Tensor | None = None, + dummy_pixel_shape: torch.Tensor | None = None, + ) -> None: + self._image_processor = image_processor + self._dtype = dtype + self._dummy_pixel_values = dummy_pixel_values + self._dummy_pixel_shape = dummy_pixel_shape + + @classmethod + def from_pretrained( + cls, + module_path: str, + *, + config_overrides: dict[str, Any] | None = None, + **kwargs: Any, + ) -> BagelVAEPreprocessor: + """Build straight from the checkpoint dir — no model instance needed. + + BAGEL ships no standalone ``preprocessor_config.json``; the image + processor is derived from the module's own ``config.json``, with + ``config_overrides`` (the module's YAML ``model_config:`` block) on top. + """ + del kwargs + # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare. + config = BagelVAEConfig.from_pretrained(module_path, model_config=config_overrides) + return cls(BagelVAEProcessor.from_config(config)) + + def bind_dummy_inputs(self, config: BagelVAEConfig, dtype: torch.dtype | None = None) -> None: + """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model.""" + size = max(int(config.image_stride), int(config.downsample) * 2) + self._dtype = dtype + self._dummy_pixel_values = torch.zeros(int(config.in_channels), size, size, dtype=dtype) + self._dummy_pixel_shape = torch.tensor([size, size], dtype=torch.long) + + def preprocess_conversations( + self, + conversation_list: list[list[ConversationItem]], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> None: + """Route image contexts, normalize VAE pixel tensors, and append per-sample dummy anchors.""" + infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type") + route_image_contexts(conversation_list, inference=inference, infer_type=infer_type) + + image_items: list[ConversationItem] = [] + missing_samples: list[list[ConversationItem]] = [] + for sample in conversation_list: + sample_image_items = list( + iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}) + ) + if sample_image_items: + for item in sample_image_items: + if item.meta.get(BAGEL_VAE_POSTERIOR) and _OMNI_TOKEN_LEN not in item.meta: + pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1) + if torch.is_tensor(pixel_shape) and int(pixel_shape.numel()) == 2: + dims = pixel_shape.detach().reshape(-1).tolist() + item.meta[_OMNI_TOKEN_LEN] = (int(dims[0]) // stride) * (int(dims[1]) // stride) + image_items.extend( + item for item in sample_image_items if not item.is_dummy and not item.meta.get(BAGEL_VAE_POSTERIOR) + ) + elif not inference: + missing_samples.append(sample) + + stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1) + if image_items: + inputs = self._image_processor( + images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype + ) + for item, pixels, pixel_shape in zip( + image_items, inputs["pixel_values"], inputs["pixel_shapes"], strict=True + ): + dims = pixel_shape.detach().reshape(-1).tolist() + height, width = int(dims[0]), int(dims[1]) + item.value = pixels[..., :height, :width].contiguous().to(dtype=self._dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_VAE_PIXEL_SHAPE] = pixel_shape.to(dtype=torch.long) + item.meta[_OMNI_TOKEN_LEN] = max(1, (height // stride) * (width // stride)) + + if not missing_samples: + return + + if image_items: + # VAE encode stacks every context item in the micro-batch, so dummies must + # match a real processed image shape whenever the batch has one. + reference_item = image_items[0] + dummy_pixel_values = torch.zeros_like(reference_item.value, dtype=self._dtype) + dummy_pixel_shape = reference_item.meta[BAGEL_VAE_PIXEL_SHAPE].to(dtype=torch.long) + else: + if self._dummy_pixel_values is None: + raise RuntimeError( + f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() " + "before training use (pure inference never reaches this branch)." + ) + dummy_pixel_values = self._dummy_pixel_values.to(dtype=self._dtype) + dummy_pixel_shape = self._dummy_pixel_shape.to(dtype=torch.long) + + dummy_dims = dummy_pixel_shape.detach().reshape(-1).tolist() + dummy_token_len = max(1, (int(dummy_dims[0]) // stride) * (int(dummy_dims[1]) // stride)) + + # The placeholder stands in for a generation target, the primary VAE row. + for sample in missing_samples: + sample.append( + ConversationItem( + type="image", + value=dummy_pixel_values.clone(), + role="assistant", + is_dummy=True, + meta={ + _IMG_TAG_KEY: "gen", + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + BAGEL_VAE_PIXEL_SHAPE: dummy_pixel_shape.clone(), + _OMNI_TOKEN_LEN: dummy_token_len, + }, + ) + ) + + +__all__ = [ + "BagelVAEProcessor", + "BagelVAEPreprocessor", + "copy_image_item", + "crop_latent_to_image_shape", + "route_image_contexts", + "unpad_vae_item_pixel_values", +] diff --git a/veomni/models/seed_omni/processing_omni.py b/veomni/models/seed_omni/processing_omni.py index 9ace3c8be2..d44926d90c 100644 --- a/veomni/models/seed_omni/processing_omni.py +++ b/veomni/models/seed_omni/processing_omni.py @@ -78,8 +78,14 @@ def _as_list(media: MediaInput | None) -> list[Any]: class OmniProcessor: """Composed SeedOmni request preprocessor (HF ``AutoProcessor``-style API).""" - def __init__(self, preprocessors: dict[str, ModulePreprocessorBase]) -> None: + def __init__( + self, + preprocessors: dict[str, ModulePreprocessorBase], + *, + infer_type: str | None = None, + ) -> None: self._preprocessors: dict[str, ModulePreprocessorBase] = dict(preprocessors) + self.infer_type: str | None = infer_type def __len__(self) -> int: """Number of worker-side preprocessors contributed by active graph modules.""" @@ -148,7 +154,7 @@ def from_config( if preprocessor is not None: preprocessors[name] = preprocessor logger.info_rank0(f"OmniProcessor: module '{name}' contributes {type(preprocessor).__name__}.") - return cls(preprocessors) + return cls(preprocessors, infer_type=config.infer_type) @classmethod def from_pretrained( @@ -253,6 +259,8 @@ def preprocess_batch( Training uses ``inference=False`` (default); single-request inference uses :meth:`__call__` with ``inference=True``. """ + if inference and self.infer_type is not None and generation_kwargs.get("infer_type") is None: + generation_kwargs["infer_type"] = self.infer_type for preprocessor in self._preprocessors.values(): preprocessor( batch, diff --git a/veomni/models/seed_omni/utils/hf_layout.py b/veomni/models/seed_omni/utils/hf_layout.py index 2206705c35..885143eece 100644 --- a/veomni/models/seed_omni/utils/hf_layout.py +++ b/veomni/models/seed_omni/utils/hf_layout.py @@ -308,12 +308,11 @@ def _write_hf_view(source: HFSource) -> str: module_configs = layout.build_module_configs(source.path) training_graphs, generation_graphs = layout.load_graphs() - infer_type = layout.infer_type if layout.infer_type in generation_graphs else None config = OmniConfig( _module_entries={name: {"model_path": name} for name in module_configs}, training_graphs=training_graphs, generation_graphs=generation_graphs, - infer_type=infer_type, + infer_type=layout.infer_type, ) config._module_configs = dict(module_configs) config.save_pretrained(view) diff --git a/veomni/ops/README.md b/veomni/ops/README.md index 431534cbe0..6b4313fa4a 100644 --- a/veomni/ops/README.md +++ b/veomni/ops/README.md @@ -43,7 +43,7 @@ when the selected implementation runs. | Op | Variants | Implementations | |---|---|---| | `attention` | `standard` | `eager`, `sdpa`, and FlashAttention/AITER/FlexAttention/MagiAttention/SageAttention names | -| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit` | `eager` orchestration | +| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit`, `bagel` | `eager` orchestration | | `rms_norm` | `standard`, `deepseek_v4`, `unweighted`, `offset` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; all accept optional `group_size` (fused weighted rows delegate grouped calls to eager) | | `layer_norm` | `standard` | `eager`, `apex` (`fused_layer_norm_cuda`); nested by async Ulysses QKV when `norm_type="layernorm"` | | `rope` | `full`, `partial`, `interleave`, `mrope`, `deepseek_v4`, `wan` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; `full` also accepts rank-3 vision layout | diff --git a/veomni/ops/kernels/async_ulysses/__init__.py b/veomni/ops/kernels/async_ulysses/__init__.py index 114f4093ef..9b065a579e 100644 --- a/veomni/ops/kernels/async_ulysses/__init__.py +++ b/veomni/ops/kernels/async_ulysses/__init__.py @@ -14,13 +14,15 @@ """Async Ulysses compound kernels. -``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard`` and ``dit`` -eager rows. Nested RMSNorm and LayerNorm use raw kernel pairs. +``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard``, ``dit``, +and ``bagel`` eager rows. Nested RMSNorm, LayerNorm, and RoPE use raw kernel pairs. """ from ...registry import register_op +from .o_proj.bagel import eager as o_bagel from .o_proj.dit import eager as o_dit from .o_proj.standard import eager as o_standard +from .qkv_proj.bagel import eager as qkv_bagel from .qkv_proj.dit import eager as qkv_dit from .qkv_proj.standard import eager as qkv_standard @@ -41,6 +43,14 @@ qkv_dit.backward, description="VeOmni asynchronous Ulysses QKV projection for diffusion transformers", ) +register_op( + "async_ulysses_qkv", + "bagel", + "eager", + qkv_bagel.forward, + qkv_bagel.backward, + description="VeOmni asynchronous Ulysses QKV projection for BAGEL two-route MoT attention", +) register_op( "async_ulysses_o", "standard", @@ -57,3 +67,11 @@ o_dit.backward, description="VeOmni asynchronous Ulysses output projection for diffusion transformers", ) +register_op( + "async_ulysses_o", + "bagel", + "eager", + o_bagel.forward, + o_bagel.backward, + description="VeOmni asynchronous Ulysses output projection for BAGEL two-route MoT attention", +) diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py new file mode 100644 index 0000000000..bc1d3afdf2 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""async_ulysses_o bagel implementations.""" diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py new file mode 100644 index 0000000000..41cdabfdc3 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py @@ -0,0 +1,158 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BAGEL two-route MoT async Ulysses output projection: inverse A2A then routed linear.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn.functional as F +from torch import Tensor + +from ......distributed.sequence_parallel.comm import get_ulysses_sequence_parallel_group +from ......distributed.sequence_parallel.ulysses import all_to_all_tensor +from ......distributed.sequence_parallel.utils import ( + padding_tensor_for_seqeunce_parallel, + unpadding_tensor_for_seqeunce_parallel, +) +from .....registry import SavedState +from ...shared.backward import linear_input_backward, linear_parameter_backward + + +@dataclass +class _BagelOMeta: + """Saved all-to-all group, input sequence length, head geometry, and bias flags.""" + + sp_group: Any + input_sequence_length: int + num_heads: int + head_dim: int + has_proj_bias_0: bool + has_proj_bias_1: bool + + +def forward( + hidden_states: Tensor, + route_0_indices: Tensor, + route_1_indices: Tensor, + proj_weight_0: Tensor, + proj_bias_0: Tensor | None, + proj_weight_1: Tensor, + proj_bias_1: Tensor | None, + *, + group: Any = None, +) -> tuple[Tensor, SavedState]: + """Inverse Ulysses exchange followed by a two-route MoT output projection.""" + sp_group = get_ulysses_sequence_parallel_group() if group is None else group + if sp_group is None: + raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.") + input_sequence_length = hidden_states.shape[0] + hidden_states = padding_tensor_for_seqeunce_parallel(hidden_states, 0, group=sp_group) + hidden_states = all_to_all_tensor( + hidden_states, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=False, + ) + num_heads = hidden_states.shape[1] + head_dim = hidden_states.shape[2] + flattened_states = hidden_states.reshape(hidden_states.shape[0], num_heads * head_dim) + + output_0 = F.linear(flattened_states.index_select(0, route_0_indices), proj_weight_0, proj_bias_0) + output_1 = F.linear(flattened_states.index_select(0, route_1_indices), proj_weight_1, proj_bias_1) + output = output_0.new_zeros((flattened_states.shape[0], proj_weight_0.shape[0])) + output.index_copy_(0, route_0_indices, output_0) + output.index_copy_(0, route_1_indices, output_1) + + return output, SavedState( + ( + flattened_states, + route_0_indices, + route_1_indices, + proj_weight_0, + proj_weight_1, + ), + _BagelOMeta( + sp_group=sp_group, + input_sequence_length=input_sequence_length, + num_heads=num_heads, + head_dim=head_dim, + has_proj_bias_0=proj_bias_0 is not None, + has_proj_bias_1=proj_bias_1 is not None, + ), + ) + + +def backward(grad_output: Tensor, saved: SavedState) -> tuple[Tensor | None, ...]: + """Compute routed input gradients, launch reverse all-to-all, and overlap parameter gradients.""" + meta = saved.metadata + assert isinstance(meta, _BagelOMeta) + flattened_states, route_0_indices, route_1_indices, proj_weight_0, proj_weight_1 = saved.tensors + grad_output_0 = grad_output.index_select(0, route_0_indices) + grad_output_1 = grad_output.index_select(0, route_1_indices) + + flattened_states_0 = flattened_states.index_select(0, route_0_indices) + flattened_states_1 = flattened_states.index_select(0, route_1_indices) + grad_flattened_states = torch.zeros_like(flattened_states) + grad_flattened_states.index_copy_( + 0, + route_0_indices, + linear_input_backward(grad_output_0, flattened_states_0, proj_weight_0), + ) + grad_flattened_states.index_copy_( + 0, + route_1_indices, + linear_input_backward(grad_output_1, flattened_states_1, proj_weight_1), + ) + grad_hidden_states = grad_flattened_states.reshape(flattened_states.shape[0], meta.num_heads, meta.head_dim) + grad_hidden_wait = all_to_all_tensor( + grad_hidden_states, + scatter_dim=1, + gather_dim=0, + group=meta.sp_group, + async_op=True, + ) + + grad_proj_weight_0, grad_proj_bias_0 = linear_parameter_backward( + grad_output_0, + flattened_states_0, + proj_weight_0, + has_bias=meta.has_proj_bias_0, + ) + grad_proj_weight_1, grad_proj_bias_1 = linear_parameter_backward( + grad_output_1, + flattened_states_1, + proj_weight_1, + has_bias=meta.has_proj_bias_1, + ) + + grad_hidden_states = unpadding_tensor_for_seqeunce_parallel( + grad_hidden_wait(), + 0, + meta.input_sequence_length, + group=meta.sp_group, + ) + return ( + grad_hidden_states, + None, + None, + grad_proj_weight_0, + grad_proj_bias_0, + grad_proj_weight_1, + grad_proj_bias_1, + ) diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py new file mode 100644 index 0000000000..693dc5493e --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""async_ulysses_qkv bagel implementations.""" diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py new file mode 100644 index 0000000000..1f0e0111c9 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py @@ -0,0 +1,424 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BAGEL two-route MoT async Ulysses QKV: routed project, QK norm, RoPE, then A2A.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import Tensor + +from ......distributed.sequence_parallel.comm import ( + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, +) +from ......distributed.sequence_parallel.ulysses import all_to_all_tensor +from ......distributed.sequence_parallel.utils import ( + padding_tensor_for_seqeunce_parallel, + unpadding_tensor_for_seqeunce_parallel, +) +from .....compound import InnerHandle, append_inner, resolve_inner_op, take_inner +from .....registry import OpEntry, SavedState +from ...shared.backward import linear_backward, reduce_repeated_kv_gradient + + +@dataclass +class _BagelQKVMeta: + """Non-tensor metadata and nested-op specs for two-route MoT QKV backward.""" + + sp_group: Any + head_dim: int + num_q_heads: int + num_kv_heads: int + repeat_kv: int + has_q_bias_0: bool + has_k_bias_0: bool + has_v_bias_0: bool + has_q_bias_1: bool + has_k_bias_1: bool + has_v_bias_1: bool + rms: OpEntry + rope: OpEntry + rms_q_0: tuple[Any, int] + rms_q_1: tuple[Any, int] + rms_k_0: tuple[Any, int] + rms_k_1: tuple[Any, int] + rope_q: tuple[Any, int] + rope_k: tuple[Any, int] + + +def _resolve_sp_group_and_size(group: Any) -> tuple[Any, int]: + """Return the active Ulysses process group and its world size.""" + sp_group = get_ulysses_sequence_parallel_group() if group is None else group + if sp_group is None: + raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.") + ulysses_size = ( + dist.get_world_size(sp_group) + if isinstance(sp_group, dist.ProcessGroup) + else get_ulysses_sequence_parallel_world_size() + ) + return sp_group, ulysses_size + + +def _project_route( + hidden_states: Tensor, + indices: Tensor, + weight: Tensor, + bias: Tensor | None, + num_heads: int, + head_dim: int, +) -> Tensor: + """Project rows selected by ``indices`` and reshape to ``[tokens, heads, head_dim]``.""" + projected = F.linear(hidden_states.index_select(0, indices), weight, bias) + return projected.reshape(-1, num_heads, head_dim) + + +def _rotate_single_forward( + rope: OpEntry, + x: Tensor, + cos: Tensor, + sin: Tensor, +) -> tuple[Tensor, SavedState]: + """Apply full RoPE to one ``[tokens, heads, head_dim]`` tensor before all-to-all.""" + q_in = x.transpose(0, 1).unsqueeze(0) + dummy_k = q_in.new_zeros((1, 1, x.shape[0], x.shape[2])) + (q_out, _), saved = rope.forward(q_in, dummy_k, cos.unsqueeze(0), sin.unsqueeze(0), None, 1) + return q_out.squeeze(0).transpose(0, 1).contiguous(), saved + + +def _rotate_single_backward( + rope: OpEntry, + grad_output: Tensor, + saved: SavedState, +) -> Tensor: + """Invert ``_rotate_single_forward`` on a ``[tokens, heads, head_dim]`` gradient.""" + grad_q = grad_output.transpose(0, 1).unsqueeze(0) + dummy_grad_k = grad_q.new_zeros((1, 1, grad_output.shape[0], grad_output.shape[2])) + dq, *_ = rope.backward((grad_q, dummy_grad_k), saved) + return dq.squeeze(0).transpose(0, 1).contiguous() + + +def forward( + hidden_states: Tensor, + route_0_indices: Tensor, + route_1_indices: Tensor, + position_cos: Tensor, + position_sin: Tensor, + q_weight_0: Tensor, + q_bias_0: Tensor | None, + k_weight_0: Tensor, + k_bias_0: Tensor | None, + v_weight_0: Tensor, + v_bias_0: Tensor | None, + q_norm_weight_0: Tensor, + k_norm_weight_0: Tensor, + q_weight_1: Tensor, + q_bias_1: Tensor | None, + k_weight_1: Tensor, + k_bias_1: Tensor | None, + v_weight_1: Tensor, + v_bias_1: Tensor | None, + q_norm_weight_1: Tensor, + k_norm_weight_1: Tensor, + *, + head_dim: int, + rms_norm_eps: float, + unpadded_dim_size: int, + group: Any = None, + rms_norm: InnerHandle = None, + rope: InnerHandle = None, +) -> tuple[tuple[Tensor, Tensor, Tensor], SavedState]: + """Two-route Q/K/V projection, per-route RMSNorm, RoPE, and async Ulysses exchange. + + Inputs are token-major ``[local_sequence, hidden_size]`` with local route + index tensors ``route_0_indices`` and ``route_1_indices``. Outputs are + ``[global_sequence, local_heads, head_dim]`` for Q, K, and V. + """ + num_q_heads = q_weight_0.shape[0] // head_dim + num_kv_heads = k_weight_0.shape[0] // head_dim + + sp_group, ulysses_size = _resolve_sp_group_and_size(group) + if num_q_heads % ulysses_size != 0: + raise ValueError(f"num_query_heads ({num_q_heads}) must be divisible by ulysses_size ({ulysses_size}).") + if ulysses_size > num_kv_heads: + if ulysses_size % num_kv_heads != 0: + raise ValueError( + f"ulysses_size ({ulysses_size}) must be divisible by num_key_value_heads ({num_kv_heads})." + ) + repeat_kv = ulysses_size // num_kv_heads + else: + if num_kv_heads % ulysses_size != 0: + raise ValueError( + f"num_key_value_heads ({num_kv_heads}) must be divisible by ulysses_size ({ulysses_size})." + ) + repeat_kv = 1 + + sequence_length = hidden_states.shape[0] + rms_entry = resolve_inner_op(rms_norm, op="rms_norm", variant="standard") + rope_entry = resolve_inner_op(rope, op="rope", variant="full") + + # Q projection + per-route RMSNorm + RoPE, followed by the first async + # exchange. K/V projection executes while earlier communication is in flight. + q_0 = _project_route(hidden_states, route_0_indices, q_weight_0, q_bias_0, num_q_heads, head_dim) + q_1 = _project_route(hidden_states, route_1_indices, q_weight_1, q_bias_1, num_q_heads, head_dim) + normalized_q_0, saved_rms_q_0 = rms_entry.forward(q_0, q_norm_weight_0, eps=rms_norm_eps) + normalized_q_1, saved_rms_q_1 = rms_entry.forward(q_1, q_norm_weight_1, eps=rms_norm_eps) + query = normalized_q_0.new_zeros((sequence_length, num_q_heads, head_dim)) + query.index_copy_(0, route_0_indices, normalized_q_0) + query.index_copy_(0, route_1_indices, normalized_q_1) + query, saved_rope_q = _rotate_single_forward(rope_entry, query, position_cos, position_sin) + query_wait = all_to_all_tensor( + query, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + k_0 = _project_route(hidden_states, route_0_indices, k_weight_0, k_bias_0, num_kv_heads, head_dim) + k_1 = _project_route(hidden_states, route_1_indices, k_weight_1, k_bias_1, num_kv_heads, head_dim) + normalized_k_0, saved_rms_k_0 = rms_entry.forward(k_0, k_norm_weight_0, eps=rms_norm_eps) + normalized_k_1, saved_rms_k_1 = rms_entry.forward(k_1, k_norm_weight_1, eps=rms_norm_eps) + key = normalized_k_0.new_zeros((sequence_length, num_kv_heads, head_dim)) + key.index_copy_(0, route_0_indices, normalized_k_0) + key.index_copy_(0, route_1_indices, normalized_k_1) + key, saved_rope_k = _rotate_single_forward(rope_entry, key, position_cos, position_sin) + if repeat_kv > 1: + key = torch.repeat_interleave(key, dim=1, repeats=repeat_kv) + key_wait = all_to_all_tensor( + key, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + v_0 = _project_route(hidden_states, route_0_indices, v_weight_0, v_bias_0, num_kv_heads, head_dim) + v_1 = _project_route(hidden_states, route_1_indices, v_weight_1, v_bias_1, num_kv_heads, head_dim) + value = v_0.new_zeros((sequence_length, num_kv_heads, head_dim)) + value.index_copy_(0, route_0_indices, v_0) + value.index_copy_(0, route_1_indices, v_1) + if repeat_kv > 1: + value = torch.repeat_interleave(value, dim=1, repeats=repeat_kv) + value_wait = all_to_all_tensor( + value, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + query = unpadding_tensor_for_seqeunce_parallel(query_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + key = unpadding_tensor_for_seqeunce_parallel(key_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + value = unpadding_tensor_for_seqeunce_parallel(value_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + + saved: list[Tensor | None] = [ + hidden_states, + route_0_indices, + route_1_indices, + q_weight_0, + k_weight_0, + v_weight_0, + q_weight_1, + k_weight_1, + v_weight_1, + ] + meta = _BagelQKVMeta( + sp_group=sp_group, + head_dim=head_dim, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + repeat_kv=repeat_kv, + has_q_bias_0=q_bias_0 is not None, + has_k_bias_0=k_bias_0 is not None, + has_v_bias_0=v_bias_0 is not None, + has_q_bias_1=q_bias_1 is not None, + has_k_bias_1=k_bias_1 is not None, + has_v_bias_1=v_bias_1 is not None, + rms=rms_entry, + rope=rope_entry, + rms_q_0=append_inner(saved, saved_rms_q_0), + rms_q_1=append_inner(saved, saved_rms_q_1), + rms_k_0=append_inner(saved, saved_rms_k_0), + rms_k_1=append_inner(saved, saved_rms_k_1), + rope_q=append_inner(saved, saved_rope_q), + rope_k=append_inner(saved, saved_rope_k), + ) + return (query, key, value), SavedState(tuple(saved), meta) + + +def backward(grad_output: tuple[Tensor, Tensor, Tensor], saved: SavedState) -> tuple[Tensor | None, ...]: + """Overlap inverse Ulysses all-to-all exchanges with routed RoPE, RMSNorm, and linear gradients.""" + meta = saved.metadata + assert isinstance(meta, _BagelQKVMeta) + ( + hidden_states, + route_0_indices, + route_1_indices, + q_weight_0, + k_weight_0, + v_weight_0, + q_weight_1, + k_weight_1, + v_weight_1, + *rest_list, + ) = saved.tensors + rest = tuple(rest_list) + saved_rms_q_0, rest = take_inner(rest, meta.rms_q_0) + saved_rms_q_1, rest = take_inner(rest, meta.rms_q_1) + saved_rms_k_0, rest = take_inner(rest, meta.rms_k_0) + saved_rms_k_1, rest = take_inner(rest, meta.rms_k_1) + saved_rope_q, rest = take_inner(rest, meta.rope_q) + saved_rope_k, rest = take_inner(rest, meta.rope_k) + + grad_query, grad_key, grad_value = grad_output + sp_group = meta.sp_group + repeat_kv = meta.repeat_kv + num_kv_heads = meta.num_kv_heads + hidden_0 = hidden_states.index_select(0, route_0_indices) + hidden_1 = hidden_states.index_select(0, route_1_indices) + + # V inverse exchange has no independent projection work before it, so + # collect it first. K then overlaps V projection gradients, and Q + # overlaps K normalization/projection gradients. + grad_value = padding_tensor_for_seqeunce_parallel(grad_value.contiguous(), 0, group=sp_group) + grad_value_wait = all_to_all_tensor( + grad_value, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + grad_value = grad_value_wait() + if repeat_kv > 1: + grad_value = reduce_repeated_kv_gradient( + grad_value, + num_kv_heads, + repeat_kv, + head_dimension=1, + ) + + grad_key = padding_tensor_for_seqeunce_parallel(grad_key.contiguous(), 0, group=sp_group) + grad_key_wait = all_to_all_tensor( + grad_key, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + + grad_hidden_v_0, grad_v_weight_0, grad_v_bias_0 = linear_backward( + grad_value.index_select(0, route_0_indices), + hidden_0, + v_weight_0, + has_bias=meta.has_v_bias_0, + ) + grad_hidden_v_1, grad_v_weight_1, grad_v_bias_1 = linear_backward( + grad_value.index_select(0, route_1_indices), + hidden_1, + v_weight_1, + has_bias=meta.has_v_bias_1, + ) + + grad_key = grad_key_wait() + if repeat_kv > 1: + grad_key = reduce_repeated_kv_gradient( + grad_key, + num_kv_heads, + repeat_kv, + head_dimension=1, + ) + + grad_query = padding_tensor_for_seqeunce_parallel(grad_query.contiguous(), 0, group=sp_group) + grad_query_wait = all_to_all_tensor( + grad_query, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + + grad_key = _rotate_single_backward(meta.rope, grad_key, saved_rope_k) + grad_k_0, grad_k_norm_weight_0 = meta.rms.backward( + grad_key.index_select(0, route_0_indices), + saved_rms_k_0, + ) + grad_k_1, grad_k_norm_weight_1 = meta.rms.backward( + grad_key.index_select(0, route_1_indices), + saved_rms_k_1, + ) + grad_hidden_k_0, grad_k_weight_0, grad_k_bias_0 = linear_backward( + grad_k_0, hidden_0, k_weight_0, has_bias=meta.has_k_bias_0 + ) + grad_hidden_k_1, grad_k_weight_1, grad_k_bias_1 = linear_backward( + grad_k_1, hidden_1, k_weight_1, has_bias=meta.has_k_bias_1 + ) + + grad_query = grad_query_wait() + grad_query = _rotate_single_backward(meta.rope, grad_query, saved_rope_q) + grad_q_0, grad_q_norm_weight_0 = meta.rms.backward( + grad_query.index_select(0, route_0_indices), + saved_rms_q_0, + ) + grad_q_1, grad_q_norm_weight_1 = meta.rms.backward( + grad_query.index_select(0, route_1_indices), + saved_rms_q_1, + ) + grad_hidden_q_0, grad_q_weight_0, grad_q_bias_0 = linear_backward( + grad_q_0, hidden_0, q_weight_0, has_bias=meta.has_q_bias_0 + ) + grad_hidden_q_1, grad_q_weight_1, grad_q_bias_1 = linear_backward( + grad_q_1, hidden_1, q_weight_1, has_bias=meta.has_q_bias_1 + ) + + grad_hidden_states = torch.zeros_like(hidden_states) + grad_hidden_states.index_add_( + 0, + route_0_indices, + grad_hidden_q_0 + grad_hidden_k_0 + grad_hidden_v_0, + ) + grad_hidden_states.index_add_( + 0, + route_1_indices, + grad_hidden_q_1 + grad_hidden_k_1 + grad_hidden_v_1, + ) + + return ( + grad_hidden_states, + None, + None, + None, + None, + grad_q_weight_0, + grad_q_bias_0, + grad_k_weight_0, + grad_k_bias_0, + grad_v_weight_0, + grad_v_bias_0, + grad_q_norm_weight_0, + grad_k_norm_weight_0, + grad_q_weight_1, + grad_q_bias_1, + grad_k_weight_1, + grad_k_bias_1, + grad_v_weight_1, + grad_v_bias_1, + grad_q_norm_weight_1, + grad_k_norm_weight_1, + ) diff --git a/veomni/trainer/omni/omni_inferencer.py b/veomni/trainer/omni/omni_inferencer.py index 15f31d47a8..b54c2d3620 100644 --- a/veomni/trainer/omni/omni_inferencer.py +++ b/veomni/trainer/omni/omni_inferencer.py @@ -242,6 +242,7 @@ def _run(self, req: InferenceRequest) -> dict[str, Any]: videos=req.videos or None, mm_configs=req.mm_configs or None, inference=True, + **req.generation_kwargs, ) if self._distributed: # Every module sits on this rank's device, as in training. An eager