From 19e28142ff7edb17f2904d57369c7ec90e46d412 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 14:02:32 +0800 Subject: [PATCH 01/22] [omni, model, ckpt, config, docs] feat: support Bagel in SeedOmni Co-authored-by: Cursor --- .../seed_omni/Bagel/bagel_7b_mot/data.yaml | 18 + .../bagel_7b_mot/infer/graph_infer_edit.yaml | 37 + .../bagel_7b_mot/infer/graph_infer_gen.yaml | 34 + .../bagel_7b_mot/infer/graph_infer_und.yaml | 19 + .../infer/modules_infer_eager.yaml | 42 + .../infer/modules_infer_fsdp.yaml | 33 + .../bagel_7b_mot/offline_cache/base.yaml | 101 ++ .../offline_cache/graph_train.yaml | 2 + .../offline_cache/modules_train.yaml | 5 + .../Bagel/bagel_7b_mot/train/base.yaml | 101 ++ .../Bagel/bagel_7b_mot/train/graph_train.yaml | 9 + .../bagel_7b_mot/train/modules_train.yaml | 21 + .../Bagel/bagel_7b_mot/with_cache/base.yaml | 99 ++ .../bagel_7b_mot/with_cache/graph_train.yaml | 9 + .../with_cache/modules_train.yaml | 21 + docs/index.md | 7 + docs/seed_omni/models/bagel.md | 352 ++++ tests/seed_omni/modules/bagel/__init__.py | 1 + tests/seed_omni/modules/bagel/helpers.py | 324 ++++ .../modules/bagel/test_bagel_accel_align.py | 179 ++ .../modules/bagel/test_bagel_eager.py | 114 ++ .../modules/bagel/test_generation.py | 542 ++++++ .../modules/bagel/test_graph_config.py | 126 ++ tests/seed_omni/modules/bagel/test_masking.py | 100 ++ .../modules/bagel/test_processing.py | 962 ++++++++++ .../seed_omni/modules/bagel/test_qwen2_mot.py | 241 +++ .../modules/bagel/test_source_routing.py | 171 ++ tests/utils/test_save_safetensor_utils.py | 22 + veomni/models/seed_omni/modules/__init__.py | 4 +- .../seed_omni/modules/bagel/__init__.py | 25 + .../seed_omni/modules/bagel/convert_model.py | 299 ++++ .../modules/bagel/flow_connector/__init__.py | 24 + .../flow_connector/accelerated/__init__.py | 10 + .../flow_connector/accelerated/accelerated.py | 367 ++++ .../bagel/flow_connector/configuration.py | 34 + .../bagel/flow_connector/generation_state.py | 156 ++ .../modules/bagel/flow_connector/modeling.py | 440 +++++ .../bagel/flow_connector/processing.py | 216 +++ .../modules/bagel/qwen2_mot/__init__.py | 24 + .../bagel/qwen2_mot/accelerated/__init__.py | 12 + .../qwen2_mot/accelerated/accelerated.py | 668 +++++++ .../bagel/qwen2_mot/checkpoint_conversion.py | 232 +++ .../modules/bagel/qwen2_mot/configuration.py | 60 + .../bagel/qwen2_mot/generation_state.py | 532 ++++++ .../modules/bagel/qwen2_mot/masking.py | 346 ++++ .../modules/bagel/qwen2_mot/modeling.py | 1557 +++++++++++++++++ .../modules/bagel/qwen2_mot/processing.py | 336 ++++ .../modules/bagel/siglip_navit/__init__.py | 31 + .../siglip_navit/accelerated/__init__.py | 10 + .../siglip_navit/accelerated/accelerated.py | 196 +++ .../bagel/siglip_navit/configuration.py | 56 + .../modules/bagel/siglip_navit/modeling.py | 564 ++++++ .../modules/bagel/siglip_navit/processing.py | 438 +++++ .../models/seed_omni/modules/bagel/sources.py | 69 + .../modules/bagel/text_encoder/__init__.py | 24 + .../text_encoder/accelerated/__init__.py | 10 + .../text_encoder/accelerated/accelerated.py | 176 ++ .../bagel/text_encoder/chat_template.py | 195 +++ .../bagel/text_encoder/configuration.py | 12 + .../modules/bagel/text_encoder/modeling.py | 193 ++ .../modules/bagel/text_encoder/processing.py | 110 ++ .../seed_omni/modules/bagel/vae/__init__.py | 31 + .../modules/bagel/vae/accelerated/__init__.py | 14 + .../bagel/vae/accelerated/accelerated.py | 332 ++++ .../modules/bagel/vae/configuration.py | 52 + .../seed_omni/modules/bagel/vae/modeling.py | 501 ++++++ .../seed_omni/modules/bagel/vae/processing.py | 563 ++++++ .../modules/text_encoder/chat_template.py | 4 +- .../modules/text_encoder/modeling.py | 2 +- veomni/utils/save_safetensor_utils.py | 40 + 70 files changed, 12653 insertions(+), 4 deletions(-) create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/data.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml create mode 100644 configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml create mode 100644 docs/seed_omni/models/bagel.md create mode 100644 tests/seed_omni/modules/bagel/__init__.py create mode 100644 tests/seed_omni/modules/bagel/helpers.py create mode 100644 tests/seed_omni/modules/bagel/test_bagel_accel_align.py create mode 100644 tests/seed_omni/modules/bagel/test_bagel_eager.py create mode 100644 tests/seed_omni/modules/bagel/test_generation.py create mode 100644 tests/seed_omni/modules/bagel/test_graph_config.py create mode 100644 tests/seed_omni/modules/bagel/test_masking.py create mode 100644 tests/seed_omni/modules/bagel/test_processing.py create mode 100644 tests/seed_omni/modules/bagel/test_qwen2_mot.py create mode 100644 tests/seed_omni/modules/bagel/test_source_routing.py create mode 100644 veomni/models/seed_omni/modules/bagel/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/convert_model.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py create mode 100644 veomni/models/seed_omni/modules/bagel/flow_connector/processing.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py create mode 100644 veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py create mode 100644 veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py create mode 100644 veomni/models/seed_omni/modules/bagel/sources.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py create mode 100644 veomni/models/seed_omni/modules/bagel/text_encoder/processing.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/configuration.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/modeling.py create mode 100644 veomni/models/seed_omni/modules/bagel/vae/processing.py diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml new file mode 100644 index 0000000000..f2636cab5b --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/data.yaml @@ -0,0 +1,18 @@ +sources: + - /mnt/hdfs/user_dir/dataset/imagenet1k_train + - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data + - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k + - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn +names: + - imagenet1k + - tulu-3-sft-mixture + - sharegpt4v_cap_100k + - seed_edit_p23_multi_turn +schedule: + - schedule_type: const + weights: [0.4, 0.2, 0.2, 0.2] +# Heterogeneous schemas (imagenet string / tulu dict / sharegpt4v list) — keep +# them in the VeOmni weighted multisource sampler. +level: token +stopping_strategy: all_exhausted +upstream_sharded: true diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml new file mode 100644 index 0000000000..e8082463e0 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml @@ -0,0 +1,37 @@ +# BAGEL-7B-MoT inference - text+image to image edit. +# Official edit builds VAE/ViT/text context, then calls image generation directly. +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_vae.encode_context, to: bagel_flow_connector.embed_context_latents } + - { from: bagel_flow_connector.embed_context_latents, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: default }, next_state: query_denoise } + + query_denoise: + body: + - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers } + - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch } + - { from: bagel_qwen2_mot.denoise_branch, to: end } + transitions: + - { condition: { type: default }, next_state: velocity_collect } + + velocity_collect: + body: + - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity } + - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise } + - { from: bagel_flow_connector.advance_denoise, to: end } + transitions: + - { condition: { type: module_signal, key: image_complete }, next_state: image_decode } + - { condition: { type: default }, next_state: query_denoise } + + image_decode: + body: + - { from: bagel_vae.decode_generated, to: end } + transitions: + - { condition: { type: default }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml new file mode 100644 index 0000000000..74f1864075 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml @@ -0,0 +1,34 @@ +# BAGEL-7B-MoT inference - text to image. +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: default }, next_state: query_denoise } + + query_denoise: + body: + - { from: bagel_flow_connector.prepare_denoise_query, to: bagel_text_encoder.encode_image_markers } + - { from: bagel_text_encoder.encode_image_markers, to: bagel_qwen2_mot.denoise_branch } + - { from: bagel_qwen2_mot.denoise_branch, to: end } + transitions: + - { condition: { type: default }, next_state: velocity_collect } + + velocity_collect: + body: + - { from: bagel_flow_connector.decode_velocity_from_hidden, to: bagel_qwen2_mot.collect_velocity } + - { from: bagel_qwen2_mot.collect_velocity, to: bagel_flow_connector.advance_denoise } + - { from: bagel_flow_connector.advance_denoise, to: end } + transitions: + - { condition: { type: module_signal, key: image_complete }, next_state: image_decode } + - { condition: { type: default }, next_state: query_denoise } + + image_decode: + body: + - { from: bagel_vae.decode_generated, to: end } + transitions: + - { condition: { type: default }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml new file mode 100644 index 0000000000..29ba541d85 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml @@ -0,0 +1,19 @@ +# BAGEL-7B-MoT inference - understanding to text (text/image -> text). +initial: prompt_encode + +states: + prompt_encode: + body: + - { from: bagel_text_encoder, to: bagel_qwen2_mot } + - { from: bagel_siglip_navit, to: bagel_qwen2_mot } + - { from: bagel_qwen2_mot, to: end } + transitions: + - { condition: { type: module_signal, key: text_done }, next_state: done } + - { condition: { type: default }, next_state: text_ar } + + text_ar: + body: + - { from: bagel_qwen2_mot, to: bagel_text_encoder } + - { from: bagel_text_encoder, to: end } + transitions: + - { condition: { type: module_signal, key: text_done }, next_state: done } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml new file mode 100644 index 0000000000..956e0e5d89 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml @@ -0,0 +1,42 @@ +# Per-module INFERENCE overrides for BAGEL-7B-MoT — all eager. +# +# Single-process load (`python tasks/omni/infer_omni.py ...`): every module +# loads via `from_pretrained(device_map='auto')`, spread across visible GPUs. + +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false + fsdp_config: + fsdp_mode: eager + +bagel_siglip_navit: + model_path: bagel_siglip_navit + accelerator: + fsdp_config: + fsdp_mode: eager + +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + fsdp_config: + fsdp_mode: eager + gradient_checkpointing: + enable: false + +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + fsdp_config: + fsdp_mode: eager + gradient_checkpointing: + enable: false + +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: sdpa + accelerator: + fsdp_config: + fsdp_mode: eager diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml new file mode 100644 index 0000000000..b6b65e8555 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_fsdp.yaml @@ -0,0 +1,33 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false + fsdp_config: + fsdp_mode: fsdp2 +bagel_siglip_navit: + model_path: bagel_siglip_navit + accelerator: + fsdp_config: + fsdp_mode: fsdp2 +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + fsdp_config: + fsdp_mode: fsdp2 + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + fsdp_config: + fsdp_mode: fsdp2 + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention + accelerator: + fsdp_config: + fsdp_mode: fsdp2 diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml new file mode 100644 index 0000000000..6b8cfdef07 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml @@ -0,0 +1,101 @@ +# VeOmni offline VAE posterior cache config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: false + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni + max_seq_len: 4096 + train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml + datasets_type: iterable + multisource_datasets_type: veomni_weighted_multisource + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: false + pin_memory: true + prefetch_factor: 2 + +train: + training_task: offline_embedding + offline_cache_dir: outputs/bagel_vae_cached_dataset + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_offline_cache + save_steps: 0 + save_epochs: 0 + save_hf_weights: false + hf_save_steps: 0 + hf_save_epochs: 0 + + wandb: + enable: false + project: VeOmni_omni + name: bagel_7b_mot_offline_cache + +infer: + generation_kwargs: + max_new_tokens: 2048 + temperature: 0.0 + top_p: 1.0 + do_sample: false + image_height: 1024 + image_width: 1024 + latent_downsample: 16 + num_timesteps: 50 + timestep_shift: 3.0 + cfg_text_scale: 1.0 + cfg_img_scale: 1.0 + cfg_interval: [0.0, 1.0] + cfg_renorm_min: 0.0 + cfg_renorm_type: global + enable_taylorseer: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml new file mode 100644 index 0000000000..82f52f7241 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/graph_train.yaml @@ -0,0 +1,2 @@ +# BAGEL-7B-MoT VAE posterior cache production DAG. +- { from: bagel_vae.offline_encode, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml new file mode 100644 index 0000000000..b1894fca61 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/modules_train.yaml @@ -0,0 +1,5 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml new file mode 100644 index 0000000000..9bc7a9417e --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml @@ -0,0 +1,101 @@ +# VeOmni training / inference config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: true + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni + max_seq_len: 4096 + train_path: configs/seed_omni/Bagel/bagel_7b_mot/data.yaml + # Streaming + VeOmni weighted multisource: no HF schema merge across sources. + datasets_type: iterable + multisource_datasets_type: veomni_weighted_multisource + # Iterable has no __len__; train_steps = ceil(train_sample / global_batch_size). + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: true + pin_memory: true + prefetch_factor: 2 + +train: + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_omni_sft + save_steps: 500 + save_epochs: 0 + save_hf_weights: true + hf_save_steps: 500 + hf_save_epochs: 0 + + wandb: + enable: true + project: VeOmni_omni + name: bagel_7b_mot_omni_sft + +infer: + generation_kwargs: + max_new_tokens: 2048 + temperature: 0.0 + top_p: 1.0 + do_sample: false + image_height: 1024 + image_width: 1024 + latent_downsample: 16 + num_timesteps: 50 + timestep_shift: 3.0 + cfg_text_scale: 1.0 + cfg_img_scale: 1.0 + cfg_interval: [0.0, 1.0] + cfg_renorm_min: 0.0 + cfg_renorm_type: global + enable_taylorseer: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml new file mode 100644 index 0000000000..3cae0e86bc --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/graph_train.yaml @@ -0,0 +1,9 @@ +# BAGEL-7B-MoT training DAG. +- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot } +- { from: bagel_siglip_navit, to: bagel_qwen2_mot } +- { from: bagel_vae.encode, to: bagel_flow_connector.embed_latent } +- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot } +- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode } +- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity } +- { from: bagel_text_encoder.decode, to: end } +- { from: bagel_flow_connector.decode_velocity, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml new file mode 100644 index 0000000000..48b58617b1 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml @@ -0,0 +1,21 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false +bagel_siglip_navit: + model_path: bagel_siglip_navit +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml new file mode 100644 index 0000000000..198c0feead --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml @@ -0,0 +1,99 @@ +# VeOmni cached VAE posterior training config for OmniModel - BAGEL-7B-MoT. + +model: + model_path: /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT + model_config: + modules: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml + train_graph: configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml + infer_graph: + infer_edit: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_edit.yaml + infer_gen: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_gen.yaml + infer_und: configs/seed_omni/Bagel/bagel_7b_mot/infer/graph_infer_und.yaml + infer_type: infer_und + ops_implementation: + attn_implementation: flash_attention_2 + + accelerator: + init_device: meta + gradient_checkpointing: + enable: true + enable_reentrant: false + fsdp_config: + fsdp_mode: fsdp2 + reshard_after_forward: true + reshard_after_backward: true + forward_prefetch: true + offload: false + max_load_broadcast_size: 20.0 + mixed_precision: + enable: true + param_dtype: bfloat16 + reduce_dtype: float32 + offload_config: + enable_activation: false + activation_gpu_limit: 0.0 + + optimizer: + type: adamw + lr: 1.0e-4 + lr_min: 1.0e-6 + lr_start: 0.0 + lr_warmup_ratio: 0.03 + lr_decay_style: cosine + lr_decay_ratio: 1.0 + weight_decay: 0.01 + max_grad_norm: 1.0 + no_decay_params: [] + no_decay_modules: [] + +data: + data_type: seedomni_cached + max_seq_len: 4096 + train_path: outputs/bagel_vae_cached_dataset + datasets_type: iterable + train_sample: 1000000 + dataloader: + type: native + num_workers: 4 + drop_last: true + pin_memory: true + prefetch_factor: 2 + +train: + training_task: offline_training + seed: 42 + num_train_epochs: 1 + global_batch_size: 32 + micro_batch_size: 2 + dyn_bsz: false + + checkpoint: + output_dir: outputs/bagel_7b_mot_omni_sft_from_cache + save_steps: 500 + save_epochs: 0 + save_hf_weights: true + hf_save_steps: 500 + hf_save_epochs: 0 + + wandb: + enable: true + project: VeOmni_omni + name: bagel_7b_mot_omni_sft_from_cache + +infer: + generation_kwargs: + max_new_tokens: 2048 + temperature: 0.0 + top_p: 1.0 + do_sample: false + image_height: 1024 + image_width: 1024 + latent_downsample: 16 + num_timesteps: 50 + timestep_shift: 3.0 + cfg_text_scale: 1.0 + cfg_img_scale: 1.0 + cfg_interval: [0.0, 1.0] + cfg_renorm_min: 0.0 + cfg_renorm_type: global + enable_taylorseer: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml new file mode 100644 index 0000000000..71951d199d --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/graph_train.yaml @@ -0,0 +1,9 @@ +# BAGEL-7B-MoT training DAG from cached VAE posterior carrier items. +- { from: bagel_text_encoder.encode, to: bagel_qwen2_mot } +- { from: bagel_siglip_navit, to: bagel_qwen2_mot } +- { from: bagel_vae.online_process, to: bagel_flow_connector.embed_latent } +- { from: bagel_flow_connector.embed_latent, to: bagel_qwen2_mot } +- { from: bagel_qwen2_mot, to: bagel_text_encoder.decode } +- { from: bagel_qwen2_mot, to: bagel_flow_connector.decode_velocity } +- { from: bagel_text_encoder.decode, to: end } +- { from: bagel_flow_connector.decode_velocity, to: end } diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml new file mode 100644 index 0000000000..48b58617b1 --- /dev/null +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml @@ -0,0 +1,21 @@ +bagel_vae: + model_path: bagel_vae + accelerator: + gradient_checkpointing: + enable: false +bagel_siglip_navit: + model_path: bagel_siglip_navit +bagel_text_encoder: + model_path: bagel_text_encoder + accelerator: + gradient_checkpointing: + enable: false +bagel_flow_connector: + model_path: bagel_flow_connector + accelerator: + gradient_checkpointing: + enable: false +bagel_qwen2_mot: + model_path: bagel_qwen2_mot + ops_implementation: + attn_implementation: flex_attention diff --git a/docs/index.md b/docs/index.md index ad69bbb332..1d6c220aea 100644 --- a/docs/index.md +++ b/docs/index.md @@ -113,6 +113,13 @@ seed_omni/mixins/metric_meter.md seed_omni/mixins/offline_encoding.md ``` +```{toctree} +:maxdepth: 1 +:caption: SeedOmni Models + +seed_omni/models/bagel.md +``` + ```{toctree} :maxdepth: 1 :caption: Design diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md new file mode 100644 index 0000000000..57dc34d2f3 --- /dev/null +++ b/docs/seed_omni/models/bagel.md @@ -0,0 +1,352 @@ +# BAGEL-7B-MoT (SeedOmni) + +End-to-end recipe for training and inferring **BAGEL-7B-MoT** as a SeedOmni +graph model. The upstream BAGEL checkpoint is split into five OmniModules: text +embedding / LM head, SigLIP-NaViT understanding tower, VAE codec, flow connector, +and Qwen2-MoT backbone. + +All paths below assume the upstream BAGEL checkpoint lives at +`/mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT`. Adjust to your +own storage. + +Config dir: `configs/seed_omni/Bagel/bagel_7b_mot/`. + +| Module | Holds | Role | +|--------|-------|------| +| `bagel_text_encoder` | tokenizer, token embedding, LM head | text/template markers, text logits, image-marker embeddings | +| `bagel_siglip_navit` | SigLIP-NaViT tower + connector | user image context for understanding and edit prompts | +| `bagel_vae` | VAE encoder/decoder | assistant image training latents, edit context latents, generated latent decode | +| `bagel_flow_connector` | VAE↔LLM projections, timestep embedding | latent patch embedding, velocity prediction, denoise state | +| `bagel_qwen2_mot` | Qwen2-MoT decoder backbone | text AR and flow-denoise hidden states | + +The [config layout](../usage/training_and_inference.md#1-config-layout) uses the +same `base.yaml` for training and inference. The training block references `train/modules_train.yaml` and `train/graph_train.yaml`; the +inference block maps each scenario to a separate generation graph. + +| File | Role | +|------|------| +| `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, train, and `infer` block. | +| `train/modules_train.yaml` | Per-module training paths. `bagel_qwen2_mot` is the accelerated class with `flex_attention`. | +| `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses SDPA. | +| `infer/modules_infer_fsdp.yaml` | Distributed inference: every module uses FSDP2; MoT uses FlexAttention. | +| `train/graph_train.yaml` | Training DAG. | +| `infer/graph_infer_und.yaml` | Image/text understanding to text. | +| `infer/graph_infer_gen.yaml` | Text to image generation. | +| `infer/graph_infer_edit.yaml` | Text+image to image edit. | +| `data.yaml` | Weighted multisource data list. | + +The BAGEL graphs cover understanding, generation, and edit. + +--- + +## 1. Convert the checkpoint + +The converter reads upstream `llm_config.json`, `vit_config.json`, +`ema.safetensors`, and `ae.safetensors`, then writes one sub-checkpoint per +module: + +```bash +python scripts/seed_omni/convert_model.py \ + --model_path /mnt/hdfs/user_dir/veomni_omni/models/transformers/BAGEL-7B-MoT \ + --output_dir /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT +``` + +The family is picked from the source `config.json` (`model_type: bagel`). The +output root becomes `model.model_path` in `base.yaml`. It contains: + +```text +BAGEL-7B-MoT/ +├── config.json # module entries, infer_type, generation_kwargs +├── training_graph.yaml +├── generation_graph.yaml # infer_und / infer_gen / infer_edit +├── bagel_text_encoder/ +├── bagel_siglip_navit/ +├── bagel_vae/ +├── bagel_flow_connector/ +└── bagel_qwen2_mot/ +``` + +`bagel_text_encoder` also stores the tokenizer, re-saved from the upstream +checkpoint via `AutoTokenizer`; SigLIP-NaViT and VAE save their processors next +to their weights. + +--- + +## 2. Prepare data + +`data.yaml` lists a weighted multisource mixture: + +```yaml +sources: + - /mnt/hdfs/user_dir/dataset/imagenet1k_train + - /mnt/hdfs/veomni/datasets/tulu-3-sft-mixture/data + - /mnt/hdfs/veomni/datasets/sharegpt4v_cap_100k + - /mnt/hdfs/user_dir/dataset/seed_edit_p23_multi_turn +names: + - imagenet1k + - tulu-3-sft-mixture + - sharegpt4v_cap_100k + - seed_edit_p23_multi_turn +schedule: + - { schedule_type: const, weights: [0.4, 0.2, 0.2, 0.2] } +``` + +`ConversationItem`s carry no module ownership: each Bagel module selects its +rows by `type` / `role` / `meta` tags. Bagel defines two private meta keys in +`veomni/models/seed_omni/modules/bagel/sources.py`: + +| Key | Values | Meaning | +|---|---|---| +| `BAGEL_CONTEXT_KEY` (`"bagel_context"`) | `BAGEL_SIGLIP_CONTEXT` (`"siglip"`), `BAGEL_VAE_CONTEXT` (`"vae"`) | Which image copy a row is: SigLIP-NaViT understanding features, or VAE latent context/target. The vision start/end marker rows around each copy carry the same tag. | +| `BAGEL_PHASE_KEY` (`"bagel_phase"`) | `start_token`, `flow_query`, `flow_hidden`, `flow_velocity`, `generated_latent` | Inference-only step of a `type="output"` row (see §4). | + +The VAE CPU preprocessor (`route_image_contexts`) expands each raw training +image by its data-layer `_img_tag` into context copies: + +- `und` → one `BAGEL_SIGLIP_CONTEXT` copy for SigLIP-NaViT. +- `gen` → one `BAGEL_VAE_CONTEXT` copy, the flow-matching target. +- `edit` → a `BAGEL_VAE_CONTEXT` copy followed by a `BAGEL_SIGLIP_CONTEXT` + copy of the same image. `_img_tag` alone cannot tell these two apart, which + is why the context tag exists. + +At inference, raw user images have no `_img_tag`; they become SigLIP context, +plus a VAE context copy when `infer_type` is `infer_edit`. + +A sample with no image for SigLIP-NaViT or the VAE gets an FSDP placeholder +flagged `is_dummy=True`. It keeps the context tag (and a representative +`role` / `_img_tag`: `user`/`und` for SigLIP, `assistant`/`gen` for the VAE), +so the encoder selects it with the same filter as real rows; packing and loss +code skip it via `item.is_dummy`. + +--- + +## 3. Train + +Training uses the accelerated Qwen2-MoT class from `train/modules_train.yaml`. The +default packed attention backend is FlexAttention. + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml +``` + +The training DAG is: + +```text +bagel_text_encoder.encode -> bagel_qwen2_mot +bagel_siglip_navit -> bagel_qwen2_mot +bagel_vae.encode -> bagel_flow_connector.embed_latent +bagel_flow_connector.embed_latent -> bagel_qwen2_mot +bagel_qwen2_mot -> bagel_text_encoder.decode +bagel_qwen2_mot -> bagel_flow_connector.decode_velocity +``` + +`bagel_flow_connector.embed_latent` patchifies VAE latents, samples the training +noise/timestep, writes the velocity target, and projects noised latent patches +into LLM space. `bagel_flow_connector.decode_velocity` consumes the MoT hidden +states for those latent query positions and computes the flow velocity loss. + +Quick smoke run: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --train.max_steps 10 \ + --train.global_batch_size 8 \ + --train.micro_batch_size 1 \ + --train.wandb.enable false +``` + +### 3.1 Attention backends + +`bagel_qwen2_mot` keeps one packed visibility metadata and materializes it as +Flex, Magi, or SDPA. Visibility is the same in all three: +`same_document & (causal | same_full_span) & ~foreign_noise_key`. Packed MoT +training does not use FlashAttention-2. + +| Backend | Class | Config | Use | +|---------|-------|--------|-----| +| FlexAttention | accelerated | `train/modules_train.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training / FSDP inference | +| MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM90+ fused training; see nfunc below | +| SDPA | eager | `infer/modules_infer_eager.yaml` | Single-process inference and the dense-mask oracle | + +Leave the YAML on `flex_attention` unless you are opting into Magi. Enable Magi +from the CLI (the launcher rewrites it to `veomni_magi_attention`): + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.modules.bagel_qwen2_mot.ops_implementation.attn_implementation magi_attention \ + --train.micro_batch_size 1 +``` + +Magi requires physical batch size 1, `cp_size == 1`, and NVIDIA SM90 or newer. +Ulysses still works. On 8 ranks, `--model.accelerator.ulysses_size 4` gives +`dp_shard=2` and SP4. Packed Magi/Flex is the training (and FSDP prefill) path; +the denoise loop still uses FlashAttention-2. The Magi adapter contract is in +[`docs/transformers_v5/veomni_attention_interface.md`](../../transformers_v5/veomni_attention_interface.md). + +#### SM90 nfunc vs dataset + +On SM90, Magi uses a precompiled CUTLASS overlay. `nfunc` is baked at install +time: it is the HSTU interval count for the worst query in a sample, not the +Magi range-list length. The installer default is `1,3,5`. A later exact +`uv sync` removes the overlay, so rerun the installer before SM90 Magi runs. + +Text-only and single-image gen samples are typically `nfunc=1`. Multi-turn +`seed_edit_p23_multi_turn` punches noise-key holes, so nfunc grows with the +number of assistant images (about `2G-1`). That mixture needs **nfunc ≥ 11**: + +```bash +bash scripts/kernel/install_magi_sm90.sh --nfunc 1,3,5,7,9,11 +``` + +If a sample's runtime nfunc is missing from the compiled matrix, the kernel +fails with `Compile-time kNFunc (...) must match runtime arbitrary_func_num (...)`. +SM100+ uses CUTE DSL/JIT and compiles unseen nfunc at runtime, so it does not +need this overlay matrix. + +### 3.2 Offline VAE posterior cache (two stages) + +Caching is not a special framework mode — it is a different `train_graph` plus a +different dataset type. Each stage has its own modules / graph YAML, and +`train.training_task` tells the trainer which stage it runs. `bagel_vae` can be +cached because it mixes in `OfflineEncodingMixin`; no config flag declares it (see +[Offline Encoding](../mixins/offline_encoding.md#how-a-run-builds-a-cached-module)). + +**Stage 1 — produce the cache** (`train.training_task: offline_embedding`). `offline_cache/modules_train.yaml` declares only +`bagel_vae`, and the DAG is a single edge, so nothing else is built: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml +``` + +```text +bagel_vae.offline_encode -> end +``` + +Posteriors are written to `train.offline_cache_dir` +(`outputs/bagel_vae_cached_dataset` by default), reading normal `seedomni` data. +Samples without a VAE image get the preprocessor's dummy row encoded too, so +every cached sample carries a VAE posterior. + +**Stage 2 — train from the cache** (`train.training_task: offline_training`). +`bagel_vae` mixes in `OfflineEncodingMixin`, so it is built on meta and never +loads its weights; the graph calls its `online_process`. `with_cache/modules_train.yaml` is the same file as +`train/modules_train.yaml`. `data.data_type` +becomes `seedomni_cached` and `data.train_path` points at the stage-1 output +directory. The cached items are marked as posteriors, so the VAE preprocessor +leaves them alone: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml +``` + +The DAG matches §3 except that `bagel_vae.online_process` replaces +`bagel_vae.encode` as the latent source — it rehydrates the cached posterior +instead of encoding pixels. + +--- + +## 4. Inference + +`tasks/omni/infer_omni.py` selects a generation graph with `--model.model_config.infer_type`. +Use `infer/modules_infer_eager.yaml` for a single-process run or +`infer/modules_infer_fsdp.yaml` with `bash train.sh` for a torchrun/FSDP2 run. + +### 4.1 Understanding + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_und \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.images /path/to/image.jpg \ + --infer.prompt "Describe this image." \ + --infer.output_dir bagel_out +``` + +`infer_und` runs SigLIP-NaViT for the prompt image, inserts image marker +embeddings via the text encoder, and then uses Qwen2-MoT + text encoder AR +decode until `text_done`. + +### 4.2 Text-to-image generation + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_gen \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.prompt "A watercolor painting of a small cabin beside a lake." \ + --infer.output_dir bagel_out \ + --infer.generation_kwargs.num_timesteps 50 \ + --infer.generation_kwargs.timestep_shift 3.0 +``` + +`infer_gen` runs prompt prefill, then loops: + +```text +flow_connector.prepare_denoise_query + -> text_encoder.encode_image_markers + -> qwen2_mot.denoise_branch + -> flow_connector.decode_velocity_from_hidden + -> qwen2_mot.collect_velocity + -> flow_connector.advance_denoise +``` + +Each step moves the tail `output` row through `BAGEL_PHASE_KEY` values +`flow_query` → `flow_hidden` → `flow_velocity`. When the denoise state emits +`image_complete`, the row is re-tagged `generated_latent` and +`bagel_vae.decode_generated` decodes it. + +### 4.3 Image edit + +```bash +python tasks/omni/infer_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.model_config.infer_type infer_edit \ + --model.model_config.modules configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml \ + --model.model_path /mnt/hdfs/user_dir/veomni_omni/models/seed_omni/BAGEL-7B-MoT \ + --infer.images /path/to/source.jpg \ + --infer.prompt "Make it look like a snowy evening." \ + --infer.output_dir bagel_out +``` + +Edit first builds context from both image branches: VAE encodes the +`BAGEL_VAE_CONTEXT` copy of the edit image, SigLIP-NaViT encodes the +`BAGEL_SIGLIP_CONTEXT` copy as visual +context, and `flow_connector.embed_context_latents` projects the VAE context into +the denoise prompt. The downstream denoise loop is shared with `infer_gen`. + +--- + +## 5. Visualize the graphs + +```bash +python scripts/seed_omni/visualize_graph.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml +# -> graphs/bagel_7b_mot_base/{training,infer_edit,infer_gen,infer_und}.mmd +``` + +The visualized graphs are generated from the same config loader path as training +and inference, so regenerate them after changing module entrypoints or graph +YAML. + +--- + +## 6. Contract checks + +The Bagel module and graph contracts cover carrier context-tag routing, generation +state transitions, packing/cache behavior, and graph config structure. Packed +MoT also compares Flex and Magi against eager SDPA on toy CE / MSE / gradients +in `tests/seed_omni/modules/bagel/test_bagel_accel_align.py`. Magi cases skip unless the +SM90 CUTLASS overlay or SM100+ CUTE JIT backend is present. + +```bash +.venv/bin/python -m pytest -q tests/seed_omni/modules/bagel +``` diff --git a/tests/seed_omni/modules/bagel/__init__.py b/tests/seed_omni/modules/bagel/__init__.py new file mode 100644 index 0000000000..54ca978df6 --- /dev/null +++ b/tests/seed_omni/modules/bagel/__init__.py @@ -0,0 +1 @@ +"""BAGEL SeedOmni tests.""" diff --git a/tests/seed_omni/modules/bagel/helpers.py b/tests/seed_omni/modules/bagel/helpers.py new file mode 100644 index 0000000000..3b8378380c --- /dev/null +++ b/tests/seed_omni/modules/bagel/helpers.py @@ -0,0 +1,324 @@ +"""Shared helpers for BAGEL tests.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Literal + +import torch +import torch.nn.functional as F + +from veomni.arguments.omni_arguments_types import ( + OmniArguments, + OmniDataArguments, + OmniInferArguments, + OmniModelRuntimeArguments, + build_omni_model_runtime_args, + build_omni_module_runtime_args, +) +from veomni.models.seed_omni import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_MODEL_REGISTRY +from veomni.models.seed_omni.configuration_omni import OmniConfig +from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY +from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import PackedConversation, preprocess_mot_inputs +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem + + +ToyCase = Literal["ce_only", "vit_ce", "mse_only", "mixed"] + +ALIGN_ATOL = 2e-2 +ALIGN_RTOL = 2e-2 +ALIGN_GRAD_ATOL = 5e-2 +ALIGN_GRAD_RTOL = 5e-2 + + +def bagel_cfg_dir() -> Path: + return Path(__file__).resolve().parents[4] / "configs" / "seed_omni" / "Bagel" / "bagel_7b_mot" + + +def config_cls(model_type: str): + return OMNI_CONFIG_REGISTRY[model_type]() + + +def model_cls(model_type: str): + """Return the VeOmni-accelerated module class (training / graph hooks).""" + return OMNI_ACCELERATED_MODEL_REGISTRY[model_type]() + + +def native_model_cls(model_type: str): + """Return the HF-native module class (eager inference weights).""" + return OMNI_MODEL_REGISTRY[model_type]() + + +def load_module_runtime_args( + *, + model_path: str = "", + modules_path: Path, + for_inference: bool = False, +) -> dict: + """Build per-module ``OmniModuleRuntimeArguments`` from ``modules_path``.""" + base = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=model_path or ".", + model_config={"modules": str(modules_path)}, + ), + data=OmniDataArguments(train_path=""), + )._to_module_global_args() + return build_omni_module_runtime_args( + global_args=base, + model_path=model_path, + modules=str(modules_path), + for_inference=for_inference, + ) + + +def load_omni_config( + *, + model_path: str = "", + modules_path: Path, + train_graph_path: Path | None = None, + infer_graph_path: Path | None = None, + generation_kwargs: dict | None = None, +) -> OmniConfig: + """Resolve and project an ``OmniConfig`` from BAGEL module and graph YAML files.""" + model_path = model_path or "." + model_config = {"modules": str(modules_path)} + if train_graph_path is not None: + model_config["train_graph"] = str(train_graph_path) + if infer_graph_path is not None: + model_config["infer_graph"] = str(infer_graph_path) + args = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=model_path, + model_config=model_config, + ), + data=OmniDataArguments(train_path="."), + infer=OmniInferArguments(generation_kwargs=dict(generation_kwargs or {})), + ) + return build_omni_model_runtime_args(args).to_hf_config() + + +def tiny_bagel_qwen2_cfg() -> dict: + return dict( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "sdpa", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + ) + + +def tiny_align_qwen2_cfg() -> dict: + config = tiny_bagel_qwen2_cfg() + config["num_key_value_heads"] = 2 + return config + + +def _clone_meta(meta: dict[str, Any]) -> dict[str, Any]: + """Deep-copy a ``ConversationItem.meta`` dict while detaching and cloning tensor values.""" + cloned: dict[str, Any] = {} + for key, value in meta.items(): + cloned[key] = value.detach().clone() if torch.is_tensor(value) else copy.deepcopy(value) + return cloned + + +def clone_conversation(conversation: list[list[ConversationItem]]) -> list[list[ConversationItem]]: + """Deep-copy a packed conversation so eager and accelerated can mutate independently.""" + cloned: list[list[ConversationItem]] = [] + for sample in conversation: + cloned_sample: list[ConversationItem] = [] + for item in sample: + value = item.value + if torch.is_tensor(value): + requires_grad = value.requires_grad + value = value.detach().clone() + if requires_grad: + value = value.requires_grad_() + cloned_sample.append( + ConversationItem( + type=item.type, + value=value, + role=item.role, + is_dummy=item.is_dummy, + meta=_clone_meta(item.meta), + ) + ) + cloned.append(cloned_sample) + return cloned + + +def _embed( + generator: torch.Generator, + length: int, + hidden_size: int, + *, + device: torch.device, + dtype: torch.dtype, + requires_grad: bool = True, +) -> torch.Tensor: + """Generate a random ``(length, hidden_size)`` leaf embedding tensor.""" + value = torch.randn(length, hidden_size, generator=generator, device=device, dtype=dtype) + if requires_grad: + value = value.requires_grad_() + return value + + +def build_toy_conversation( + case: ToyCase, + *, + hidden_size: int, + vocab_size: int, + patch_latent_dim: int, + device: torch.device, + dtype: torch.dtype, + seed: int = 17, +) -> list[list[ConversationItem]]: + """Build one packed sample covering CE-only, vit-CE, MSE-only, or mixed.""" + generator = torch.Generator(device=device).manual_seed(seed) + items: list[ConversationItem] = [] + + if case in {"vit_ce", "mixed"}: + items.append( + ConversationItem( + type="image", + value=_embed(generator, 4, hidden_size, device=device, dtype=dtype), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "und"}, + ) + ) + + if case in {"ce_only", "vit_ce", "mixed"}: + labels = torch.randint(3, vocab_size, (8,), generator=generator, device=device) + items.append( + ConversationItem( + type="text", + value=_embed(generator, 8, hidden_size, device=device, dtype=dtype), + role="assistant", + meta={"labels": labels}, + ) + ) + + if case in {"mse_only", "mixed"}: + items.append( + ConversationItem( + type="image", + value=_embed(generator, 6, hidden_size, device=device, dtype=dtype), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: "gen", + "flow_velocity_target": torch.randn( + 6, + patch_latent_dim, + generator=generator, + device=device, + dtype=dtype, + ), + }, + ) + ) + + return [items] + + +def scatter_mot_hidden( + conversation: list[list[ConversationItem]], + packed: PackedConversation, + hidden_states: torch.Tensor, +) -> None: + """Write packed MoT hidden states back onto the original carrier items.""" + for span in packed.spans: + span_hidden = hidden_states[span.start : span.start + span.length] + offset = 0 + for item, length in zip(span.items, span.lengths, strict=True): + item.value = span_hidden[offset : offset + length] + offset += length + + +def run_eager_mot( + model: torch.nn.Module, + conversation: list[list[ConversationItem]], +) -> torch.Tensor: + """Pack ``conversation``, run eager ``BagelQwen2MoT``, and scatter hidden states back.""" + packed = preprocess_mot_inputs( + conversation, + device=model.device, + dtype=model.dtype, + hidden_size=int(model.config.hidden_size), + ) + if packed is None: + raise ValueError("toy conversation produced no packable tokens") + outputs = model( + packed_sequence=packed.packed_sequence, + packed_position_ids=packed.packed_position_ids, + packed_token_type_ids=packed.packed_token_type_ids, + packed_attention_metadata=packed.packed_attention_metadata, + ) + hidden_states = outputs["hidden_states"] + scatter_mot_hidden(conversation, packed, hidden_states) + return hidden_states + + +def run_accelerated_mot( + model: torch.nn.Module, + conversation: list[list[ConversationItem]], +) -> torch.Tensor: + inputs = model.forward_pre(conversation_list=conversation) + outputs = model(**inputs) + hidden_states = outputs["hidden_states"] + model.forward_post(**outputs) + return hidden_states + + +def conversation_ce_loss( + conversation: list[list[ConversationItem]], + lm_head: torch.nn.Linear, +) -> torch.Tensor | None: + """Compute shifted causal LM cross-entropy loss over text items in ``conversation``.""" + hidden_parts: list[torch.Tensor] = [] + label_parts: list[torch.Tensor] = [] + for item in (item for sample in conversation for item in sample): + labels = item.meta.get("labels") + if item.type != "text" or not torch.is_tensor(item.value) or not torch.is_tensor(labels): + continue + hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value + hidden_parts.append(hidden) + label_parts.append(labels.to(device=hidden.device)) + if not hidden_parts: + return None + hidden = torch.cat(hidden_parts, dim=0) + labels = torch.cat(label_parts, dim=0) + shift_labels = F.pad(labels[..., 1:].contiguous(), (0, 1), value=-100) + logits = lm_head(hidden.float()) + return F.cross_entropy(logits, shift_labels, ignore_index=-100) + + +def conversation_mse_loss( + conversation: list[list[ConversationItem]], + llm2vae: torch.nn.Linear, +) -> torch.Tensor | None: + """Compute flow-matching MSE velocity loss over target image items in ``conversation``.""" + velocity_parts: list[torch.Tensor] = [] + target_parts: list[torch.Tensor] = [] + for item in (item for sample in conversation for item in sample): + target = item.meta.get("flow_velocity_target") + if not torch.is_tensor(item.value) or not torch.is_tensor(target): + continue + hidden = item.value.squeeze(0) if item.value.dim() == 3 else item.value + velocity_parts.append(llm2vae(hidden.float())) + target_parts.append(target.to(device=hidden.device, dtype=torch.float32)) + if not velocity_parts: + return None + mse = (torch.cat(velocity_parts, dim=0) - torch.cat(target_parts, dim=0)).square() + return mse.mean(dim=-1).mean() diff --git a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py new file mode 100644 index 0000000000..a531d7be6c --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py @@ -0,0 +1,179 @@ +"""Accelerated Bagel Qwen2-MoT vs eager CE / MSE alignment on toy data.""" + +from __future__ import annotations + +import copy +import importlib.util + +import pytest +import torch + +from tests.seed_omni.modules.bagel.helpers import ( + ALIGN_ATOL, + ALIGN_GRAD_ATOL, + ALIGN_GRAD_RTOL, + ALIGN_RTOL, + ToyCase, + build_toy_conversation, + clone_conversation, + config_cls, + conversation_ce_loss, + conversation_mse_loss, + native_model_cls, + run_accelerated_mot, + run_eager_mot, + tiny_align_qwen2_cfg, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated +from veomni.ops.kernels.attention.standard.magi import _fa4_cuda as magi_fa4_backend +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + + +_PATCH_LATENT_DIM = 4 +_CASES: tuple[ToyCase, ...] = ("ce_only", "vit_ce", "mse_only", "mixed") +_MAGI_IMPLEMENTATION = "veomni_magi_attention" +_FLEX_IMPLEMENTATION = "veomni_flex_attention" + + +def _is_magi_ffa_available() -> bool: + """Return whether a runnable MagiAttention CUTLASS or CUTE backend is installed on this GPU.""" + if not IS_CUDA_AVAILABLE or importlib.util.find_spec("magi_attention") is None: + return False + + kernel_mode = magi_fa4_backend._get_magi_kernel_mode(torch.device(get_device_type())) + if kernel_mode == magi_fa4_backend._MAGI_KERNEL_CUTE_JIT: + return importlib.util.find_spec("flash_attn_cute") is not None + if kernel_mode != magi_fa4_backend._MAGI_KERNEL_CUTLASS: + return False + + try: + from flash_attn_cute.ffa_fa3 import flash_attn_interface + except (ImportError, OSError, RuntimeError): + return False + + return all( + callable(getattr(flash_attn_interface, name, None)) for name in ("_flash_attn_forward", "_flash_attn_backward") + ) + + +_MAGI_FFA_AVAILABLE = _is_magi_ffa_available() +_MAGI_FFA_REASON = ( + "MagiAttention alignment requires a supported NVIDIA GPU with its CUTLASS overlay or CUTE DSL/JIT backend" +) + + +def _eager_config(): + cfg = tiny_align_qwen2_cfg() + cfg.pop("ops_implementation", None) + return config_cls("bagel_qwen2_mot")(**cfg, attn_implementation="sdpa") + + +def _accelerated_config(attn_implementation: str): + cfg = tiny_align_qwen2_cfg() + cfg.pop("ops_implementation", None) + return config_cls("bagel_qwen2_mot")( + **cfg, + attn_implementation=attn_implementation, + ) + + +def _shared_heads(hidden_size: int, vocab_size: int, device: torch.device) -> tuple[torch.nn.Linear, torch.nn.Linear]: + lm_head = torch.nn.Linear(hidden_size, vocab_size, bias=False).to(device=device, dtype=torch.float32) + llm2vae = torch.nn.Linear(hidden_size, _PATCH_LATENT_DIM, bias=False).to(device=device, dtype=torch.float32) + return lm_head, llm2vae + + +def _build_aligned_models(device: torch.device, dtype: torch.dtype, *, attn_implementation: str): + torch.manual_seed(29) + eager = native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device=device, dtype=dtype).train() + accelerated = ( + BagelQwen2MoTAccelerated(_accelerated_config(attn_implementation)).to(device=device, dtype=dtype).train() + ) + accelerated.load_state_dict(copy.deepcopy(eager.state_dict())) + return eager, accelerated + + +def _assert_accelerated_matches_eager(case: ToyCase, attn_implementation: str) -> None: + """Verify that ``BagelQwen2MoTAccelerated`` matches eager SDPA outputs, losses, and gradients.""" + device = torch.device(get_device_type()) + dtype = torch.bfloat16 + eager, accelerated = _build_aligned_models(device, dtype, attn_implementation=attn_implementation) + hidden_size = int(eager.config.hidden_size) + vocab_size = int(eager.config.vocab_size) + conversation = build_toy_conversation( + case, + hidden_size=hidden_size, + vocab_size=vocab_size, + patch_latent_dim=_PATCH_LATENT_DIM, + device=device, + dtype=dtype, + ) + eager_conversation = clone_conversation(conversation) + accelerated_conversation = clone_conversation(conversation) + lm_head, llm2vae = _shared_heads(hidden_size, vocab_size, device) + + eager_hidden = run_eager_mot(eager, eager_conversation) + accelerated_hidden = run_accelerated_mot(accelerated, accelerated_conversation) + + assert torch.isfinite(eager_hidden).all() + assert torch.isfinite(accelerated_hidden).all() + assert eager_hidden.shape == accelerated_hidden.shape + + eager_ce = conversation_ce_loss(eager_conversation, lm_head) + accelerated_ce = conversation_ce_loss(accelerated_conversation, lm_head) + eager_mse = conversation_mse_loss(eager_conversation, llm2vae) + accelerated_mse = conversation_mse_loss(accelerated_conversation, llm2vae) + + if case in {"ce_only", "vit_ce", "mixed"}: + assert eager_ce is not None and accelerated_ce is not None + assert torch.isfinite(eager_ce) and torch.isfinite(accelerated_ce) + torch.testing.assert_close(accelerated_ce, eager_ce, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + else: + assert eager_ce is None and accelerated_ce is None + + if case in {"mse_only", "mixed"}: + assert eager_mse is not None and accelerated_mse is not None + assert torch.isfinite(eager_mse) and torch.isfinite(accelerated_mse) + torch.testing.assert_close(accelerated_mse, eager_mse, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + else: + assert eager_mse is None and accelerated_mse is None + + eager_loss = eager_hidden.float().square().mean() + accelerated_loss = accelerated_hidden.float().square().mean() + if eager_ce is not None: + eager_loss = eager_loss + eager_ce + accelerated_loss = accelerated_loss + accelerated_ce + if eager_mse is not None: + eager_loss = eager_loss + eager_mse + accelerated_loss = accelerated_loss + accelerated_mse + eager_loss.backward() + accelerated_loss.backward() + + for name in ( + "model.layers.0.self_attn.o_proj.weight", + "model.layers.0.mlp.gate_proj.weight", + "model.norm.weight", + ): + eager_grad = dict(eager.named_parameters())[name].grad + accelerated_grad = dict(accelerated.named_parameters())[name].grad + assert eager_grad is not None and accelerated_grad is not None + assert torch.isfinite(eager_grad).all() and torch.isfinite(accelerated_grad).all() + torch.testing.assert_close( + accelerated_grad, + eager_grad, + atol=ALIGN_GRAD_ATOL, + rtol=ALIGN_GRAD_RTOL, + msg=f"gradient mismatch for {name}", + ) + + +@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="FlexAttention alignment requires CUDA") +@pytest.mark.parametrize("case", _CASES) +def test_accelerated_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: + _assert_accelerated_matches_eager(case, _FLEX_IMPLEMENTATION) + + +@pytest.mark.skipif(not _MAGI_FFA_AVAILABLE, reason=_MAGI_FFA_REASON) +@pytest.mark.parametrize("case", _CASES) +def test_magi_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: + _assert_accelerated_matches_eager(case, _MAGI_IMPLEMENTATION) diff --git a/tests/seed_omni/modules/bagel/test_bagel_eager.py b/tests/seed_omni/modules/bagel/test_bagel_eager.py new file mode 100644 index 0000000000..551c203e81 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_bagel_eager.py @@ -0,0 +1,114 @@ +"""Smoke coverage for native eager Bagel Qwen2-MoT train and sequential infer.""" + +from __future__ import annotations + +import pytest +import torch + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + build_toy_conversation, + config_cls, + native_model_cls, + run_eager_mot, + tiny_bagel_qwen2_cfg, +) +from veomni.models.seed_omni.modules.bagel.sources import ( + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_PHASE_KEY, + BAGEL_START_TOKEN, +) +from veomni.models.seed_omni.utils.conversation import ConversationItem +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + + +def _eager_config(**overrides): + return config_cls("bagel_qwen2_mot")(**{**tiny_bagel_qwen2_cfg(), **overrides, "attn_implementation": "sdpa"}) + + +@pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) +def test_eager_packed_train_forward_runs() -> None: + """Verify eager packed MoT forward and backward populate both und and gen expert gradients.""" + model = native_model_cls("bagel_qwen2_mot")(_eager_config()).train() + conversation = build_toy_conversation( + "mixed", + hidden_size=int(model.config.hidden_size), + vocab_size=int(model.config.vocab_size), + patch_latent_dim=4, + device=model.device, + dtype=model.dtype, + ) + + hidden_states = run_eager_mot(model, conversation) + assert hidden_states.shape[-1] == int(model.config.hidden_size) + assert torch.isfinite(hidden_states).all() + + hidden_states.float().square().mean().backward() + query_grad = model.model.layers[0].self_attn.q_proj.weight.grad + gen_query_grad = model.model.layers[0].self_attn.q_proj_moe_gen.weight.grad + assert query_grad is not None and torch.isfinite(query_grad).all() + assert gen_query_grad is not None and torch.isfinite(gen_query_grad).all() + assert query_grad.abs().max() > 0 + assert gen_query_grad.abs().max() > 0 + + +def _eager_cuda_model(): + device = torch.device(get_device_type()) + return native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device=device, dtype=torch.bfloat16).eval() + + +@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") +def test_eager_und_generate_runs() -> None: + """Verify eager understanding generation populates the KV cache and emits a finite output embedding.""" + model = _eager_cuda_model() + prompt = ConversationItem( + type="text", + value=torch.randn(4, int(model.config.hidden_size), device=model.device, dtype=model.dtype), + role="user", + ) + bos = ConversationItem( + type="output", + value=torch.randn(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype), + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, + ) + + outputs = model.generate([prompt, bos], generation_kwargs={"infer_type": "infer_und"}) + conversation = outputs["conversation_list"] + tail = conversation[-1] + assert tail.type == "output" + assert torch.is_tensor(tail.value) + assert tail.value.shape == (1, int(model.config.hidden_size)) + assert torch.isfinite(tail.value).all() + assert model._generation_state.main.cache is not None + + +@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") +def test_eager_gen_denoise_runs() -> None: + """Verify eager ``denoise_branch`` transitions ``flow_query`` into ``flow_hidden``.""" + model = _eager_cuda_model() + hidden_size = int(model.config.hidden_size) + prompt = ConversationItem( + type="text", + value=torch.randn(3, hidden_size, device=model.device, dtype=model.dtype), + role="user", + ) + model.generate([prompt], generation_kwargs={"infer_type": "infer_gen"}) + + query = ConversationItem( + type="output", + value=torch.randn(5, hidden_size, device=model.device, dtype=model.dtype), + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, + ) + outputs = model.denoise_branch( + [query], + generation_kwargs={"infer_type": "infer_gen", "cfg_text_scale": 1.0, "cfg_img_scale": 1.0}, + ) + tail = outputs["conversation_list"][-1] + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN + assert torch.is_tensor(tail.value) + assert tail.value.shape[0] == 5 + assert tail.value.shape[-1] == hidden_size + assert torch.isfinite(tail.value).all() diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py new file mode 100644 index 0000000000..3850c4f5c2 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -0,0 +1,542 @@ +"""BAGEL generation graph smoke and eager/accelerated denoise structure.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn +from PIL import Image + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + bagel_cfg_dir, + config_cls, + load_omni_config, + native_model_cls, + tiny_bagel_qwen2_cfg, +) +from veomni.models.seed_omni.accelerated import OmniModelRuntime +from veomni.models.seed_omni.graphs.generation_graph import FSM_SIGNAL_KEY +from veomni.models.seed_omni.mixins.base_mixin import BaseMixin +from veomni.models.seed_omni.mixins.inference_module_mixin import InferenceModuleMixin +from veomni.models.seed_omni.modeling_omni import OmniModel +from veomni.models.seed_omni.modules.bagel.sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_FLOW_VELOCITY, + BAGEL_GENERATED_LATENT, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_START_TOKEN, + BAGEL_VAE_CONTEXT, +) +from veomni.models.seed_omni.modules.module_configuration_base import OmniModuleConfig +from veomni.models.seed_omni.modules.module_modeling_base import PretrainedOmniModule +from veomni.models.seed_omni.utils.conversation import ConversationItem +from veomni.models.seed_omni.utils.graph_profiler import GraphProfiler + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _make_veomni_runtime(cfg, modules): + # The stubs implement generation endpoints only; OmniModel checks every graph's methods. + cfg.training_graphs = {} + model = OmniModel(cfg, modules).eval() + return OmniModelRuntime(model), model + + +def test_bagel_text_encoder_injects_raw_bos_for_understanding() -> None: + """Verify ``BagelTextEncoder.generate`` appends an embedded BOS output item on the first understanding step.""" + BagelTextEncoder = native_model_cls("bagel_text_encoder") + BagelTextEncoderConfig = config_cls("bagel_text_encoder") + model = BagelTextEncoder(BagelTextEncoderConfig(vocab_size=8, hidden_size=4)).eval() + model._chat_template = SimpleNamespace(bos_token_id=3) + with torch.no_grad(): + model.embed_tokens.weight.copy_(torch.arange(32, dtype=torch.float32).reshape(8, 4)) + + prompt = ConversationItem( + type="text", + value=torch.tensor([1, 2]), + role="user", + meta={"_omni_tokenized": True, "input_ids": torch.tensor([1, 2])}, + ) + conversation = model.generate([prompt], generation_kwargs={"infer_type": "infer_und"})["conversation_list"] + + assert len(conversation) == 2 + assert conversation[-1].type == "output" + assert conversation[-1].meta[BAGEL_PHASE_KEY] == BAGEL_START_TOKEN + assert conversation[-1].meta["input_ids"].tolist() == [3] + torch.testing.assert_close(conversation[-1].value, model.embed_tokens.weight[3].reshape(1, 4)) + + +def test_bagel_qwen_first_understanding_step_prefills_prompt_then_decodes_bos(monkeypatch) -> None: + """Verify ``BagelQwen2MoT.generate`` prefills the prompt before decoding the tail BOS token.""" + BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).eval() + hidden_size = int(model.config.hidden_size) + prompt = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") + bos = ConversationItem( + type="output", + value=torch.full((1, hidden_size), 2.0), + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, + ) + calls: list[tuple[str, list[ConversationItem]]] = [] + + def _prefill(self, conversation_list, generation_kwargs): + del self, generation_kwargs + calls.append(("prefill", list(conversation_list))) + return torch.full((2, hidden_size), 3.0) + + def _decode(self, conversation_list): + del self + calls.append(("decode", list(conversation_list))) + return torch.full((1, hidden_size), 4.0) + + monkeypatch.setattr(type(model), "_prefill_prompt", _prefill) + monkeypatch.setattr(type(model), "_decode_next_token", _decode) + + conversation = model.generate( + [prompt, bos], + generation_kwargs={"infer_type": "infer_und"}, + )["conversation_list"] + + assert calls == [("prefill", [prompt]), ("decode", [prompt, bos])] + assert conversation == [prompt, bos] + assert BAGEL_PHASE_KEY not in bos.meta + torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 4.0)) + + +def test_bagel_infer_gen_denoise_signal_smoke(): + """Verify the ``infer_gen`` FSM transitions through ``prompt_encode``, ``query_denoise``, ``velocity_collect``, and ``image_decode``.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", + ) + runtime, _model = _make_veomni_runtime( + cfg, + { + "bagel_text_encoder": _InferGenTextEncoder(), + "bagel_siglip_navit": _NoopBagelSiglip(), + "bagel_qwen2_mot": _InferGenBagelQwen(), + "bagel_flow_connector": _InferGenBagelFlow(), + "bagel_vae": _InferGenBagelVAE(), + }, + ) + profiler = GraphProfiler() + request = {"conversation_list": [ConversationItem(type="text", value="prompt", role="user")]} + generated = runtime.generate( + request, + profiler=profiler, + generation_kwargs={ + "max_new_tokens": 8, + "do_sample": False, + "image_height": 64, + "image_width": 64, + }, + ) + + trace = profiler.save_records() + assert any("transition: prompt_encode -> query_denoise" in entry for entry in trace) + assert any("transition: query_denoise -> velocity_collect" in entry for entry in trace) + assert any("transition: velocity_collect -> image_decode" in entry for entry in trace) + assert any("transition: image_decode -> done" in entry for entry in trace) + assert any(item["type"] == "image" for item in generated) + assert "timestep" not in request["conversation_list"][-1].meta + + +def test_bagel_infer_gen_user_image_runs_siglip_context_only(): + """Verify ``infer_gen`` encodes user images through SigLIP without creating VAE context items.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", + ) + siglip = _CountingInferGenBagelSiglip() + runtime, _model = _make_veomni_runtime( + cfg, + { + "bagel_text_encoder": _InferGenTextEncoder(), + "bagel_siglip_navit": siglip, + "bagel_qwen2_mot": _InferGenBagelQwen(), + "bagel_flow_connector": _InferGenBagelFlow(), + "bagel_vae": _InferGenBagelVAE(), + }, + ) + request = { + "conversation_list": [ + ConversationItem( + type="image", + value=Image.new("RGB", (1, 1)), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ), + ConversationItem(type="text", value="prompt", role="user"), + ] + } + generated = runtime.generate( + request, + generation_kwargs={ + "max_new_tokens": 8, + "do_sample": False, + "image_height": 64, + "image_width": 64, + }, + ) + + assert siglip.calls == 1 + assert all(item.meta.get(BAGEL_CONTEXT_KEY) != BAGEL_VAE_CONTEXT for item in request["conversation_list"]) + assert torch.equal(request["conversation_list"][0].value[0], torch.zeros(8)) + assert torch.equal(request["conversation_list"][0].value[1:], torch.ones(3, 8)) + assert any(item["type"] == "image" for item in generated) + + +def test_bagel_infer_edit_defaults_to_denoise_signal_smoke(): + """Verify ``infer_edit`` encodes both VAE and SigLIP context and runs the denoise loop to completion.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_edit.yaml", + ) + runtime, _model = _make_veomni_runtime( + cfg, + { + "bagel_text_encoder": _InferGenTextEncoder(), + "bagel_siglip_navit": _NoopBagelSiglip(), + "bagel_qwen2_mot": _InferGenBagelQwen(), + "bagel_flow_connector": _InferEditBagelFlow(), + "bagel_vae": _InferEditBagelVAE(), + }, + ) + profiler = GraphProfiler() + request = { + "conversation_list": [ + ConversationItem( + type="image", + value=Image.new("RGB", (1, 1)), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + ), + ConversationItem( + type="image", + value=Image.new("RGB", (1, 1)), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ), + ConversationItem(type="text", value="prompt", role="user"), + ] + } + generated = runtime.generate( + request, + profiler=profiler, + generation_kwargs={ + "max_new_tokens": 8, + "do_sample": False, + "image_height": 64, + "image_width": 64, + }, + ) + + trace = profiler.save_records() + assert any("transition: prompt_encode -> query_denoise" in entry for entry in trace) + assert not any("transition: prompt_encode -> text_ar" in entry for entry in trace) + assert any("transition: image_decode -> done" in entry for entry in trace) + assert any(item["type"] == "image" for item in generated) + assert "timestep" not in request["conversation_list"][-1].meta + + +def _fake_cache(model: nn.Module, values: torch.Tensor): + cache = model._new_empty_cache() + cache.key_cache[0] = values.reshape(-1, 1, 1) + cache.value_cache[0] = (values + 100.0).reshape(-1, 1, 1) + return cache + + +def _install_three_branch_caches(model: nn.Module) -> None: + state = model._generation_state + state.main.install_cache( + cache=_fake_cache(model, torch.tensor([10.0, 11.0])), + cache_len=2, + next_position_id=torch.tensor(3), + device=model.device, + ) + state.cfg_text.install_cache( + cache=model._new_empty_cache(), + cache_len=0, + next_position_id=torch.tensor(7), + device=model.device, + ) + state.cfg_img.install_cache( + cache=_fake_cache(model, torch.tensor([20.0, 21.0, 22.0])), + cache_len=3, + next_position_id=torch.tensor(11), + device=model.device, + ) + + +def test_bagel_qwen2_mot_eager_denoise_branch_runs_serial_forward_inference(monkeypatch): + """Verify eager ``denoise_branch`` runs one serial ``forward_inference`` call per active CFG branch.""" + BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) + _install_three_branch_caches(model) + query = torch.zeros(5, int(model.config.hidden_size)) + calls: list[dict[str, object]] = [] + + def _capture_forward_inference(self, **kwargs): + del self + calls.append(kwargs) + return {"hidden_states": kwargs["packed_query_sequence"]} + + monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference) + tail = ConversationItem( + type="output", + value=query, + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, + ) + model.denoise_branch([tail], generation_kwargs={"cfg_text_scale": 2.0, "cfg_img_scale": 1.5}) + + assert len(calls) == 3 + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN + assert int(tail.value.shape[0]) == 15 + assert calls[0]["query_lens"].tolist() == [5] + assert calls[1]["query_lens"].tolist() == [5] + assert calls[2]["query_lens"].tolist() == [5] + + +def test_bagel_qwen2_mot_accelerated_denoise_branch_packs_cfg_branches(monkeypatch): + """Verify accelerated ``denoise_branch`` stacks all active CFG branches into one packed ``forward_inference`` call.""" + from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated + + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoTAccelerated(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) + _install_three_branch_caches(model) + query = torch.zeros(5, int(model.config.hidden_size)) + captured: dict[str, object] = {} + + def _capture_forward_inference(self, **kwargs): + del self + captured.update(kwargs) + return {"hidden_states": kwargs["packed_query_sequence"]} + + monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference) + tail = ConversationItem( + type="output", + value=query, + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, + ) + model.denoise_branch([tail], generation_kwargs={"cfg_text_scale": 2.0, "cfg_img_scale": 1.5}) + + assert captured["query_lens"].tolist() == [5, 5, 5] + assert int(captured["packed_query_sequence"].shape[0]) == 15 + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN + assert int(tail.value.shape[0]) == 15 + + +def _fake_cfg_branch_count(generation_kwargs: dict | None) -> int: + """Return the number of active CFG branches implied by ``generation_kwargs``.""" + branch_count = 1 + if float((generation_kwargs or {}).get("cfg_text_scale", 1.0)) > 1.0: + branch_count += 1 + if float((generation_kwargs or {}).get("cfg_img_scale", 1.0)) > 1.0: + branch_count += 1 + return branch_count + + +class _StubConfig(OmniModuleConfig): + model_type = "bagel_generation_stub" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + +class _StubModule(PretrainedOmniModule, BaseMixin, InferenceModuleMixin): + """Weightless graph participant: ``OmniModel`` only takes ``PretrainedOmniModule``.""" + + config_class = _StubConfig + + def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): + return attn_implementation + + def __init__(self): + super().__init__(_StubConfig()) + + +class _NoopBagelSiglip(_StubModule): + def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + return {"conversation_list": conversation_list} + + +class _CountingInferGenBagelSiglip(_StubModule): + def __init__(self): + super().__init__() + self.calls = 0 + + def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + self.calls += 1 + assert not any( + item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + for item in conversation_list + ) + for item in conversation_list: + if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT: + item.value = torch.ones(2, 8) + return {"conversation_list": conversation_list} + + +class _InferGenTextEncoder(_StubModule): + def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + return {"conversation_list": conversation_list} + + def encode_image_markers(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + for item in conversation_list: + if item.type not in {"image", "output"} or not torch.is_tensor(item.value) or item.value.dim() != 2: + continue + item.value = torch.cat([torch.zeros(1, 8), item.value, torch.ones(1, 8)], dim=0) + return {"conversation_list": conversation_list} + + +class _InferGenBagelQwen(_StubModule): + def generate( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict | None = None, + **kwargs, + ): + del kwargs + assert conversation_list is not None + if not conversation_list or conversation_list[-1].type != "output": + return {"conversation_list": conversation_list} + tail = conversation_list[-1] + if tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_QUERY: + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = tail.value.repeat(_fake_cfg_branch_count(generation_kwargs), 1) + return {"conversation_list": conversation_list} + if tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_VELOCITY: + tail.value = torch.zeros(16, 4) + return {"conversation_list": conversation_list} + return {"conversation_list": conversation_list} + + def denoise_branch( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict | None = None, + **kwargs, + ): + del kwargs + assert conversation_list is not None + tail = conversation_list[-1] + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_QUERY + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = tail.value.repeat(_fake_cfg_branch_count(generation_kwargs), 1) + return {"conversation_list": conversation_list} + + def collect_velocity( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict | None = None, + **kwargs, + ): + del kwargs + assert conversation_list is not None + tail = conversation_list[-1] + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_VELOCITY + assert tail.value.shape[0] == 18 * _fake_cfg_branch_count(generation_kwargs) + tail.value = torch.zeros(16, 4) + return {"conversation_list": conversation_list} + + +class _InferGenBagelFlow(_StubModule): + def prepare_denoise_query(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + if ( + conversation_list + and conversation_list[-1].type == "output" + and not torch.is_tensor(conversation_list[-1].value) + ): + item = conversation_list[-1] + else: + item = conversation_list[-1] if conversation_list and conversation_list[-1].type == "output" else None + if item is None: + conversation_list.append( + ConversationItem( + type="output", + value=torch.zeros(16, 8), + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": torch.tensor(0.5)}, + ) + ) + else: + item.value = torch.zeros(16, 8) + item.meta = {BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": torch.tensor(0.5)} + return {"conversation_list": conversation_list} + + def decode_velocity_from_hidden(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + item = conversation_list[-1] + assert item.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN + assert item.value.shape[0] % 18 == 0 + item.value = torch.zeros(item.value.shape[0], 4) + item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY + return {"conversation_list": conversation_list} + + def advance_denoise(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + item = conversation_list[-1] + item.value = torch.zeros(1, 4, 4) + item.meta[BAGEL_PHASE_KEY] = BAGEL_GENERATED_LATENT + item.meta.pop("timestep", None) + return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: "image_complete"} + + +class _InferGenBagelVAE(_StubModule): + def decode_generated(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + assert "timestep" not in conversation_list[-1].meta + return { + "conversation_list": conversation_list, + "generated": {"type": "image", "value": Image.new("RGB", (1, 1)), "meta": {}}, + } + + +class _InferEditBagelVAE(_InferGenBagelVAE): + def encode_context(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + for item in conversation_list: + if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT: + item.value = torch.zeros(4, 4, 4) + return {"conversation_list": conversation_list} + + +class _InferEditBagelFlow(_InferGenBagelFlow): + def embed_context_latents(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + for item in conversation_list: + if ( + item.type == "image" + and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + and torch.is_tensor(item.value) + ): + item.value = torch.zeros(16, 8) + item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} + return {"conversation_list": conversation_list} diff --git a/tests/seed_omni/modules/bagel/test_graph_config.py b/tests/seed_omni/modules/bagel/test_graph_config.py new file mode 100644 index 0000000000..9247e0c37d --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_graph_config.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import pytest +import yaml + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import bagel_cfg_dir, load_module_runtime_args, load_omni_config +from veomni.models.seed_omni.graphs.generation_graph import GenerationGraph + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def test_bagel_train_yaml_loads_with_v2_module_names(): + """Verify that the BAGEL training YAML resolves all five module entries and graph endpoints.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", + ) + + assert set(cfg._module_entries) == { + "bagel_text_encoder", + "bagel_siglip_navit", + "bagel_qwen2_mot", + "bagel_flow_connector", + "bagel_vae", + } + assert isinstance(cfg.training_graph, list) and cfg.training_graph + endpoints = {e["from"] for e in cfg.training_graph} | {e["to"] for e in cfg.training_graph} + assert "bagel_text_encoder.encode" in endpoints + assert "bagel_siglip_navit" in endpoints + assert "bagel_vae.encode" in endpoints + assert "bagel_flow_connector.embed_latent" in endpoints + assert "bagel_text_encoder.decode" in endpoints + assert "bagel_flow_connector.decode_velocity" in endpoints + assert "end" in endpoints + runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") + assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") + + +def test_bagel_train_graph_fan_in_execution_order(): + """Verify topological execution order of the BAGEL training DAG.""" + from veomni.models.seed_omni.graphs.training_graph import TrainingGraph + + graph = TrainingGraph(yaml.safe_load((bagel_cfg_dir() / "train/graph_train.yaml").read_text())) + order = graph.execution_order + assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_text_encoder.encode") + assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_siglip_navit.forward") + assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_flow_connector.embed_latent") + assert order.index("bagel_flow_connector.embed_latent") > order.index("bagel_vae.encode") + assert set(graph.sources) == { + "bagel_text_encoder.encode", + "bagel_siglip_navit.forward", + "bagel_vae.encode", + } + + +def test_bagel_infer_gen_graph_uses_siglip_context_without_vae_context(): + """Verify ``graph_infer_gen.yaml`` routes prompt encoding through text and SigLIP only.""" + data = yaml.safe_load((bagel_cfg_dir() / "infer/graph_infer_gen.yaml").read_text()) + prompt_body = data["states"]["prompt_encode"]["body"] + + assert {"from": "bagel_text_encoder", "to": "bagel_qwen2_mot"} in prompt_body + assert {"from": "bagel_siglip_navit", "to": "bagel_qwen2_mot"} in prompt_body + assert all(edge["from"] != "bagel_vae.encode_context" for edge in prompt_body) + assert all(edge["from"] != "bagel_flow_connector.embed_context_latents" for edge in prompt_body) + + +def test_bagel_edit_prompt_graph_exposes_independent_prompt_producers() -> None: + """Verify ``graph_infer_edit.yaml`` wires VAE context and SigLIP context as independent prompt producers.""" + graph_config = yaml.safe_load((bagel_cfg_dir() / "infer/graph_infer_edit.yaml").read_text()) + prompt_body = graph_config["states"]["prompt_encode"]["body"] + + assert {"from": "bagel_text_encoder", "to": "bagel_qwen2_mot"} in prompt_body + assert {"from": "bagel_siglip_navit", "to": "bagel_qwen2_mot"} in prompt_body + assert {"from": "bagel_vae.encode_context", "to": "bagel_flow_connector.embed_context_latents"} in prompt_body + assert {"from": "bagel_flow_connector.embed_context_latents", "to": "bagel_qwen2_mot"} in prompt_body + assert all(edge["to"] != "bagel_text_encoder.encode_image_markers" for edge in prompt_body) + + forbidden_edges = { + ("bagel_vae.encode_context", "bagel_siglip_navit"), + ("bagel_siglip_navit", "bagel_vae.encode_context"), + ("bagel_siglip_navit", "bagel_flow_connector.embed_context_latents"), + } + assert not forbidden_edges.intersection({(edge["from"], edge["to"]) for edge in prompt_body}) + + graph = GenerationGraph(graph_config) + sequence = graph.state_node_sequence("prompt_encode") + assert "bagel_vae.encode_context" in sequence + assert "bagel_siglip_navit.generate" in sequence + + +@pytest.mark.parametrize( + "infer_graph", + ["infer/graph_infer_und.yaml", "infer/graph_infer_gen.yaml", "infer/graph_infer_edit.yaml"], +) +def test_bagel_train_plus_infer_merges_generation_graph(infer_graph: str): + """Verify that each BAGEL inference graph merges cleanly onto the training config and reaches ``done``.""" + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / infer_graph, + ) + assert set(cfg._module_entries) == { + "bagel_text_encoder", + "bagel_siglip_navit", + "bagel_qwen2_mot", + "bagel_flow_connector", + "bagel_vae", + } + assert cfg.generation_graph is not None + runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") + assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") + assert cfg.generation_graph["initial"] == "prompt_encode" + assert "done" not in cfg.generation_graph["states"] + assert any( + t.get("next_state") == "done" + for state in cfg.generation_graph["states"].values() + for t in state.get("transitions", []) + ), f"{infer_graph} has no transition to `done`." + for state_name, state in cfg.generation_graph["states"].items(): + for e in state.get("body", []): + assert isinstance(e, dict) and "from" in e and "to" in e, ( + f"state '{state_name}' body item must be a `{{from, to}}` dict: {e!r}" + ) diff --git a/tests/seed_omni/modules/bagel/test_masking.py b/tests/seed_omni/modules/bagel/test_masking.py new file mode 100644 index 0000000000..838cdc5364 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_masking.py @@ -0,0 +1,100 @@ +"""BAGEL Qwen2-MoT mask materializers: Flex, Magi, and SDPA share one metadata.""" + +from __future__ import annotations + +import pytest +import torch +from torch.nn.attention.flex_attention import BlockMask + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAttentionAccelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + build_mot_magi_mask, + build_mot_sdpa_mask, + pad_mot_attention_metadata, +) +from veomni.ops.mask import MagiAttentionMask + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _expand_magi_mask(mask: MagiAttentionMask, sequence_length: int) -> torch.Tensor: + """Materialize a ``MagiAttentionMask`` into a dense ``[S, S]`` boolean visibility matrix.""" + visible = torch.zeros(sequence_length, sequence_length, dtype=torch.bool, device=mask.q_ranges.device) + attn_types = mask.attn_type_map + if attn_types is None: + attn_types = torch.zeros(mask.q_ranges.shape[0], dtype=torch.int32, device=mask.q_ranges.device) + for range_index in range(mask.q_ranges.shape[0]): + query_start, query_end = (int(mask.q_ranges[range_index, 0]), int(mask.q_ranges[range_index, 1])) + key_start, key_end = (int(mask.k_ranges[range_index, 0]), int(mask.k_ranges[range_index, 1])) + attn_type = int(attn_types[range_index]) + if attn_type == 0: + visible[query_start:query_end, key_start:key_end] = True + continue + if attn_type != 1: + raise AssertionError(f"Unexpected Magi attn type {attn_type} in BAGEL mask tests.") + query_idx = torch.arange(query_start, query_end, device=visible.device) + key_idx = torch.arange(key_start, key_end, device=visible.device) + visible[query_start:query_end, key_start:key_end] |= query_idx[:, None] >= key_idx[None, :] + return visible + + +@pytest.mark.parametrize( + ("sample_splits", "sample_attn_modes"), + [ + ([[8]], [["causal"]]), + ([[4, 8]], [["full", "causal"]]), + ([[8, 6]], [["causal", "noise"]]), + ([[4, 8, 6]], [["full", "causal", "noise"]]), + ([[3, 4, 5]], [["causal", "noise", "causal"]]), + ([[4], [5]], [["causal"], ["causal"]]), + ([[2, 3], [4, 2]], [["causal", "full"], ["causal", "noise"]]), + ], +) +def test_magi_mask_matches_sdpa_visibility( + sample_splits: list[list[int]], + sample_attn_modes: list[list[str]], +) -> None: + metadata = build_mot_attention_metadata(sample_splits, sample_attn_modes, device=torch.device("cpu")) + sdpa = build_mot_sdpa_mask(metadata) + magi = build_mot_magi_mask(metadata) + visible = _expand_magi_mask(magi, int(metadata.shape[1])) + torch.testing.assert_close(visible[None, None], sdpa) + + +def test_magi_mask_keeps_padding_as_isolated_full_document() -> None: + """Verify that SP-padded tokens form an isolated document invisible to real query tokens.""" + metadata = build_mot_attention_metadata([[4]], [["causal"]], device=torch.device("cpu")) + padded = pad_mot_attention_metadata(metadata, padded_length=8) + sdpa = build_mot_sdpa_mask(padded) + magi = build_mot_magi_mask(padded) + visible = _expand_magi_mask(magi, 8) + torch.testing.assert_close(visible[None, None], sdpa) + assert not bool(visible[:4, 4:].any()) + assert bool(visible[4:, 4:].all()) + + +def test_accelerated_build_attention_mask_dispatches_flex_and_magi() -> None: + """Verify that ``BagelQwen2MoTAttentionAccelerated.build_attention_mask`` dispatches Flex and Magi.""" + metadata = build_mot_attention_metadata([[4, 4]], [["full", "causal"]], device=torch.device("cpu")) + flex_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask( + metadata, + attn_implementation="veomni_flex_attention", + ) + magi_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask( + metadata, + attn_implementation="veomni_magi_attention", + ) + assert isinstance(flex_mask, BlockMask) + assert isinstance(magi_mask, MagiAttentionMask) + default_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask(metadata) + assert isinstance(default_mask, BlockMask) + + +def test_accelerated_build_attention_mask_rejects_unsupported_backend() -> None: + """Verify that ``build_attention_mask`` rejects non-packed attention backends.""" + metadata = build_mot_attention_metadata([[2]], [["causal"]], device=torch.device("cpu")) + with pytest.raises(ValueError, match="packed fused attention"): + BagelQwen2MoTAttentionAccelerated.build_attention_mask(metadata, attn_implementation="sdpa") diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py new file mode 100644 index 0000000000..4cf4896753 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -0,0 +1,962 @@ +from __future__ import annotations + +import copy +import pickle +from inspect import Parameter, signature +from types import SimpleNamespace + +import pytest +import torch + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import ( + config_cls, + model_cls, + native_model_cls, + tiny_bagel_qwen2_cfg, +) +from veomni.models.seed_omni import ( + OMNI_ACCELERATED_MODEL_REGISTRY, + OMNI_CONFIG_REGISTRY, + OMNI_MODEL_REGISTRY, + OMNI_PROCESSOR_REGISTRY, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import TrainingMixin +from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import preprocess_mot_inputs +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + _OMNI_POSITION_IDS as BAGEL_SIGLIP_POSITION_IDS, +) +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + _OMNI_TOKEN_LEN as BAGEL_SIGLIP_TOKEN_LEN, +) +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + BagelSiglipNavitPreprocessor, + BagelSiglipNavitProcessor, +) +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT +from veomni.models.seed_omni.modules.bagel.text_encoder.chat_template import BagelChatTemplate +from veomni.models.seed_omni.modules.bagel.text_encoder.modeling import ( + _OMNI_TOKENIZED as BAGEL_TOK, +) +from veomni.models.seed_omni.modules.bagel.text_encoder.processing import ( + BagelTextEncoderPreprocessor, +) +from veomni.models.seed_omni.modules.bagel.vae.processing import ( + BAGEL_VAE_PIXEL_SHAPE, + BAGEL_VAE_POSTERIOR, + BagelVAEPreprocessor, + BagelVAEProcessor, +) +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +@pytest.mark.parametrize("conversation_list", [None, []]) +def test_bagel_mot_packing_rejects_missing_conversation_list(conversation_list): + """Verify ``preprocess_mot_inputs`` rejects ``None`` or empty ``conversation_list``.""" + with pytest.raises(ValueError, match="requires a non-empty conversation_list"): + preprocess_mot_inputs( + conversation_list, + device=torch.device("cpu"), + dtype=torch.float32, + hidden_size=4, + ) + + +def test_bagel_mot_packing_rejects_incompatible_vae_img_tag(): + """Verify ``preprocess_mot_inputs`` rejects VAE context items tagged with ``_img_tag='und'``.""" + item = ConversationItem( + type="image", + value=torch.ones(2, 4), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "und"}, + ) + + with pytest.raises(ValueError, match="_img_tag"): + preprocess_mot_inputs([[item]], device=torch.device("cpu"), dtype=torch.float32, hidden_size=4) + + +def test_bagel_mot_packing_routes_tagged_edit_vae_through_generation_expert(): + """Verify ``edit`` and ``gen`` VAE context items route through the generation expert while SigLIP routes through und.""" + edit_context = ConversationItem( + type="image", + value=torch.ones(2, 4), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, + ) + gen_target = ConversationItem( + type="image", + value=torch.full((2, 4), 2), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + siglip_context = ConversationItem( + type="image", + value=torch.full((1, 4), 3), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "edit"}, + ) + + packed = preprocess_mot_inputs( + [[edit_context, gen_target, siglip_context]], + device=torch.device("cpu"), + dtype=torch.float32, + hidden_size=4, + ) + + assert packed is not None + assert torch.equal(packed.packed_token_type_ids, torch.tensor([1, 1, 1, 1, 0])) + assert packed.sample_splits == [[2, 2, 1]] + + +def test_bagel_mot_forward_pre_returns_sample_local_tensor_contract(): + """Verify ``BagelQwen2MoTAccelerated.forward_pre`` returns the packed tensor contract expected by ``forward``.""" + BagelQwen2MoT = model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train() + hidden_size = int(model.config.hidden_size) + text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") + gen_target = ConversationItem( + type="image", + value=torch.ones(3, hidden_size), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + + inputs = model.forward_pre(conversation_list=[[text, gen_target]]) + + # The accelerated class inherits the graph's generic ``forward(**kwargs)`` + # trampoline, so the explicit packed-tensor contract lives on the native one. + forward_parameters = signature(native_model_cls("bagel_qwen2_mot").forward).parameters + assert "packed_token_type_ids" in forward_parameters + assert "packed_und_token_indexes" not in forward_parameters + assert "packed_gen_token_indexes" not in forward_parameters + assert all(parameter.kind is not Parameter.VAR_KEYWORD for parameter in forward_parameters.values()) + assert set(inputs) == { + "packed_sequence", + "packed_position_ids", + "packed_token_type_ids", + "packed_attention_metadata", + } + assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1])) + assert inputs["packed_attention_metadata"].shape == (3, 5) + assert inputs["packed_attention_metadata"].dtype == torch.int32 + assert inputs["packed_attention_metadata"].is_contiguous() + + +def test_bagel_mot_forward_pre_keeps_metadata_full_and_marks_sequence_padding(monkeypatch): + """Verify ``forward_pre`` under Ulysses SP pads sequence tensors with ``-1`` token types and keeps full metadata.""" + from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated + + BagelQwen2MoT = model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train() + monkeypatch.setattr( + accelerated, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=4, cp_size=1, ulysses_size=4, sp_group=object()), + ) + + def fake_sp_pad(tensor, dim, pad_value): + pad_shape = list(tensor.shape) + pad_shape[dim] = 3 + padding = torch.full(pad_shape, pad_value, device=tensor.device, dtype=tensor.dtype) + return torch.cat((tensor, padding), dim=dim) + + monkeypatch.setattr(accelerated, "sp_pad", fake_sp_pad) + monkeypatch.setattr(accelerated, "slice_input_tensor", lambda tensor, **kwargs: tensor) + hidden_size = int(model.config.hidden_size) + text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") + image = ConversationItem( + type="image", + value=torch.ones(3, hidden_size), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + + inputs = model.forward_pre(conversation_list=[[text, image]]) + + assert inputs["packed_attention_metadata"].shape == (3, 8) + assert inputs["packed_sequence"].shape[0] == 8 + assert set(inputs) == { + "packed_sequence", + "packed_position_ids", + "packed_token_type_ids", + "packed_attention_metadata", + } + assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1, -1, -1, -1])) + assert torch.equal(inputs["packed_attention_metadata"][0, 5:], torch.tensor([5, 5, 5], dtype=torch.int32)) + assert torch.equal(inputs["packed_attention_metadata"][1, 5:], torch.tensor([5, 5, 5], dtype=torch.int32)) + assert torch.equal(inputs["packed_attention_metadata"][2, 5:], torch.tensor([-1, -1, -1], dtype=torch.int32)) + + +def test_bagel_flow_training_embed_treats_edit_context_as_clean_and_gen_as_target(): + """Verify ``embed_latent_pre`` leaves ``edit`` context at timestep 0 without targets and noises ``gen`` targets.""" + BagelFlowConnector = model_cls("bagel_flow_connector") + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + model = BagelFlowConnector( + BagelFlowConnectorConfig( + hidden_size=4, + z_channels=1, + latent_patch_size=1, + patch_latent_dim=1, + max_latent_size=4, + timestep_frequency_embedding_size=4, + timestep_shift=1.0, + ) + ) + edit_context = ConversationItem( + type="image", + value=torch.ones(1, 1, 2), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, + ) + gen_target = ConversationItem( + type="image", + value=torch.full((1, 1, 2), 2.0), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + + inputs = model.embed_latent_pre(conversation_list=[[edit_context, gen_target]]) + + assert model._embed_items == [edit_context, gen_target] + assert model._embed_lengths == [2, 2] + assert torch.equal(inputs["latents"][:2], torch.ones(2, 1)) + assert torch.equal(inputs["timesteps"][:2], torch.zeros(2)) + assert "flow_velocity_target" not in edit_context.meta + assert "timestep" not in edit_context.meta + assert gen_target.meta["flow_velocity_target"].shape == (2, 1) + assert gen_target.meta["timestep"].shape == (2,) + + +def test_mot_forward_post_scatters_virtual_marker_triplet_hidden_states() -> None: + """Verify ``forward_post`` scatters packed hidden states back across ``(start, image, end)`` triplet items.""" + hidden_size = 4 + markers = torch.tensor([[101.0, 102.0, 103.0, 104.0], [201.0, 202.0, 203.0, 204.0]]) + + def marker(row: int, context: str) -> ConversationItem: + return ConversationItem( + type="text", + value=markers[row : row + 1], + role="user", + meta={"labels": torch.full((1,), -100), BAGEL_CONTEXT_KEY: context}, + ) + + latent_item = ConversationItem( + type="image", + value=torch.arange(8, dtype=torch.float32).reshape(2, hidden_size) + 20, + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + ) + siglip_item = ConversationItem( + type="image", + value=torch.arange(12, dtype=torch.float32).reshape(3, hidden_size), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ) + text_item = ConversationItem( + type="text", + value=torch.arange(8, dtype=torch.float32).reshape(2, hidden_size) + 40, + role="user", + ) + sample = [ + marker(0, BAGEL_VAE_CONTEXT), + latent_item, + marker(1, BAGEL_VAE_CONTEXT), + marker(0, BAGEL_SIGLIP_CONTEXT), + siglip_item, + marker(1, BAGEL_SIGLIP_CONTEXT), + text_item, + ] + packed = preprocess_mot_inputs( + [sample], + device=torch.device("cpu"), + dtype=torch.float32, + hidden_size=hidden_size, + ) + assert packed is not None + + model = SimpleNamespace( + device=torch.device("cpu"), + _conversation_carrier=[sample], + _packed_training=packed, + ) + hidden_states = torch.arange(11 * hidden_size, dtype=torch.float32).reshape(11, hidden_size) + + out = TrainingMixin.forward_post(model, hidden_states) + + assert out["conversation_list"] == [sample] + assert torch.equal(sample[0].value, hidden_states[0:1]) + assert torch.equal(latent_item.value, hidden_states[1:3]) + assert torch.equal(sample[2].value, hidden_states[3:4]) + assert torch.equal(sample[3].value, hidden_states[4:5]) + assert torch.equal(siglip_item.value, hidden_states[5:8]) + assert torch.equal(sample[5].value, hidden_states[8:9]) + assert torch.equal(text_item.value, hidden_states[9:11]) + + +def test_bagel_vae_online_process_runs_on_a_meta_built_model() -> None: + """``offline_training`` builds the VAE on meta with no weights, because it mixes in ``OfflineEncodingMixin``.""" + encode_model = _tiny_vae() + process_model = _tiny_vae(meta=True) + encoded_cache = encode_model.offline_encode(pixel_values=torch.zeros(1, 3, 8, 8))["encoded_cache"] + item_cache = encoded_cache[0].reshape(2, process_model.config.z_channels, *encoded_cache.shape[-2:]) + + assert process_model.encoder.conv_in.weight.is_meta + latents = process_model.online_process(encoded_cache=item_cache)["latents"] + assert isinstance(latents, list) + assert latents[0].shape == item_cache.shape[1:] + assert not latents[0].is_meta + + +def test_bagel_vae_online_process_consumes_variable_size_cache_items_without_padding() -> None: + """Verify ``online_process`` samples variable-size cached posteriors without spatial batch padding.""" + model = _tiny_vae(meta=True) + first = ConversationItem( + type="image", + value=torch.zeros(2, 2, 2, 1), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + ) + second = ConversationItem( + type="image", + value=torch.zeros(2, 2, 2, 2), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + ) + conversation = [[first], [second]] + + pre = model.online_process_pre(conversation_list=conversation) + out = model.online_process(encoded_cache=pre["encoded_cache"]) + post = model.post_forward("online_process", latents=out["latents"]) + + assert isinstance(pre["encoded_cache"], list) + assert [tuple(cache.shape) for cache in pre["encoded_cache"]] == [(2, 2, 2, 1), (2, 2, 2, 2)] + assert post["conversation_list"] is conversation + assert first.value.shape == (2, 2, 1) + assert second.value.shape == (2, 2, 2) + + +def test_bagel_vae_offline_cache_replay_keeps_img_tag_for_flow_connector(tmp_path) -> None: + """Verify offline VAE cache parquet round-trip preserves ``_img_tag`` for downstream flow connector routing.""" + from datasets import load_dataset + + from veomni.data.seed_omni.seedomni_transform import process_seedomni_cached_example + from veomni.models.seed_omni.utils.offline_cache import SeedOmniOfflineCacheWriter + + encode_model = _tiny_vae() + process_model = _tiny_vae(meta=True) + + def vae_image(tag: str) -> ConversationItem: + return ConversationItem( + type="image", + value=torch.zeros(3, 4, 4), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: tag, + BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 4]), + }, + ) + + pre = encode_model.pre_forward("offline_encode", conversation_list=[[vae_image("edit"), vae_image("gen")]]) + out = encode_model.offline_encode(**pre) + encoded = encode_model.post_forward("offline_encode", **out)["conversation_list"] + + writer = SeedOmniOfflineCacheWriter(str(tmp_path)) + writer.open() + writer.save_conversation_list(encoded) + writer.flush() + dataset = load_dataset("parquet", data_files=[str(tmp_path / "shard_000000.parquet")], split="train") + replayed = [process_seedomni_cached_example(dataset[0])[0]["conversation_list"]] + assert all(item.meta[BAGEL_VAE_POSTERIOR] for item in replayed[0]) + + pre = process_model.pre_forward("online_process", conversation_list=replayed) + out = process_model.online_process(**pre) + processed = process_model.post_forward("online_process", **out)["conversation_list"] + + assert [item.meta[_IMG_TAG_KEY] for item in processed[0]] == ["edit", "gen"] + assert all(item.meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT for item in processed[0]) + + BagelFlowConnector = model_cls("bagel_flow_connector") + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + flow_connector = BagelFlowConnector( + BagelFlowConnectorConfig( + hidden_size=4, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + timestep_shift=1.0, + ) + ) + edit_item, gen_item = processed[0] + inputs = flow_connector.embed_latent_pre(conversation_list=processed) + + assert flow_connector._embed_lengths == [16, 16] + assert torch.equal(inputs["timesteps"][:16], torch.zeros(16)) + assert "flow_velocity_target" not in edit_item.meta + assert gen_item.meta["flow_velocity_target"].shape == (16, 2) + + +def _tiny_vae(meta=False, **config_overrides): + """Construct a tiny ``BagelVAE`` on CPU or meta device with a bound processor.""" + BagelVAE = model_cls("bagel_vae") + BagelVAEConfig = config_cls("bagel_vae") + config_kwargs = dict( + resolution=8, + ch=32, + ch_mult=[1], + num_res_blocks=1, + z_channels=2, + max_image_size=8, + min_image_size=4, + image_stride=4, + max_pixels=64, + downsample=1, + ) + config_kwargs.update(config_overrides) + with torch.device("meta" if meta else "cpu"): + model = BagelVAE(BagelVAEConfig(**config_kwargs)) + model._image_processor = BagelVAEProcessor.from_config(model.config) + return model + + +def _worker_dummies(conversation_list, role): + """Return worker-appended ``is_dummy`` image placeholders standing in for ``role`` images.""" + return [it for it in iter_desired_items(conversation_list, types=["image"], roles=[role]) if it.is_dummy] + + +class _FakeBagelTokenizer: + """Char-ordinal tokenizer with the marker tokens/ids that ``BagelChatTemplate`` resolves.""" + + bos_token = "" + eos_token = "" + boi_token = "" + eoi_token = "" + bos_token_id = 1 + eos_token_id = 2 + boi_token_id = 3 + eoi_token_id = 4 + pad_token_id = 0 + unk_token_id = -1 + special_tokens = { + "": 1, + "": 2, + "": 3, + "": 4, + "<|im_start|>": 5, + "<|im_end|>": 2, + "<|vision_start|>": 7, + "<|vision_end|>": 8, + } + + def convert_tokens_to_ids(self, token): + return self.special_tokens.get(token, self.unk_token_id) + + def __call__(self, text, add_special_tokens=False): + del add_special_tokens + input_ids = [] + index = 0 + while index < len(text): + matched = False + for token, token_id in sorted(self.special_tokens.items(), key=lambda item: len(item[0]), reverse=True): + if text.startswith(token, index): + input_ids.append(token_id) + index += len(token) + matched = True + break + if matched: + continue + input_ids.append(ord(text[index])) + index += 1 + return {"input_ids": input_ids} + + +def _bagel_template() -> BagelChatTemplate: + return BagelChatTemplate(_FakeBagelTokenizer()) + + +def _b(conversation_list): + return {"conversation_list": conversation_list} + + +def test_bagel_registry_entries_and_processors() -> None: + """Verify all five BAGEL modules and image processors are registered in the SeedOmni registries.""" + expected = { + "bagel_text_encoder": ("BagelTextEncoder", "BagelTextEncoderAccelerated"), + "bagel_siglip_navit": ("BagelSiglipNavit", "BagelSiglipNavitAccelerated"), + "bagel_qwen2_mot": ("BagelQwen2MoT", "BagelQwen2MoTAccelerated"), + "bagel_flow_connector": ("BagelFlowConnector", "BagelFlowConnectorAccelerated"), + "bagel_vae": ("BagelVAE", "BagelVAEAccelerated"), + } + for model_type, (native_name, accel_name) in expected.items(): + assert model_type in OMNI_CONFIG_REGISTRY.valid_keys() + assert OMNI_MODEL_REGISTRY[model_type]().__name__ == native_name + assert OMNI_ACCELERATED_MODEL_REGISTRY[model_type]().__name__ == accel_name + assert {"bagel_siglip_navit", "bagel_vae"}.issubset(set(OMNI_PROCESSOR_REGISTRY.valid_keys())) + + +def test_bagel_text_preprocessor_tokenizes_plain_items_and_is_idempotent() -> None: + """Verify ``BagelTextEncoderPreprocessor`` tokenizes text items, wraps images with markers, and is idempotent.""" + pre = BagelTextEncoderPreprocessor(_bagel_template()) + batch = [ + [ + ConversationItem(type="text", value="hi", role="user"), + ConversationItem( + type="image", value=torch.zeros(3, 4, 4), role="user", meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT} + ), + ConversationItem(type="text", value="ok", role="assistant"), + ] + ] + + pre(_b(batch)) + user_text, image_start, image, image_end, assistant_text = batch[0] + + assert image.type == "image" + assert image.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert image_start.type == "text" + assert image_start.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert torch.equal(image_start.value, torch.tensor([7])) + assert image_end.type == "text" + assert image_end.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert torch.equal(image_end.value, torch.tensor([8])) + assert torch.equal(user_text.value, torch.tensor([5, ord("h"), ord("i"), 2])) + assert user_text.value.device.type == "cpu" + assert user_text.meta[BAGEL_TOK] is True + assert torch.equal(user_text.meta["labels"], torch.full_like(user_text.value, -100)) + assert torch.equal(user_text.meta["attention_mask"], torch.ones_like(user_text.value)) + + assert torch.equal(assistant_text.value, torch.tensor([5, ord("o"), ord("k"), 2])) + assert assistant_text.meta[BAGEL_TOK] is True + assert torch.equal(assistant_text.meta["labels"], assistant_text.value) + + snapshot = copy.deepcopy(batch) + pre(_b(batch)) + assert len(batch[0]) == len(snapshot[0]) + for actual, expected in zip(batch[0], snapshot[0]): + if isinstance(actual.value, torch.Tensor): + assert torch.equal(actual.value, expected.value) + + +def test_bagel_siglip_preprocessor_patchifies_and_tags_context() -> None: + """Verify ``BagelSiglipNavitPreprocessor`` patchifies user images and populates position/length metadata.""" + pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ) + batch = [ + [ + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 7, dtype=torch.uint8), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ) + ] + ] + + pre(_b(batch)) + item = batch[0][0] + assert item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert item.meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 + assert item.meta[BAGEL_SIGLIP_POSITION_IDS].tolist() == [0, 1, 2, 3] + assert item.value.shape == (4, 2 * 2 * 3) + assert item.value.dtype == torch.bfloat16 + + +def test_bagel_siglip_preprocessor_appends_per_sample_dummy_for_missing_context() -> None: + """Verify ``BagelSiglipNavitPreprocessor`` appends a dummy SigLIP image to samples without one.""" + pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ) + batch = [ + [ + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 7, dtype=torch.uint8), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ) + ], + [ConversationItem(type="text", value="text-only", role="user")], + ] + + pre(_b(batch)) + + assert len(_worker_dummies(batch, "user")) == 1 + assert len(batch[0]) == 1 + dummy = batch[1][-1] + assert dummy.type == "image" + assert dummy.role == "user" and dummy.is_dummy + assert dummy.meta[_IMG_TAG_KEY] == "und" + assert dummy.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT + assert dummy.meta[BAGEL_SIGLIP_TOKEN_LEN] == 1 + assert dummy.meta[BAGEL_SIGLIP_POSITION_IDS].tolist() == [0] + assert dummy.value.shape == (1, 2 * 2 * 3) + assert dummy.value.dtype == torch.bfloat16 + + +def test_bagel_siglip_preprocessor_keeps_bs4_sample_aligned_for_0_2_4_images() -> None: + """Verify ``BagelSiglipNavitPreprocessor`` maintains one SigLIP item per sample across mixed 4-sample batches.""" + pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ) + + for real_count in (0, 2, 4): + batch = [] + for index in range(4): + if index < real_count: + batch.append( + [ + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 7, dtype=torch.uint8), + role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ) + ] + ) + else: + batch.append([ConversationItem(type="text", value=f"text-{index}", role="user")]) + + pre(_b(batch)) + + assert len(_worker_dummies(batch, "user")) == 4 - real_count + for sample in batch: + context_items = [ + item + for item in sample + if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + ] + assert len(context_items) == 1 + + +def test_bagel_vae_preprocessor_leaves_cached_posteriors_alone() -> None: + """An ``offline_training`` batch: ``offline_encode`` already turned every VAE + row (its dummies included) into a posterior, so nothing is re-processed or added.""" + pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + posterior = torch.randn(2, 4, 1, 1) + meta = {_IMG_TAG_KEY: "gen", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, BAGEL_VAE_POSTERIOR: True} + batch = [ + [ConversationItem(type="image", value=posterior, role="assistant", meta=dict(meta))], + [ConversationItem(type="image", value=posterior.clone(), role="assistant", meta=dict(meta), is_dummy=True)], + ] + + pre(_b(batch)) + + assert [len(sample) for sample in batch] == [1, 1] + assert batch[0][0].value is posterior + + +def test_bagel_preprocessors_route_inference_edit_prompt_context() -> None: + """Verify ``infer_edit`` preprocessing expands a user image into both VAE and SigLIP context triplets.""" + text_pre = BagelTextEncoderPreprocessor(_bagel_template()) + siglip_pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ) + vae_pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) + assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) + batch = [ + [ + ConversationItem(type="text", value="hi", role="user"), + ConversationItem(type="image", value=user_image.clone(), role="user"), + ConversationItem( + type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} + ), + ] + ] + + for preprocessor in (vae_pre, siglip_pre, text_pre): + preprocessor(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_edit"}) + + sample = batch[0] + assert [item.type for item in sample] == [ + "text", + "text", + "image", + "text", + "text", + "image", + "text", + "text", + "image", + "text", + ] + assert torch.equal(sample[0].value, torch.tensor([5, ord("h"), ord("i"), 2])) + assert sample[2].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + assert sample[5].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert sample[8].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + assert sample[2].value.shape == (3, 4, 4) + assert sample[2].value.dtype == torch.bfloat16 + assert sample[5].value.shape == (4, 2 * 2 * 3) + assert sample[5].value.dtype == torch.bfloat16 + assert sample[5].meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 + assert sample[8].value.shape == (3, 4, 4) + assert sample[8].value.dtype == torch.bfloat16 + assert [sample[i].meta.get(BAGEL_CONTEXT_KEY) for i in [1, 3, 4, 6, 7, 9]] == [ + BAGEL_VAE_CONTEXT, + BAGEL_VAE_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + BAGEL_VAE_CONTEXT, + ] + assert [int(sample[i].value.numel()) for i in [1, 3, 4, 6, 7, 9]] == [1, 1, 1, 1, 1, 1] + + +def test_bagel_preprocessors_route_tagged_edit_without_infer_type() -> None: + """Verify training preprocessing routes ``_img_tag='edit'`` images into VAE and SigLIP copies.""" + text_pre = BagelTextEncoderPreprocessor(_bagel_template()) + siglip_pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 12, dtype=torch.bfloat16), + ) + vae_pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) + assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) + batch = [ + [ + ConversationItem(type="text", value="edit", role="user"), + ConversationItem(type="image", value=user_image.clone(), role="user", meta={_IMG_TAG_KEY: "edit"}), + ConversationItem( + type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} + ), + ] + ] + + for preprocessor in (vae_pre, siglip_pre, text_pre): + preprocessor(_b(batch)) + + sample = batch[0] + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in sample if item.type == "image"] == [ + BAGEL_VAE_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + ] + assert [item.meta.get(_IMG_TAG_KEY) for item in sample if item.type == "image"] == ["edit", "edit", "gen"] + vae_source, siglip_source, vae_target = (item for item in sample if item.type == "image") + assert vae_source.value.shape == (3, 4, 4) + assert vae_source.value.dtype == torch.bfloat16 + assert siglip_source.value.shape == (4, 2 * 2 * 3) + assert siglip_source.value.dtype == torch.bfloat16 + assert vae_target.value.shape == (3, 4, 4) + assert vae_target.value.dtype == torch.bfloat16 + + +def test_bagel_preprocessors_route_inference_und_user_image_to_siglip_only() -> None: + """Verify ``infer_und`` preprocessing routes user images to SigLIP context only.""" + vae_pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + text_pre = BagelTextEncoderPreprocessor(_bagel_template()) + image = torch.full((3, 4, 4), 7, dtype=torch.uint8) + batch = [[ConversationItem(type="image", value=image.clone(), role="user")]] + + vae_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) + text_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) + + assert [item.type for item in batch[0]] == ["text", "image", "text"] + assert batch[0][1].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT + assert torch.equal(batch[0][1].value, image) + + +def test_bagel_vae_preprocessor_appends_per_sample_dummy_for_missing_context() -> None: + """Verify ``BagelVAEPreprocessor`` appends a shape-matched dummy VAE image to samples without one.""" + pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + batch = [ + [ + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 9, dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + ], + [ConversationItem(type="text", value="text-only", role="user")], + ] + + pre(_b(batch)) + + real = batch[0][0] + assert real.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + assert real.value.shape == (3, 4, 4) + assert real.value.dtype == torch.bfloat16 + assert real.meta[BAGEL_VAE_PIXEL_SHAPE].tolist() == [4, 4] + + assert len(_worker_dummies(batch, "assistant")) == 1 + dummy = batch[1][-1] + assert dummy.type == "image" + assert dummy.role == "assistant" and dummy.is_dummy + assert dummy.meta[_IMG_TAG_KEY] == "gen" + assert dummy.meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT + assert dummy.meta[BAGEL_VAE_PIXEL_SHAPE].tolist() == [4, 4] + assert dummy.value.shape == real.value.shape + assert dummy.value.dtype == torch.bfloat16 + torch.stack([real.value, dummy.value], dim=0) + + +def test_bagel_vae_preprocessor_keeps_bs4_sample_aligned_for_0_2_4_images() -> None: + """Verify ``BagelVAEPreprocessor`` maintains uniform tensor shapes across mixed 4-sample batches.""" + pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + + for real_count in (0, 2, 4): + batch = [] + for index in range(4): + if index < real_count: + batch.append( + [ + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 9, dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + ] + ) + else: + batch.append([ConversationItem(type="text", value=f"text-{index}", role="user")]) + + pre(_b(batch)) + + assert len(_worker_dummies(batch, "assistant")) == 4 - real_count + for sample in batch: + context_items = [ + item + for item in sample + if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT + ] + assert len(context_items) == 1 + shapes = [tuple(sample[-1].value.shape) for sample in batch] + assert len(set(shapes)) == 1 + expected_shape = (3, 2, 2) if real_count == 0 else (3, 4, 4) + assert shapes[0] == expected_shape + + +def test_bagel_preprocessors_are_picklable() -> None: + """Verify all three BAGEL worker preprocessors round-trip through ``pickle``.""" + for pre in ( + BagelTextEncoderPreprocessor(_bagel_template()), + BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ), + BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ), + ): + restored = pickle.loads(pickle.dumps(pre)) + assert type(restored) is type(pre) + + +def test_bagel_preprocessors_populate_omni_token_len_for_dyn_bsz() -> None: + """Verify BAGEL text, SigLIP, and VAE preprocessors populate ``_omni_token_len`` on real and dummy items.""" + from veomni.data.dataset import get_length_by_labels_fn, get_length_fn + from veomni.models.seed_omni.utils.conversation import _OMNI_TOKEN_LEN + + text_pre = BagelTextEncoderPreprocessor(_bagel_template()) + siglip_pre = BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), + ) + vae_pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + gen_sample = [ + ConversationItem(type="text", value="draw", role="user"), + ConversationItem( + type="image", + value=torch.full((3, 4, 4), 9, dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ), + ] + batch = [gen_sample] + for pre in (vae_pre, siglip_pre, text_pre): + pre(_b(batch)) + + assert all(isinstance(item.meta.get(_OMNI_TOKEN_LEN), int) and item.meta[_OMNI_TOKEN_LEN] > 0 for item in batch[0]) + # User text (6 tokens) + VAE start/end markers (2 tokens) + 4 VAE tokens + 1 dummy SigLIP token = 13 total + assert get_length_fn({"conversation_list": batch[0]}) == 13 + # Assistant VAE image (4 tokens) is supervised; user text and dummy SigLIP are not + assert get_length_by_labels_fn({"conversation_list": batch[0]}) == 4 + + +def test_bagel_siglip_preprocessor_from_pretrained_applies_config_overrides(tmp_path) -> None: + """Verify ``BagelSiglipNavitPreprocessor.from_pretrained`` applies ``config_overrides`` to processor geometry.""" + from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig + + BagelSiglipNavitConfig(image_size=28, min_image_size=14, patch_size=14).save_pretrained(tmp_path) + pre = BagelSiglipNavitPreprocessor.from_pretrained( + str(tmp_path), + config_overrides={"image_size": 56, "min_image_size": 28}, + ) + assert pre._image_processor.image_size == 56 + assert pre._image_processor.min_image_size == 28 diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py new file mode 100644 index 0000000000..7e32069fb4 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -0,0 +1,241 @@ +"""BAGEL Qwen2-MoT attention dispatch and QKV checkpoint schema.""" + +from __future__ import annotations + +import pytest +import torch +from safetensors import safe_open +from transformers.utils import SAFE_WEIGHTS_NAME + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import config_cls, tiny_bagel_qwen2_cfg +from veomni.models.checkpoint.weights import load_model_weights +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.checkpoint_conversion import ( + BagelQwen2MoTCheckpointTensorConverter, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + build_mot_block_mask, +) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import _sdpa_packed_attention + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def _flex_config() -> BagelQwen2MoTConfig: + return BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=1, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "veomni_flex_attention", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + ) + + +def test_cached_sdpa_causal_attention_is_bottom_right_aligned() -> None: + query = torch.zeros(1, 1, 1) + key = torch.zeros(4, 1, 1) + value = torch.tensor([1.0, 2.0, 4.0, 8.0]).reshape(4, 1, 1) + + output = _sdpa_packed_attention( + query, + key, + value, + is_causal=True, + cu_seq_lens_q=torch.tensor([0, 1], dtype=torch.int32), + cu_seq_lens_k=torch.tensor([0, 4], dtype=torch.int32), + scale=1.0, + enable_gqa=False, + ) + + # A single cached decode query is the final logical row, so it can attend + # to all four historical keys. Upper-left causal alignment would return 1. + torch.testing.assert_close(output, torch.tensor([[[3.75]]])) + + +def test_accelerated_training_attention_rejects_non_flex_backend() -> None: + """Verify ``BagelQwen2MoTAttentionAccelerated.forward_packed_train`` rejects non-packed backends.""" + config = config_cls("bagel_qwen2_mot")(**tiny_bagel_qwen2_cfg(), attn_implementation="sdpa") + attention = accelerated.BagelQwen2MoTAttentionAccelerated(config, layer_idx=0) + sequence_length = 2 + metadata = build_mot_attention_metadata([[2]], [["causal"]], device=torch.device("cpu")) + + with pytest.raises(ValueError, match="requires packed fused attention"): + attention.forward_packed_train( + packed_sequence=torch.randn(sequence_length, config.hidden_size), + attention_mask=build_mot_block_mask(metadata), + packed_position_cos=torch.ones(sequence_length, attention.head_dim), + packed_position_sin=torch.zeros(sequence_length, attention.head_dim), + packed_und_token_indexes=torch.arange(sequence_length), + packed_gen_token_indexes=torch.empty(0, dtype=torch.long), + ) + + +def test_native_gqa_rejects_invalid_global_head_ratio_at_config_init() -> None: + """Verify ``BagelQwen2MoTConfig`` rejects query head counts not divisible by KV head counts.""" + with pytest.raises(ValueError, match="query heads must be divisible"): + config_cls("bagel_qwen2_mot")( + **{ + **tiny_bagel_qwen2_cfg(), + "num_attention_heads": 6, + "num_key_value_heads": 4, + } + ) + + +def test_checkpoint_tensor_converter_combines_legacy_qkv_groups() -> None: + """Verify ``BagelQwen2MoTCheckpointTensorConverter`` merges split Q/K/V weights and biases.""" + converter = BagelQwen2MoTCheckpointTensorConverter() + prefix = "model.layers.3.self_attn." + tensors = { + f"{prefix}q_proj.weight": torch.full((64, 64), 1.0), + f"{prefix}k_proj.weight": torch.full((16, 64), 2.0), + f"{prefix}v_proj.weight": torch.full((16, 64), 3.0), + f"{prefix}q_proj_moe_gen.bias": torch.full((64,), 4.0), + f"{prefix}k_proj_moe_gen.bias": torch.full((16,), 5.0), + f"{prefix}v_proj_moe_gen.bias": torch.full((16,), 6.0), + } + + converted = [] + for name in reversed(tensors): + result = converter.convert(name, tensors[name]) + if result is not None: + converted.append(result) + assert converter.finalize() == [] + assert {result.name for result in converted} == { + f"{prefix}qkv_proj_und.weight", + f"{prefix}qkv_proj_gen.bias", + } + + by_name = {result.name: result.tensor for result in converted} + und_weight = by_name[f"{prefix}qkv_proj_und.weight"] + gen_bias = by_name[f"{prefix}qkv_proj_gen.bias"] + torch.testing.assert_close(und_weight[:64], tensors[f"{prefix}q_proj.weight"]) + torch.testing.assert_close(und_weight[64:80], tensors[f"{prefix}k_proj.weight"]) + torch.testing.assert_close(und_weight[80:], tensors[f"{prefix}v_proj.weight"]) + torch.testing.assert_close(gen_bias[:64], tensors[f"{prefix}q_proj_moe_gen.bias"]) + torch.testing.assert_close(gen_bias[64:80], tensors[f"{prefix}k_proj_moe_gen.bias"]) + torch.testing.assert_close(gen_bias[80:], tensors[f"{prefix}v_proj_moe_gen.bias"]) + + +def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> None: + """Verify fused runtime QKV parameters round-trip through split HF safetensors and ``load_model_weights``.""" + torch.manual_seed(3014) + reference = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + internal_parameter_names = set(dict(reference.named_parameters())) + assert "model.layers.0.self_attn.qkv_proj_und.weight" in internal_parameter_names + assert "model.layers.0.self_attn.qkv_proj_gen.weight" in internal_parameter_names + assert "model.layers.0.self_attn.q_proj.weight" not in internal_parameter_names + assert "model.layers.0.self_attn.q_proj_moe_gen.weight" not in internal_parameter_names + + checkpoint_state = reference.state_dict() + assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_state + assert "model.layers.0.self_attn.k_proj.weight" in checkpoint_state + assert "model.layers.0.self_attn.v_proj.weight" in checkpoint_state + assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in checkpoint_state + assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_state + assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in checkpoint_state + + state_dict_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + state_dict_loaded.load_state_dict(checkpoint_state) + for name in ("qkv_proj_und", "qkv_proj_gen"): + expected = getattr(reference.model.layers[0].self_attn, name) + actual = getattr(state_dict_loaded.model.layers[0].self_attn, name) + torch.testing.assert_close(actual.weight, expected.weight) + torch.testing.assert_close(actual.bias, expected.bias) + + reference.save_pretrained(tmp_path) + with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint: + checkpoint_keys = set(checkpoint.keys()) + assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_keys + assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in checkpoint_keys + assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_keys + assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in checkpoint_keys + + hf_loaded = accelerated.BagelQwen2MoTAccelerated.from_pretrained(tmp_path, config=_flex_config()) + veomni_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + load_model_weights(veomni_loaded, str(tmp_path), init_device="cpu") + for name in ("qkv_proj_und", "qkv_proj_gen"): + expected = getattr(reference.model.layers[0].self_attn, name) + torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).weight, expected.weight) + torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).bias, expected.bias) + torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).weight, expected.weight) + torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).bias, expected.bias) + + +def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config() -> None: + """Verify ``BagelQwen2MoTAccelerated`` binds ``VeomniOp`` handles from ``config.ops_config``.""" + cfg = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=1, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "flex_attention", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + ) + model = accelerated.BagelQwen2MoTAccelerated(cfg) + + layer = model.model.layers[0] + assert layer.input_layernorm.veomni_rms_norm.impl == "eager" + assert layer.mlp.veomni_swiglu_mlp.impl == "eager" + assert layer.self_attn.veomni_rope.impl == "eager" + assert layer.self_attn.veomni_attn is not None + assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention" + assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2" + + +def test_bagel_siglip_navit_selects_module_level_ops_and_forwards() -> None: + """Verify ``BagelSiglipNavit`` binds vision RoPE and attention ops from ``config.ops_config``.""" + from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig + from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit + + cfg = BagelSiglipNavitConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=4, + image_size=28, + patch_size=14, + ops_implementation={ + "attn_implementation": "sdpa", + "rotary_pos_emb_vision_implementation": "eager", + }, + ) + model = BagelSiglipNavit(cfg).eval() + attn = model.vision_model.encoder.layers[0].self_attn + assert attn.veomni_attn.impl == "veomni_sdpa" + assert attn.veomni_rope.impl == "eager" + + packed_patches = torch.randn(4, 3 * 14 * 14) + position_ids = torch.arange(4, dtype=torch.long) + cu_seqlens = torch.tensor([0, 4], dtype=torch.int32) + out = model.vision_model( + patchified_pixel_values=packed_patches, + patchified_position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=4, + ) + assert out.shape == (4, 32) + assert torch.isfinite(out).all() diff --git a/tests/seed_omni/modules/bagel/test_source_routing.py b/tests/seed_omni/modules/bagel/test_source_routing.py new file mode 100644 index 0000000000..ab12387ab7 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_source_routing.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.seed_omni.modules.bagel.helpers import config_cls, model_cls, tiny_bagel_qwen2_cfg +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT +from veomni.models.seed_omni.modules.bagel.vae.processing import route_image_contexts +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) + + +def test_bagel_route_image_contexts_prefers_img_tag() -> None: + """Verify ``route_image_contexts`` routes training images according to ``_img_tag``.""" + und = ConversationItem( + type="image", + value=torch.zeros(3, 2, 2), + role="user", + meta={_IMG_TAG_KEY: "und"}, + ) + gen = ConversationItem( + type="image", + value=torch.ones(3, 2, 2), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + edit = ConversationItem( + type="image", + value=torch.full((3, 2, 2), 2), + role="user", + meta={_IMG_TAG_KEY: "edit"}, + ) + sample = [und, gen, edit] + + route_image_contexts([sample], inference=False, infer_type=None) + + assert sample[0] is und + assert sample[0].meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT + assert sample[1] is gen + assert sample[1].meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT + assert sample[3] is edit + assert sample[2].meta == {_IMG_TAG_KEY: "edit", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} + assert sample[3].meta == {_IMG_TAG_KEY: "edit", BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT} + assert sample[2] is not sample[3] + sample[2].value[0, 0, 0] = 99 + assert int(sample[3].value[0, 0, 0]) == 2 + + +def test_bagel_route_image_contexts_requires_img_tag_for_raw_training_images() -> None: + """Verify ``route_image_contexts`` raises when a raw training image lacks ``_img_tag``.""" + image = ConversationItem(type="image", value=torch.zeros(3, 2, 2), role="assistant") + + with pytest.raises(ValueError, match="_img_tag"): + route_image_contexts([[image]], inference=False, infer_type=None) + + +def test_bagel_route_image_contexts_uses_infer_type_for_raw_inference_images() -> None: + """Verify ``route_image_contexts`` routes raw inference user images based on ``infer_type``.""" + edit_user = ConversationItem(type="image", value=torch.full((3, 2, 2), 3), role="user") + und_user = ConversationItem(type="image", value=torch.ones(3, 2, 2), role="user") + gen_user = ConversationItem(type="image", value=torch.zeros(3, 2, 2), role="user") + edit_sample = [edit_user] + und_sample = [und_user] + gen_sample = [gen_user] + + route_image_contexts([edit_sample], inference=True, infer_type="infer_edit") + route_image_contexts([und_sample], inference=True, infer_type="infer_und") + route_image_contexts([gen_sample], inference=True, infer_type="infer_gen") + + assert edit_sample[0].meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT + assert edit_sample[1] is edit_user + assert edit_sample[1].meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT + assert und_sample[0] is und_user + assert und_user.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT + assert gen_sample[0] is gen_user + assert gen_user.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT + + +def test_bagel_qwen2_mot_forward_pre_rejects_only_upstream_dummy_anchor(monkeypatch) -> None: + """Verify ``forward_pre`` raises when the batch contains only dummy anchors and no real tokens.""" + model = _tiny_qwen2_mot() + hidden_size = int(model.config.hidden_size) + siglip_dummy = ConversationItem( + type="image", + value=torch.ones(1, hidden_size, requires_grad=True), + role="user", + is_dummy=True, + meta={_IMG_TAG_KEY: "und", BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ) + monkeypatch.setattr(model, "_has_valid_upstream_embeddings", lambda conversation_list: (True, False)) + + with pytest.raises(ValueError, match="got no packable tokens"): + model.forward_pre(conversation_list=[[siglip_dummy]]) + + +def test_bagel_qwen2_mot_rejects_context_parallel_training(monkeypatch) -> None: + """Verify ``BagelQwen2MoTAccelerated.forward_pre`` rejects ``cp_size > 1``.""" + from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated + + model = _tiny_qwen2_mot() + monkeypatch.setattr( + accelerated, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=2, cp_size=2), + ) + conversation = [ + [ + ConversationItem( + type="text", + value=torch.zeros(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype), + role="user", + ) + ] + ] + + with pytest.raises(ValueError, match="Ulysses sequence parallelism only"): + model.forward_pre(conversation_list=conversation) + + +def test_bagel_flow_dummy_embed_anchors_to_vae_dummy_output() -> None: + """Verify dummy flow embedding preserves autograd connectivity to the upstream VAE dummy output.""" + model = _tiny_flow_connector() + vae_dummy = ConversationItem( + type="image", + value=torch.ones(1, 1, 1, requires_grad=True), + role="assistant", + is_dummy=True, + meta={_IMG_TAG_KEY: "gen", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + ) + conversation = [[vae_dummy]] + + inputs = model.embed_latent_pre(conversation_list=conversation) + latent_embeds = torch.ones(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype) + latent_embeds = latent_embeds + inputs["latents"].sum() * 0.0 + out = model.embed_latent_post(latent_embeds) + + assert inputs["latents"].requires_grad + assert out["conversation_list"] is conversation + flow_dummy = conversation[0][-1] + assert flow_dummy.type == "output" + assert flow_dummy.is_dummy + assert flow_dummy.meta == {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} + assert torch.is_tensor(flow_dummy.value) + assert flow_dummy.value.requires_grad + + +def _tiny_flow_connector(): + BagelFlowConnector = model_cls("bagel_flow_connector") + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + return BagelFlowConnector( + BagelFlowConnectorConfig( + hidden_size=4, + z_channels=1, + latent_patch_size=1, + patch_latent_dim=1, + max_latent_size=4, + timestep_frequency_embedding_size=4, + resolution=16, + ) + ) + + +def _tiny_qwen2_mot(): + BagelQwen2MoT = model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + return BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) diff --git a/tests/utils/test_save_safetensor_utils.py b/tests/utils/test_save_safetensor_utils.py index 6f2da449c8..01900e7865 100644 --- a/tests/utils/test_save_safetensor_utils.py +++ b/tests/utils/test_save_safetensor_utils.py @@ -443,6 +443,28 @@ def test_none_mapping_passed_through(self): ) self.assertIsNone(writer_cls.call_args.kwargs["fqn_to_index_mapping"]) + def test_revert_model_weight_conversions_unwraps_ddp_wrapper(self): + from veomni.utils.save_safetensor_utils import _revert_model_weight_conversions_for_hf + + class _Inner(nn.Module): + _export_hf_checkpoint_with_weight_conversions = True + + def __init__(self): + super().__init__() + self._weight_conversions = [] + + class _Wrapper(nn.Module): + def __init__(self, inner): + super().__init__() + self.module = inner + + inner = _Inner() + wrapped = _Wrapper(inner) + with patch("transformers.core_model_loading.revert_weight_conversion") as mock_revert: + mock_revert.side_effect = lambda m, sd: {"unwrapped_cls": type(m).__name__, **sd} + out = _revert_model_weight_conversions_for_hf(wrapped, {"w": torch.ones(1)}) + self.assertEqual(out["unwrapped_cls"], "_Inner") + def test_naflatten_and_unflatten_roundtrip_keeps_shape_on_cpu(self): from veomni.utils.tensor_utils import naflatten, unflatten diff --git a/veomni/models/seed_omni/modules/__init__.py b/veomni/models/seed_omni/modules/__init__.py index 615cbc36a7..fa7c2bbaea 100644 --- a/veomni/models/seed_omni/modules/__init__.py +++ b/veomni/models/seed_omni/modules/__init__.py @@ -109,8 +109,8 @@ def read_model_type(model_path: str) -> str: return model_type -# Side-effect only: attach @register factories under text_encoder/ and fake_model/. -from . import fake_model, text_encoder # noqa: F401 E402 +# Side-effect only: attach @register factories under bagel/, text_encoder/, and fake_model/. +from . import bagel, fake_model, text_encoder # noqa: F401 E402 __all__ = [ diff --git a/veomni/models/seed_omni/modules/bagel/__init__.py b/veomni/models/seed_omni/modules/bagel/__init__.py new file mode 100644 index 0000000000..07280c870b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/__init__.py @@ -0,0 +1,25 @@ +"""BAGEL SeedOmni modules. + +The SeedOmni graph keeps BAGEL modules split by producer/consumer boundaries: + +- training: text/SigLIP/VAE/flow hooks write embedded carrier items, then + ``bagel_qwen2_mot.forward`` packs them into the MoT backbone and scatters + hidden states back to the same items. +- prompt inference: ``bagel_text_encoder`` and optional VAE/SigLIP context + feed ``bagel_qwen2_mot.generate`` for prompt prefill or text decoding. +- image denoise: ``flow_connector.prepare_denoise_query`` writes + ``bagel_flow_query``; text marker wrapping feeds ``qwen2_mot.denoise_branch``; + ``flow_connector.decode_velocity_from_hidden`` writes ``bagel_flow_velocity``; + ``qwen2_mot.collect_velocity`` strips branch markers and handles CFG merge; finally + ``flow_connector.advance_denoise`` either loops or emits generated latents + for VAE decode. +""" + +from . import ( + convert_model, # noqa: F401 + flow_connector, # noqa: F401 + qwen2_mot, # noqa: F401 + siglip_navit, # noqa: F401 + text_encoder, # noqa: F401 + vae, # noqa: F401 +) diff --git a/veomni/models/seed_omni/modules/bagel/convert_model.py b/veomni/models/seed_omni/modules/bagel/convert_model.py new file mode 100644 index 0000000000..cb5923f115 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/convert_model.py @@ -0,0 +1,299 @@ +"""Split a BAGEL checkpoint into SeedOmni modules. + +Registered under ``OMNI_CONVERT_REGISTRY["bagel"]``; :func:`convert_checkpoint` +writes the omni checkpoint. Graphs come from ``configs/seed_omni/Bagel/bagel_7b_mot/``. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable + +import torch +from safetensors import safe_open +from transformers import AutoTokenizer +from transformers.initialization import no_init_weights + +from veomni.models.checkpoint.weights import init_empty_weights +from veomni.models.seed_omni.utils.convert_registry import ( + OMNI_CONVERT_REGISTRY, + attach_module_assets, + load_family_graphs, +) + + +BAGEL_GENERATION_KWARGS = { + "max_new_tokens": 2048, + "temperature": 0.0, + "top_p": 1.0, + "do_sample": False, + "image_height": 1024, + "image_width": 1024, + "latent_downsample": 16, + "num_timesteps": 50, + "timestep_shift": 3.0, + "cfg_text_scale": 1.0, + "cfg_img_scale": 1.0, + "cfg_interval": [0.0, 1.0], + "cfg_renorm_min": 0.0, + "cfg_renorm_type": "global", + "enable_taylorseer": False, +} + + +def _read_json(path: Path) -> dict[str, Any]: + """Read and parse a UTF-8 JSON file.""" + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def _load_prefixed_safetensors( + path: Path, + remap: Callable[[str], str | None], + consumed_keys: set[str] | None = None, +) -> dict[str, Any]: + """Load safetensors keys selected by ``remap`` onto CPU.""" + state_dict: dict[str, Any] = {} + with safe_open(path, framework="pt", device="cpu") as f: + for key in f.keys(): + target_key = remap(key) + if target_key is None: + continue + if consumed_keys is not None: + consumed_keys.add(key) + state_dict[target_key] = f.get_tensor(key) + return state_dict + + +def _safetensor_keys(path: Path) -> set[str]: + """Return the set of tensor keys stored in ``path``.""" + with safe_open(path, framework="pt", device="cpu") as f: + return set(f.keys()) + + +def _assert_no_unhandled_keys(path: Path, consumed_keys: set[str], ignored_keys: set[str]) -> None: + """Raise if any safetensors key in ``path`` is neither consumed nor ignored.""" + unhandled = sorted(_safetensor_keys(path) - consumed_keys - ignored_keys) + if unhandled: + preview = ", ".join(unhandled[:20]) + raise RuntimeError(f"Unhandled source keys in {path}: {preview}") + + +def _instantiate(model_type: str, config: Any): + """Instantiate an empty-weight SeedOmni module for ``model_type``.""" + from veomni.models.seed_omni.modules import OMNI_MODEL_REGISTRY + + model_cls = OMNI_MODEL_REGISTRY[model_type]() + with no_init_weights(), init_empty_weights(): + return model_cls._from_config(config) + + +def _materialize_allowed_missing_parameters(model: Any, missing: list[str]) -> None: + """Materialize and initialize deterministic missing position-embedding parameters.""" + for key in missing: + if not key.endswith(".pos_embed"): + continue + module_name, _, param_name = key.rpartition(".") + module = model.get_submodule(module_name) + parameter = getattr(module, param_name) + setattr( + module, + param_name, + torch.nn.Parameter(torch.empty(tuple(parameter.shape)), requires_grad=parameter.requires_grad), + ) + if not hasattr(module, "_init_weights"): + raise RuntimeError(f"Cannot initialize deterministic missing parameter: {key}") + module._init_weights() + + +def _build_module( + model_type: str, + config: Any, + state_dict: dict[str, Any], + *, + allowed_missing: set[str] | None = None, +) -> Any: + """Build ``model_type`` from ``config`` and assign ``state_dict`` tensors.""" + model = _instantiate(model_type, config) + allowed_missing = set() if allowed_missing is None else allowed_missing + missing, unexpected = model.load_state_dict(state_dict, strict=False, assign=True) + unexpected = list(unexpected) + missing = list(missing) + unexpected_missing = [key for key in missing if key not in allowed_missing] + if unexpected_missing or unexpected: + raise RuntimeError(f"{model_type} load mismatch: missing={unexpected_missing}, unexpected={unexpected}") + _materialize_allowed_missing_parameters(model, [key for key in missing if key in allowed_missing]) + print(f"[bagel] extracted {model_type}") + return model + + +def convert_bagel_checkpoint( + model_path: str, + *, + max_latent_size: int = 64, + **kwargs: Any, +) -> dict[str, Any]: + """Split an upstream BAGEL checkpoint into five SeedOmni modules.""" + training_graphs, generation_graphs = load_family_graphs( + "configs/seed_omni/Bagel/bagel_7b_mot", + training="train/graph_train.yaml", + generation={ + "infer_edit": "infer/graph_infer_edit.yaml", + "infer_gen": "infer/graph_infer_gen.yaml", + "infer_und": "infer/graph_infer_und.yaml", + }, + training_graph=kwargs.pop("training_graph", None), + generation_graph=kwargs.pop("generation_graph", None), + ) + del kwargs + max_latent_size = int(max_latent_size) + + model_root = Path(model_path) + + ema_path = model_root / "ema.safetensors" + ae_path = model_root / "ae.safetensors" + if not ema_path.exists(): + raise FileNotFoundError(f"Missing BAGEL EMA weights: {ema_path}") + if not ae_path.exists(): + raise FileNotFoundError(f"Missing BAGEL VAE weights: {ae_path}") + + consumed_ema_keys: set[str] = set() + consumed_ae_keys: set[str] = set() + + llm_dict = _read_json(model_root / "llm_config.json") + vit_dict = _read_json(model_root / "vit_config.json") + hidden_size = int(llm_dict["hidden_size"]) + vit_hidden_size = int(vit_dict["hidden_size"]) + + from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY + + text_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_text_encoder"]() + text_cfg = text_cfg_cls( + vocab_size=int(llm_dict["vocab_size"]), + hidden_size=hidden_size, + tie_word_embeddings=False, + lm_head_bias=False, + ) + text_state = _load_prefixed_safetensors( + ema_path, + lambda key: { + "language_model.model.embed_tokens.weight": "embed_tokens.weight", + "language_model.lm_head.weight": "lm_head.weight", + }.get(key), + consumed_ema_keys, + ) + text_encoder = _build_module("bagel_text_encoder", text_cfg, text_state) + attach_module_assets(text_encoder, tokenizer=AutoTokenizer.from_pretrained(model_path)) + + qwen_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_qwen2_mot"]() + qwen_cfg = qwen_cfg_cls( + **llm_dict, + qk_norm=True, + layer_module="Qwen2MoTDecoderLayer", + freeze_und=False, + ) + qwen_state = _load_prefixed_safetensors( + ema_path, + lambda key: ( + key.removeprefix("language_model.") + if key.startswith("language_model.") + and not key.startswith("language_model.model.embed_tokens.") + and not key.startswith("language_model.lm_head.") + else None + ), + consumed_ema_keys, + ) + qwen2_mot = _build_module("bagel_qwen2_mot", qwen_cfg, qwen_state) + + siglip_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_siglip_navit"]() + siglip_cfg = siglip_cfg_cls( + hidden_size=vit_hidden_size, + output_size=hidden_size, + image_size=int(vit_dict["image_size"]), + intermediate_size=int(vit_dict["intermediate_size"]), + num_attention_heads=int(vit_dict["num_attention_heads"]), + # Official inference/training uses one fewer active layer than vit_config.json. + num_hidden_layers=int(vit_dict["num_hidden_layers"]) - 1, + patch_size=int(vit_dict["patch_size"]), + connector_act="gelu_pytorch_tanh", + vit_max_num_patch_per_side=int(vit_dict["image_size"]) // int(vit_dict["patch_size"]), + rope=False, + ) + siglip_state = _load_prefixed_safetensors( + ema_path, + lambda key: ( + key.removeprefix("vit_model.") + if key.startswith("vit_model.") + else key + if key.startswith("connector.") + else None + ), + consumed_ema_keys, + ) + siglip = _build_module( + "bagel_siglip_navit", + siglip_cfg, + siglip_state, + allowed_missing={"vit_pos_embed.pos_embed"}, + ) + from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import BagelSiglipNavitProcessor + + attach_module_assets(siglip, image_processor=BagelSiglipNavitProcessor.from_config(siglip_cfg)) + + flow_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]() + patch_latent_dim = 2 * 2 * 16 + flow_cfg = flow_cfg_cls( + hidden_size=hidden_size, + patch_latent_dim=patch_latent_dim, + max_latent_size=max_latent_size, + timestep_shift=1.0, + ) + flow_prefixes = ("vae2llm.", "llm2vae.", "time_embedder.") + flow_state = _load_prefixed_safetensors( + ema_path, + lambda key: key if key.startswith(flow_prefixes) else None, + consumed_ema_keys, + ) + flow_connector = _build_module( + "bagel_flow_connector", + flow_cfg, + flow_state, + allowed_missing={"latent_pos_embed.pos_embed"}, + ) + + vae_cfg_cls = OMNI_CONFIG_REGISTRY["bagel_vae"]() + vae_cfg = vae_cfg_cls() + vae_state = _load_prefixed_safetensors(ae_path, lambda key: key, consumed_ae_keys) + vae = _build_module("bagel_vae", vae_cfg, vae_state) + from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEProcessor + + attach_module_assets(vae, image_processor=BagelVAEProcessor.from_config(vae_cfg)) + + # Position embeddings are deterministic sin-cos buffers regenerated from config. + ignored_ema_keys = {"latent_pos_embed.pos_embed", "vit_pos_embed.pos_embed"} + _assert_no_unhandled_keys(ema_path, consumed_ema_keys, ignored_ema_keys) + _assert_no_unhandled_keys(ae_path, consumed_ae_keys, set()) + + return { + "modules": { + "bagel_vae": vae, + "bagel_siglip_navit": siglip, + "bagel_text_encoder": text_encoder, + "bagel_flow_connector": flow_connector, + "bagel_qwen2_mot": qwen2_mot, + }, + "training_graphs": training_graphs, + "generation_graphs": generation_graphs, + "infer_type": "infer_und", + "generation_kwargs": deepcopy(BAGEL_GENERATION_KWARGS), + } + + +@OMNI_CONVERT_REGISTRY.register("bagel") +def _register_bagel_convert(): + return convert_bagel_checkpoint + + +__all__ = ["convert_bagel_checkpoint"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py new file mode 100644 index 0000000000..a84b9c7f23 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL VAE/LLM flow connector module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_config(): + from .configuration import BagelFlowConnectorConfig + + return BagelFlowConnectorConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_model(): + from .modeling import BagelFlowConnector + + return BagelFlowConnector + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_flow_connector") +def register_bagel_flow_connector_accelerated_model(): + from .accelerated import BagelFlowConnectorAccelerated + + return BagelFlowConnectorAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py new file mode 100644 index 0000000000..d1c9c0a21c --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/flow_connector OmniModule (conversation graph).""" + +from .accelerated import ( + BagelFlowConnectorAccelerated, +) + + +__all__ = [ + "BagelFlowConnectorAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py new file mode 100644 index 0000000000..f9146848eb --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py @@ -0,0 +1,367 @@ +"""SeedOmni carrier hooks for BAGEL's flow connector — training-graph hooks only. + +``embed_context_latents()`` / ``prepare_denoise_query()`` / ``decode_velocity_from_hidden()`` +/ ``advance_denoise()`` and the shared :class:`~.generation_state.FlowGenerationState` +live on the native :class:`~.modeling.BagelFlowConnector` — this file only carries +the training pre/forward/post hooks (carrier selection, SP token slicing, dummy +alignment). +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import gather_outputs, slice_input_tensor, sp_pad +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ...sources import BAGEL_FLOW_HIDDEN, BAGEL_FLOW_VELOCITY, BAGEL_PHASE_KEY, bagel_phase +from ..configuration import BagelFlowConnectorConfig +from ..modeling import BagelFlowConnector, scatter_flow_latent_embeds, select_vae_context_latent_items +from ..processing import preprocess_context_latent_embed, preprocess_decode_velocity, preprocess_latent_embed + + +def slice_sp_token_inputs(method: str, **inputs: torch.Tensor) -> tuple[int, dict[str, torch.Tensor]]: + """Pad and slice token-level ``inputs`` across the active Ulysses SP group.""" + ps = get_parallel_state() + if ps.cp_size != 1: + raise ValueError(f"BAGEL flow connector {method} supports Ulysses groups only; got cp_size={ps.cp_size}.") + + lengths = {name: int(tensor.shape[0]) for name, tensor in inputs.items()} + if not lengths or next(iter(lengths.values())) == 0: + raise ValueError(f"BAGEL flow connector {method} SP requires at least one token.") + if len(set(lengths.values())) != 1: + raise ValueError(f"BAGEL flow connector {method} SP inputs must have matching token lengths: {lengths}.") + + full_length = next(iter(lengths.values())) + local_inputs = { + name: slice_input_tensor( + sp_pad(tensor, dim=0, pad_value=0), + dim=0, + padding=False, + group=ps.sp_group, + ) + for name, tensor in inputs.items() + } + return full_length, local_inputs + + +def gather_sp_token_output(method: str, output: torch.Tensor, full_length: int | None) -> torch.Tensor: + """All-gather token-level ``output`` across the SP group and trim SP padding.""" + if full_length is None: + raise RuntimeError(f"BAGEL flow connector {method} SP token length was not initialized.") + output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group) + return output.narrow(0, 0, full_length) + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph carrier hooks — depends on :class:`BagelFlowConnector` modeling APIs.""" + + config: BagelFlowConnectorConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._embed_items: list[ConversationItem] = [] + self._embed_lengths: list[int] = [] + self._sp_embed_length: int | None = None + self._decode_target_groups: list[list[ConversationItem]] = [] + self._decode_lengths: list[int] = [] + self._decode_target: torch.Tensor | None = None + self._sp_decode_length: int | None = None + + @pre_forward("embed_latent") + def embed_latent_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Prepare patchified latent tokens, position IDs, and timesteps for ``embed_latent``.""" + del kwargs + self._conversation_carrier = conversation_list + self._embed_lengths = [] + self._sp_embed_length = None + + self._embed_items = select_vae_context_latent_items(conversation_list) + if not self._embed_items: + raise ValueError("BAGEL flow connector requires per-sample VAE context carriers before embed_latent.") + + parts: list[dict[str, torch.Tensor]] = [] + meter_lengths: list[int] = [] + for item in self._embed_items: + if item.is_dummy: + inputs, lengths = preprocess_context_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + ) + anchor = item.value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + inputs["latents"] = inputs["latents"] + anchor + parts.append(inputs) + self._embed_lengths.extend(lengths) + meter_lengths.extend(int(v) for v in lengths) + continue + + tag = item.meta.get(_IMG_TAG_KEY) + if tag == "gen": + inputs, lengths = preprocess_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + timestep_shift=float(self.config.timestep_shift), + ) + elif tag == "edit": + inputs, lengths = preprocess_context_latent_embed( + [item], + config=self.config, + device=self.device, + dtype=self.dtype, + ) + else: + raise ValueError( + f"BAGEL flow connector training expects VAE image {_IMG_TAG_KEY} to be 'edit' or 'gen', got {tag!r}." + ) + + parts.append(inputs) + self._embed_lengths.extend(lengths) + meter_lengths.extend(int(v) for v in lengths) + + self.metric_meter_set_seqlens("embed_latent", meter_lengths) + + inputs = { + "latents": torch.cat([part["latents"] for part in parts], dim=0), + "position_ids": torch.cat([part["position_ids"] for part in parts], dim=0), + "timesteps": torch.cat([part["timesteps"] for part in parts], dim=0), + } + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_embed_length = None + return inputs + + self._sp_embed_length, inputs = slice_sp_token_inputs("embed_latent", **inputs) + return inputs + + @post_forward("embed_latent") + def embed_latent_post(self, latent_embeds: torch.Tensor) -> dict[str, Any]: + """Gather SP shards if needed and scatter ``latent_embeds`` back onto conversation items.""" + if get_parallel_state().sp_size > 1: + latent_embeds = gather_sp_token_output("embed_latent", latent_embeds, self._sp_embed_length) + + conversation = self._conversation_carrier + embed_items = self._embed_items + embed_lengths = self._embed_lengths + self._conversation_carrier = None + self._embed_items = [] + self._embed_lengths = [] + self._sp_embed_length = None + + scatter_flow_latent_embeds( + embed_items, + embed_lengths, + latent_embeds, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation} + + @pre_forward("decode_velocity") + def decode_velocity_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Collect latent-token hidden states and velocity targets for ``decode_velocity``.""" + del kwargs + self._conversation_carrier = conversation_list + self._decode_lengths = [] + self._decode_target = None + self._sp_decode_length = None + + self._decode_target_groups = self._select_velocity_target_groups(conversation_list) + + inputs_parts: list[dict[str, torch.Tensor]] = [] + target_parts: list[torch.Tensor] = [] + for sample, group in zip(conversation_list, self._decode_target_groups, strict=True): + if not group: + # No velocity target for this sample; run an anchored dummy decode + # so the decode head stays in the graph with zero loss. + dummy = self._anchor_dummy_decode_velocity_inputs([sample]) + inputs_parts.append(dummy) + self._decode_lengths.append(1) + continue + + inputs, lengths, target = preprocess_decode_velocity( + group, + config=self.config, + device=self.device, + dtype=self.dtype, + ) + inputs_parts.append(inputs) + self._decode_lengths.extend(lengths) + target_parts.append(target) + if target_parts: + self._decode_target = torch.cat(target_parts, dim=0) + + inputs = {"hidden_states": torch.cat([part["hidden_states"] for part in inputs_parts], dim=0)} + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_decode_length = None + return inputs + + self._sp_decode_length, inputs = slice_sp_token_inputs("decode_velocity", **inputs) + return inputs + + @post_forward("decode_velocity") + def decode_velocity_post(self, velocity: torch.Tensor) -> dict[str, Any]: + """Gather predicted velocity tokens, scatter onto items, and compute MSE flow loss.""" + if get_parallel_state().sp_size > 1: + velocity = gather_sp_token_output("decode_velocity", velocity, self._sp_decode_length) + + conversation = self._conversation_carrier + decode_target_groups = self._decode_target_groups + decode_lengths = self._decode_lengths + target = self._decode_target + self._conversation_carrier = None + self._decode_target_groups = [] + self._decode_lengths = [] + self._decode_target = None + self._sp_decode_length = None + + real_velocity_parts = self._scatter_velocity(decode_target_groups, decode_lengths, velocity) + loss = velocity.sum() * 0.0 + if target is not None and real_velocity_parts: + real_velocity = torch.cat(real_velocity_parts, dim=0) + mse = (real_velocity - target.to(device=velocity.device, dtype=velocity.dtype)).square() + token_count = torch.tensor(float(mse.shape[0]), device=mse.device, dtype=mse.dtype) + loss = loss + mse.mean(dim=-1).sum() / token_count + return {"conversation_list": conversation, "_loss": loss} + + def _anchor_dummy_decode_velocity_inputs( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> dict[str, torch.Tensor]: + """Tie dummy flow loss to MoT hidden states without changing its value.""" + dummy = { + "hidden_states": torch.zeros( + 1, + int(self.config.hidden_size), + device=self.device, + dtype=self.dtype, + ) + } + + if conversation_list is None: + return dummy + + anchor = None + for item in iter_desired_items( + conversation_list, + types=["text", "image", "output"], + roles=["user", "assistant"], + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size): + anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + break + if anchor is None: + return dummy + + return {"hidden_states": dummy["hidden_states"] + anchor} + + def _select_velocity_target_groups( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> list[list[ConversationItem]]: + """Group non-dummy image items carrying ``flow_velocity_target`` per sample.""" + if conversation_list is None: + raise ValueError("BAGEL flow connector requires conversation_list to select velocity targets.") + + target_groups: list[list[ConversationItem]] = [] + for sample in conversation_list: + # Keep one target group per sample. An empty group means the hook + # should run a sample-level dummy decode anchored to this sample's + # MoT hidden states. + sample_target_items: list[ConversationItem] = [] + for item in iter_desired_items([sample], types=["image"]): + if not item.is_dummy and torch.is_tensor(item.meta.get("flow_velocity_target")): + sample_target_items.append(item) + target_groups.append(sample_target_items) + return target_groups + + def _scatter_velocity( + self, + target_groups: list[list[ConversationItem]], + decode_lengths: list[int], + velocity: torch.Tensor, + ) -> list[torch.Tensor]: + """Write predicted velocity spans onto target items and return non-dummy spans.""" + real_velocity_parts: list[torch.Tensor] = [] + offset = 0 + length_iter = iter(decode_lengths) + for group in target_groups: + if not group: + length = next(length_iter) + offset += length + continue + for item in group: + length = next(length_iter) + span = velocity[offset : offset + length] + offset += length + item.value = span.to(device=self.device, dtype=self.dtype) + if bagel_phase(item) == BAGEL_FLOW_HIDDEN: + item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY + real_velocity_parts.append(span) + if offset != int(velocity.shape[0]): + raise RuntimeError("BAGEL flow connector token count mismatch during velocity scatter.") + return real_velocity_parts + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's flow connector.""" + + config: BagelFlowConnectorConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + # Only ``embed_latent`` stashes lengths. Per latent token it runs vae2llm + # and the per-token timestep MLP (freq -> hidden -> hidden); the hidden x + # hidden layer dominates. ``decode_velocity``'s llm2vae is folded in per + # embedded token, which slightly over-counts edit-context tokens that are + # never decoded (llm2vae is ~2% of the per-token cost). + cfg = self.config + hidden = int(cfg.hidden_size) + latent = int(cfg.patch_latent_dim) + freq = int(cfg.timestep_frequency_embedding_size) + per_token_n = latent * hidden + freq * hidden + hidden * hidden + hidden * latent + return 6 * per_token_n * sum(seqlens) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks for latent embedding and velocity projection. + + ``embed_context_latents()`` / ``prepare_denoise_query()`` / + ``decode_velocity_from_hidden()`` / ``advance_denoise()`` and the shared + flow-matching generation-FSM state already live on the native + :class:`~.modeling.BagelFlowConnector` (via its own :class:`~.modeling.InferenceMixin`), + so no ``InferenceMixin`` is needed here. + """ + + +class BagelFlowConnectorAccelerated(VeOmniMixin, BagelFlowConnector): + pass + + +__all__ = ["BagelFlowConnectorAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py new file mode 100644 index 0000000000..5d6eb4000f --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/configuration.py @@ -0,0 +1,34 @@ +"""Configuration for BAGEL's VAE/LLM flow connector.""" + +from ...module_configuration_base import OmniModuleConfig + + +class BagelFlowConnectorConfig(OmniModuleConfig): + """BAGEL latent-token connector config.""" + + model_type = "bagel_flow_connector" + + def __init__( + self, + hidden_size: int = 3584, + z_channels: int = 16, + latent_patch_size: int = 2, + patch_latent_dim: int | None = None, + max_latent_size: int = 64, + timestep_frequency_embedding_size: int = 256, + timestep_shift: float = 1.0, + **kwargs, + ) -> None: + self.hidden_size = hidden_size + self.z_channels = z_channels + self.latent_patch_size = latent_patch_size + self.patch_latent_dim = ( + latent_patch_size * latent_patch_size * z_channels if patch_latent_dim is None else patch_latent_dim + ) + self.max_latent_size = max_latent_size + self.timestep_frequency_embedding_size = timestep_frequency_embedding_size + self.timestep_shift = timestep_shift + super().__init__(**kwargs) + + +__all__ = ["BagelFlowConnectorConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py new file mode 100644 index 0000000000..fb483a7408 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/generation_state.py @@ -0,0 +1,156 @@ +"""Generation-time denoise state for BAGEL flow connector.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + + +@dataclass +class FlowGenerationState: + """Flow-side latent denoise state. + + The flow connector owns the numerical denoise state: noisy latent tokens, + timesteps, dt values, and step progress. Branch/cache/controller state + belongs to the MoT module and is intentionally not represented here. + """ + + _latents: torch.Tensor | None = None + _timestep_values: torch.Tensor | None = None + _dt_values: torch.Tensor | None = None + _step_index: int = 0 + _image_shape: tuple[int, int] | None = None + _grid_shape: tuple[int, int] | None = None + _token_count: int = 0 + + def reset(self) -> None: + self._latents = None + self._timestep_values = None + self._dt_values = None + self._step_index = 0 + self._image_shape = None + self._grid_shape = None + self._token_count = 0 + + def initialize( + self, + generation_kwargs: dict[str, Any], + *, + resolution: int, + patch_latent_dim: int, + device: torch.device, + ) -> None: + """Initialize initial Gaussian latent tokens and shifted Euler timestep schedule.""" + if bool(generation_kwargs.get("enable_taylorseer", False)): + raise NotImplementedError("BAGEL infer_gen TaylorSeer is not implemented in the SeedOmni graph path.") + + image_height = int(generation_kwargs.get("image_height", resolution)) + image_width = int(generation_kwargs.get("image_width", image_height)) + latent_downsample = int(generation_kwargs.get("latent_downsample", 16)) + if image_height % latent_downsample != 0 or image_width % latent_downsample != 0: + raise ValueError("BAGEL infer_gen image size must be divisible by latent_downsample.") + + grid_shape = (image_height // latent_downsample, image_width // latent_downsample) + token_count = int(grid_shape[0] * grid_shape[1]) + num_timesteps = int(generation_kwargs.get("num_timesteps", 50)) + if num_timesteps < 2: + raise ValueError("BAGEL infer_gen num_timesteps must be at least 2.") + + timestep_shift = float(generation_kwargs.get("timestep_shift", 1.0)) + timesteps = torch.linspace(1.0, 0.0, num_timesteps, device=device, dtype=torch.float32) + timesteps = timestep_shift * timesteps / (1.0 + (timestep_shift - 1.0) * timesteps) + + self._timestep_values = timesteps[:-1] + self._dt_values = timesteps[:-1] - timesteps[1:] + self._latents = torch.randn( + token_count, + int(patch_latent_dim), + device=device, + dtype=torch.float32, + ) + self._step_index = 0 + self._image_shape = (image_height, image_width) + self._grid_shape = grid_shape + self._token_count = token_count + + @property + def initialized(self) -> bool: + return self._latents is not None + + @property + def step_index(self) -> int: + return self._step_index + + @property + def image_shape(self) -> tuple[int, int] | None: + return self._image_shape + + @property + def token_count(self) -> int: + return self._token_count + + @property + def latents(self) -> torch.Tensor: + """Return the current noisy latent tokens ``x_t``.""" + if self._latents is None: + raise RuntimeError("BAGEL flow denoise latent state is not initialized.") + return self._latents + + @property + def timestep_values(self) -> torch.Tensor: + """Return the shifted timestep schedule tensor.""" + if self._timestep_values is None: + raise RuntimeError("BAGEL flow denoise timesteps are not initialized.") + return self._timestep_values + + @property + def dt_values(self) -> torch.Tensor: + """Return the step size deltas ``dt`` between consecutive timesteps.""" + if self._dt_values is None: + raise RuntimeError("BAGEL flow denoise dt values are not initialized.") + return self._dt_values + + @property + def grid_shape(self) -> tuple[int, int]: + """Return the ``(height, width)`` patch grid dimensions.""" + if self._grid_shape is None: + raise RuntimeError("BAGEL flow denoise latent grid shape is not initialized.") + return self._grid_shape + + def current_timestep(self) -> torch.Tensor: + """Return the scalar timestep tensor for the current Euler step.""" + timesteps = self.timestep_values + if self._step_index >= int(timesteps.numel()): + raise RuntimeError("BAGEL flow denoise timestep index is past the final step.") + return timesteps[self._step_index] + + def current_timestep_tokens(self) -> torch.Tensor: + x_t = self.latents + return self.current_timestep().reshape(1).expand(int(x_t.shape[0])) + + def is_complete(self) -> bool: + timesteps = self.timestep_values + return self._step_index >= int(timesteps.numel()) + + def advance(self, velocity: torch.Tensor) -> bool: + """Apply one Euler step ``x_t <- x_t - v_t * dt`` and return whether denoising finished.""" + x_t = self.latents + dts = self.dt_values + # Official generate_image does `x_t - v_t.to(x_t.device) * dts[i]` without + # pre-casting velocity to fp32. bf16 * fp32 is not always the same as + # fp32(v) * fp32(dt); keep the mixed-dtype multiply. + velocity = velocity.to(device=x_t.device) + if velocity.shape != x_t.shape: + raise ValueError( + f"BAGEL flow velocity shape mismatch: got {tuple(velocity.shape)}, expected {tuple(x_t.shape)}." + ) + + dt = dts[self._step_index].to(device=x_t.device) + self._latents = x_t - velocity * dt + self._step_index += 1 + return self.step_index >= int(dts.numel()) + + +__all__ = ["FlowGenerationState"] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py new file mode 100644 index 0000000000..fe3a47afe7 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py @@ -0,0 +1,440 @@ +"""BAGEL flow connector. + +``embed_latent`` projects patchified VAE latent tokens, timestep embeddings, +and latent position embeddings to MoT hidden width. ``decode_velocity`` projects +MoT hidden states back to patch-latent velocity tokens. Carrier selection, +dummy alignment, and loss computation live in the SeedOmni module mixin. + +The flow-matching FSM ``generate``-family endpoints (``embed_context_latents`` +/ ``prepare_denoise_query`` / ``decode_velocity_from_hidden`` / ``advance_denoise``) +and their shared :class:`~.generation_state.FlowGenerationState` live here too — +pure inference over an already-loaded ``BagelFlowConnector`` needs no VeOmni +training-graph machinery. ``accelerated.py`` only carries the training-graph +pre/forward/post hooks (carrier selection, SP token slicing, dummy alignment). +""" + +from __future__ import annotations + +import math +from typing import Any + +import numpy as np +import torch +import torch.nn as nn + +from ....graphs.generation_graph import FSM_SIGNAL_KEY +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_FLOW_VELOCITY, + BAGEL_GENERATED_LATENT, + BAGEL_PHASE_KEY, + BAGEL_VAE_CONTEXT, + get_tail_phase_item, +) +from .configuration import BagelFlowConnectorConfig +from .generation_state import FlowGenerationState +from .processing import flattened_position_ids, preprocess_context_latent_embed, unpatchify_latent_tokens + + +SIGNAL_IMAGE_COMPLETE = "image_complete" + + +def select_vae_context_latent_items( + conversation_list: list[list[ConversationItem]] | None, +) -> list[ConversationItem]: + """Select per-sample VAE context latents shared by training and inference hooks.""" + if conversation_list is None: + raise ValueError("BAGEL flow connector requires conversation_list to select VAE context latents.") + + # VAE preprocessing already makes BAGEL_VAE_CONTEXT per-sample by + # appending a dummy carrier only for samples without real VAE context. + # Reuse those existing carriers here; do not append another dummy. + return list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]})) + + +def scatter_flow_latent_embeds( + embed_items: list[ConversationItem], + embed_lengths: list[int], + latent_embeds: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Write flow latent embeds back onto carrier items (training + inference).""" + offset = 0 + for item, length in zip(embed_items, embed_lengths, strict=True): + item.value = latent_embeds[offset : offset + length].to(device=device, dtype=dtype) + offset += length + + # Hand the upstream VAE dummy carrier off as a flow dummy anchor; + # downstream MoT should not treat it as another VAE image span. Only + # the context tag survives: it is how MoT recognizes the flow anchor. + if item.is_dummy: + item.type = "output" + item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} + + if offset != int(latent_embeds.shape[0]): + raise RuntimeError("BAGEL flow connector latent count mismatch during embed scatter.") + + +class InferenceMixin: + """Flow-matching denoise FSM (``embed_context_latents`` / ``prepare_denoise_query`` + / ``decode_velocity_from_hidden`` / ``advance_denoise``) — HF ``GenerationMixin`` + analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelFlowConnector`'s bases: ``PretrainedOmniModule`` ships a no-op + ``reset_local_inference_state`` default (kept as a safety net for modules + that don't need real inference state), and MRO resolves left-to-right — + put second, that no-op would shadow the real one below. + """ + + def reset_local_inference_state(self) -> None: + self._generation_state.reset() + + def embed_context_latents( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Embed clean VAE context latents into LLM hidden space during prompt encoding.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + embed_items = [item for item in select_vae_context_latent_items(batched) if not item.is_dummy] + if not embed_items: + return {"conversation_list": conversation_list} + + inputs, embed_lengths = preprocess_context_latent_embed( + embed_items, + config=self.config, + device=self.device, + dtype=self.dtype, + ) + outputs = self.embed_latent(**inputs) + scatter_flow_latent_embeds( + embed_items, + embed_lengths, + outputs["latent_embeds"], + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation_list} + + def prepare_denoise_query( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Embed the current noisy latent tokens and write a ``flow_query`` output item.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + state = self._generation_state + if not state.initialized: + self._generation_state.initialize( + generation_kwargs or {}, + resolution=int(getattr(self.config, "resolution", 1024)), + patch_latent_dim=int(self.config.patch_latent_dim), + device=self._vae2llm_device, + ) + + x_t = state.latents + timestep = state.current_timestep() + timestep_tokens = state.current_timestep_tokens() + position_ids = flattened_position_ids( + state.grid_shape, + max_latent_size=int(self.config.max_latent_size), + device=x_t.device, + ) + outputs = self.embed_latent( + latents=x_t, + position_ids=position_ids, + timesteps=timestep_tokens, + ) + query = outputs["latent_embeds"].to(device=self.device, dtype=self.dtype) + timestep_meta = timestep.detach().to(device=query.device, dtype=torch.float32) + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if item is None: + conversation_list.append( + ConversationItem( + type="output", + value=query, + role="assistant", + meta={"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY}, + ) + ) + else: + item.type = "output" + item.role = "assistant" + item.value = query + item.meta = {"timestep": timestep_meta, BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY} + return {"conversation_list": conversation_list} + + def decode_velocity_from_hidden( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Project MoT denoise hidden states to patch-latent velocity tokens.""" + del generation_kwargs, kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_HIDDEN) + if item is None or not torch.is_tensor(item.value): + raise ValueError(f"BAGEL flow decode_velocity requires an output in phase {BAGEL_FLOW_HIDDEN!r}.") + + hidden = item.value + if hidden.dim() == 3 and hidden.shape[0] == 1: + hidden = hidden.squeeze(0) + if hidden.dim() != 2: + raise ValueError(f"BAGEL flow decode_velocity expected rank-2 hidden states, got {tuple(hidden.shape)}.") + if int(hidden.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + "BAGEL flow decode_velocity hidden-size mismatch: " + f"got {hidden.shape[-1]}, expected {self.config.hidden_size}." + ) + + # Official llm2vae is one GEMM per CFG branch. A stacked GEMM over + # concatenated branches is not bitwise even when each branch hidden + # matches, so split back to the per-branch query length. + query_len = int(self._generation_state.token_count) + 2 + if ( + self._generation_state.initialized + and query_len >= 3 + and int(hidden.shape[0]) >= query_len + and int(hidden.shape[0]) % query_len == 0 + ): + velocity = torch.cat( + [self.decode_velocity(hidden_states=chunk)["velocity"] for chunk in hidden.split(query_len, dim=0)], + dim=0, + ) + else: + velocity = self.decode_velocity(hidden_states=hidden)["velocity"] + item.type = "output" + item.role = "assistant" + item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY + item.value = velocity.to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + def advance_denoise( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Advance the Euler denoise state by one step or emit the final unpatchified latent.""" + del generation_kwargs, kwargs + if conversation_list is None: + raise ValueError("BAGEL flow inference requires conversation_list.") + + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if item is None or not torch.is_tensor(item.value): + raise ValueError(f"BAGEL flow advance requires an output in phase {BAGEL_FLOW_VELOCITY!r}.") + + velocity = item.value + if velocity.dim() == 3 and velocity.shape[0] == 1: + velocity = velocity.squeeze(0) + complete = self._generation_state.advance(velocity) + + if complete: + return self._emit_final_latent(conversation_list) + item.meta.pop("timestep", None) + return {"conversation_list": conversation_list} + + def _emit_final_latent( + self, + conversation_list: list[ConversationItem], + ) -> dict[str, Any]: + """Unpatchify the completed latent grid and emit ``image_complete``.""" + x_t = self._generation_state.latents + item = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + latent = unpatchify_latent_tokens( + x_t, + self._generation_state.grid_shape, + z_channels=int(self.config.z_channels), + latent_patch_size=int(self.config.latent_patch_size), + ) + if item is None: + conversation_list.append( + ConversationItem( + type="output", + value=latent, + role="assistant", + meta={BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT}, + ) + ) + else: + item.type = "output" + item.role = "assistant" + item.value = latent.to(device=self.device, dtype=self.dtype) + item.meta = {BAGEL_PHASE_KEY: BAGEL_GENERATED_LATENT} + + self._generation_state.reset() + return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: SIGNAL_IMAGE_COMPLETE} + + +class BagelFlowConnector(InferenceMixin, PretrainedOmniModule): + config_class = BagelFlowConnectorConfig + base_model_prefix = "bagel_flow_connector" + main_input_name = "hidden_states" + _no_split_modules: list[str] = [] + + def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): + # The flow connector projects latent tokens and timesteps; it does not execute attention. + return attn_implementation + + def __init__(self, config: BagelFlowConnectorConfig) -> None: + super().__init__(config) + self.time_embedder = TimestepEmbedder(config.hidden_size, config.timestep_frequency_embedding_size) + self.vae2llm = nn.Linear(config.patch_latent_dim, config.hidden_size) + self.llm2vae = nn.Linear(config.hidden_size, config.patch_latent_dim) + self.latent_pos_embed = PositionEmbedding(config.max_latent_size, config.hidden_size) + self._generation_state = FlowGenerationState() + self.post_init() + nn.init.constant_(self.llm2vae.weight, 0) + nn.init.constant_(self.llm2vae.bias, 0) + + def _init_weights(self, module: nn.Module) -> None: + if isinstance(module, PositionEmbedding): + module.reset_parameters() + return + super()._init_weights(module) + + @property + def _vae2llm_device(self) -> torch.device: + return self.vae2llm.weight.device + + @property + def _llm2vae_device(self) -> torch.device: + return self.llm2vae.weight.device + + @property + def _pos_embed_device(self) -> torch.device: + return self.latent_pos_embed.pos_embed.device + + @property + def _time_embedder_device(self) -> torch.device: + return self.time_embedder.mlp[0].weight.device + + def embed_latent( + self, + latents: torch.Tensor, + position_ids: torch.LongTensor, + timesteps: torch.Tensor, + ) -> dict[str, torch.Tensor]: + """Project patchified latents and add timestep and 2D sinusoidal position embeddings.""" + latents = latents.to(device=self._vae2llm_device, dtype=self.vae2llm.weight.dtype) + position_ids = position_ids.to(device=self._pos_embed_device, dtype=torch.long).reshape(-1) + timesteps = timesteps.to(device=self._time_embedder_device, dtype=torch.float32).reshape(-1) + if position_ids.numel() != latents.shape[0]: + raise ValueError("position_ids must have one value per latent token.") + + latent_embeds = self.vae2llm(latents) + time_embeds = self.time_embedder(timesteps) + pos_embeds = self.latent_pos_embed(position_ids) + return { + "latent_embeds": latent_embeds + + time_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype) + + pos_embeds.to(device=latent_embeds.device, dtype=latent_embeds.dtype) + } + + def decode_velocity(self, hidden_states: torch.Tensor) -> dict[str, torch.Tensor]: + hidden_states = hidden_states.to(device=self._llm2vae_device) + return {"velocity": self.llm2vae(hidden_states)} + + +class TimestepEmbedder(nn.Module): + def __init__(self, hidden_size: int, frequency_embedding_size: int = 256) -> None: + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + """Create sinusoidal timestep embeddings of width ``dim``.""" + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(half, dtype=torch.float32, device=t.device) / half) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + first_weight = self.mlp[0].weight + return self.mlp(t_freq.to(device=first_weight.device, dtype=first_weight.dtype)) + + +class PositionEmbedding(nn.Module): + def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: + super().__init__() + self.max_num_patch_per_side = max_num_patch_per_side + self.hidden_size = hidden_size + self.pos_embed = nn.Parameter( + torch.zeros(max_num_patch_per_side**2, hidden_size), + requires_grad=False, + ) + self.reset_parameters() + + def reset_parameters(self) -> None: + new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) + if hasattr(self.pos_embed.data, "to_local"): + self.pos_embed.data.to_local().copy_(new_data) + else: + self.pos_embed.data.copy_(new_data) + + def _init_weights(self) -> None: + self.reset_parameters() + + def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: + return self.pos_embed[position_ids.to(device=self.pos_embed.device)] + + +def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0).reshape(2, 1, grid_size, grid_size) + return torch.from_numpy(_get_2d_sincos_pos_embed_from_grid(embed_dim, grid)).float() + + +def _get_2d_sincos_pos_embed_from_grid(embed_dim: int, grid: np.ndarray) -> np.ndarray: + emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + return np.concatenate([emb_h, emb_w], axis=1) + + +def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + out = np.einsum("m,d->md", pos.reshape(-1), omega) + return np.concatenate([np.sin(out), np.cos(out)], axis=1) + + +__all__ = [ + "BagelFlowConnector", + "BagelFlowConnectorConfig", + "InferenceMixin", + "SIGNAL_IMAGE_COMPLETE", + "select_vae_context_latent_items", + "scatter_flow_latent_embeds", +] diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py new file mode 100644 index 0000000000..c19124e84b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py @@ -0,0 +1,216 @@ +"""Latent patch and timestep helpers for BAGEL flow connector.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from ....utils.conversation import ConversationItem + + +def preprocess_latent_embed( + embed_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, + timestep_shift: float, +) -> tuple[dict[str, torch.Tensor], list[int]]: + """Build noised latent embed inputs and training velocity targets.""" + latents: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + timesteps: list[torch.Tensor] = [] + embed_lengths: list[int] = [] + for item in embed_items: + clean, grid_shape = _patchify_latent_grid( + item.value, + z_channels=int(config.z_channels), + latent_patch_size=int(config.latent_patch_size), + ) + clean = clean.to(device=device, dtype=dtype) + token_count = int(clean.shape[0]) + + noise = torch.randn_like(clean) + timestep = torch.sigmoid(torch.randn(token_count, device=device, dtype=torch.float32)) + timestep = float(timestep_shift) * timestep / (1.0 + (float(timestep_shift) - 1.0) * timestep) + timestep_values = timestep.reshape(-1, 1).to(device=clean.device, dtype=torch.float32) + noised = (1.0 - timestep_values) * clean + timestep_values * noise + noised = noised.to(dtype=dtype) + + item.meta["timestep"] = timestep.detach() + item.meta["noise"] = noise.detach() + item.meta["flow_velocity_target"] = (noise - clean).detach() + latents.append(noised) + position_ids.append( + flattened_position_ids( + grid_shape, + max_latent_size=int(config.max_latent_size), + device=device, + ) + ) + timesteps.append(timestep) + embed_lengths.append(token_count) + + return ( + { + "latents": torch.cat(latents, dim=0), + "position_ids": torch.cat(position_ids, dim=0), + "timesteps": torch.cat(timesteps, dim=0), + }, + embed_lengths, + ) + + +def preprocess_context_latent_embed( + embed_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, +) -> tuple[dict[str, torch.Tensor], list[int]]: + """Build clean latent embed inputs for inference context latents.""" + latents: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + embed_lengths: list[int] = [] + for item in embed_items: + clean, grid_shape = _patchify_latent_grid( + item.value, + z_channels=int(config.z_channels), + latent_patch_size=int(config.latent_patch_size), + ) + clean = clean.to(device=device) + token_count = int(clean.shape[0]) + + latents.append(clean) + position_ids.append( + flattened_position_ids( + grid_shape, + max_latent_size=int(config.max_latent_size), + device=device, + ) + ) + embed_lengths.append(token_count) + + return ( + { + "latents": torch.cat(latents, dim=0), + "position_ids": torch.cat(position_ids, dim=0), + "timesteps": torch.zeros(sum(embed_lengths), device=device, dtype=torch.float32), + }, + embed_lengths, + ) + + +def preprocess_decode_velocity( + decode_items: list[ConversationItem], + *, + config: Any, + device: torch.device, + dtype: torch.dtype, +) -> tuple[dict[str, torch.Tensor], list[int], torch.Tensor]: + """Build packed hidden states and velocity targets for training decode.""" + hidden_states: list[torch.Tensor] = [] + targets: list[torch.Tensor] = [] + decode_lengths: list[int] = [] + for item in decode_items: + hidden = item.value + if not torch.is_tensor(hidden): + raise ValueError("BAGEL flow decode_velocity expects tensor hidden states.") + if hidden.dim() == 3 and hidden.shape[0] == 1: + hidden = hidden.squeeze(0) + if hidden.dim() != 2: + raise ValueError( + f"BAGEL flow decode_velocity expects rank-2 hidden states, got {tuple(item.value.shape)}." + ) + if int(hidden.shape[-1]) != int(config.hidden_size): + raise ValueError( + "BAGEL flow decode_velocity hidden-size mismatch: " + f"got {hidden.shape[-1]}, expected {config.hidden_size}." + ) + hidden = hidden.to(device=device) + hidden_states.append(hidden) + decode_lengths.append(int(hidden.shape[0])) + + target = item.meta.get("flow_velocity_target").to(device=device, dtype=dtype) + expected_shape = (hidden.shape[0], int(config.patch_latent_dim)) + if target.shape != expected_shape: + raise ValueError( + f"BAGEL flow_velocity_target shape mismatch: got {tuple(target.shape)}, expected {expected_shape}." + ) + targets.append(target) + + return {"hidden_states": torch.cat(hidden_states, dim=0)}, decode_lengths, torch.cat(targets, dim=0) + + +def _patchify_latent_grid( + value: torch.Tensor, + *, + z_channels: int, + latent_patch_size: int, +) -> tuple[torch.Tensor, tuple[int, int]]: + """Patchify a latent grid into a list of tokens.""" + latent = value.detach() + if latent.dim() == 4 and latent.shape[0] == 1: + latent = latent.squeeze(0) + if latent.dim() != 3: + raise ValueError(f"BAGEL flow connector expects latent grid tensors, got shape {tuple(value.shape)}.") + if int(latent.shape[0]) != int(z_channels): + raise ValueError( + f"BAGEL flow connector latent channel mismatch: got {int(latent.shape[0])}, expected {int(z_channels)}." + ) + channels, height, width = latent.shape + + patch_size = int(latent_patch_size) + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError( + "BAGEL flow connector latent grid is not divisible by latent_patch_size: " + f"shape={tuple(latent.shape)}, latent_patch_size={patch_size}." + ) + + grid_h, grid_w = height // patch_size, width // patch_size + tokens = latent.reshape(channels, grid_h, patch_size, grid_w, patch_size) + tokens = torch.einsum("chpwq->hwpqc", tokens) + return tokens.reshape(-1, patch_size * patch_size * channels), (grid_h, grid_w) + + +def unpatchify_latent_tokens( + tokens: torch.Tensor, + grid_shape: tuple[int, int], + *, + z_channels: int, + latent_patch_size: int, +) -> torch.Tensor: + grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1]) + patch = int(latent_patch_size) + channels = int(z_channels) + latent = tokens.reshape(grid_h, grid_w, patch, patch, channels) + latent = torch.einsum("hwpqc->chpwq", latent) + return latent.reshape(channels, grid_h * patch, grid_w * patch) + + +def flattened_position_ids( + grid_shape: tuple[int, int], + *, + max_latent_size: int, + device: torch.device | None = None, +) -> torch.Tensor: + """Compute flattened 2D grid position IDs bounded by ``max_latent_size``.""" + grid_h, grid_w = int(grid_shape[0]), int(grid_shape[1]) + if grid_h > max_latent_size or grid_w > max_latent_size: + raise ValueError( + "BAGEL flow connector latent grid exceeds max_latent_size: " + f"grid={grid_shape}, max_latent_size={max_latent_size}." + ) + coords_h = torch.arange(grid_h, device=device, dtype=torch.long) + coords_w = torch.arange(grid_w, device=device, dtype=torch.long) + return (coords_h[:, None] * int(max_latent_size) + coords_w).flatten() + + +__all__ = [ + "flattened_position_ids", + "preprocess_context_latent_embed", + "preprocess_decode_velocity", + "preprocess_latent_embed", + "unpatchify_latent_tokens", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py new file mode 100644 index 0000000000..8c774dee1a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL Qwen2 MoT backbone module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_config(): + from .configuration import BagelQwen2MoTConfig + + return BagelQwen2MoTConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_model(): + from .modeling import BagelQwen2MoT + + return BagelQwen2MoT + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_qwen2_mot") +def register_bagel_qwen2_mot_accelerated_model(): + from .accelerated import BagelQwen2MoTAccelerated + + return BagelQwen2MoTAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py new file mode 100644 index 0000000000..30fc075956 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/__init__.py @@ -0,0 +1,12 @@ +"""Accelerated runtime for the bagel/qwen2_mot OmniModule (conversation graph).""" + +from .accelerated import ( + BagelQwen2MoTAccelerated, + InferenceMixinAccelerated, +) + + +__all__ = [ + "BagelQwen2MoTAccelerated", + "InferenceMixinAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py new file mode 100644 index 0000000000..5c7cecb203 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -0,0 +1,668 @@ +"""VeOmni-accelerated BAGEL Qwen2-MoT — instance-local VeomniOp ops plus training-graph hooks. + +Native weights and packed ``forward`` live on :class:`BagelQwen2MoTCore`. +This module adds packed CFG denoise, fused RMSNorm / SwiGLU / RoPE, +attention ``VeomniOp`` dispatch, and the training pre/forward/post hooks. +""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.distributed as dist +from torch.nn.attention.flex_attention import BlockMask +from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import ( + gather_outputs, + slice_input_tensor, + sp_pad, +) +from .......ops import VeomniOp +from .......ops.kernels.swiglu_mlp import has_plain_linear_projections +from .......ops.mask import MagiAttentionMask +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ...sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_FLOW_HIDDEN, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + bagel_context, +) +from ..checkpoint_conversion import ( + BagelQwen2MoTCheckpointTensorConverter, + combine_qkv_state_dict_pre_hook, + register_accelerated_qkv_conversion_mapping, + split_qkv_state_dict_post_hook, +) +from ..configuration import BagelQwen2MoTConfig +from ..masking import build_mot_block_mask, build_mot_magi_mask, pad_mot_attention_metadata +from ..modeling import ( + BagelQwen2MoTAttention, + BagelQwen2MoTCore, + InferenceMixin, +) +from ..modeling import ( + _apply_rotary_pos_emb as _apply_rotary_pos_emb_eager, +) +from ..processing import PackedConversation, preprocess_mot_inputs + + +_FLASH_ATTENTION_2 = "veomni_flash_attention_2" + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelQwen2MoTCore` modeling APIs.""" + + config: BagelQwen2MoTConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._packed_training: PackedConversation | None = None + self._sp_full_sequence_length: int | None = None + self._validated_ulysses_size: int | None = None + + @pre_forward("forward") + def forward_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Pack multimodal conversation items into MoT training tensors and slice for Ulysses SP.""" + del kwargs + self._conversation_carrier = conversation_list + self._packed_training = preprocess_mot_inputs( + conversation_list, + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if self._packed_training is None: + raise ValueError( + "BAGEL Qwen2-MoT forward requires a non-empty conversation_list with real tokens; " + "got no packable tokens across the whole batch." + ) + + # Meter the full replicated batch before SP slices its packed sequence. + self.metric_meter_set_seqlens( + "forward", + [int(sum(splits)) for splits in self._packed_training.sample_splits], + ) + + packed_sequence = self._fold_dummy_anchors(self._packed_training.packed_sequence, conversation_list) + sequence_length = int(packed_sequence.shape[0]) + attention_metadata = self._packed_training.packed_attention_metadata + expected_metadata_shape = (3, sequence_length) + if tuple(attention_metadata.shape) != expected_metadata_shape: + raise ValueError( + "BAGEL Qwen2-MoT attention metadata must match the full packed sample: " + f"expected {expected_metadata_shape}, got {tuple(attention_metadata.shape)}." + ) + + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_full_sequence_length = None + return { + "packed_sequence": packed_sequence, + "packed_position_ids": self._packed_training.packed_position_ids, + "packed_token_type_ids": self._packed_training.packed_token_type_ids, + "packed_attention_metadata": attention_metadata, + } + + if ps.cp_size != 1: + raise ValueError( + f"BAGEL Qwen2-MoT training supports Ulysses sequence parallelism only; got cp_size={ps.cp_size}." + ) + if self._validated_ulysses_size != ps.ulysses_size: + num_heads = int(self.config.num_attention_heads) + num_key_value_heads = int(self.config.num_key_value_heads) + if num_heads % ps.ulysses_size != 0: + raise ValueError( + "BAGEL Qwen2-MoT attention heads must be divisible by " + f"ulysses_size: num_heads={num_heads}, ulysses_size={ps.ulysses_size}." + ) + if num_key_value_heads % ps.ulysses_size != 0: + raise ValueError( + "BAGEL Qwen2-MoT KV heads must be divisible by " + f"ulysses_size for native GQA: num_key_value_heads={num_key_value_heads}, " + f"ulysses_size={ps.ulysses_size}." + ) + self._validated_ulysses_size = ps.ulysses_size + + # Tensor inputs are padded then sliced for Ulysses. Attention metadata + # stays full because the Flex adapter reconstructs the full sequence + # before building/using its BlockMask. + self._sp_full_sequence_length = sequence_length + packed_sequence = sp_pad(packed_sequence, dim=0, pad_value=0) + packed_position_ids = sp_pad(self._packed_training.packed_position_ids, dim=0, pad_value=0) + packed_token_type_ids = sp_pad(self._packed_training.packed_token_type_ids, dim=0, pad_value=-1) + attention_metadata = pad_mot_attention_metadata( + attention_metadata, + padded_length=int(packed_sequence.shape[0]), + ) + packed_sequence = slice_input_tensor(packed_sequence, dim=0, padding=False, group=ps.sp_group) + packed_position_ids = slice_input_tensor( + packed_position_ids, + dim=0, + padding=False, + group=ps.sp_group, + ) + packed_token_type_ids = slice_input_tensor( + packed_token_type_ids, + dim=0, + padding=False, + group=ps.sp_group, + ) + + return { + "packed_sequence": packed_sequence, + "packed_position_ids": packed_position_ids, + "packed_token_type_ids": packed_token_type_ids, + # FlexAttention's generic Ulysses adapter reconstructs the complete + # padded sequence, so every rank keeps the same full metadata. + "packed_attention_metadata": attention_metadata, + } + + @post_forward("forward") + def forward_post(self, hidden_states: torch.Tensor, **kwargs: Any) -> dict[str, Any]: + """Gather SP sequence shards if active and scatter MoT hidden states onto carrier items.""" + del kwargs + ps = get_parallel_state() + if ps.sp_size > 1: + if self._sp_full_sequence_length is None: + raise RuntimeError("BAGEL Qwen2-MoT SP sequence length was not initialized.") + hidden_states = gather_outputs(hidden_states, gather_dim=0, group=ps.sp_group) + hidden_states = hidden_states.narrow(0, 0, self._sp_full_sequence_length) + + conversation = self._conversation_carrier + packed = self._packed_training + self._conversation_carrier = None + self._packed_training = None + self._sp_full_sequence_length = None + + for span in packed.spans: + span_hidden = hidden_states[span.start : span.start + span.length].to(device=self.device) + offset = 0 + for item, length in zip(span.items, span.lengths, strict=True): + item.value = span_hidden[offset : offset + length] + offset += length + return {"conversation_list": conversation} + + def _fold_dummy_anchors( + self, + packed_sequence: torch.Tensor, + conversation_list: list[list[ConversationItem]] | None, + ) -> torch.Tensor: + """Fold zero-valued FSDP dummy anchors from upstream SigLIP and flow encoders.""" + # Dummy encoder outputs must still touch MoT on ranks where another rank + # has the real branch, otherwise FSDP sees different gradient buckets. + anchor = packed_sequence.new_zeros(()) + has_anchor = False + include_siglip_dummy, include_flow_dummy = self._has_valid_upstream_embeddings(conversation_list) + + for item in iter_desired_items(conversation_list or []): + if not item.is_dummy or not torch.is_tensor(item.value): + continue + + # The flow connector hands its embedded VAE placeholder on as an + # ``output`` row; a VAE placeholder still of type ``image`` never + # reached the flow connector and holds no MoT-width embedding. + context = bagel_context(item) + is_flow_dummy = item.type == "output" and context == BAGEL_VAE_CONTEXT + if not is_flow_dummy and context != BAGEL_SIGLIP_CONTEXT: + continue + if is_flow_dummy and not include_flow_dummy: + continue + if context == BAGEL_SIGLIP_CONTEXT and not include_siglip_dummy: + continue + + has_anchor = True + anchor = ( + anchor + + item.value.to( + device=packed_sequence.device, + dtype=packed_sequence.dtype, + ).sum() + * 0.0 + ) + + if not has_anchor: + return packed_sequence + return packed_sequence + anchor + + def _has_valid_upstream_embeddings( + self, + conversation_list: list[list[ConversationItem]] | None, + ) -> tuple[bool, bool]: + """Return whether any rank in the distributed group has real SigLIP or flow embeddings.""" + has_siglip = int( + self._has_valid_upstream_embedding( + conversation_list, + label="SigLIP", + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}, + ) + ) + has_flow = int( + self._has_valid_upstream_embedding( + conversation_list, + label="flow", + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}, + meta_keys=["flow_velocity_target"], + ) + ) + + if not dist.is_available() or not dist.is_initialized(): + return bool(has_siglip), bool(has_flow) + flags = torch.tensor([has_siglip, has_flow], device=self.device, dtype=torch.int32) + dist.all_reduce(flags, op=dist.ReduceOp.MAX) + return bool(flags[0].item()), bool(flags[1].item()) + + def _has_valid_upstream_embedding( + self, + conversation_list: list[list[ConversationItem]] | None, + *, + label: str, + types: list[str], + meta: dict[str, list[Any]] | None = None, + meta_keys: list[str] | None = None, + ) -> bool: + """Check whether ``conversation_list`` contains a non-dummy rank-2 embedding matching the filter.""" + for item in iter_desired_items( + conversation_list or [], + types=types, + roles=["user", "assistant"], + meta_keys=meta_keys, + meta=meta, + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT {label} alignment expects rank-2 embeddings, got {tuple(value.shape)}." + ) + if int(value.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + f"BAGEL Qwen2-MoT {label} alignment hidden-size mismatch: " + f"got {value.shape[-1]}, expected {self.config.hidden_size}." + ) + return True + return False + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's Qwen2-MoT backbone (transformer layers only).""" + + config: BagelQwen2MoTConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + """Estimate TFLOPs for the Qwen2-MoT transformer layers over ``seqlens``.""" + cfg = self.config + hidden = cfg.hidden_size + num_layers = cfg.num_hidden_layers + num_heads = cfg.num_attention_heads + num_kv_heads = cfg.num_key_value_heads + # Attention uses head_dim = hidden // num_heads (see BagelQwen2MoTAttention). + head_dim = hidden // num_heads + + # SwiGLU MLP (gate/up/down) + attention projections (q, o over num_heads; + # k, v over num_kv_heads). Biases are O(hidden) → negligible, ignored. + mlp_n = hidden * cfg.intermediate_size * 3 + attn_linear_n = hidden * head_dim * (num_heads * 2 + num_kv_heads * 2) + dense_n = (mlp_n + attn_linear_n) * num_layers + + tokens = sum(seqlens) + seqlen_sq = sum(s * s for s in seqlens) + dense_flops = 6 * dense_n * tokens + attn_flops = 12 * seqlen_sq * head_dim * num_heads * num_layers + return (dense_flops + attn_flops) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks and graph entrypoints for BAGEL's packed MoT backbone. + + Packed CFG denoise is mixed in separately via :class:`InferenceMixinAccelerated` + so this mixin stays runtime-only. + """ + + +class Qwen2RMSNormAccelerated(Qwen2RMSNorm): + """Qwen2 RMSNorm with instance-local VeomniOp fused dispatch.""" + + def __init__(self, hidden_size: int, eps: float = 1e-6, *, config: BagelQwen2MoTConfig | None = None) -> None: + super().__init__(hidden_size, eps=eps) + self.config = config + rms_impl = config.ops_config.rms_norm_implementation if config is not None else "eager" + self.veomni_rms_norm = VeomniOp("rms_norm", "standard", rms_impl) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Apply RMSNorm, falling back to eager on empty modality branches.""" + # Empty modality branches retain zero-gradient FSDP anchors. Liger's + # Triton kernel cannot launch the resulting zero-row grid. + if hidden_states.numel() == 0: + return super().forward(hidden_states) + + if self.veomni_rms_norm.impl != "eager": + return self.veomni_rms_norm(hidden_states, self.weight, eps=self.variance_epsilon) + + return super().forward(hidden_states) + + +class Qwen2MLPAccelerated(Qwen2MLP): + """Qwen2 SwiGLU MLP with instance-local VeomniOp fused dispatch.""" + + def __init__(self, config: BagelQwen2MoTConfig) -> None: + super().__init__(config) + self.veomni_swiglu_mlp = VeomniOp("swiglu_mlp", "standard", config.ops_config.swiglu_mlp_implementation) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply SwiGLU MLP, falling back to eager on empty modality branches.""" + # Empty modality branches retain zero-gradient FSDP anchors. Liger's + # Triton kernel cannot launch the resulting zero-row grid. + if x.numel() == 0: + return super().forward(x) + + if self.veomni_swiglu_mlp.impl != "eager": + if self.config.hidden_act not in {"silu", "swish"}: + raise ValueError( + f"Liger SwiGLU requires hidden_act='silu' or 'swish', got {self.config.hidden_act!r}. " + "Set model.ops_implementation.swiglu_mlp_implementation='eager' " + "to use the Transformers reference implementation." + ) + if has_plain_linear_projections(self): + return self.veomni_swiglu_mlp( + x, + self.gate_proj.weight, + self.gate_proj.bias if self.gate_proj.bias is not None else self.gate_proj.weight.new_empty(0), + self.up_proj.weight, + self.up_proj.bias if self.up_proj.bias is not None else self.gate_proj.weight.new_empty(0), + self.down_proj.weight, + self.down_proj.bias if self.down_proj.bias is not None else self.down_proj.weight.new_empty(0), + ) + + return super().forward(x) + + +def _apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + *, + rope_op: VeomniOp | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Packed RoPE with optional instance-local VeomniOp dispatch.""" + if rope_op is None: + rope_op = VeomniOp("rope", "full", "eager") + if q.numel() != 0 and k.numel() != 0 and rope_op.impl != "eager": + if unsqueeze_dim != 1: + raise NotImplementedError("BAGEL packed fused RoPE requires unsqueeze_dim=1.") + if q.ndim != 3 or k.ndim != 3 or cos.ndim != 2 or sin.ndim != 2: + raise NotImplementedError( + "BAGEL fused RoPE requires packed q/k tensors shaped [tokens, heads, head_dim] " + "and cos/sin tensors shaped [tokens, head_dim]." + ) + if not (q.shape[0] == k.shape[0] == cos.shape[0] == sin.shape[0]): + raise ValueError("BAGEL packed q/k/cos/sin tensors must share the token dimension.") + if not (q.shape[-1] == k.shape[-1] == cos.shape[-1] == sin.shape[-1]): + raise NotImplementedError("Liger full RoPE does not support partial rotary dimensions.") + + # The shared full-RoPE kernels consume [batch, heads, sequence, head_dim]. + # BAGEL stores packed Q/K as [tokens, heads, head_dim], so adapt through + # a synthetic batch dimension and restore the packed layout afterwards. + q_embed, k_embed = rope_op( + q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), + cos.unsqueeze(0), + sin.unsqueeze(0), + unsqueeze_dim=1, + ) + return ( + q_embed.squeeze(0).transpose(0, 1).contiguous(), + k_embed.squeeze(0).transpose(0, 1).contiguous(), + ) + + return _apply_rotary_pos_emb_eager(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim) + + +_PACKED_FUSED_ATTN_IMPLEMENTATIONS = ( + "veomni_flex_attention", + "veomni_magi_attention", +) +_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"}) +_MAGI_ATTN_IMPLEMENTATIONS = frozenset({"magi_attention", "veomni_magi_attention"}) + + +def _normalize_packed_attn_impl(attn_implementation: str | None) -> str | None: + """Normalize Flex/Magi attention aliases to their canonical ``veomni_*`` names.""" + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS: + return "veomni_flex_attention" + if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS: + return "veomni_magi_attention" + return attn_implementation + + +class BagelQwen2MoTAttentionAccelerated(BagelQwen2MoTAttention): + """MoT attention with instance-local RoPE and attention VeomniOps.""" + + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__(config, layer_idx, rms_norm_cls=rms_norm_cls) + self.veomni_rope = VeomniOp("rope", "full", config.ops_config.rotary_pos_emb_implementation) + attn_impl = _normalize_packed_attn_impl(config.ops_config.attn_implementation) + if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: + self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl) + elif attn_impl in _MAGI_ATTN_IMPLEMENTATIONS: + try: + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + except RuntimeError: + self.veomni_attn = None + else: + self.veomni_attn = None + self.veomni_denoise_attn = VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + self.register_load_state_dict_pre_hook(combine_qkv_state_dict_pre_hook) + self.register_state_dict_post_hook(split_qkv_state_dict_post_hook) + + def _build_qkv_proj(self) -> None: + qkv_size = sum(self.qkv_split_sizes) + self.qkv_proj_und = torch.nn.Linear(self.hidden_size, qkv_size, bias=True) + self.qkv_proj_gen = torch.nn.Linear(self.hidden_size, qkv_size, bias=True) + + def _project_qkv( + self, + hidden: torch.Tensor, + *, + is_gen: bool, + split: bool = False, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project ``hidden`` through the fused und or gen QKV linear layer.""" + projected = (self.qkv_proj_gen if is_gen else self.qkv_proj_und)(hidden) + if split: + return projected.split(self.qkv_split_sizes, dim=-1) + return projected + + @staticmethod + def build_attention_mask( + packed_attention_metadata: torch.Tensor, + *, + attn_implementation: str | None = None, + ) -> BlockMask | MagiAttentionMask: + """Materialize a Flex ``BlockMask`` or ``MagiAttentionMask`` from ``packed_attention_metadata``.""" + if attn_implementation in _MAGI_ATTN_IMPLEMENTATIONS: + return build_mot_magi_mask(packed_attention_metadata) + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS or attn_implementation is None: + return build_mot_block_mask(packed_attention_metadata) + raise ValueError( + "BAGEL Qwen2-MoT packed fused attention requires " + f"{', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}, got {attn_implementation!r}." + ) + + def apply_rotary_pos_emb( + self, + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + ) -> tuple[torch.Tensor, torch.Tensor]: + return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim, rope_op=self.veomni_rope) + + def _resolve_packed_attn_op(self, *, context: str) -> VeomniOp: + """Resolve the packed fused attention ``VeomniOp`` for ``context``.""" + attn_impl = _normalize_packed_attn_impl(self.config._attn_implementation) + if attn_impl not in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: + raise ValueError( + f"BAGEL Qwen2-MoT {context} requires packed fused attention " + f"({', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}), got " + f"{self.config._attn_implementation!r}." + ) + if self.veomni_attn is not None and self.veomni_attn.impl == attn_impl: + return self.veomni_attn + return VeomniOp("attention", "standard", attn_impl) + + def attend( + self, + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + attention_mask: BlockMask | MagiAttentionMask, + ) -> torch.Tensor: + attn_op = self._resolve_packed_attn_op(context="training") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + ) + packed_attn_output = packed_attn_output.squeeze(0) + return packed_attn_output.reshape(-1, self.num_heads * self.head_dim) + + def attend_inference( + self, + packed_query_states: torch.Tensor, + merged_key_states: torch.Tensor, + merged_value_states: torch.Tensor, + *, + attention_mask: BlockMask | MagiAttentionMask | None, + is_causal: bool, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + ) -> torch.Tensor: + """Run packed inference attention using Flex/Magi for prefill and FlashAttention-2 for decode/denoise.""" + if attention_mask is not None: + attn_op = self._resolve_packed_attn_op(context="packed prefill") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + # Inference packs complete logical documents locally and does + # not enter the training-only Ulysses redistribution. + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + + packed_attn_output, _ = self.veomni_denoise_attn( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask=None, + dropout=0.0, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + # Inference owns its packed KV-cache layout and does not enter the + # training-only module Ulysses redistribution. + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + + +class InferenceMixinAccelerated(InferenceMixin): + """Packed CFG denoise for the FSDP/runtime MoT class. + + Stacks active CFG branches into one ``forward_inference`` call so decoder + layers unshard once per denoise step. ``generate`` and ``collect_velocity`` + stay on :class:`~.modeling.InferenceMixin`. + """ + + def denoise_branch( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Run one stacked CFG denoise forward across all active branches.""" + del kwargs + tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs) + inputs = self._generation_state.preprocess_parallel_denoise_inputs( + query, + generation_kwargs or {}, + timestep=tail.meta.get("timestep"), + empty_cache_factory=self._new_empty_cache, + device=self.device, + dtype=self.dtype, + ) + outputs = self.forward_inference( + **inputs, + update_past_key_values=False, + is_causal=False, + mode="gen", + ) + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = outputs["hidden_states"].to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + +class BagelQwen2MoTAccelerated(VeOmniMixin, InferenceMixinAccelerated, BagelQwen2MoTCore): + _supports_flex_attn = True + _supports_magi_attn = True + _export_hf_checkpoint_with_weight_conversions = True + attention_cls = BagelQwen2MoTAttentionAccelerated + mlp_cls = Qwen2MLPAccelerated + rms_norm_cls = Qwen2RMSNormAccelerated + + @staticmethod + def _create_checkpoint_tensor_converter( + model: BagelQwen2MoTCore, + ) -> BagelQwen2MoTCheckpointTensorConverter: + del model + return BagelQwen2MoTCheckpointTensorConverter() + + +register_accelerated_qkv_conversion_mapping() + + +__all__ = ["BagelQwen2MoTAccelerated", "InferenceMixinAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py new file mode 100644 index 0000000000..44a8199290 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py @@ -0,0 +1,232 @@ +"""Checkpoint compatibility for accelerated BAGEL fused QKV projections. + +Eager attention owns official split ``q_proj`` / ``k_proj`` / ``v_proj`` +parameters. Accelerated attention stores one merged ``qkv_proj_und`` or +``qkv_proj_gen`` parameter per MoT branch. Existing checkpoints keep their +separate Q/K/V keys, so only the accelerated runtime needs an adapter: +PyTorch state-dict hooks, Transformers weight converters, or VeOmni's streaming +checkpoint converter. +""" + +from __future__ import annotations + +import re +from typing import Any + +import torch +from transformers.conversion_mapping import register_checkpoint_conversion_mapping +from transformers.core_model_loading import ConversionOps, WeightConverter + +from ......models.checkpoint.convert import ConvertedCheckpointTensor + + +_QKV_COMPONENTS = ("q", "k", "v") +_QKV_BRANCHES = ( + ("qkv_proj_und", ("q_proj", "k_proj", "v_proj")), + ("qkv_proj_gen", ("q_proj_moe_gen", "k_proj_moe_gen", "v_proj_moe_gen")), +) +_QKV_CHECKPOINT_RE = re.compile( + r"^(?P.*\.self_attn\.)" + r"(?P[qkv]_proj(?:_moe_gen)?)\." + r"(?Pweight|bias)$" +) + + +def combine_qkv_state_dict_pre_hook( + module: torch.nn.Module, + state_dict: dict[str, torch.Tensor], + prefix: str, + local_metadata: dict[str, Any], + strict: bool, + missing_keys: list[str], + unexpected_keys: list[str], + error_msgs: list[str], +) -> None: + """Accept the legacy split Q/K/V schema in direct PyTorch and DCP loads.""" + del module, local_metadata, strict, missing_keys, unexpected_keys + for combined_name, checkpoint_names in _QKV_BRANCHES: + for kind in ("weight", "bias"): + combined_key = f"{prefix}{combined_name}.{kind}" + checkpoint_keys = [f"{prefix}{name}.{kind}" for name in checkpoint_names] + if combined_key in state_dict or not any(key in state_dict for key in checkpoint_keys): + continue + if not all(key in state_dict for key in checkpoint_keys): + present = [key for key in checkpoint_keys if key in state_dict] + error_msgs.append( + f"Incomplete BAGEL QKV checkpoint group for {combined_key}: found {present}, " + f"expected {checkpoint_keys}." + ) + continue + + state_dict[combined_key] = torch.cat([state_dict.pop(key) for key in checkpoint_keys], dim=0) + + +def split_qkv_state_dict_post_hook( + module: torch.nn.Module, + state_dict: dict[str, torch.Tensor], + prefix: str, + local_metadata: dict[str, Any], +) -> None: + """Expose legacy split Q/K/V keys while keeping combined runtime parameters.""" + del local_metadata + split_sizes = module.qkv_split_sizes + for combined_name, checkpoint_names in _QKV_BRANCHES: + for kind in ("weight", "bias"): + combined_key = f"{prefix}{combined_name}.{kind}" + tensor = state_dict.get(combined_key) + if tensor is None or tensor.is_meta or hasattr(tensor, "device_mesh"): + # Transformers inspects a meta model's state dict to discover the + # runtime parameter schema before applying its weight converters. + # DCP likewise requires DTensor keys to remain resolvable to the + # live combined parameters before the HF export layer converts them. + continue + + state_dict.pop(combined_key) + for checkpoint_name, chunk in zip( + checkpoint_names, + tensor.split(split_sizes, dim=0), + strict=True, + ): + state_dict[f"{prefix}{checkpoint_name}.{kind}"] = chunk + + +class _ConcatenateQKV(ConversionOps): + """Concatenate checkpoint Q/K/V tensors along the projection dimension.""" + + @torch.no_grad() + def convert( + self, + input_dict: dict[str, list[torch.Tensor]], + source_patterns: list[str], + target_patterns: list[str], + **kwargs: Any, + ) -> dict[str, torch.Tensor]: + """Concatenate ``source_patterns`` tensors into a single fused target tensor.""" + del kwargs + if len(target_patterns) != 1: + raise ValueError("Combined BAGEL QKV conversion requires one target pattern.") + + tensors: list[torch.Tensor] = [] + for source_pattern in source_patterns: + values = input_dict.pop(source_pattern) + tensors.extend(values if isinstance(values, list) else [values]) + return {target_patterns[0]: torch.cat(tensors, dim=0)} + + @property + def reverse_op(self) -> ConversionOps: + return _SplitQKV() + + +class _SplitQKV(ConversionOps): + """Restore combined QKV tensors to BAGEL's external checkpoint schema.""" + + @torch.no_grad() + def convert( + self, + input_dict: dict[str, list[torch.Tensor]], + source_patterns: list[str], + target_patterns: list[str], + **kwargs: Any, + ) -> dict[str, torch.Tensor]: + """Split a fused QKV tensor into separate Q, K, and V checkpoint tensors.""" + del source_patterns + if len(input_dict) != 1 or len(target_patterns) != 3: + raise ValueError("BAGEL QKV export requires one combined source and three target patterns.") + + values = next(iter(input_dict.values())) + tensor = values[0] if isinstance(values, list) else values + # FSDP2 / DCP keeps a dim-0 DTensor on the fused parameter. Q/K/V row + # sizes do not line up with those shards, so gather before splitting. + full_tensor = getattr(tensor, "full_tensor", None) + if callable(full_tensor) and getattr(tensor, "device_mesh", None) is not None: + tensor = full_tensor() + config = kwargs["config"] + head_dim = config.hidden_size // config.num_attention_heads + split_sizes = ( + config.num_attention_heads * head_dim, + config.num_key_value_heads * head_dim, + config.num_key_value_heads * head_dim, + ) + return dict(zip(target_patterns, tensor.split(split_sizes, dim=0), strict=True)) + + @property + def reverse_op(self) -> ConversionOps: + return _ConcatenateQKV() + + +def _qkv_weight_converters() -> list[WeightConverter]: + """Build Transformers ``WeightConverter`` rules for both MoT QKV branches.""" + converters: list[WeightConverter] = [] + for branch, source_suffix in (("und", ""), ("gen", "_moe_gen")): + for kind in ("weight", "bias"): + converters.append( + WeightConverter( + source_patterns=[ + f"self_attn.{component}_proj{source_suffix}.{kind}" for component in _QKV_COMPONENTS + ], + target_patterns=f"self_attn.qkv_proj_{branch}.{kind}", + operations=[_ConcatenateQKV()], + ) + ) + return converters + + +def register_accelerated_qkv_conversion_mapping() -> None: + """Register fused-QKV converters for the accelerated class only. + + Lookups prefer class name over ``model_type``. Do not register + ``bagel_qwen2_mot`` here: eager ``BagelQwen2MoT`` uses native split keys. + """ + register_checkpoint_conversion_mapping( + "BagelQwen2MoTAccelerated", + _qkv_weight_converters(), + overwrite=True, + ) + + +class BagelQwen2MoTCheckpointTensorConverter: + """Stream legacy Q/K/V tensors into accelerated combined runtime parameters. + + VeOmni's ``load_model_weights`` assigns tensors directly instead of calling + ``load_state_dict``, so the PyTorch pre-hook cannot perform this merge. + """ + + def __init__(self) -> None: + self._pending: dict[tuple[str, str, str], dict[str, torch.Tensor]] = {} + + def can_handle(self, name: str) -> bool: + return _QKV_CHECKPOINT_RE.fullmatch(name) is not None + + def convert(self, name: str, tensor: torch.Tensor) -> ConvertedCheckpointTensor | None: + """Buffer split Q/K/V tensors and emit the fused ``qkv_proj_{und,gen}`` tensor once complete.""" + match = _QKV_CHECKPOINT_RE.fullmatch(name) + if match is None: + return ConvertedCheckpointTensor(name=name, tensor=tensor) + + projection = match.group("projection") + component = projection[0] + branch = "gen" if projection.endswith("_moe_gen") else "und" + group_key = (match.group("prefix"), branch, match.group("kind")) + group = self._pending.setdefault(group_key, {}) + if component in group: + raise ValueError(f"Duplicate BAGEL QKV checkpoint tensor: {name}.") + group[component] = tensor + if group.keys() != set(_QKV_COMPONENTS): + return None + + self._pending.pop(group_key) + combined = torch.cat([group[component] for component in _QKV_COMPONENTS], dim=0) + prefix, branch, kind = group_key + return ConvertedCheckpointTensor( + name=f"{prefix}qkv_proj_{branch}.{kind}", + tensor=combined, + ) + + def finalize(self) -> list[ConvertedCheckpointTensor]: + """Verify no partial QKV groups remain buffered at the end of checkpoint loading.""" + if self._pending: + incomplete = ", ".join( + f"{prefix}qkv_proj_{branch}.{kind}" for prefix, branch, kind in sorted(self._pending) + ) + raise ValueError(f"Incomplete BAGEL QKV checkpoint groups: {incomplete}.") + return [] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py new file mode 100644 index 0000000000..bf2905f746 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py @@ -0,0 +1,60 @@ +"""Configuration for BAGEL's Qwen2 MoT backbone module.""" + +from transformers.models.qwen2.configuration_qwen2 import Qwen2Config + +from ...module_configuration_base import OmniModuleConfig + + +class BagelQwen2MoTConfig(OmniModuleConfig, Qwen2Config): + """Qwen2 config with BAGEL's runtime MoT defaults. + + The upstream ``llm_config.json`` is a plain Qwen2 config. BAGEL's loader + mutates it at runtime with ``qk_norm=True`` and + ``layer_module="Qwen2MoTDecoderLayer"``; this config bakes those defaults + into the split module so a SeedOmni checkpoint is self-describing. + """ + + model_type = "bagel_qwen2_mot" + + def __init__( + self, + vocab_size: int = 152064, + hidden_size: int = 3584, + intermediate_size: int = 18944, + num_hidden_layers: int = 28, + num_attention_heads: int = 28, + num_key_value_heads: int = 4, + max_position_embeddings: int = 32768, + rms_norm_eps: float = 1e-6, + rope_theta: float = 1000000.0, + bos_token_id: int = 151643, + eos_token_id: int = 151645, + tie_word_embeddings: bool = False, + qk_norm: bool = True, + layer_module: str = "Qwen2MoTDecoderLayer", + freeze_und: bool = False, + **kwargs, + ): + super().__init__( + vocab_size=vocab_size, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + num_key_value_heads=num_key_value_heads, + max_position_embeddings=max_position_embeddings, + rms_norm_eps=rms_norm_eps, + rope_theta=rope_theta, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + if self.num_attention_heads % self.num_key_value_heads != 0: + raise ValueError( + "BAGEL Qwen2-MoT query heads must be divisible by KV heads, " + f"got query={self.num_attention_heads}, key_value={self.num_key_value_heads}." + ) + self.qk_norm = qk_norm + self.layer_module = layer_module + self.freeze_und = freeze_und diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py new file mode 100644 index 0000000000..fd903c8950 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py @@ -0,0 +1,532 @@ +"""Internal generation state for BAGEL Qwen2-MoT inference.""" + +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Any, Callable, Iterator + +import torch + +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + + +def _build_denoise_index_layout( + key_values_lens: torch.Tensor, + query_len: int, + *, + device: torch.device, +) -> dict[str, torch.Tensor]: + """Build attention-space and query-space indexes for stacked CFG branches.""" + query_indexes: list[torch.Tensor] = [] + key_value_indexes: list[torch.Tensor] = [] + text_indexes: list[torch.Tensor] = [] + vae_token_indexes: list[torch.Tensor] = [] + # Attention indexes address the merged [branch cache, branch query] + # layout. Expert indexes address the query-only tensor, so they use a + # separate cursor that does not include cached rows. + attention_offset = 0 + query_offset = 0 + for key_values_len in key_values_lens.tolist(): + key_values_len = int(key_values_len) + key_value_indexes.append( + torch.arange(attention_offset, attention_offset + key_values_len, device=device, dtype=torch.long) + ) + query_indexes.append( + torch.arange( + attention_offset + key_values_len, + attention_offset + key_values_len + query_len, + device=device, + dtype=torch.long, + ) + ) + text_indexes.append( + torch.tensor([query_offset, query_offset + query_len - 1], device=device, dtype=torch.long) + ) + vae_token_indexes.append( + torch.arange(query_offset + 1, query_offset + query_len - 1, device=device, dtype=torch.long) + ) + attention_offset += key_values_len + query_len + query_offset += query_len + + return { + "packed_query_indexes": torch.cat(query_indexes, dim=0), + "packed_key_value_indexes": ( + torch.cat(key_value_indexes, dim=0) + if key_value_indexes + else torch.empty(0, device=device, dtype=torch.long) + ), + "packed_text_indexes": torch.cat(text_indexes, dim=0), + "packed_vae_token_indexes": torch.cat(vae_token_indexes, dim=0), + } + + +@dataclass +class MotCacheContext: + """KV-cache state for one logical inference branch. + + FA receives all active branches in one packed call. Each branch therefore + keeps its own cache plus the indexes needed to place that cache in the + temporary packed ``[past, query]`` attention layout. + """ + + name: str + _cache: Any | None = None + _key_values_lens: torch.Tensor | None = None + _packed_key_value_indexes: torch.Tensor | None = None + _next_position_ids: torch.Tensor | None = None + + # ── Cache state ───────────────────────────────────────────── + + def reset(self) -> None: + self._cache = None + self._key_values_lens = None + self._packed_key_value_indexes = None + self._next_position_ids = None + + @property + def cache(self) -> Any | None: + return self._cache + + @property + def key_values_lens(self) -> torch.Tensor | None: + return self._key_values_lens + + @property + def packed_key_value_indexes(self) -> torch.Tensor | None: + return self._packed_key_value_indexes + + @property + def next_position_ids(self) -> torch.Tensor | None: + return self._next_position_ids + + # ── Cache mutation ────────────────────────────────── + + def _set_cache_state( + self, + *, + cache: Any, + key_values_lens: torch.Tensor | None, + packed_key_value_indexes: torch.Tensor | None, + next_position_ids: torch.Tensor | None, + ) -> None: + self._cache = cache + self._key_values_lens = key_values_lens + self._packed_key_value_indexes = packed_key_value_indexes + self._next_position_ids = next_position_ids + + def install_cache( + self, + *, + cache: Any, + cache_len: int, + next_position_id: torch.Tensor, + device: torch.device, + ) -> None: + """Install one contiguous cache extracted from the packed Flex prefill.""" + if cache_len < 0: + raise ValueError(f"BAGEL {self.name} branch cache length must be non-negative.") + self._set_cache_state( + cache=cache, + key_values_lens=torch.tensor([cache_len], device=device, dtype=torch.int32), + packed_key_value_indexes=torch.arange(cache_len, device=device, dtype=torch.long), + next_position_ids=next_position_id.detach().reshape(1).to(device=device, dtype=torch.long), + ) + + def append_packed_query( + self, + *, + cache: Any, + query_lens: torch.Tensor, + device: torch.device, + ) -> None: + """Advance the main AR cache after a successful one-branch decode.""" + self.require_ready() + next_key_values_lens = self._key_values_lens + query_lens + + self._set_cache_state( + cache=cache, + key_values_lens=next_key_values_lens, + packed_key_value_indexes=torch.arange( + int(next_key_values_lens.sum().item()), + device=device, + dtype=torch.long, + ), + next_position_ids=self._next_position_ids + query_lens, + ) + + # ── Cache accessors ────────────────────────────────── + + def require_ready(self) -> None: + """Raise if this branch's KV-cache state has not been initialized.""" + if ( + self._cache is None + or self._key_values_lens is None + or self._packed_key_value_indexes is None + or self._next_position_ids is None + ): + raise RuntimeError(f"BAGEL {self.name} branch context is not initialized.") + + def cache_len(self) -> int: + """Return the total cached sequence length for this branch.""" + if self._key_values_lens is None: + raise RuntimeError(f"BAGEL {self.name} branch cache lengths are not initialized.") + return int(self._key_values_lens.sum().item()) + + def repeated_position_ids(self, query_len: int, *, device: torch.device) -> torch.Tensor: + """Use one logical position for every token in a denoise image span.""" + if self._next_position_ids is None: + raise RuntimeError(f"BAGEL {self.name} branch position ids are not initialized.") + return self._next_position_ids.reshape(1).expand(query_len).to(device=device) + + def packed_query_args( + self, + query_len: int, + *, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build sequential positions and cache-relative indexes for AR decode.""" + self.require_ready() + cache_len = self.cache_len() + query_lens = torch.tensor([query_len], device=device, dtype=torch.int32) + packed_query_indexes = torch.arange(cache_len, cache_len + query_len, device=device, dtype=torch.long) + packed_position_ids = self._next_position_ids.reshape(1) + torch.arange( + query_len, + device=device, + dtype=torch.long, + ) + + return query_lens, packed_query_indexes, packed_position_ids + + +@dataclass +class MotGenerationState: + """Scheduling state shared by Qwen2-MoT inference graph invocations. + + ``main`` is the conditional prompt, ``cfg_text`` is a prefix of ``main``, + and ``cfg_img`` is the text-only document used for image CFG. The state + also remembers the branch layout between denoise and velocity collection. + """ + + _infer_mode: str = "und" + _velocity_buffer: dict[str, torch.Tensor] = field(default_factory=dict) + _pending_denoise_branches: tuple[str, ...] | None = None + _pending_denoise_query_len: int | None = None + _main: MotCacheContext = field(default_factory=lambda: MotCacheContext("main")) + _cfg_text: MotCacheContext = field(default_factory=lambda: MotCacheContext("text CFG")) + _cfg_img: MotCacheContext = field(default_factory=lambda: MotCacheContext("image CFG")) + + def reset(self) -> None: + self._infer_mode = "und" + self._velocity_buffer.clear() + self._pending_denoise_branches = None + self._pending_denoise_query_len = None + self._main.reset() + self._cfg_text.reset() + self._cfg_img.reset() + + def update_infer_mode(self, generation_kwargs: dict[str, Any]) -> str: + """Update and return the sticky inference mode (``'und'`` or ``'gen'``) from ``generation_kwargs``.""" + # Later graph steps omit infer_type, so the mode is sticky until the + # graph resets this module's local inference state. + if "infer_type" in generation_kwargs: + infer_type = str(generation_kwargs["infer_type"]) + if infer_type not in {"infer_und", "infer_gen", "infer_edit"}: + raise ValueError(f"Unsupported BAGEL infer_type: {infer_type!r}.") + infer_mode = "und" if infer_type == "infer_und" else "gen" + else: + infer_mode = self._infer_mode or "und" + self._infer_mode = infer_mode + return self._infer_mode + + # ── CFG cache contexts ────────────────────────────────── + + @property + def main(self) -> MotCacheContext: + return self._main + + @property + def cfg_text(self) -> MotCacheContext: + return self._cfg_text + + @property + def cfg_img(self) -> MotCacheContext: + return self._cfg_img + + def _cache_for_branch(self, branch: str) -> MotCacheContext: + """Return the ``MotCacheContext`` for ``branch`` (``'main'``, ``'cfg_text'``, or ``'cfg_img'``).""" + if branch == "main": + return self._main + if branch == "cfg_text": + return self._cfg_text + if branch == "cfg_img": + return self._cfg_img + raise RuntimeError(f"Unsupported BAGEL denoise branch {branch!r}.") + + # ── CFG request policy ────────────────────────────────── + + def validate_cfg_request(self, generation_kwargs: dict[str, object]) -> None: + """Validate that image CFG is only enabled when text CFG is also active.""" + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0)) + if cfg_img_scale > 1.0 and cfg_text_scale <= 1.0: + raise ValueError("cfg_img_scale > 1.0 requires cfg_text_scale > 1.0") + + def cfg_img_requested(self, generation_kwargs: dict[str, object]) -> bool: + return float(generation_kwargs.get("cfg_img_scale", 1.0)) > 1.0 + + # ── Denoise velocity round ────────────────────────────────── + + def iter_serial_denoise_inputs( + self, + query: torch.Tensor, + generation_kwargs: dict[str, object], + *, + timestep: object, + device: torch.device, + dtype: torch.dtype, + ) -> Iterator[dict[str, Any]]: + """Yield one ``forward_inference`` input dict per active CFG branch.""" + query = query.to(device=device, dtype=dtype) + query_len = int(query.shape[0]) + if query_len < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + + branches = self._denoise_branches_for_timestep(generation_kwargs, timestep) + self._pending_denoise_branches = branches + self._pending_denoise_query_len = query_len + for branch in branches: + context = self._cache_for_branch(branch) + context.require_ready() + cache_len = context.cache_len() + packed_key_value_indexes = context.packed_key_value_indexes + if packed_key_value_indexes is None: + packed_key_value_indexes = torch.empty(0, device=device, dtype=torch.long) + yield { + "packed_query_sequence": query, + "query_lens": torch.tensor([query_len], device=device, dtype=torch.int32), + "packed_query_position_ids": context.repeated_position_ids(query_len, device=device), + "packed_query_indexes": torch.arange( + cache_len, cache_len + query_len, device=device, dtype=torch.long + ), + "past_key_values": context.cache, + "key_values_lens": context.key_values_lens, + "packed_key_value_indexes": packed_key_value_indexes, + "packed_text_indexes": torch.tensor([0, query_len - 1], device=device, dtype=torch.long), + "packed_vae_token_indexes": torch.arange(1, query_len - 1, device=device, dtype=torch.long), + } + + def preprocess_parallel_denoise_inputs( + self, + query: torch.Tensor, + generation_kwargs: dict[str, object], + *, + timestep: object, + empty_cache_factory: Callable[[], Any], + device: torch.device, + dtype: torch.dtype, + ) -> dict[str, Any]: + """Stack active CFG branches into one packed ``forward_inference`` call.""" + query = query.to(device=device, dtype=dtype) + query_len = int(query.shape[0]) + if query_len < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + + branches = self._denoise_branches_for_timestep(generation_kwargs, timestep) + contexts = [self._cache_for_branch(branch) for branch in branches] + for context in contexts: + context.require_ready() + + query_lens = torch.full((len(branches),), query_len, device=device, dtype=torch.int32) + key_values_lens = torch.tensor([context.cache_len() for context in contexts], device=device, dtype=torch.int32) + packed_query_sequence = torch.cat([query] * len(branches), dim=0) + packed_query_position_ids = torch.cat( + [context.repeated_position_ids(query_len, device=device) for context in contexts], + dim=0, + ) + index_layout = _build_denoise_index_layout(key_values_lens, query_len, device=device) + + # The flow connector preserves row order but does not carry branch + # labels, so collect_velocity must replay this exact layout. + self._pending_denoise_branches = branches + self._pending_denoise_query_len = query_len + return { + "packed_query_sequence": packed_query_sequence, + "query_lens": query_lens, + "key_values_lens": key_values_lens, + "packed_query_position_ids": packed_query_position_ids, + "past_key_values": self._merged_branch_cache(contexts, key_values_lens, empty_cache_factory), + **index_layout, + } + + def collect_velocity( + self, + velocity: torch.Tensor, + generation_kwargs: dict[str, object], + *, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + """Split stacked velocities by branch and apply BAGEL CFG equations.""" + branches = self._pending_denoise_branches + query_len = self._pending_denoise_query_len + if branches is None or query_len is None: + raise RuntimeError("BAGEL MoT velocity collection requires a pending denoise branch layout.") + expected_rows = len(branches) * query_len + if int(velocity.shape[0]) != expected_rows: + raise ValueError( + f"BAGEL MoT velocity collection expected {expected_rows} stacked rows, got {velocity.shape[0]}." + ) + + self._velocity_buffer.clear() + offset = 0 + for branch in branches: + branch_velocity = velocity[offset : offset + query_len].to(device=device, dtype=dtype) + self._velocity_buffer[branch] = self._strip_denoise_query_markers(branch_velocity) + offset += query_len + + merged_velocity = ( + self._merged_velocity_without_cfg(device=device) + if "cfg_text" not in self._velocity_buffer + else self._merged_velocity_with_cfg(generation_kwargs, device=device) + ) + self._velocity_buffer.clear() + self._pending_denoise_branches = None + self._pending_denoise_query_len = None + return merged_velocity + + def _denoise_branches_for_timestep( + self, generation_kwargs: dict[str, object], timestep: object + ) -> tuple[str, ...]: + """Return the ordered tuple of active CFG branch names for ``timestep``.""" + self.validate_cfg_request(generation_kwargs) + cfg_text_active = self._cfg_text_active(generation_kwargs, timestep) + branches = ["main"] + if cfg_text_active: + branches.append("cfg_text") + if cfg_text_active and self.cfg_img_requested(generation_kwargs): + branches.append("cfg_img") + return tuple(branches) + + def _cfg_text_active(self, generation_kwargs: dict[str, object], timestep: object) -> bool: + """Return whether text CFG guidance is active at ``timestep``.""" + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + if cfg_text_scale <= 1.0: + return False + + # CFG applies only inside the configured open-left, closed-right + # diffusion-time interval. + interval = generation_kwargs.get("cfg_interval", (0.0, 1.0)) + if interval is None: + lower, upper = 0.0, 1.0 + elif isinstance(interval, (list, tuple)) and len(interval) == 2: + lower, upper = float(interval[0]), float(interval[1]) + else: + lower, upper = float(interval), 1.0 + + if torch.is_tensor(timestep): + if timestep.numel() == 0: + raise ValueError("BAGEL CFG timestep tensor must not be empty.") + t = float(timestep.detach().reshape(-1)[0].item()) + elif timestep is None: + raise ValueError("BAGEL CFG requires current-round meta['timestep'].") + else: + t = float(timestep) + return t > lower and t <= upper + + def _strip_denoise_query_markers(self, velocity: torch.Tensor) -> torch.Tensor: + """Strip the leading and trailing vision marker rows from a branch velocity tensor.""" + if int(velocity.shape[0]) < 3: + raise ValueError("BAGEL MoT denoise velocity must include start/end marker rows.") + return velocity[1:-1] + + def _merged_velocity_without_cfg(self, *, device: torch.device) -> torch.Tensor: + """Return the unguided ``main`` branch velocity.""" + main_velocity = self._velocity_buffer.get("main") + if main_velocity is None: + raise RuntimeError("BAGEL CFG merge requires a collected main branch velocity.") + return main_velocity.to(device=device) + + def _merged_velocity_with_cfg(self, generation_kwargs: dict[str, object], *, device: torch.device) -> torch.Tensor: + """Combine ``main``, ``cfg_text``, and optional ``cfg_img`` velocities with CFG renormalization.""" + main_velocity = self._merged_velocity_without_cfg(device=device) + + cfg_text_scale = float(generation_kwargs.get("cfg_text_scale", 1.0)) + cfg_text_velocity = self._velocity_buffer.get("cfg_text") + if cfg_text_velocity is None: + raise RuntimeError("BAGEL CFG merge requires a collected cfg_text branch velocity.") + cfg_text_velocity = cfg_text_velocity.to(device=device, dtype=main_velocity.dtype) + + cfg_img_scale = float(generation_kwargs.get("cfg_img_scale", 1.0)) + cfg_img_velocity = self._velocity_buffer.get("cfg_img") + if cfg_img_scale > 1.0 and cfg_img_velocity is None: + raise ValueError("BAGEL image CFG merge requires a collected cfg_img velocity.") + if cfg_img_velocity is not None: + cfg_img_velocity = cfg_img_velocity.to(device=device, dtype=main_velocity.dtype) + + cfg_renorm_min = float(generation_kwargs.get("cfg_renorm_min", 0.0)) + cfg_renorm_type = str(generation_kwargs.get("cfg_renorm_type", "global")) + + # Official InterleaveInferencer wraps generate_image in bf16 autocast, so + # the CFG scale/norm/renorm math is not the same as eager bf16 * python float. + autocast = ( + torch.amp.autocast(device.type, dtype=torch.bfloat16) + if IS_CUDA_AVAILABLE and device.type == get_device_type() + else nullcontext() + ) + with autocast: + guided = cfg_text_velocity + cfg_text_scale * (main_velocity - cfg_text_velocity) + if cfg_renorm_type == "text_channel": + norm_main = torch.norm(main_velocity, dim=-1, keepdim=True) + norm_text = torch.norm(guided, dim=-1, keepdim=True) + scale = (norm_main / (norm_text + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) + merged = guided * scale + if cfg_img_scale > 1.0: + assert cfg_img_velocity is not None + merged = cfg_img_velocity + cfg_img_scale * (merged - cfg_img_velocity) + return merged.to(device=device) + + if cfg_img_scale > 1.0: + assert cfg_img_velocity is not None + guided = cfg_img_velocity + cfg_img_scale * (guided - cfg_img_velocity) + + if cfg_renorm_type == "global": + norm_main = torch.norm(main_velocity) + norm_guided = torch.norm(guided) + elif cfg_renorm_type == "channel": + norm_main = torch.norm(main_velocity, dim=-1, keepdim=True) + norm_guided = torch.norm(guided, dim=-1, keepdim=True) + else: + raise NotImplementedError(f"BAGEL infer_gen CFG renorm type {cfg_renorm_type!r} is not implemented.") + scale = (norm_main / (norm_guided + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) + return (guided * scale).to(device=device) + + def _merged_branch_cache( + self, + contexts: list[MotCacheContext], + key_values_lens: torch.Tensor, + empty_cache_factory: Callable[[], Any], + ) -> Any: + """Concatenate per-branch KV caches into one packed ``NaiveCache``.""" + # Concatenation order must match preprocess_parallel_denoise_inputs: + # main, then cfg_text, then cfg_img when those branches are active. + merged_cache = empty_cache_factory() + for layer_idx in merged_cache.key_cache: + key_parts = [] + value_parts = [] + for context, key_values_len in zip(contexts, key_values_lens.tolist(), strict=True): + key_values_len = int(key_values_len) + if key_values_len == 0: + continue + cache = context.cache + if cache is None or cache.key_cache[layer_idx] is None or cache.value_cache[layer_idx] is None: + raise RuntimeError(f"BAGEL {context.name} branch cache is missing layer {layer_idx}.") + key_parts.append(cache.key_cache[layer_idx]) + value_parts.append(cache.value_cache[layer_idx]) + if key_parts: + merged_cache.key_cache[layer_idx] = torch.cat(key_parts, dim=0) + merged_cache.value_cache[layer_idx] = torch.cat(value_parts, dim=0) + return merged_cache + + +__all__ = ["MotCacheContext", "MotGenerationState"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py new file mode 100644 index 0000000000..52270435de --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/masking.py @@ -0,0 +1,346 @@ +"""Attention visibility metadata and mask materializers for BAGEL Qwen2-MoT. + +`packed_attention_metadata` is the source of truth. `build_mot_block_mask` +materializes a Flex `BlockMask` for accelerated training. `build_mot_magi_mask` +materializes MagiAttention ranges. `build_mot_sdpa_mask` materializes a dense +boolean mask for eager SDPA (``True`` means attend). +""" + +from __future__ import annotations + +import torch +from torch.nn.attention.flex_attention import BlockMask + +from ......ops.mask import MagiAttentionMask + + +_MOT_BLOCK_SIZE = 128 +_MAGI_ATTN_FULL = 0 +_MAGI_ATTN_CAUSAL = 1 + + +def build_mot_attention_metadata( + sample_splits: list[list[int]], + sample_attn_modes: list[list[str]], + *, + device: torch.device, +) -> torch.Tensor: + """Encode BAGEL packed visibility in three O(sequence) int32 rows. + + Row 0 identifies independent packed documents. Row 1 identifies spans + whose tokens can attend bidirectionally within the span (``full`` and + ``noise``). Row 2 identifies noise spans so their keys remain invisible + outside that same noise span. ``-1`` means that a row does not apply. + + Callers materialize this compact representation as a Flex ``BlockMask``, + MagiAttention ranges, or a dense SDPA mask, without changing the metadata. + """ + total_length = sum(sum(split_lens) for split_lens in sample_splits) + metadata = torch.full((3, total_length), -1, device=device, dtype=torch.int32) + cursor = 0 + span_id = 0 + for document_id, (split_lens, attn_modes) in enumerate(zip(sample_splits, sample_attn_modes, strict=True)): + for length, mode in zip(split_lens, attn_modes, strict=True): + if mode not in {"causal", "full", "noise"}: + raise ValueError(f"Unsupported BAGEL attention mode: {mode!r}.") + span_end = cursor + length + metadata[0, cursor:span_end] = document_id + if mode != "causal": + metadata[1, cursor:span_end] = span_id + if mode == "noise": + metadata[2, cursor:span_end] = span_id + cursor = span_end + span_id += 1 + return metadata.contiguous() + + +def _validate_mot_attention_metadata(packed_attention_metadata: torch.Tensor) -> None: + """Validate that ``packed_attention_metadata`` is a ``[3, S]`` int32 tensor.""" + if packed_attention_metadata.ndim != 2 or packed_attention_metadata.shape[0] != 3: + raise ValueError( + "BAGEL Qwen2-MoT attention metadata must have shape [3, sequence], " + f"got {tuple(packed_attention_metadata.shape)}." + ) + if packed_attention_metadata.dtype != torch.int32: + raise ValueError( + f"BAGEL Qwen2-MoT attention metadata must use torch.int32, got {packed_attention_metadata.dtype}." + ) + + +def pad_mot_attention_metadata( + packed_attention_metadata: torch.Tensor, + padded_length: int, +) -> torch.Tensor: + """Pad metadata with one isolated full-attention span for SP alignment. + + Giving padding its own document prevents real tokens from attending to it; + leaving its noise ID at ``-1`` makes the padding block safe to classify as + a regular clean block. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + + sequence_length = int(packed_attention_metadata.shape[1]) + if padded_length < sequence_length: + raise ValueError( + "BAGEL Qwen2-MoT padded attention length cannot shrink the sequence: " + f"sequence_length={sequence_length}, padded_length={padded_length}." + ) + if padded_length == sequence_length: + return packed_attention_metadata + + padded = torch.full( + (3, padded_length), + -1, + device=packed_attention_metadata.device, + dtype=packed_attention_metadata.dtype, + ) + padded[:, :sequence_length] = packed_attention_metadata + padding_id = sequence_length + padded[0, sequence_length:] = padding_id + padded[1, sequence_length:] = padding_id + return padded.contiguous() + + +def _metadata_block_ranges( + token_ids: torch.Tensor, + *, + block_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Summarize monotonic token IDs for conservative block classification.""" + padding = (-int(token_ids.numel())) % block_size + if padding: + token_ids = torch.nn.functional.pad(token_ids, (0, padding), value=-1) + token_ids = token_ids.view(-1, block_size) + valid = token_ids >= 0 + minimum = torch.where(valid, token_ids, torch.iinfo(token_ids.dtype).max).amin(dim=-1) + maximum = token_ids.amax(dim=-1) + return minimum, maximum, valid.any(dim=-1), valid.all(dim=-1) + + +def _ordered_block_indices(blocks: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Convert a dense block-level predicate to FlexAttention ordered rows.""" + block_count = int(blocks.shape[-1]) + block_ids = torch.arange(block_count, device=blocks.device, dtype=torch.int32).expand(block_count, -1) + sentinel = torch.full_like(block_ids, block_count) + indices = torch.where(blocks, block_ids, sentinel).sort(dim=-1).values + num_blocks = blocks.sum(dim=-1, dtype=torch.int32) + return num_blocks.unsqueeze(0).unsqueeze(0), indices.unsqueeze(0).unsqueeze(0) + + +def build_mot_block_mask(packed_attention_metadata: torch.Tensor) -> BlockMask: + """Build BAGEL's head-independent BlockMask from compact metadata. + + Token visibility is: + ``same_document & (causal | same_full_span) & ~foreign_noise_key``. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + + def mask_mod(batch_idx, head_idx, query_idx, key_idx): + # MoT visibility is identical across batch and attention heads, but + # FlexAttention requires the four-argument callback signature. + del batch_idx, head_idx + same_document = document_ids[query_idx] == document_ids[key_idx] + causal = query_idx >= key_idx + same_full_or_noise_span = (full_span_ids[query_idx] >= 0) & ( + full_span_ids[query_idx] == full_span_ids[key_idx] + ) + foreign_noise_key = (noise_span_ids[key_idx] >= 0) & (noise_span_ids[query_idx] != noise_span_ids[key_idx]) + return same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key + + doc_min, doc_max, doc_valid, doc_complete = _metadata_block_ranges( + document_ids, + block_size=_MOT_BLOCK_SIZE, + ) + full_min, full_max, full_valid, full_complete = _metadata_block_ranges( + full_span_ids, + block_size=_MOT_BLOCK_SIZE, + ) + _, noise_max, _, _ = _metadata_block_ranges( + noise_span_ids, + block_size=_MOT_BLOCK_SIZE, + ) + + # Range overlap produces a conservative candidate set: mixed boundary + # blocks may be included here and are filtered token-by-token by mask_mod. + same_document_block = ( + doc_valid[:, None] + & doc_valid[None, :] + & (doc_min[:, None] <= doc_max[None, :]) + & (doc_min[None, :] <= doc_max[:, None]) + ) + shared_full_span_block = ( + full_valid[:, None] + & full_valid[None, :] + & (full_min[:, None] <= full_max[None, :]) + & (full_min[None, :] <= full_max[:, None]) + ) + block_count = int(doc_min.numel()) + block_ids = torch.arange(block_count, device=packed_attention_metadata.device) + causal_block = block_ids[:, None] >= block_ids[None, :] + candidate_blocks = same_document_block & (causal_block | shared_full_span_block) + + # Full blocks bypass mask_mod. Mark a block pair full only when every token + # is provably in one document and visibility is unconditional: either the + # key block is strictly earlier and clean, or both blocks share one + # bidirectional span. + single_document_block = doc_complete & (doc_min == doc_max) + same_single_document = ( + single_document_block[:, None] & single_document_block[None, :] & (doc_min[:, None] == doc_min[None, :]) + ) + single_full_span_block = full_complete & full_valid & (full_min == full_max) + same_single_full_span = ( + single_full_span_block[:, None] & single_full_span_block[None, :] & (full_min[:, None] == full_min[None, :]) + ) + strictly_earlier_clean_key_block = (block_ids[:, None] > block_ids[None, :]) & (noise_max[None, :] < 0) + full_blocks = candidate_blocks & same_single_document & (strictly_earlier_clean_key_block | same_single_full_span) + partial_blocks = candidate_blocks & ~full_blocks + + partial_num_blocks, partial_indices = _ordered_block_indices(partial_blocks) + full_num_blocks, full_indices = _ordered_block_indices(full_blocks) + return BlockMask.from_kv_blocks( + partial_num_blocks, + partial_indices, + full_num_blocks, + full_indices, + BLOCK_SIZE=_MOT_BLOCK_SIZE, + mask_mod=mask_mod, + seq_lengths=(sequence_length, sequence_length), + ) + + +def _mot_span_mode(full_span_id: int, noise_span_id: int) -> str: + """Classify a token span as ``'noise'``, ``'full'``, or ``'causal'`` from its span IDs.""" + if noise_span_id >= 0: + return "noise" + if full_span_id >= 0: + return "full" + return "causal" + + +def _iter_mot_spans(packed_attention_metadata: torch.Tensor) -> list[tuple[int, int, int, str]]: + """Recover contiguous (start, end, document_id, mode) spans from metadata.""" + sequence_length = int(packed_attention_metadata.shape[1]) + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + changed = document_ids.new_ones(sequence_length, dtype=torch.bool) + changed[1:] = ( + (document_ids[1:] != document_ids[:-1]) + | (full_span_ids[1:] != full_span_ids[:-1]) + | (noise_span_ids[1:] != noise_span_ids[:-1]) + ) + starts = torch.nonzero(changed, as_tuple=False).flatten() + ends = torch.cat((starts[1:], starts.new_tensor([sequence_length]))) + spans: list[tuple[int, int, int, str]] = [] + for start, end, document_id, full_span_id, noise_span_id in zip( + starts.tolist(), + ends.tolist(), + document_ids[starts].tolist(), + full_span_ids[starts].tolist(), + noise_span_ids[starts].tolist(), + strict=True, + ): + spans.append((start, end, int(document_id), _mot_span_mode(int(full_span_id), int(noise_span_id)))) + return spans + + +def _merge_adjacent_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]: + """Coalesce contiguous ``(start, end)`` intervals into maximal ranges.""" + if not ranges: + return [] + merged = [ranges[0]] + for start, end in ranges[1:]: + last_start, last_end = merged[-1] + if start == last_end: + merged[-1] = (last_start, end) + else: + merged.append((start, end)) + return merged + + +def build_mot_magi_mask(packed_attention_metadata: torch.Tensor) -> MagiAttentionMask: + """Build BAGEL's MagiAttention ranges from compact metadata. + + Token visibility is the same as Flex/SDPA: + ``same_document & (causal | same_full_span) & ~foreign_noise_key``. + + Each document is encoded as: + - a self range per span (causal inside ``causal`` spans, full inside ``full``/``noise``) + - full ranges from each query span onto earlier non-noise spans in that document + + Noise keys are omitted from every other span's key ranges. Metadata is + already the full packed sequence, including Ulysses padding, so this + builder does not rescale ranges by ``ulysses_size``. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + + q_ranges: list[tuple[int, int]] = [] + k_ranges: list[tuple[int, int]] = [] + attn_types: list[int] = [] + document_spans: list[tuple[int, int, str]] = [] + current_document_id: int | None = None + + def flush_document(spans: list[tuple[int, int, str]]) -> None: + """Append Magi Q/K range pairs and attention types for one document's spans.""" + for query_index, (query_start, query_end, query_mode) in enumerate(spans): + earlier_clean = [ + (key_start, key_end) for key_start, key_end, key_mode in spans[:query_index] if key_mode != "noise" + ] + for key_start, key_end in _merge_adjacent_ranges(earlier_clean): + q_ranges.append((query_start, query_end)) + k_ranges.append((key_start, key_end)) + attn_types.append(_MAGI_ATTN_FULL) + q_ranges.append((query_start, query_end)) + k_ranges.append((query_start, query_end)) + attn_types.append(_MAGI_ATTN_FULL if query_mode != "causal" else _MAGI_ATTN_CAUSAL) + + for start, end, document_id, mode in _iter_mot_spans(packed_attention_metadata): + if current_document_id is not None and document_id != current_document_id: + flush_document(document_spans) + document_spans = [] + current_document_id = document_id + document_spans.append((start, end, mode)) + flush_document(document_spans) + + device = packed_attention_metadata.device + return MagiAttentionMask( + q_ranges=torch.tensor(q_ranges, device=device, dtype=torch.int32), + k_ranges=torch.tensor(k_ranges, device=device, dtype=torch.int32), + attn_type_map=torch.tensor(attn_types, device=device, dtype=torch.int32), + ) + + +def build_mot_sdpa_mask(packed_attention_metadata: torch.Tensor) -> torch.Tensor: + """Materialize a dense ``[1, 1, S, S]`` boolean SDPA mask. + + ``True`` means the query may attend that key. Do not store this as float32 + ``0 / -inf``; PyTorch SDPA already treats boolean ``False`` as masked out. + """ + _validate_mot_attention_metadata(packed_attention_metadata) + sequence_length = int(packed_attention_metadata.shape[1]) + if sequence_length == 0: + raise ValueError("BAGEL Qwen2-MoT attention metadata must contain at least one token.") + + document_ids, full_span_ids, noise_span_ids = packed_attention_metadata + query_idx = torch.arange(sequence_length, device=packed_attention_metadata.device) + same_document = document_ids[:, None] == document_ids[None, :] + causal = query_idx[:, None] >= query_idx[None, :] + same_full_or_noise_span = (full_span_ids[:, None] >= 0) & (full_span_ids[:, None] == full_span_ids[None, :]) + foreign_noise_key = (noise_span_ids[None, :] >= 0) & (noise_span_ids[:, None] != noise_span_ids[None, :]) + visible = same_document & (causal | same_full_or_noise_span) & ~foreign_noise_key + return visible.to(dtype=torch.bool)[None, None].contiguous() + + +__all__ = [ + "build_mot_attention_metadata", + "build_mot_block_mask", + "build_mot_magi_mask", + "build_mot_sdpa_mask", + "pad_mot_attention_metadata", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py new file mode 100644 index 0000000000..870ca1e7e0 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -0,0 +1,1557 @@ +"""BAGEL Qwen2 MoT backbone.""" + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +import torch.nn as nn +from torch.nn.attention import SDPBackend, sdpa_kernel +from torch.nn.functional import scaled_dot_product_attention +from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm +from transformers.utils import ModelOutput + +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + +from ....utils.conversation import ConversationItem +from ...module_modeling_base import PretrainedOmniModule +from ..sources import ( + BAGEL_FLOW_HIDDEN, + BAGEL_FLOW_QUERY, + BAGEL_FLOW_VELOCITY, + BAGEL_PHASE_KEY, + BAGEL_START_TOKEN, + bagel_phase, + get_tail_phase_item, +) +from .configuration import BagelQwen2MoTConfig +from .generation_state import MotGenerationState +from .masking import build_mot_sdpa_mask +from .processing import PackedConversation, preprocess_mot_inputs + + +class InferenceMixin: + """FSM ``generate`` / serial denoise / velocity-collection — HF ``GenerationMixin`` analog. + + Listed *before* :class:`BagelQwen2MoTCore` (and therefore before + :class:`~...module_modeling_base.PretrainedOmniModule`): the pretrained + base ships a no-op ``reset_local_inference_state``, and MRO resolves + left-to-right — put second, that no-op would shadow the real one below. + """ + + def forward_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_ids: torch.Tensor, + packed_query_indexes: torch.Tensor, + past_key_values: Optional["NaiveCache"] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + packed_attention_metadata: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + ) -> Dict[str, Any]: + """Run one packed inference step through the MoT backbone and return hidden states and KV cache.""" + call_kwargs: Dict[str, Any] = { + "packed_query_sequence": packed_query_sequence, + "query_lens": query_lens, + "packed_query_position_ids": packed_query_position_ids, + "packed_query_indexes": packed_query_indexes, + "past_key_values": past_key_values, + "key_values_lens": key_values_lens, + "packed_key_value_indexes": packed_key_value_indexes, + "update_past_key_values": update_past_key_values, + "is_causal": is_causal, + "mode": mode, + "packed_attention_metadata": packed_attention_metadata, + } + + is_gen = _check_packed_inference_mode(mode) + if is_gen: + call_kwargs["packed_vae_token_indexes"] = packed_vae_token_indexes + call_kwargs["packed_text_indexes"] = packed_text_indexes + + output = self.model.forward_packed_inference(**call_kwargs) + + return { + "hidden_states": output.packed_query_sequence, + "past_key_values": output.past_key_values, + } + + def reset_local_inference_state(self) -> None: + self._generation_state.reset() + + def generate( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Prefill prompt caches or decode the next understanding token in the generation FSM.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT generate requires conversation_list.") + + generation_kwargs = generation_kwargs or {} + infer_mode = self._generation_state.update_infer_mode(generation_kwargs) + # Generation/edit rebuilds all CFG prompt caches. Understanding reuses + # the main cache and starts AR from the raw assistant BOS embedding. + if infer_mode == "gen": + hidden_states = self._prefill_prompt(conversation_list, generation_kwargs) + elif self._generation_state.main.cache is None: + tail = conversation_list[-1] + if tail.type != "output" or bagel_phase(tail) != BAGEL_START_TOKEN: + raise ValueError("BAGEL understanding prefill requires a tail assistant BOS embedding.") + self._prefill_prompt(conversation_list[:-1], generation_kwargs) + hidden_states = self._decode_next_token(conversation_list) + else: + hidden_states = self._decode_next_token(conversation_list) + + if infer_mode != "gen": + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + if hidden_states.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}.") + tail = conversation_list[-1] + if tail.type != "output": + raise ValueError(f"BAGEL understanding decode expects tail output item, got {tail.type!r}.") + tail.value = hidden_states[-1:].contiguous() + tail.meta.pop(BAGEL_PHASE_KEY, None) + return {"conversation_list": conversation_list} + + def _prepare_denoise_query( + self, + conversation_list: list[ConversationItem] | None, + generation_kwargs: dict[str, Any] | None, + ) -> tuple[ConversationItem, torch.Tensor]: + """Validate CFG parameters and extract the tail ``flow_query`` tensor.""" + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT denoise_branch requires conversation_list.") + + self._generation_state.validate_cfg_request(generation_kwargs or {}) + self._generation_state.main.require_ready() + tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_QUERY) + if tail is None or not torch.is_tensor(tail.value): + raise ValueError(f"BAGEL Qwen2-MoT denoise branch requires an output in phase {BAGEL_FLOW_QUERY!r}.") + + query = tail.value + if query.dim() == 3 and query.shape[0] == 1: + query = query.squeeze(0) + if query.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT denoise branch expects rank-2 query tensor, got {tuple(query.shape)}.") + if int(query.shape[-1]) != int(self.config.hidden_size): + raise ValueError( + "BAGEL Qwen2-MoT denoise branch hidden-size mismatch: " + f"got {query.shape[-1]}, expected {self.config.hidden_size}." + ) + if int(query.shape[0]) < 3: + raise ValueError("BAGEL Qwen2-MoT denoise query must include start/end marker embeddings.") + return tail, query + + def denoise_branch( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Run serial CFG denoise forwards over active branches and write ``flow_hidden``.""" + del kwargs + tail, query = self._prepare_denoise_query(conversation_list, generation_kwargs) + + hiddens: list[torch.Tensor] = [] + for inputs in self._generation_state.iter_serial_denoise_inputs( + query, + generation_kwargs or {}, + timestep=tail.meta.get("timestep"), + device=self.device, + dtype=self.dtype, + ): + outputs = self.forward_inference( + **inputs, + update_past_key_values=False, + is_causal=False, + mode="gen", + ) + hiddens.append(outputs["hidden_states"]) + if not hiddens: + raise RuntimeError("BAGEL Qwen2-MoT denoise branch produced no branch hidden states.") + + tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN + tail.value = torch.cat(hiddens, dim=0).to(device=self.device, dtype=self.dtype) + return {"conversation_list": conversation_list} + + def collect_velocity( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Strip marker rows from ``flow_velocity`` and apply CFG combination.""" + del kwargs + if conversation_list is None: + raise ValueError("BAGEL Qwen2-MoT collect_velocity requires conversation_list.") + + self._generation_state.validate_cfg_request(generation_kwargs or {}) + tail = get_tail_phase_item(conversation_list, BAGEL_FLOW_VELOCITY) + if tail is None or not torch.is_tensor(tail.value): + raise ValueError( + f"BAGEL Qwen2-MoT velocity collection requires an output in phase {BAGEL_FLOW_VELOCITY!r}." + ) + + velocity = tail.value + if velocity.dim() == 3 and velocity.shape[0] == 1: + velocity = velocity.squeeze(0) + if velocity.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT velocity collection expects rank-2 velocity, got {tuple(velocity.shape)}." + ) + + # Restore branch identity from the layout recorded by denoise_branch, + # strip marker rows, and replace the stacked tensor with guided velocity. + tail.value = self._generation_state.collect_velocity( + velocity, + generation_kwargs or {}, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation_list} + + def _prefill_prompt( + self, + conversation_list: list[ConversationItem], + generation_kwargs: dict[str, Any], + ) -> torch.Tensor: + """Pack and prefill the main prompt and optional CFG branch caches.""" + main_packed = preprocess_mot_inputs( + [conversation_list], + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if main_packed is None: + raise ValueError("BAGEL Qwen2-MoT generate requires at least one embedded text/image item.") + + state = self._generation_state + state.main.reset() + state.cfg_text.reset() + state.cfg_img.reset() + + # Text CFG reuses the latest valid prefix of the main prompt: its cache + # boundary is before a text span and after an image span. + cfg_text_slice, cfg_text_next_position_id = self._cfg_text_prefix_slice(main_packed) + use_cfg_img = state.cfg_img_requested(generation_kwargs) + + # Image CFG keeps logical text spans only. Inspect the grouped spans so + # marker tokens belonging to image triplets are not treated as text. + cfg_img_items = [] + if use_cfg_img: + cfg_img_items = [item for span in main_packed.spans if span.item.type == "text" for item in span.items] + + # Pack main and cfg_img as independent logical documents. The attention + # metadata prevents cross-document attention inside the Flex kernel. + packed = main_packed + if cfg_img_items: + packed = preprocess_mot_inputs( + [conversation_list, cfg_img_items], + device=self.device, + dtype=self.dtype, + hidden_size=int(self.config.hidden_size), + ) + if packed is None: + raise RuntimeError("BAGEL Qwen2-MoT CFG prompt packing produced no tokens.") + + main_length = sum(main_packed.sample_splits[0]) + total_length = int(packed.packed_sequence.shape[0]) + main_slice = slice(0, main_length) + cfg_img_slice = slice(main_length, total_length) + + # This is a fresh-cache prefill: forward_inference allocates the cache, + # while packed attention metadata selects FlexAttention. + outputs = self.forward_packed_prefill(packed) + packed_cache = outputs["past_key_values"] + + # Detach main and cfg_img from the temporary packed cache allocation. + # cfg_text is a read-only prefix view of main and can share its storage. + main_cache = self._slice_prefill_cache(packed_cache, main_slice, clone=True) + cfg_text_cache = self._slice_prefill_cache(main_cache, cfg_text_slice, clone=False) + + state.main.install_cache( + cache=main_cache, + cache_len=main_length, + next_position_id=self._next_position_id(packed.packed_position_ids[main_slice]), + device=self.device, + ) + state.cfg_text.install_cache( + cache=cfg_text_cache, + cache_len=cfg_text_slice.stop, + next_position_id=cfg_text_next_position_id, + device=self.device, + ) + if use_cfg_img: + cfg_img_cache = self._slice_prefill_cache(packed_cache, cfg_img_slice, clone=True) + state.cfg_img.install_cache( + cache=cfg_img_cache, + cache_len=cfg_img_slice.stop - cfg_img_slice.start, + next_position_id=self._next_position_id(packed.packed_position_ids[cfg_img_slice]), + device=self.device, + ) + + # cfg_img is an auxiliary packed document; graph consumers should only + # observe hidden states from the main prompt. + hidden_states = outputs["hidden_states"] + if int(hidden_states.shape[0]) != total_length: + raise RuntimeError( + "BAGEL Qwen2-MoT FlexAttention prefill returned an unexpected sequence length: " + f"expected {total_length}, got {hidden_states.shape[0]}." + ) + return hidden_states[main_slice] + + def forward_packed_prefill(self, packed: PackedConversation) -> dict[str, Any]: + """Run one fresh-cache Flex prefill over all logical documents.""" + total_length = int(packed.packed_sequence.shape[0]) + document_lens = [sum(split_lens) for split_lens in packed.sample_splits] + query_lens = torch.tensor(document_lens, device=self.device, dtype=torch.int32) + + # A packed prefill may contain both understanding and generation expert + # tokens even though it executes as one model forward. + packed_gen_token_indexes = torch.nonzero( + packed.packed_token_type_ids == 1, + as_tuple=False, + ).flatten() + mode = "gen" if int(packed_gen_token_indexes.numel()) > 0 else "und" + call_kwargs: dict[str, Any] = { + "packed_query_sequence": packed.packed_sequence, + "query_lens": query_lens, + "packed_query_position_ids": packed.packed_position_ids, + "packed_query_indexes": torch.arange(total_length, device=self.device, dtype=torch.long), + "update_past_key_values": True, + "is_causal": False, + "mode": mode, + "packed_attention_metadata": packed.packed_attention_metadata, + } + if mode == "gen": + call_kwargs["packed_text_indexes"] = torch.nonzero( + packed.packed_token_type_ids == 0, + as_tuple=False, + ).flatten() + call_kwargs["packed_vae_token_indexes"] = packed_gen_token_indexes + return self.forward_inference(**call_kwargs) + + def _decode_next_token(self, conversation_list: list[ConversationItem]) -> torch.Tensor: + """Run one AR decode step against the cached ``main`` prompt context.""" + # AR has one query token and an existing contiguous cache, so the + # dedicated FlashAttention path is preferable to rebuilding a packed mask. + main_context = self._generation_state.main + main_context.require_ready() + tail = conversation_list[-1] + if tail.type != "output": + raise ValueError(f"BAGEL Qwen2-MoT decode expects tail output item, got {tail.type!r}.") + + packed_query_sequence = tail.value + if not torch.is_tensor(packed_query_sequence): + raise ValueError("BAGEL Qwen2-MoT decode expects tail output.value to be an embedding tensor.") + if packed_query_sequence.dim() == 3 and packed_query_sequence.shape[0] == 1: + packed_query_sequence = packed_query_sequence.squeeze(0) + if packed_query_sequence.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT expected tail output embedding rank 2, got {tuple(packed_query_sequence.shape)}." + ) + packed_query_sequence = packed_query_sequence[-1:].contiguous().to(device=self.device, dtype=self.dtype) + + query_lens, packed_query_indexes, packed_position_ids = main_context.packed_query_args( + 1, + device=self.device, + ) + outputs = self.forward_inference( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_ids=packed_position_ids, + packed_query_indexes=packed_query_indexes, + past_key_values=main_context.cache, + key_values_lens=main_context.key_values_lens, + packed_key_value_indexes=main_context.packed_key_value_indexes, + update_past_key_values=True, + is_causal=True, + mode="und", + ) + main_context.append_packed_query( + cache=outputs["past_key_values"], + query_lens=query_lens, + device=self.device, + ) + + return outputs["hidden_states"] + + def _cfg_text_prefix_slice(self, packed: PackedConversation) -> tuple[slice, torch.Tensor]: + """Compute the ``main`` cache prefix slice and next position ID for text CFG.""" + # Moving across an image keeps the image in text-CFG conditioning; + # encountering text moves the boundary to just before that text span. + prefix_end = 0 + next_position_id = torch.zeros(1, device=self.device, dtype=torch.long) + for span in packed.spans: + span_end = span.start + span.length + if span.item.type == "text": + prefix_end = span.start + next_position_id = packed.packed_position_ids[span.start].reshape(1) + elif span.item.type == "image": + prefix_end = span_end + next_position_id = self._next_position_id(packed.packed_position_ids[:span_end]) + return slice(0, prefix_end), next_position_id + + def _slice_prefill_cache(self, packed_cache: Any, rows: slice, *, clone: bool) -> Any: + """Extract a row slice from ``packed_cache`` into a standalone ``NaiveCache``.""" + # main/cfg_img outlive the temporary packed cache and own their storage. + # cfg_text is an immutable prefix of main, so it may use cheap views. + cache = self._new_empty_cache() + if rows.stop <= rows.start: + return cache + for layer_idx in cache.key_cache: + key = packed_cache.key_cache[layer_idx][rows] + value = packed_cache.value_cache[layer_idx][rows] + cache.key_cache[layer_idx] = key.clone() if clone else key + cache.value_cache[layer_idx] = value.clone() if clone else value + return cache + + def _next_position_id(self, position_ids: torch.Tensor) -> torch.Tensor: + """Return ``max(position_ids) + 1`` or ``0`` when ``position_ids`` is empty.""" + if int(position_ids.numel()) == 0: + return torch.zeros(1, device=self.device, dtype=torch.long) + return position_ids.max().reshape(1).to(device=self.device, dtype=torch.long) + 1 + + def _new_empty_cache(self) -> Any: + return NaiveCache(len(self.model.layers)) + + +class BagelQwen2MoTCore(PretrainedOmniModule): + """Shared weights, packed train/infer forward, and generation-state shell. + + Public eager and accelerated classes mix their inference mixins onto this + core. It is not registered on ``OMNI_*_REGISTRY``. + """ + + config_class = BagelQwen2MoTConfig + base_model_prefix = "bagel_qwen2_mot" + main_input_name = "inputs_embeds" + _no_split_modules = ["BagelQwen2MoTDecoderLayer"] + supports_gradient_checkpointing = True + _supports_sdpa = True + _supports_flex_attn = False + _export_hf_checkpoint_with_weight_conversions = False + + def __init__(self, config: BagelQwen2MoTConfig): + super().__init__(config) + self.model = BagelQwen2MoTBackbone( + config, + attention_cls=type(self).attention_cls, + mlp_cls=type(self).mlp_cls, + rms_norm_cls=type(self).rms_norm_cls, + ) + self._generation_state = MotGenerationState() + self.post_init() + + def forward( # type: ignore[override] + self, + packed_sequence: torch.Tensor, + packed_position_ids: torch.Tensor, + packed_token_type_ids: torch.Tensor, + packed_attention_metadata: torch.Tensor, + ) -> Dict[str, Any]: + packed_und_token_indexes = torch.nonzero(packed_token_type_ids == 0, as_tuple=False).flatten() + packed_gen_token_indexes = torch.nonzero(packed_token_type_ids == 1, as_tuple=False).flatten() + output = self.model( + packed_sequence=packed_sequence, + packed_position_ids=packed_position_ids, + packed_attention_metadata=packed_attention_metadata, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + return {"hidden_states": output.packed_query_sequence} + + +class BagelQwen2MoT(InferenceMixin, BagelQwen2MoTCore): + """HF-native eager Qwen2-MoT with serial CFG denoise.""" + + +class NaiveCache: + """Official BAGEL packed KV cache.""" + + def __init__(self, num_layers: int): + self.key_cache = dict.fromkeys(range(num_layers)) + self.value_cache = dict.fromkeys(range(num_layers)) + + +@dataclass +class BaseNavitOutputWithPast(ModelOutput): + packed_query_sequence: torch.FloatTensor | None = None + past_key_values: Optional[NaiveCache] = None + + +def _rotate_half(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def _apply_rotary_pos_emb( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, +) -> tuple[torch.Tensor, torch.Tensor]: + """Eager packed RoPE. Fused dispatch lives on the accelerated attention class.""" + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (_rotate_half(q) * sin) + k_embed = (k * cos) + (_rotate_half(k) * sin) + return q_embed, k_embed + + +def _sdpa_context(device: torch.device): + """Pin mem-efficient SDPA on CUDA, matching official dense-train Bagel. + + Official 2D-mask training forces ``SDPBackend.EFFICIENT_ATTENTION`` so the + dispatcher cannot pick MATH (OOM) or a newer CUDNN kernel. CPU/NPU keep the + default dispatcher because Efficient is CUDA-only. + """ + if IS_CUDA_AVAILABLE and device.type == get_device_type(): + return sdpa_kernel(backends=[SDPBackend.EFFICIENT_ATTENTION]) + return nullcontext() + + +def _repeat_kv_heads(packed_states: torch.Tensor, num_query_heads: int) -> torch.Tensor: + """Expand packed KV from ``[tokens, kv_heads, dim]`` to query heads. + + Official dense-train Bagel repeats KV before Efficient SDPA. The CUDA + Efficient kernel does not accept ``enable_gqa`` with a 2D mask. + """ + num_kv_heads = int(packed_states.shape[1]) + if num_kv_heads == num_query_heads: + return packed_states + if num_query_heads % num_kv_heads != 0: + raise ValueError( + "BAGEL Qwen2-MoT GQA requires query heads divisible by KV heads: " + f"num_query_heads={num_query_heads}, num_kv_heads={num_kv_heads}." + ) + groups = num_query_heads // num_kv_heads + packed_states = packed_states[:, :, None, :].repeat(1, 1, groups, 1) + return packed_states.reshape(-1, num_query_heads, packed_states.shape[-1]) + + +def _sdpa_packed_attention( + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + *, + attention_mask: Optional[torch.Tensor] = None, + is_causal: bool = False, + cu_seq_lens_q: Optional[torch.Tensor] = None, + cu_seq_lens_k: Optional[torch.Tensor] = None, + scale: float, + enable_gqa: bool, +) -> torch.Tensor: + """Packed SDPA returning ``[tokens, heads, head_dim]``. + + The 2D-mask path pins Efficient SDPA on CUDA. The varlen / ``is_causal`` + path is left unpinned: official inference uses ``flash_attn_varlen``, not + Efficient. + """ + if packed_query_states.shape[0] == 0: + return packed_query_states.new_zeros(packed_query_states.shape) + + if attention_mask is not None: + if enable_gqa: + num_query_heads = int(packed_query_states.shape[1]) + packed_key_states = _repeat_kv_heads(packed_key_states, num_query_heads) + packed_value_states = _repeat_kv_heads(packed_value_states, num_query_heads) + query = packed_query_states.transpose(0, 1).unsqueeze(0) + key = packed_key_states.transpose(0, 1).unsqueeze(0) + value = packed_value_states.transpose(0, 1).unsqueeze(0) + with _sdpa_context(packed_query_states.device): + output = scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=False, + scale=scale, + ) + return output.squeeze(0).transpose(0, 1).contiguous() + + if cu_seq_lens_q is None or cu_seq_lens_k is None: + raise ValueError("BAGEL Qwen2-MoT SDPA without an attention mask requires cu_seq_lens_q/k.") + + outputs: list[torch.Tensor] = [] + for seq_idx in range(int(cu_seq_lens_q.shape[0]) - 1): + query_start = int(cu_seq_lens_q[seq_idx].item()) + query_end = int(cu_seq_lens_q[seq_idx + 1].item()) + key_start = int(cu_seq_lens_k[seq_idx].item()) + key_end = int(cu_seq_lens_k[seq_idx + 1].item()) + if query_end <= query_start: + continue + + query = packed_query_states[query_start:query_end].transpose(0, 1).unsqueeze(0) + key = packed_key_states[key_start:key_end].transpose(0, 1).unsqueeze(0) + value = packed_value_states[key_start:key_end].transpose(0, 1).unsqueeze(0) + + query_length = query_end - query_start + key_length = key_end - key_start + causal_mask = None + sdpa_is_causal = is_causal + if is_causal and query_length != key_length: + # FlashAttention varlen uses bottom-right causal alignment. PyTorch + # SDPA's ``is_causal=True`` is upper-left aligned when Q and K have + # different lengths, which makes a one-token cached decode see only + # the first key. The common Q=1 case needs no mask because the final + # logical query can see the entire cache; use an explicit lower-right + # mask only for multi-token cached decode. + sdpa_is_causal = False + if query_length != 1: + causal_mask = torch.ones( + (query_length, key_length), + dtype=torch.bool, + device=query.device, + ).tril(diagonal=key_length - query_length) + + output = scaled_dot_product_attention( + query, + key, + value, + attn_mask=causal_mask, + dropout_p=0.0, + is_causal=sdpa_is_causal, + scale=scale, + enable_gqa=enable_gqa, + ) + outputs.append(output.squeeze(0).transpose(0, 1)) + if not outputs: + return packed_query_states.new_zeros(packed_query_states.shape) + return torch.cat(outputs, dim=0) + + +def _fold_zero_anchors(target: torch.Tensor, *anchors: torch.Tensor) -> torch.Tensor: + """Add ``0 * sum(anchor)`` to ``target`` so empty-route parameters remain in the autograd graph.""" + # When a packed batch has no generation tokens, the MoT "gen" expert weights still + # run on an empty slice and produce zero-sized outputs. Folding ``sum() * 0.0`` of + # those outputs into ``target`` keeps the gen-expert parameters in the autograd graph + # (with zero gradient) so FSDP/DP gradient reduction sees the same parameter set on + # every rank regardless of which modalities a micro-batch contains. + anchor = target.new_zeros(()) + has_anchor = False + for value in anchors: + if torch.is_tensor(value): + anchor = anchor + value.sum() * 0.0 + has_anchor = True + if not has_anchor: + return target + return target + anchor + + +def _scatter_moe( + packed: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + und_value: torch.Tensor, + gen_value: torch.Tensor, + *, + extra_und_anchors: tuple[torch.Tensor, ...] = (), + extra_gen_anchors: tuple[torch.Tensor, ...] = (), +) -> torch.Tensor: + """Write MoT und/gen expert outputs into a packed buffer and keep empty-route FSDP anchors.""" + packed[packed_und_token_indexes] = und_value + packed[packed_gen_token_indexes] = gen_value + if int(packed_und_token_indexes.numel()) == 0: + packed = _fold_zero_anchors(packed, und_value, *extra_und_anchors) + if int(packed_gen_token_indexes.numel()) == 0: + packed = _fold_zero_anchors(packed, gen_value, *extra_gen_anchors) + return packed + + +def _check_packed_inference_mode( + mode: str, +) -> bool: + """Validate ``mode`` (``'und'`` or ``'gen'``) and return ``True`` for ``'gen'``.""" + if mode == "und": + return False + if mode == "gen": + return True + raise ValueError(f"Unsupported BAGEL Qwen2 MoT inference mode: {mode!r}") + + +class BagelQwen2RotaryEmbedding(nn.Module): + """Official-compatible Qwen2 RoPE for BAGEL parity.""" + + def __init__(self, config: BagelQwen2MoTConfig, device: Optional[torch.device] = None): + super().__init__() + self.config = config + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + self.rope_type = "default" + self.attention_scaling = 1.0 + inv_freq, _ = self.compute_default_rope_parameters(config, device) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) + + def _apply(self, fn: Any, recurse: bool = True) -> nn.Module: + # Snapshot the true fp32 table before ``fn`` can quantize it (``.to(bfloat16)``). + # Casting first and then ``.float()`` keeps bf16 rounding, which is not official + # mixed-precision train (buffers stay fp32) and is not bitwise with a fresh table. + inv_freq = self.inv_freq.detach().to(dtype=torch.float32).clone() + original_inv_freq = self.original_inv_freq.detach().to(dtype=torch.float32).clone() + module = super()._apply(fn, recurse=recurse) + device = self.inv_freq.device + self.inv_freq = inv_freq.to(device=device) + self.original_inv_freq = original_inv_freq.to(device=device) + return module + + @staticmethod + def compute_default_rope_parameters( + config: BagelQwen2MoTConfig, + device: Optional[torch.device] = None, + seq_len: Optional[int] = None, + ) -> tuple[torch.Tensor, float]: + """Compute default inverse frequencies and attention scaling factor for Qwen2 RoPE.""" + del seq_len + head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) + dim = int(head_dim * partial_rotary_factor) + base = getattr(config, "rope_theta", None) + if base is None: + base = config.rope_parameters["rope_theta"] + inv_freq = 1.0 / ( + base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) + ) + return inv_freq, 1.0 + + @torch.no_grad() + def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + inv_freq = self.inv_freq.to(device=x.device, dtype=torch.float32) + inv_freq_expanded = inv_freq[None, :, None].expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + cos = cos * self.attention_scaling + sin = sin * self.attention_scaling + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +def _build_rms_norm( + rms_norm_cls: type[Qwen2RMSNorm], + hidden_size: int, + config: BagelQwen2MoTConfig, +) -> Qwen2RMSNorm: + """Instantiate ``rms_norm_cls``, forwarding ``config`` when the subclass accepts it.""" + try: + return rms_norm_cls(hidden_size, eps=config.rms_norm_eps, config=config) + except TypeError: + return rms_norm_cls(hidden_size, eps=config.rms_norm_eps) + + +class BagelQwen2MoTAttention(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.num_key_value_heads = config.num_key_value_heads + self.head_dim = self.hidden_size // self.num_heads + + self.query_size = self.num_heads * self.head_dim + self.key_value_size = self.num_key_value_heads * self.head_dim + self.qkv_split_sizes = (self.query_size, self.key_value_size, self.key_value_size) + + self._build_qkv_proj() + self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.q_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.k_norm = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.o_proj_moe_gen = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) + self.q_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) + self.k_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) + + def _build_qkv_proj(self) -> None: + self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True) + self.k_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.v_proj = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.q_proj_moe_gen = nn.Linear(self.hidden_size, self.query_size, bias=True) + self.k_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + self.v_proj_moe_gen = nn.Linear(self.hidden_size, self.key_value_size, bias=True) + + def _project_qkv( + self, + hidden: torch.Tensor, + *, + is_gen: bool, + split: bool = False, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project ``hidden`` through the und or gen Q/K/V linear layers.""" + query_proj, key_proj, value_proj = ( + (self.q_proj_moe_gen, self.k_proj_moe_gen, self.v_proj_moe_gen) + if is_gen + else (self.q_proj, self.k_proj, self.v_proj) + ) + query_states = query_proj(hidden) + key_states = key_proj(hidden) + value_states = value_proj(hidden) + if split: + return query_states, key_states, value_states + return torch.cat((query_states, key_states, value_states), dim=-1) + + @staticmethod + def build_attention_mask( + packed_attention_metadata: torch.Tensor, + *, + attn_implementation: str | None = None, + ) -> torch.Tensor: + del attn_implementation + return build_mot_sdpa_mask(packed_attention_metadata) + + def apply_rotary_pos_emb( + self, + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, + ) -> tuple[torch.Tensor, torch.Tensor]: + return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim) + + def project_qkv( + self, + packed_sequence: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + packed_qkv_states = _scatter_moe( + packed_sequence.new_zeros((packed_sequence.shape[0], sum(self.qkv_split_sizes))), + packed_und_token_indexes, + packed_gen_token_indexes, + self._project_qkv(packed_sequence[packed_und_token_indexes], is_gen=False), + self._project_qkv(packed_sequence[packed_gen_token_indexes], is_gen=True), + ) + packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split( + self.qkv_split_sizes, dim=-1 + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + + packed_query_states = _scatter_moe( + packed_query_states.new_zeros(packed_query_states.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.q_norm(packed_query_states[packed_und_token_indexes]), + self.q_norm_moe_gen(packed_query_states[packed_gen_token_indexes]), + ) + packed_key_states = _scatter_moe( + packed_key_states.new_zeros(packed_key_states.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.k_norm(packed_key_states[packed_und_token_indexes]), + self.k_norm_moe_gen(packed_key_states[packed_gen_token_indexes]), + ) + packed_query_states, packed_key_states = self.apply_rotary_pos_emb( + packed_query_states, + packed_key_states, + packed_position_cos, + packed_position_sin, + unsqueeze_dim=1, + ) + return packed_query_states, packed_key_states, packed_value_states + + def attend( + self, + packed_query_states: torch.Tensor, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + attention_mask: torch.Tensor, + ) -> torch.Tensor: + packed_attn_output = _sdpa_packed_attention( + packed_query_states, + packed_key_states, + packed_value_states, + attention_mask=attention_mask, + scale=self.head_dim**-0.5, + enable_gqa=self.num_heads != self.num_key_value_heads, + ) + return packed_attn_output.reshape(-1, self.num_heads * self.head_dim) + + def project_o( + self, + packed_attn_output: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + return _scatter_moe( + packed_attn_output.new_zeros(packed_attn_output.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.o_proj(packed_attn_output[packed_und_token_indexes]), + self.o_proj_moe_gen(packed_attn_output[packed_gen_token_indexes]), + ) + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + packed_query_states, packed_key_states, packed_value_states = self.project_qkv( + packed_sequence, + packed_und_token_indexes, + packed_gen_token_indexes, + packed_position_cos, + packed_position_sin, + ) + packed_attn_output = self.attend( + packed_query_states, + packed_key_states, + packed_value_states, + attention_mask, + ) + return self.project_o( + packed_attn_output, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + + def project_qkv_inference( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project packed inference tokens through the matching MoT attention expert.""" + if not is_gen: + packed_query_states, packed_key_states, packed_value_states = self._project_qkv( + packed_query_sequence, is_gen=False, split=True + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_query_states = self.q_norm(packed_query_states) + packed_key_states = self.k_norm(packed_key_states) + return packed_query_states, packed_key_states, packed_value_states + + packed_query_sequence = packed_query_sequence.to(torch.bfloat16) + packed_qkv_states = packed_query_sequence.new_zeros( + (packed_query_sequence.shape[0], sum(self.qkv_split_sizes)) + ) + packed_qkv_states[packed_text_indexes] = self._project_qkv( + packed_query_sequence[packed_text_indexes], is_gen=False + ) + packed_qkv_states[packed_vae_token_indexes] = self._project_qkv( + packed_query_sequence[packed_vae_token_indexes], is_gen=True + ) + packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split( + self.qkv_split_sizes, dim=-1 + ) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim).to(torch.float32) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim).to(torch.float32) + packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) + packed_query_states[packed_text_indexes] = self.q_norm(packed_query_states[packed_text_indexes]) + packed_query_states[packed_vae_token_indexes] = self.q_norm_moe_gen( + packed_query_states[packed_vae_token_indexes] + ) + packed_key_states[packed_text_indexes] = self.k_norm(packed_key_states[packed_text_indexes]) + packed_key_states[packed_vae_token_indexes] = self.k_norm_moe_gen(packed_key_states[packed_vae_token_indexes]) + return packed_query_states, packed_key_states, packed_value_states + + def merge_kv_cache( + self, + packed_key_states: torch.Tensor, + packed_value_states: torch.Tensor, + *, + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache], + key_values_lens: Optional[torch.Tensor], + packed_key_value_indexes: Optional[torch.Tensor], + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Merge the current query span with this layer's packed KV cache.""" + if past_key_values is None or past_key_values.key_cache[self.layer_idx] is None: + return packed_key_states, packed_value_states + if key_values_lens is None or packed_key_value_indexes is None: + raise ValueError("key_values_lens and packed_key_value_indexes are required when cache is non-empty.") + + past_key_states = past_key_values.key_cache[self.layer_idx] + past_value_states = past_key_values.value_cache[self.layer_idx] + merged_key_states = past_key_states.new_zeros( + (total_key_value_tokens, self.num_key_value_heads, self.head_dim) + ) + merged_value_states = past_key_states.new_zeros( + (total_key_value_tokens, self.num_key_value_heads, self.head_dim) + ) + merged_key_states[packed_query_indexes] = packed_key_states + merged_key_states[packed_key_value_indexes] = past_key_states + merged_value_states[packed_query_indexes] = packed_value_states + merged_value_states[packed_key_value_indexes] = past_value_states + return merged_key_states, merged_value_states + + def attend_inference( + self, + packed_query_states: torch.Tensor, + merged_key_states: torch.Tensor, + merged_value_states: torch.Tensor, + *, + attention_mask: Optional[torch.Tensor], + is_causal: bool, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + ) -> torch.Tensor: + del max_length_q, max_length_k + return _sdpa_packed_attention( + packed_query_states, + merged_key_states, + merged_value_states, + attention_mask=attention_mask, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + scale=self.head_dim**-0.5, + enable_gqa=self.num_heads != self.num_key_value_heads, + ) + + def project_o_inference( + self, + packed_attn_output: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Project packed attention outputs through the matching MoT output expert.""" + packed_attn_output = packed_attn_output.reshape(-1, self.hidden_size) + if not is_gen: + return self.o_proj(packed_attn_output) + + packed_attn_output[packed_text_indexes] = self.o_proj(packed_attn_output[packed_text_indexes]) + packed_attn_output[packed_vae_token_indexes] = self.o_proj_moe_gen( + packed_attn_output[packed_vae_token_indexes] + ) + return packed_attn_output + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor], + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + attention_mask: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + *, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, Optional[NaiveCache]]: + is_gen = _check_packed_inference_mode(mode) + packed_query_states, packed_key_states, packed_value_states = self.project_qkv_inference( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + + packed_cos, packed_sin = packed_query_position_embeddings + packed_query_states, packed_key_states = self.apply_rotary_pos_emb( + packed_query_states, + packed_key_states, + packed_cos, + packed_sin, + unsqueeze_dim=1, + ) + + packed_query_states = packed_query_states.to(torch.bfloat16) + packed_key_states = packed_key_states.to(torch.bfloat16) + packed_value_states = packed_value_states.to(torch.bfloat16) + + merged_key_states, merged_value_states = self.merge_kv_cache( + packed_key_states, + packed_value_states, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + total_key_value_tokens=total_key_value_tokens, + ) + packed_attn_output = self.attend_inference( + packed_query_states, + merged_key_states, + merged_value_states, + attention_mask=attention_mask, + is_causal=is_causal, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + ) + packed_attn_output = self.project_o_inference( + packed_attn_output, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + + if update_past_key_values: + if past_key_values is None: + raise ValueError("past_key_values is required when update_past_key_values=True.") + past_key_values.key_cache[self.layer_idx] = merged_key_states + past_key_values.value_cache[self.layer_idx] = merged_value_states + + return packed_attn_output, past_key_values + + def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]: + if self.training: + return self.forward_packed_train(*args, **kwargs), None + return self.forward_packed_inference(*args, **kwargs) + + +class BagelQwen2MoTDecoderLayer(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + layer_idx: int, + *, + attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention, + mlp_cls: type[Qwen2MLP] = Qwen2MLP, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.self_attn = attention_cls(config, layer_idx, rms_norm_cls=rms_norm_cls) + self.mlp = mlp_cls(config) + self.mlp_moe_gen = mlp_cls(config) + self.input_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.input_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.post_attention_layernorm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.post_attention_layernorm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + + def _apply_moe_norm( + self, + packed_sequence: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + und_norm: nn.Module, + gen_norm: nn.Module, + ) -> torch.Tensor: + return _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + und_norm(packed_sequence[packed_und_token_indexes]), + gen_norm(packed_sequence[packed_gen_token_indexes]), + ) + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: torch.Tensor, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + residual = packed_sequence + packed_sequence = self._apply_moe_norm( + packed_sequence, + packed_und_token_indexes, + packed_gen_token_indexes, + self.input_layernorm, + self.input_layernorm_moe_gen, + ) + packed_sequence, _ = self.self_attn( + packed_sequence=packed_sequence, + attention_mask=attention_mask, + packed_position_cos=packed_position_cos, + packed_position_sin=packed_position_sin, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + packed_sequence = residual + packed_sequence + + residual = packed_sequence + post_attn_und = self.post_attention_layernorm(packed_sequence[packed_und_token_indexes]) + post_attn_gen = self.post_attention_layernorm_moe_gen(packed_sequence[packed_gen_token_indexes]) + packed_sequence = _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.mlp(post_attn_und), + self.mlp_moe_gen(post_attn_gen), + extra_und_anchors=(post_attn_und,), + extra_gen_anchors=(post_attn_gen,), + ) + return residual + packed_sequence + + def _apply_inference_input_norm( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Normalize inference tokens with the matching MoT expert.""" + if not is_gen: + return self.input_layernorm(packed_query_sequence) + + normed_sequence = torch.zeros_like(packed_query_sequence) + normed_sequence[packed_text_indexes] = self.input_layernorm(packed_query_sequence[packed_text_indexes]) + normed_sequence[packed_vae_token_indexes] = self.input_layernorm_moe_gen( + packed_query_sequence[packed_vae_token_indexes] + ) + return normed_sequence + + def _apply_inference_mlp( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Apply the inference MLP with text and VAE tokens routed independently.""" + if not is_gen: + return self.mlp(self.post_attention_layernorm(packed_query_sequence)) + + packed_text_query_sequence = self.post_attention_layernorm(packed_query_sequence[packed_text_indexes]).to( + torch.bfloat16 + ) + packed_vae_query_sequence = self.post_attention_layernorm_moe_gen( + packed_query_sequence[packed_vae_token_indexes] + ).to(torch.bfloat16) + mlp_output = torch.zeros_like(packed_query_sequence).to(torch.bfloat16) + mlp_output[packed_text_indexes] = self.mlp(packed_text_query_sequence) + mlp_output[packed_vae_token_indexes] = self.mlp_moe_gen(packed_vae_query_sequence) + return mlp_output + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_embeddings: tuple[torch.Tensor, torch.Tensor], + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + attention_mask: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + *, + cu_seq_lens_q: torch.Tensor, + cu_seq_lens_k: torch.Tensor, + max_length_q: int, + max_length_k: int, + total_key_value_tokens: int, + ) -> tuple[torch.Tensor, Optional[NaiveCache]]: + is_gen = _check_packed_inference_mode(mode) + residual = packed_query_sequence + packed_query_sequence = self._apply_inference_input_norm( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + packed_query_sequence, past_key_values = self.self_attn( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_embeddings=packed_query_position_embeddings, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + update_past_key_values=update_past_key_values, + is_causal=is_causal, + mode=mode, + attention_mask=attention_mask, + packed_vae_token_indexes=packed_vae_token_indexes, + packed_text_indexes=packed_text_indexes, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + total_key_value_tokens=total_key_value_tokens, + ) + packed_query_sequence = residual + packed_query_sequence + + residual = packed_query_sequence + packed_query_sequence = self._apply_inference_mlp( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + packed_query_sequence = residual + packed_query_sequence + + return packed_query_sequence, past_key_values + + def forward(self, *args: Any, **kwargs: Any) -> tuple[torch.Tensor, Optional[NaiveCache]]: + if self.training: + return self.forward_packed_train(*args, **kwargs), None + return self.forward_packed_inference(*args, **kwargs) + + +class BagelQwen2MoTBackbone(nn.Module): + def __init__( + self, + config: BagelQwen2MoTConfig, + *, + attention_cls: type[BagelQwen2MoTAttention] = BagelQwen2MoTAttention, + mlp_cls: type[Qwen2MLP] = Qwen2MLP, + rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, + ): + super().__init__() + self.config = config + self.gradient_checkpointing = False + self.layers = nn.ModuleList( + [ + BagelQwen2MoTDecoderLayer( + config, + layer_idx, + attention_cls=attention_cls, + mlp_cls=mlp_cls, + rms_norm_cls=rms_norm_cls, + ) + for layer_idx in range(config.num_hidden_layers) + ] + ) + self.norm = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.norm_moe_gen = _build_rms_norm(rms_norm_cls, config.hidden_size, config) + self.rotary_emb = BagelQwen2RotaryEmbedding(config=config) + self.use_moe = "Mo" in config.layer_module + self.attention_cls = attention_cls + + def _build_inference_attention_mask( + self, + packed_query_sequence: torch.Tensor, + packed_attention_metadata: Optional[torch.Tensor], + *, + cache_has_values: bool, + ) -> Any: + """Validate packed-prefill invariants and materialize its attention mask.""" + if packed_attention_metadata is None: + return None + if cache_has_values: + raise ValueError("BAGEL packed prefill requires an empty KV cache.") + + expected_metadata_shape = (3, int(packed_query_sequence.shape[0])) + if tuple(packed_attention_metadata.shape) != expected_metadata_shape: + raise ValueError( + "BAGEL packed prefill metadata must match the packed query sequence: " + f"expected {expected_metadata_shape}, got {tuple(packed_attention_metadata.shape)}." + ) + return self.attention_cls.build_attention_mask( + packed_attention_metadata.to(device=packed_query_sequence.device), + attn_implementation=self.config._attn_implementation, + ) + + def _apply_inference_final_norm( + self, + packed_query_sequence: torch.Tensor, + *, + is_gen: bool, + packed_text_indexes: Optional[torch.Tensor], + packed_vae_token_indexes: Optional[torch.Tensor], + ) -> torch.Tensor: + """Apply the final backbone norm with the matching MoT expert.""" + if not is_gen: + return self.norm(packed_query_sequence) + + normed_sequence = torch.zeros_like(packed_query_sequence) + normed_sequence[packed_text_indexes] = self.norm(packed_query_sequence[packed_text_indexes]) + normed_sequence[packed_vae_token_indexes] = self.norm_moe_gen(packed_query_sequence[packed_vae_token_indexes]) + return normed_sequence + + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + packed_position_ids: torch.Tensor, + packed_attention_metadata: torch.Tensor, + packed_und_token_indexes: Optional[torch.Tensor] = None, + packed_gen_token_indexes: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Run packed training forward across all MoT decoder layers and final expert norms.""" + attention_mask = self.attention_cls.build_attention_mask( + packed_attention_metadata, + attn_implementation=self.config._attn_implementation, + ) + cos, sin = self.rotary_emb(packed_sequence, packed_position_ids.unsqueeze(0)) + packed_position_cos = cos.squeeze(0) + packed_position_sin = sin.squeeze(0) + + if self.use_moe: + if packed_und_token_indexes is None: + raise ValueError("packed_und_token_indexes is required for BAGEL MoT training.") + if packed_gen_token_indexes is None: + packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0]) + else: + packed_und_token_indexes = torch.arange(packed_sequence.shape[0], device=packed_sequence.device) + packed_gen_token_indexes = packed_und_token_indexes.new_ones(size=[0]) + + for decoder_layer in self.layers: + if self.gradient_checkpointing and self.training: + packed_sequence, _ = self._gradient_checkpointing_func( + decoder_layer.__call__, + packed_sequence, + attention_mask, + packed_position_cos, + packed_position_sin, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + else: + packed_sequence, _ = decoder_layer( + packed_sequence=packed_sequence, + attention_mask=attention_mask, + packed_position_cos=packed_position_cos, + packed_position_sin=packed_position_sin, + packed_und_token_indexes=packed_und_token_indexes, + packed_gen_token_indexes=packed_gen_token_indexes, + ) + + if self.use_moe: + return _scatter_moe( + packed_sequence.new_zeros(packed_sequence.shape), + packed_und_token_indexes, + packed_gen_token_indexes, + self.norm(packed_sequence[packed_und_token_indexes]), + self.norm_moe_gen(packed_sequence[packed_gen_token_indexes]), + ) + return self.norm(packed_sequence) + + def forward_packed_inference( + self, + packed_query_sequence: torch.Tensor, + query_lens: torch.Tensor, + packed_query_position_ids: torch.Tensor, + packed_query_indexes: torch.Tensor, + past_key_values: Optional[NaiveCache] = None, + key_values_lens: Optional[torch.Tensor] = None, + packed_key_value_indexes: Optional[torch.Tensor] = None, + update_past_key_values: bool = True, + is_causal: bool = True, + mode: str = "und", + packed_attention_metadata: Optional[torch.Tensor] = None, + packed_vae_token_indexes: Optional[torch.Tensor] = None, + packed_text_indexes: Optional[torch.Tensor] = None, + ) -> BaseNavitOutputWithPast: + """Execute all MoT decoder layers for packed inference prefill, decode, or denoise.""" + is_gen = _check_packed_inference_mode(mode) + query_device = packed_query_sequence.device + packed_query_indexes = packed_query_indexes.to(device=query_device) + packed_query_position_ids = packed_query_position_ids.to(device=query_device) + if packed_key_value_indexes is not None: + packed_key_value_indexes = packed_key_value_indexes.to(device=query_device) + if packed_vae_token_indexes is not None: + packed_vae_token_indexes = packed_vae_token_indexes.to(device=query_device) + if packed_text_indexes is not None: + packed_text_indexes = packed_text_indexes.to(device=query_device) + if past_key_values is None: + past_key_values = NaiveCache(len(self.layers)) + + cos, sin = self.rotary_emb(packed_query_sequence, packed_query_position_ids.unsqueeze(0)) + packed_query_position_embeddings = (cos.squeeze(0), sin.squeeze(0)) + + cache_has_values = past_key_values.key_cache[0] is not None + if cache_has_values and key_values_lens is None: + raise ValueError("key_values_lens is required when cache is non-empty.") + attention_mask = self._build_inference_attention_mask( + packed_query_sequence, + packed_attention_metadata, + cache_has_values=cache_has_values, + ) + + effective_key_values_lens = key_values_lens + query_lens if cache_has_values else query_lens + cu_seq_lens_q = torch.nn.functional.pad(torch.cumsum(query_lens, dim=0), (1, 0)).to(torch.int32) + cu_seq_lens_k = torch.nn.functional.pad(torch.cumsum(effective_key_values_lens, dim=0), (1, 0)).to(torch.int32) + max_length_q = int(query_lens.max().item()) + max_length_k = int(effective_key_values_lens.max().item()) + total_key_value_tokens = int(effective_key_values_lens.sum().item()) + + for decoder_layer in self.layers: + packed_query_sequence, past_key_values = decoder_layer( + packed_query_sequence=packed_query_sequence, + query_lens=query_lens, + packed_query_position_embeddings=packed_query_position_embeddings, + packed_query_indexes=packed_query_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + update_past_key_values=update_past_key_values, + is_causal=is_causal, + mode=mode, + attention_mask=attention_mask, + packed_vae_token_indexes=packed_vae_token_indexes, + packed_text_indexes=packed_text_indexes, + cu_seq_lens_q=cu_seq_lens_q, + cu_seq_lens_k=cu_seq_lens_k, + max_length_q=max_length_q, + max_length_k=max_length_k, + total_key_value_tokens=total_key_value_tokens, + ) + + packed_query_sequence = self._apply_inference_final_norm( + packed_query_sequence, + is_gen=is_gen, + packed_text_indexes=packed_text_indexes, + packed_vae_token_indexes=packed_vae_token_indexes, + ) + return BaseNavitOutputWithPast( + packed_query_sequence=packed_query_sequence, + past_key_values=past_key_values, + ) + + def forward(self, *args: Any, **kwargs: Any) -> BaseNavitOutputWithPast: + if self.training: + return BaseNavitOutputWithPast(packed_query_sequence=self.forward_packed_train(*args, **kwargs)) + return self.forward_packed_inference(*args, **kwargs) + + +BagelQwen2MoTCore.attention_cls = BagelQwen2MoTAttention +BagelQwen2MoTCore.mlp_cls = Qwen2MLP +BagelQwen2MoTCore.rms_norm_cls = Qwen2RMSNorm + + +__all__ = [ + "BaseNavitOutputWithPast", + "BagelQwen2MoT", + "BagelQwen2MoTCore", + "InferenceMixin", + "NaiveCache", +] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py new file mode 100644 index 0000000000..5795f8656b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/processing.py @@ -0,0 +1,336 @@ +"""Conversation packing helpers for BAGEL Qwen2-MoT.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context +from .masking import build_mot_attention_metadata + + +_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"}) + + +@dataclass(frozen=True) +class PackedSpan: + """Describe one logical span in the packed sequence. + + Most spans contain one carrier item. A VAE image span can group its leading + marker, image, and trailing marker; ``primary_index`` identifies the image, + while ``lengths`` preserves each item's subrange for routing and writeback. + """ + + start: int + items: tuple[ConversationItem, ...] + lengths: tuple[int, ...] + primary_index: int = 0 + + @property + def item(self) -> ConversationItem: + return self.items[self.primary_index] + + @property + def length(self) -> int: + return sum(self.lengths) + + @property + def primary_start(self) -> int: + return sum(self.lengths[: self.primary_index]) + + @property + def primary_length(self) -> int: + return self.lengths[self.primary_index] + + @property + def is_image_triplet(self) -> bool: + return ( + len(self.items) == 3 + and self.primary_index == 1 + and self.items[0].type == "text" + and self.items[1].type == "image" + and self.items[2].type == "text" + ) + + +@dataclass +class PackedConversation: + """Result of packing an embedded conversation batch for MoT. + + Packed tensors concatenate all samples. ``sample_splits`` preserves their + document/span boundaries for attention metadata. ``spans`` uses global + packed offsets to map model outputs back to the original carrier items. + Token type 0 selects the understanding expert and type 1 the generation + expert. + """ + + packed_sequence: torch.Tensor + sample_splits: list[list[int]] + packed_attention_metadata: torch.Tensor + packed_position_ids: torch.Tensor + packed_token_type_ids: torch.Tensor + spans: list[PackedSpan] + + +def preprocess_mot_inputs( + conversation_list: list[list[ConversationItem]] | None, + *, + device: torch.device, + dtype: torch.dtype, + hidden_size: int, +) -> PackedConversation | None: + """Preprocess already-embedded carrier items into MoT packed inputs. + + Text and SigLIP spans route through the understanding expert. VAE latent + spans and output spans route through the generation expert. Attention mode + and expert routing are deliberately computed separately: a clean VAE + conditioning image still uses the generation expert. Packed layout stays + local to MoT. + """ + + if not conversation_list: + raise ValueError("BAGEL Qwen2-MoT preprocessing requires a non-empty conversation_list.") + + sequence_parts: list[torch.Tensor] = [] + sample_splits_by_sample: list[list[int]] = [] + sample_attn_modes_by_sample: list[list[str]] = [] + position_parts: list[torch.Tensor] = [] + token_type_parts: list[torch.Tensor] = [] + spans: list[PackedSpan] = [] + sequence_cursor = 0 + + for sample in conversation_list: + sample_start = sequence_cursor + sample_splits: list[int] = [] + sample_attn_modes: list[str] = [] + sample_position_cursor = 0 + + # Dummy carrier items are handled separately as zero-gradient anchors; + # only real user/assistant embeddings participate in packed attention. + items = [ + item + for item in iter_desired_items( + [sample], + types=["text", "image", "output"], + roles=["user", "assistant"], + ) + if not item.is_dummy + ] + item_index = 0 + while item_index < len(items): + if _is_vision_marker_triplet_at(items, item_index): + # Treat , image tokens, and as one + # logical attention span while preserving three writeback + # ranges and routing only the image tokens to the gen expert. + span_items = (items[item_index], items[item_index + 1], items[item_index + 2]) + span_values = [ + _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size) + for item in span_items + ] + span_lengths = tuple(int(value.shape[0]) for value in span_values) + item = span_items[1] + value = torch.cat(span_values, dim=0) + length = int(value.shape[0]) + primary_index = 1 + item_index += 3 + else: + item = items[item_index] + value = _mot_value_for_item(item, device=device, dtype=dtype, hidden_size=hidden_size) + length = int(value.shape[0]) + span_items = (item,) + span_lengths = (length,) + primary_index = 0 + item_index += 1 + + if length == 0: + continue + + # sequence_cursor is global across the packed batch; positions + # restart independently for each logical document. + indexes = torch.arange(sequence_cursor, sequence_cursor + length, device=device, dtype=torch.long) + position_ids, sample_position_cursor = _mot_position_ids_for_span( + item, + start=sample_position_cursor, + length=length, + device=device, + ) + mode = _mot_attn_mode_for_item(item) + + sequence_parts.append(value) + position_parts.append(position_ids) + sample_splits.append(length) + sample_attn_modes.append(mode) + span = PackedSpan( + start=sequence_cursor, + items=span_items, + lengths=span_lengths, + primary_index=primary_index, + ) + spans.append(span) + sequence_cursor += length + + gen_token_indexes = _mot_gen_token_indexes_for_span(span, indexes) + token_type_ids = torch.zeros(length, device=device, dtype=torch.long) + if gen_token_indexes.numel() > 0: + token_type_ids[(gen_token_indexes - indexes[0]).to(device=device, dtype=torch.long)] = 1 + token_type_parts.append(token_type_ids) + + sample_len = sequence_cursor - sample_start + if sample_len > 0: + sample_splits_by_sample.append(list(sample_splits)) + sample_attn_modes_by_sample.append(list(sample_attn_modes)) + + if not sequence_parts: + return None + + packed_sequence = torch.cat(sequence_parts, dim=0).to(device=device, dtype=dtype) + return PackedConversation( + packed_sequence=packed_sequence, + sample_splits=sample_splits_by_sample, + packed_attention_metadata=build_mot_attention_metadata( + sample_splits_by_sample, + sample_attn_modes_by_sample, + device=device, + ), + packed_position_ids=torch.cat(position_parts, dim=0).to(device=device, dtype=torch.long), + packed_token_type_ids=torch.cat(token_type_parts, dim=0).to(device=device, dtype=torch.long), + spans=spans, + ) + + +def _mot_value_for_item( + item: ConversationItem, + *, + device: torch.device, + dtype: torch.dtype, + hidden_size: int, +) -> torch.Tensor: + """Validate and cast an item's rank-2 embedding tensor to ``(device, dtype)``.""" + value = item.value + if not torch.is_tensor(value): + raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got {type(value).__name__}.") + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT expects embedded item tensors, got shape {tuple(item.value.shape)}.") + if int(value.shape[-1]) != int(hidden_size): + raise ValueError(f"BAGEL Qwen2-MoT item hidden-size mismatch: got {value.shape[-1]}, expected {hidden_size}.") + return value.to(device=device, dtype=dtype) + + +def _mot_position_ids_for_span( + item: ConversationItem, + *, + start: int, + length: int, + device: torch.device, +) -> tuple[torch.Tensor, int]: + """Assign BAGEL positions and return the next document-local position.""" + meta_position_ids = item.meta.get("position_ids") + if torch.is_tensor(meta_position_ids): + position_ids = meta_position_ids.detach().to(device=device, dtype=torch.long).reshape(-1) + if int(position_ids.numel()) != length: + raise ValueError("BAGEL Qwen2-MoT position_ids length must match the item span length.") + next_start = max(start + 1, int(position_ids.max().item()) + 1) if length else start + return position_ids, next_start + + # All latent/image tokens in one span share a position; text remains AR + # sequential. Thus a whole image advances the logical position only once. + if item.type in {"image", "output"}: + return torch.full((length,), start, device=device, dtype=torch.long), start + 1 + return torch.arange(start, start + length, device=device, dtype=torch.long), start + length + + +def _mot_attn_mode_for_item(item: ConversationItem) -> str: + """Classify visibility independently from MoT expert routing.""" + if item.type == "output": + return "noise" + if item.type == "image": + if bagel_context(item) != BAGEL_VAE_CONTEXT: + # SigLIP context (and any non-VAE image) is clean/full-attention + # regardless of ``_img_tag``; the tag only disambiguates VAE roles. + return "full" + tag = item.meta.get(_IMG_TAG_KEY) + if tag is None: + # Untagged VAE images come from inference prompt routing. Training + # targets must carry ``_img_tag="gen"``. + return "full" + if tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.") + if tag == "gen": + return "noise" + if tag == "edit": + # Edit VAE context is a clean conditioning image, not a noised target. + return "full" + raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.") + return "causal" + + +def _mot_gen_token_indexes_for_span(span: PackedSpan, indexes: torch.Tensor) -> torch.Tensor: + """Return packed rows routed through the generation expert.""" + item = span.item + if item.type == "output": + return indexes + if item.type != "image" or bagel_context(item) != BAGEL_VAE_CONTEXT: + return indexes.new_empty(0) + + tag = item.meta.get(_IMG_TAG_KEY) + if tag is not None and tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL Qwen2-MoT received image with unknown {_IMG_TAG_KEY}: {tag!r}.") + if tag == "und": + raise ValueError(f"BAGEL Qwen2-MoT received VAE image with incompatible {_IMG_TAG_KEY}: {tag!r}.") + + # Expert routing follows token modality, independently of whether the VAE + # latent is clean conditioning (untagged/edit) or a noised generation + # target. Marker tokens remain on the understanding path. + if span.is_image_triplet: + primary_start = span.primary_start + primary_end = primary_start + span.primary_length + return indexes[primary_start:primary_end] + return indexes + + +def _is_vision_marker_triplet_at(items: list[ConversationItem], index: int) -> bool: + """Return whether ``items[index : index + 3]`` forms a ``(start, image, end)`` triplet.""" + # Context-tag equality prevents unrelated one-token text items around an + # image from being mistaken for BAGEL's encoder-specific marker pair. + if index + 2 >= len(items): + return False + start, image, end = items[index], items[index + 1], items[index + 2] + context = bagel_context(image) + if image.type != "image" or context not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + return False + return _is_bagel_vision_marker(start, context=context) and _is_bagel_vision_marker(end, context=context) + + +def _is_bagel_vision_marker(item: ConversationItem, *, context: str) -> bool: + """Return whether ``item`` is a single-token vision marker for ``context``.""" + return item.type == "text" and bagel_context(item) == context and _text_item_length(item) == 1 + + +def _text_item_length(item: ConversationItem) -> int | None: + """Return the token length of a text ``ConversationItem``, or ``None`` if unavailable.""" + value = item.value + if torch.is_tensor(value): + if value.dim() == 0: + return 1 + if value.dim() == 1: + return int(value.shape[0]) + if value.dim() == 2: + return int(value.shape[0]) + if value.dim() == 3 and int(value.shape[0]) == 1: + return int(value.shape[1]) + return None + input_ids = item.meta.get("input_ids") + if torch.is_tensor(input_ids): + return int(input_ids.reshape(-1).shape[0]) + return None + + +__all__ = [ + "PackedConversation", + "PackedSpan", + "preprocess_mot_inputs", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py new file mode 100644 index 0000000000..23fabe4009 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/__init__.py @@ -0,0 +1,31 @@ +"""BAGEL SigLIP NaViT vision encoder module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_config(): + from .configuration import BagelSiglipNavitConfig + + return BagelSiglipNavitConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_model(): + from .modeling import BagelSiglipNavit + + return BagelSiglipNavit + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_accelerated_model(): + from .accelerated import BagelSiglipNavitAccelerated + + return BagelSiglipNavitAccelerated + + +@OMNI_PROCESSOR_REGISTRY.register("bagel_siglip_navit") +def register_bagel_siglip_navit_processor(): + from .processing import BagelSiglipNavitProcessor + + return BagelSiglipNavitProcessor diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py new file mode 100644 index 0000000000..f4ac46ddca --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/siglip_navit OmniModule (conversation graph).""" + +from .accelerated import ( + BagelSiglipNavitAccelerated, +) + + +__all__ = [ + "BagelSiglipNavitAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py new file mode 100644 index 0000000000..9941e82d0a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py @@ -0,0 +1,196 @@ +"""SeedOmni training-graph hooks for BAGEL's SigLIP NaViT vision encoder. + +``generate()`` lives natively on :class:`~.modeling.BagelSiglipNavit` — this +file only carries the SP-aware training pre/forward/post hooks. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import gather_outputs +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ...sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from ..configuration import BagelSiglipNavitConfig +from ..modeling import ( + BagelSiglipNavit, + scatter_siglip_image_embeds, + select_siglip_image_items, + siglip_inputs_from_preprocessed_items, +) +from ..processing import _OMNI_TOKEN_LEN + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelSiglipNavit` modeling APIs.""" + + config: BagelSiglipNavitConfig + device: torch.device + dtype: torch.dtype + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._image_items: list[ConversationItem] = [] + self._sp_image_count: int | None = None + self._sp_token_count: int | None = None + + @pre_forward("forward") + def forward_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Select preprocessed SigLIP image items and slice across Ulysses SP ranks if enabled.""" + del kwargs + self._conversation_carrier = conversation_list + + self._image_items = select_siglip_image_items(conversation_list) + out = siglip_inputs_from_preprocessed_items(self._image_items, device=self.device, dtype=self.dtype) + self.metric_meter_set_seqlens("forward", self._metric_sample_token_lens(conversation_list)) + + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_image_count = None + self._sp_token_count = None + return out + + if ps.cp_size != 1: + raise ValueError(f"BAGEL SigLIP NaViT training supports Ulysses groups only; got cp_size={ps.cp_size}.") + + token_lens = out["token_lens"] + image_count = int(token_lens.numel()) + if image_count == 0: + raise ValueError("BAGEL SigLIP NaViT SP requires at least one real or dummy image.") + self._sp_image_count = image_count + self._sp_token_count = int(token_lens.sum().item()) + + pad_images = (-image_count) % ps.sp_size + if pad_images: + token_lens = torch.cat((token_lens, token_lens.new_ones(pad_images)), dim=0) + out["patchified_pixel_values"] = torch.cat( + ( + out["patchified_pixel_values"], + out["patchified_pixel_values"].new_zeros((pad_images, *out["patchified_pixel_values"].shape[1:])), + ), + dim=0, + ) + out["patchified_position_ids"] = torch.cat( + ( + out["patchified_position_ids"], + out["patchified_position_ids"].new_zeros((pad_images, *out["patchified_position_ids"].shape[1:])), + ), + dim=0, + ) + + images_per_rank = int(token_lens.numel()) // ps.sp_size + image_start = ps.sp_rank * images_per_rank + image_end = image_start + images_per_rank + full_cu_seqlens = torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0)) + token_start = int(full_cu_seqlens[image_start].item()) + token_end = int(full_cu_seqlens[image_end].item()) + local_token_lens = token_lens[image_start:image_end].contiguous() + return { + "patchified_pixel_values": out["patchified_pixel_values"][token_start:token_end].contiguous(), + "patchified_position_ids": out["patchified_position_ids"][token_start:token_end].contiguous(), + "cu_seqlens": torch.nn.functional.pad(torch.cumsum(local_token_lens, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(local_token_lens.max().item()), + "token_lens": local_token_lens, + } + + @post_forward("forward") + def forward_post( + self, + image_embeds: torch.Tensor, + token_lens: torch.Tensor | None = None, + ) -> dict[str, Any]: + """Gather SP image feature shards and scatter ``image_embeds`` onto conversation items.""" + if token_lens is None: + raise ValueError("BagelSiglipNavit.forward_post requires token_lens for non-dummy outputs.") + + ps = get_parallel_state() + if ps.sp_size > 1: + if self._sp_image_count is None or self._sp_token_count is None: + raise RuntimeError("BAGEL SigLIP NaViT SP image shape was not initialized.") + image_embeds = gather_outputs(image_embeds, gather_dim=0, group=ps.sp_group) + token_lens = gather_outputs(token_lens, gather_dim=0, group=ps.sp_group) + image_embeds = image_embeds.narrow(0, 0, self._sp_token_count) + token_lens = token_lens.narrow(0, 0, self._sp_image_count) + + conversation = self._conversation_carrier + image_items = self._image_items + self._conversation_carrier = None + self._image_items = [] + self._sp_image_count = None + self._sp_token_count = None + + scatter_siglip_image_embeds( + image_items, + image_embeds, + token_lens, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": conversation} + + def _metric_sample_token_lens( + self, + conversation_list: list[list[ConversationItem]], + ) -> list[int]: + """Sum SigLIP patch token lengths per sample for FLOPs metering.""" + sample_lens: list[int] = [] + for sample in conversation_list: + sample_len = 0 + # SigLIP encodes per image, including dummy carriers, but + # multi-source metering needs one aggregated length per sample. + for item in iter_desired_items( + [sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]} + ): + sample_len += int(item.meta[_OMNI_TOKEN_LEN]) + sample_lens.append(sample_len) + return sample_lens + + +class MeterMixin(MetricMeterMixin): + """Per-module training trace for BAGEL SigLIP NaViT.""" + + config: BagelSiglipNavitConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + """Estimate TFLOPs for the SigLIP-NaViT encoder and MLP connector over ``seqlens``.""" + cfg = self.config + dim = cfg.hidden_size + heads = cfg.num_attention_heads + head_dim = dim // heads + patch_embed_n = dim * cfg.num_channels * cfg.patch_size * cfg.patch_size + attn_linear_n = dim * 4 * dim + mlp_n = dim * cfg.intermediate_size * 2 + connector_n = dim * cfg.output_size + cfg.output_size * cfg.output_size + dense_n = patch_embed_n + (attn_linear_n + mlp_n) * cfg.num_hidden_layers + connector_n + tokens = sum(seqlens) + seqlen_sq = sum(length * length for length in seqlens) + dense_flops = 6 * dense_n * tokens + attn_flops = 12 * seqlen_sq * head_dim * heads * cfg.num_hidden_layers + return (dense_flops + attn_flops) / 1e12 + + +class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): + """Carrier hooks for BAGEL visual-understanding image features. + + ``generate()`` already lives on the native :class:`~.modeling.BagelSiglipNavit` + (via its own :class:`~.modeling.InferenceMixin`), so no ``InferenceMixin`` is + needed here. + """ + + +class BagelSiglipNavitAccelerated(VeOmniMixin, BagelSiglipNavit): + pass + + +__all__ = ["BagelSiglipNavitAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py new file mode 100644 index 0000000000..f59232e2f6 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/configuration.py @@ -0,0 +1,56 @@ +"""Configuration for :class:`BagelSiglipNavit`.""" + +from typing import List, Optional + +from ...module_configuration_base import OmniModuleConfig + + +class BagelSiglipNavitConfig(OmniModuleConfig): + """BAGEL SigLIP NaViT vision encoder config.""" + + model_type = "bagel_siglip_navit" + + def __init__( + self, + hidden_size: int = 1152, + output_size: int = 3584, + image_size: int = 980, + min_image_size: int = 378, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: Optional[List[float]] = None, + image_std: Optional[List[float]] = None, + intermediate_size: int = 4304, + num_attention_heads: int = 16, + num_hidden_layers: int = 26, + num_channels: int = 3, + patch_size: int = 14, + hidden_act: str = "gelu_pytorch_tanh", + connector_act: str = "gelu_pytorch_tanh", + vit_max_num_patch_per_side: int = 70, + layer_norm_eps: float = 1e-6, + attention_dropout: float = 0.0, + rope: bool = False, + **kwargs, + ) -> None: + self.hidden_size = hidden_size + self.output_size = output_size + self.image_size = image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.intermediate_size = intermediate_size + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_hidden_layers + self.num_channels = num_channels + self.patch_size = patch_size + self.hidden_act = hidden_act + self.connector_act = connector_act + self.vit_max_num_patch_per_side = vit_max_num_patch_per_side + self.layer_norm_eps = layer_norm_eps + self.attention_dropout = attention_dropout + self.rope = rope + super().__init__(**kwargs) + + +__all__ = ["BagelSiglipNavitConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py new file mode 100644 index 0000000000..a15c0fc55b --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -0,0 +1,564 @@ +"""BAGEL SigLIP NaViT vision encoder. + +The training node is ``bagel_siglip_navit.forward``. It receives local +patchified image tensors from ``accelerated.py``, runs SigLIP/NaViT attention, +projects features to the Bagel MoT hidden size, adds visual position embeddings, +and returns ``{"image_embeds": ..., "token_lens": ...}``. The post-hook writes +those feature spans back to the image ``ConversationItem.value`` fields. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch +import torch.nn as nn +from transformers.activations import ACT2FN + +from veomni.ops import VeomniOp + +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from .configuration import BagelSiglipNavitConfig +from .processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN, BagelSiglipNavitPreprocessor, BagelSiglipNavitProcessor + + +def select_siglip_image_items( + conversation_list: list[list[ConversationItem]] | None, +) -> list[ConversationItem]: + """Select BAGEL SigLIP image carriers (training + inference).""" + if conversation_list is None: + raise ValueError("BagelSiglipNavit requires conversation_list to select image items.") + + return list( + iter_desired_items( + conversation_list, + types=["image"], + meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}, + ) + ) + + +def siglip_inputs_from_preprocessed_items( + image_items: list[ConversationItem], + *, + device: torch.device, + dtype: torch.dtype, +) -> dict[str, Any]: + """Build NaViT forward inputs from CPU-preprocessed image items.""" + token_lens = torch.tensor( + [int(item.meta[_OMNI_TOKEN_LEN]) for item in image_items], + dtype=torch.int32, + device=device, + ) + return { + "patchified_pixel_values": torch.cat([item.value for item in image_items], dim=0).to( + device=device, dtype=dtype, non_blocking=True + ), + "patchified_position_ids": torch.cat( + [item.meta[_OMNI_POSITION_IDS] for item in image_items], + dim=0, + ).to(device=device, dtype=torch.long, non_blocking=True), + "cu_seqlens": torch.nn.functional.pad(torch.cumsum(token_lens, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(token_lens.max().item()), + "token_lens": token_lens, + } + + +def scatter_siglip_image_embeds( + image_items: list[ConversationItem], + image_embeds: torch.Tensor, + token_lens: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Scatter NaViT outputs back onto conversation image items.""" + offset = 0 + lengths = token_lens.detach().cpu().reshape(-1).tolist() + for item, length in zip(image_items, lengths, strict=True): + item.value = image_embeds[offset : offset + int(length)].to(device=device, dtype=dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + offset += int(length) + + if offset != int(image_embeds.shape[0]): + raise RuntimeError("BAGEL SigLIP token count mismatch during feature scatter.") + + +class InferenceMixin: + """FSM ``generate`` — HF ``GenerationMixin`` analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelSiglipNavit`'s bases for consistency with every other + module's native / accelerated split (see ``janus/llama/modeling.py`` for + the full MRO rationale where a competing no-op default exists). + """ + + def generate( + self, + conversation_list: list[ConversationItem] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Encode SigLIP context images and scatter visual embeddings onto ``conversation_list``.""" + del kwargs + batched = [conversation_list] + image_items = select_siglip_image_items(batched) + if not image_items: + return {"conversation_list": conversation_list} + + inputs = siglip_inputs_from_preprocessed_items(image_items, device=self.device, dtype=self.dtype) + outputs = self.forward(**inputs) + token_lens = outputs.get("token_lens", inputs["token_lens"]) + scatter_siglip_image_embeds( + image_items, + outputs["image_embeds"], + token_lens, + device=self.device, + dtype=self.dtype, + ) + return {"conversation_list": batched[0]} + + +class BagelSiglipNavit(InferenceMixin, PretrainedOmniModule): + config_class = BagelSiglipNavitConfig + image_processor_class = BagelSiglipNavitProcessor + preprocessor_class = BagelSiglipNavitPreprocessor + base_model_prefix = "bagel_siglip_navit" + main_input_name = "patchified_pixel_values" + _no_split_modules = ["BagelSiglipEncoderLayer"] + supports_gradient_checkpointing = True + _supports_sdpa = True + _supports_flash_attn = True + + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__(config) + self.vision_model = BagelSiglipVisionTransformer(config) + self.connector = MLPConnector(config.hidden_size, config.output_size, config.connector_act) + self.vit_pos_embed = PositionEmbedding(config.vit_max_num_patch_per_side, config.output_size) + self._image_processor: BagelSiglipNavitProcessor = None + self.post_init() + + def _init_weights(self, module: nn.Module) -> None: + if isinstance(module, PositionEmbedding): + module.reset_parameters() + return + super()._init_weights(module) + + @property + def _vit_device(self) -> torch.device: + return self.vision_model.embeddings.patch_embedding.weight.device + + @property + def _connector_device(self) -> torch.device: + return self.connector.fc1.weight.device + + @property + def _pos_embed_device(self) -> torch.device: + return self.vit_pos_embed.pos_embed.device + + def forward( # type: ignore[override] + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + token_lens: torch.IntTensor, + **kwargs: Any, + ) -> dict[str, Any]: + del kwargs + vit_hidden = self._encode_patches( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + image_embeds = self._project_with_position( + vit_hidden=vit_hidden, + patchified_position_ids=patchified_position_ids, + ) + return {"image_embeds": image_embeds, "token_lens": token_lens} + + def _encode_patches( + self, + *, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + ) -> torch.Tensor: + patchified_pixel_values = patchified_pixel_values.to(device=self._vit_device, dtype=self.dtype) + patchified_position_ids = patchified_position_ids.to(device=self._vit_device, dtype=torch.long) + cu_seqlens = cu_seqlens.to(device=self._vit_device, dtype=torch.int32) + return self.vision_model( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + + def _project_with_position( + self, + *, + vit_hidden: torch.Tensor, + patchified_position_ids: torch.LongTensor, + ) -> torch.Tensor: + vit_hidden = vit_hidden.to(device=self._connector_device, dtype=self.connector.fc1.weight.dtype) + patchified_position_ids = patchified_position_ids.to(device=self._pos_embed_device) + image_embeds = self.connector(vit_hidden) + return image_embeds + self.vit_pos_embed(patchified_position_ids).to( + device=image_embeds.device, + dtype=image_embeds.dtype, + ) + + +class RotaryEmbedding2D(nn.Module): + def __init__(self, dim: int, max_h: int, max_w: int, base: int = 10000) -> None: + super().__init__() + freq = torch.arange(0, dim, 2, dtype=torch.int64).float() / dim + inv_freq = 1.0 / (base**freq) + + grid_h = torch.arange(0, max_h).to(inv_freq.dtype) + grid_h = grid_h[:, None].repeat(1, max_w) + + grid_w = torch.arange(0, max_w).to(inv_freq.dtype) + grid_w = grid_w[None, :].repeat(max_h, 1) + + cos_h, sin_h = self._forward_one_side(grid_h, inv_freq) + cos_w, sin_w = self._forward_one_side(grid_w, inv_freq) + + self.register_buffer("cos_h", cos_h, persistent=False) + self.register_buffer("sin_h", sin_h, persistent=False) + self.register_buffer("cos_w", cos_w, persistent=False) + self.register_buffer("sin_w", sin_w, persistent=False) + + @staticmethod + def _forward_one_side(grid: torch.Tensor, inv_freq: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + freqs = grid[..., None] * inv_freq[None, None, :] + emb = torch.cat((freqs, freqs), dim=-1).flatten(0, 1) + return emb.cos(), emb.sin() + + +class PositionEmbedding(nn.Module): + def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: + super().__init__() + self.max_num_patch_per_side = max_num_patch_per_side + self.hidden_size = hidden_size + self.pos_embed = nn.Parameter( + torch.zeros(max_num_patch_per_side**2, hidden_size), + requires_grad=False, + ) + self.reset_parameters() + + def reset_parameters(self) -> None: + new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) + if hasattr(self.pos_embed.data, "to_local"): + self.pos_embed.data.to_local().copy_(new_data) + else: + self.pos_embed.data.copy_(new_data) + + def _init_weights(self) -> None: + self.reset_parameters() + + def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: + return self.pos_embed[position_ids.to(device=self.pos_embed.device)] + + +def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0).reshape([2, 1, grid_size, grid_size]) + emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + return torch.from_numpy(np.concatenate([emb_h, emb_w], axis=1)).float() + + +def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + out = np.einsum("m,d->md", pos.reshape(-1), omega) + return np.concatenate([np.sin(out), np.cos(out)], axis=1) + + +class MLPConnector(nn.Module): + def __init__(self, in_dim: int, out_dim: int, hidden_act: str) -> None: + super().__init__() + self.activation_fn = ACT2FN[hidden_act] + self.fc1 = nn.Linear(in_dim, out_dim) + self.fc2 = nn.Linear(out_dim, out_dim) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.fc2(self.activation_fn(self.fc1(hidden_states))) + + +class BagelSiglipVisionEmbeddings(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + patch_dim = config.num_channels * config.patch_size * config.patch_size + self.patch_embedding = nn.Linear(patch_dim, config.hidden_size, bias=True) + self.position_embedding = ( + None if config.rope else nn.Embedding((config.image_size // config.patch_size) ** 2, config.hidden_size) + ) + + def forward( + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + ) -> torch.Tensor: + patchified_pixel_values = patchified_pixel_values.to( + device=self.patch_embedding.weight.device, + dtype=self.patch_embedding.weight.dtype, + ) + patch_embeds = self.patch_embedding(patchified_pixel_values) + if self.position_embedding is None: + return patch_embeds + position_ids = patchified_position_ids.to(device=self.position_embedding.weight.device, dtype=torch.long) + position_embeds = self.position_embedding(position_ids) + return patch_embeds + position_embeds.to(device=patch_embeds.device, dtype=patch_embeds.dtype) + + +def _is_flash_attention_impl(impl: str | None) -> bool: + """Return whether ``impl`` names a FlashAttention backend.""" + return impl is not None and "flash_attention" in impl + + +class BagelSiglipAttention(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + """Initialize NaViT self-attention projections and instance-local ``VeomniOp`` handles.""" + super().__init__() + self.config = config + self.num_heads = config.num_attention_heads + self.head_dim = config.hidden_size // config.num_attention_heads + self.scaling = self.head_dim**-0.5 + self.attention_dropout = config.attention_dropout + self.is_causal = False + self.q_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.k_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.v_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + self.out_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=True) + + rope_impl = ( + config.ops_config.rotary_pos_emb_vision_implementation + if "rotary_pos_emb_vision_implementation" in config.ops_implementation + or "rotary_pos_emb_implementation" not in config.ops_implementation + else config.ops_config.rotary_pos_emb_implementation + ) + self.veomni_rope = VeomniOp("rope", "full", rope_impl) + + attn_impl = config.ops_config.attn_implementation + if not attn_impl or attn_impl == "eager": + attn_impl = "veomni_sdpa" + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + """Run varlen non-causal self-attention over packed NaViT patch tokens.""" + total_q_len, _ = hidden_states.size() + query_states = self.q_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + key_states = self.k_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + value_states = self.v_proj(hidden_states).view(total_q_len, self.num_heads, self.head_dim) + + if self.config.rope: + if cos_h is None or sin_h is None or cos_w is None or sin_w is None: + raise ValueError("2D RoPE tensors are required when BagelSiglipNavitConfig.rope=True.") + qh, qw = query_states[:, :, : self.head_dim // 2], query_states[:, :, self.head_dim // 2 :] + kh, kw = key_states[:, :, : self.head_dim // 2], key_states[:, :, self.head_dim // 2 :] + cos_h = cos_h.to(device=query_states.device, dtype=query_states.dtype) + sin_h = sin_h.to(device=query_states.device, dtype=query_states.dtype) + cos_w = cos_w.to(device=query_states.device, dtype=query_states.dtype) + sin_w = sin_w.to(device=query_states.device, dtype=query_states.dtype) + qh, kh = self.veomni_rope(qh, kh, cos_h, sin_h) + qw, kw = self.veomni_rope(qw, kw, cos_w, sin_w) + query_states = torch.cat([qh, qw], dim=-1) + key_states = torch.cat([kh, kw], dim=-1) + + query_states = query_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) + key_states = key_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) + value_states = value_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) + dropout = 0.0 if not self.training else self.attention_dropout + + if _is_flash_attention_impl(self.veomni_attn.impl): + attn_output, _ = self.veomni_attn( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + skip_ulysses=True, + ) + else: + lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + splits = [torch.split(tensor, lengths, dim=2) for tensor in (query_states, key_states, value_states)] + attn_outputs = [ + self.veomni_attn( + self, + q, + k, + v, + attention_mask=None, + scaling=self.scaling, + dropout=dropout, + is_causal=False, + skip_ulysses=True, + )[0] + for q, k, v in zip(*splits, strict=True) + ] + attn_output = torch.cat(attn_outputs, dim=1) + + return self.out_proj(attn_output.reshape(total_q_len, -1).to(hidden_states.dtype)) + + +class BagelSiglipMLP(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size, bias=True) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size, bias=True) + self.activation_fn = ACT2FN[config.hidden_act] + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return self.fc2(self.activation_fn(self.fc1(hidden_states))) + + +class BagelSiglipEncoderLayer(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.self_attn = BagelSiglipAttention(config) + self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.mlp = BagelSiglipMLP(config) + self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states = self.self_attn( + hidden_states=hidden_states, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + cos_h=cos_h, + sin_h=sin_h, + cos_w=cos_w, + sin_w=sin_w, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + return residual + hidden_states + + +class BagelSiglipEncoder(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.gradient_checkpointing = False + self.layers = nn.ModuleList([BagelSiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + inputs_embeds: torch.Tensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + cos_h: torch.Tensor | None = None, + sin_h: torch.Tensor | None = None, + cos_w: torch.Tensor | None = None, + sin_w: torch.Tensor | None = None, + ) -> torch.Tensor: + hidden_states = inputs_embeds + for encoder_layer in self.layers: + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func( + encoder_layer.__call__, + hidden_states, + cu_seqlens, + max_seqlen, + cos_h, + sin_h, + cos_w, + sin_w, + ) + else: + hidden_states = encoder_layer( + hidden_states, + cu_seqlens, + max_seqlen, + cos_h=cos_h, + sin_h=sin_h, + cos_w=cos_w, + sin_w=sin_w, + ) + return hidden_states + + +class BagelSiglipVisionTransformer(nn.Module): + def __init__(self, config: BagelSiglipNavitConfig) -> None: + super().__init__() + self.config = config + self.embeddings = BagelSiglipVisionEmbeddings(config) + if config.rope: + max_size = config.image_size // config.patch_size + dim_head = config.hidden_size // config.num_attention_heads + self.rope = RotaryEmbedding2D(dim_head // 2, max_size, max_size) + self.encoder = BagelSiglipEncoder(config) + self.post_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward( + self, + patchified_pixel_values: torch.Tensor, + patchified_position_ids: torch.LongTensor, + cu_seqlens: torch.IntTensor, + max_seqlen: int, + ) -> torch.Tensor: + hidden_states = self.embeddings( + patchified_pixel_values=patchified_pixel_values, + patchified_position_ids=patchified_position_ids, + ) + extra_inputs: dict[str, torch.Tensor] = {} + if self.config.rope: + rope_position_ids = patchified_position_ids.to(device=self.rope.cos_h.device, dtype=torch.long) + extra_inputs.update( + cos_h=self.rope.cos_h[rope_position_ids], + sin_h=self.rope.sin_h[rope_position_ids], + cos_w=self.rope.cos_w[rope_position_ids], + sin_w=self.rope.sin_w[rope_position_ids], + ) + last_hidden_state = self.encoder( + inputs_embeds=hidden_states, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + **extra_inputs, + ) + return self.post_layernorm(last_hidden_state) + + +__all__ = [ + "BagelSiglipNavit", + "BagelSiglipNavitConfig", + "InferenceMixin", +] diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py new file mode 100644 index 0000000000..ba7ac3fe72 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/processing.py @@ -0,0 +1,438 @@ +"""Image processor + worker-side preprocessor for BAGEL SigLIP NaViT. + +:class:`BagelSiglipNavitPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) — +built straight off the checkpoint dir, with no model instance involved. Unlike +Janus, BAGEL ships no separate ``preprocessor_config.json``: the image +processor is fully derived from the module's own ``config.json`` via +:meth:`BagelSiglipNavitProcessor.from_config`. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TVF +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature + +from ....utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from ....utils.conversation import _OMNI_TOKEN_LEN as _CONVERSATION_TOKEN_LEN +from ...module_processing_base import ModulePreprocessorBase +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from .configuration import BagelSiglipNavitConfig + + +class BagelSiglipNavitProcessor(BaseImageProcessor): + """BAGEL SigLIP NaViT image processor. + + Converts raw images into flattened patch rows plus the varlen metadata that + the NaViT tower consumes. Carrier selection and embed scatter stay in the + module mixin because they are SeedOmni conversation semantics. + """ + + model_input_names = [ + "patchified_pixel_values", + "patchified_position_ids", + "cu_seqlens", + "max_seqlen", + "token_lens", + ] + + def __init__( + self, + patch_size: int = 14, + image_size: int = 980, + min_image_size: int = 378, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + vit_max_num_patch_per_side: int = 70, + num_channels: int = 3, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.patch_size = patch_size + self.image_size = image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.vit_max_num_patch_per_side = vit_max_num_patch_per_side + self.num_channels = num_channels + + @classmethod + def from_config(cls, config: Any) -> BagelSiglipNavitProcessor: + return cls( + patch_size=int(config.patch_size), + image_size=int(config.image_size), + min_image_size=int(config.min_image_size), + max_pixels=int(config.max_pixels), + image_mean=list(config.image_mean), + image_std=list(config.image_std), + vit_max_num_patch_per_side=int(config.vit_max_num_patch_per_side), + num_channels=int(config.num_channels), + ) + + def preprocess( + self, + images: Any, + *, + return_tensors: str | None = None, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + **kwargs: Any, + ) -> BatchFeature: + """Resize, normalize, and patchify ``images`` into NaViT varlen batch tensors.""" + del return_tensors, kwargs + image_list = images if isinstance(images, list) else [images] + + pixel_values: list[torch.Tensor] = [] + position_ids: list[torch.Tensor] = [] + token_lens: list[int] = [] + for image in image_list: + image_tensor = _preprocess_image( + image, + patch_size=self.patch_size, + image_size=self.image_size, + min_image_size=self.min_image_size, + max_pixels=self.max_pixels, + image_mean=self.image_mean, + image_std=self.image_std, + ) + patches = _patchify_image(image_tensor, self.patch_size) + positions = _flattened_position_ids( + image_tensor.shape[-2], + image_tensor.shape[-1], + patch_size=self.patch_size, + max_num_patches_per_side=self.vit_max_num_patch_per_side, + ) + pixel_values.append(patches) + position_ids.append(positions) + token_lens.append(int(patches.shape[0])) + + tensor_device = torch.device("cpu") if device is None else device + token_lens_tensor = torch.tensor(token_lens, dtype=torch.int32, device=tensor_device) + pixel_tensor = torch.cat(pixel_values, dim=0) + if dtype is not None: + pixel_tensor = pixel_tensor.to(dtype=dtype) + return BatchFeature( + data={ + "patchified_pixel_values": pixel_tensor.to(device=tensor_device), + "patchified_position_ids": torch.cat(position_ids, dim=0).to(device=tensor_device, dtype=torch.long), + "cu_seqlens": F.pad(torch.cumsum(token_lens_tensor, dim=0), (1, 0)).to(torch.int32), + "max_seqlen": int(token_lens_tensor.max().item()), + "token_lens": token_lens_tensor, + } + ) + + +def _preprocess_image( + image: Any, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + image_mean: list[float], + image_std: list[float], +) -> torch.Tensor: + """Convert ``image`` to a normalized ``[3, H, W]`` float32 tensor aligned to ``patch_size``.""" + if isinstance(image, (Image.Image, np.ndarray)): + pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image)) + pil_image = _resize_pil( + pil_image, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + tensor = _pil_to_rgb_tensor(pil_image) + else: + tensor = _to_rgb_tensor(image) + tensor = _resize_tensor( + tensor, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + return tensor.sub(mean).div(std) + + +def _to_rgb_tensor(image: Any) -> torch.Tensor: + """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``.""" + if isinstance(image, Image.Image): + pil_image = _to_rgb_pil(image) + array = np.array(pil_image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + if isinstance(image, np.ndarray): + return _to_rgb_tensor(Image.fromarray(image)) + if torch.is_tensor(image): + tensor = image.detach().to(dtype=torch.float32) + if tensor.dim() != 3: + raise TypeError(f"BAGEL image tensor must be 3-D, got shape {tuple(tensor.shape)}.") + if tensor.shape[0] in (1, 3, 4): + pass + elif tensor.shape[-1] in (1, 3, 4): + tensor = tensor.permute(2, 0, 1).contiguous() + else: + raise TypeError(f"Unable to infer channel dimension for BAGEL image tensor {tuple(tensor.shape)}.") + if tensor.shape[0] == 1: + tensor = tensor.repeat(3, 1, 1) + elif tensor.shape[0] == 4: + alpha = tensor[3:4] + if alpha.max() > 1: + alpha = alpha / 255.0 + rgb = tensor[:3] + if rgb.max() > 1: + rgb = rgb / 255.0 + tensor = rgb * alpha + (1.0 - alpha) + if tensor.max() > 1: + tensor = tensor / 255.0 + return tensor[:3].clamp(0.0, 1.0) + raise TypeError(f"BAGEL image item value must be PIL, numpy, or tensor, got {type(image).__name__}.") + + +def _resize_tensor( + tensor: torch.Tensor, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> torch.Tensor: + """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to the target patch grid.""" + height, width = tensor.shape[-2:] + new_width, new_height = _target_size( + width, + height, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + img_num=img_num, + ) + if new_width == width and new_height == height: + return tensor + resized = F.interpolate( + tensor.unsqueeze(0), + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + antialias=True, + ) + return resized.squeeze(0).clamp(0.0, 1.0) + + +def _resize_pil( + image: Image.Image, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> Image.Image: + """Resize a PIL image with bicubic interpolation to the target patch grid.""" + width, height = image.size + new_width, new_height = _target_size( + width, + height, + patch_size=patch_size, + image_size=image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + img_num=img_num, + ) + if new_width == width and new_height == height: + return image + return TVF.resize( + image, + (new_height, new_width), + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ) + + +def _target_size( + width: int, + height: int, + *, + patch_size: int, + image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int, +) -> tuple[int, int]: + """Compute patch-stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds.""" + scale = min(image_size / max(width, height), 1.0) + scale = max(scale, min_image_size / min(width, height)) + new_width, new_height = _apply_scale(width, height, scale, patch_size) + if new_width * new_height > max_pixels / img_num: + scale = max_pixels / img_num / (new_width * new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size) + if max(new_width, new_height) > image_size: + scale = image_size / max(new_width, new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, patch_size) + return new_width, new_height + + +def _to_rgb_pil(image: Image.Image) -> Image.Image: + """Convert ``image`` to RGB, compositing transparency over a white background.""" + if image.mode == "RGBA" or image.info.get("transparency", None) is not None: + rgba = image.convert("RGBA") + white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255)) + white.paste(rgba, mask=rgba.split()[3]) + return white + return image.convert("RGB") + + +def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor: + array = np.array(image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + + +def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]: + new_width = round(width * scale) + new_height = round(height * scale) + return ( + max(stride, int(round(new_width / stride) * stride)), + max(stride, int(round(new_height / stride) * stride)), + ) + + +def _patchify_image(image: torch.Tensor, patch_size: int) -> torch.Tensor: + """Flatten a ``[C, H, W]`` image tensor into ``[(H/P)*(W/P), P*P*C]`` patch rows.""" + channels, height, width = image.shape + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError("BAGEL preprocessed image height and width must be divisible by patch_size.") + image = image.reshape(channels, height // patch_size, patch_size, width // patch_size, patch_size) + image = torch.einsum("chpwq->hwpqc", image) + return image.reshape(-1, patch_size**2 * channels) + + +def _flattened_position_ids( + height: int, + width: int, + *, + patch_size: int, + max_num_patches_per_side: int, +) -> torch.Tensor: + num_patches_h = height // patch_size + num_patches_w = width // patch_size + coords_h = torch.arange(0, num_patches_h, dtype=torch.long) + coords_w = torch.arange(0, num_patches_w, dtype=torch.long) + return (coords_h[:, None] * max_num_patches_per_side + coords_w).flatten() + + +_OMNI_POSITION_IDS = "bagel_siglip_navit_position_ids" +_OMNI_TOKEN_LEN = "bagel_siglip_navit_token_len" + + +class BagelSiglipNavitPreprocessor(ModulePreprocessorBase): + """Worker-side image patchify for BAGEL SigLIP NaViT context images.""" + + def __init__( + self, + image_processor: Any, + dtype: torch.dtype | None = None, + dummy_pixel_values: torch.Tensor | None = None, + ) -> None: + self._image_processor = image_processor + self._dtype = dtype + self._dummy_pixel_values = dummy_pixel_values + + @classmethod + def from_pretrained( + cls, module_path: str, *, config_overrides: dict[str, Any] | None = None, **kwargs: Any + ) -> BagelSiglipNavitPreprocessor: + """Build straight from the checkpoint dir — no model instance needed. + + BAGEL ships no standalone ``preprocessor_config.json``; the image + processor is derived from the module's own ``config.json``. + + ``config_overrides`` (the module's YAML ``model_config:`` block) is + applied on top of the on-disk default before deriving the image + processor's geometry, so it agrees with the live model's own config. + """ + del kwargs + # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare. + config = BagelSiglipNavitConfig.from_pretrained(module_path, model_config=config_overrides) + return cls(BagelSiglipNavitProcessor.from_config(config)) + + def bind_dummy_inputs(self, config: BagelSiglipNavitConfig, dtype: torch.dtype | None = None) -> None: + """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model.""" + patch_dim = int(config.num_channels) * int(config.patch_size) * int(config.patch_size) + self._dtype = dtype + self._dummy_pixel_values = torch.zeros(1, patch_dim, dtype=dtype) + + def preprocess_conversations( + self, + conversation_list: list[list[ConversationItem]], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> None: + """Patchify SigLIP context images in place and append per-sample dummy anchors in training.""" + del generation_kwargs + + image_items: list[ConversationItem] = [] + for sample in conversation_list: + sample_image_items = list( + iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_SIGLIP_CONTEXT]}) + ) + if sample_image_items: + image_items.extend(item for item in sample_image_items if not item.is_dummy) + elif not inference: + if self._dummy_pixel_values is None: + raise RuntimeError( + f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() " + "before training use (pure inference never reaches this branch)." + ) + sample.append( + ConversationItem( + type="image", + value=self._dummy_pixel_values.to(dtype=self._dtype).clone(), + role="user", + is_dummy=True, + meta={ + _IMG_TAG_KEY: "und", + BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, + _OMNI_POSITION_IDS: torch.zeros(1, dtype=torch.long), + _OMNI_TOKEN_LEN: 1, + _CONVERSATION_TOKEN_LEN: 1, + }, + ) + ) + + if not image_items: + return + + inputs = self._image_processor( + images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype + ) + lengths = inputs["token_lens"].detach().cpu().reshape(-1).tolist() + pixel_chunks = torch.split(inputs["patchified_pixel_values"], lengths, dim=0) + position_chunks = torch.split(inputs["patchified_position_ids"], lengths, dim=0) + for item, pixels, position_ids, length in zip( + image_items, pixel_chunks, position_chunks, lengths, strict=True + ): + item.value = pixels.to(dtype=self._dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + item.meta[_OMNI_POSITION_IDS] = position_ids.to(dtype=torch.long) + item.meta[_OMNI_TOKEN_LEN] = int(length) + item.meta[_CONVERSATION_TOKEN_LEN] = int(length) + + +__all__ = [ + "BagelSiglipNavitProcessor", + "BagelSiglipNavitPreprocessor", +] diff --git a/veomni/models/seed_omni/modules/bagel/sources.py b/veomni/models/seed_omni/modules/bagel/sources.py new file mode 100644 index 0000000000..e58bdd1f9a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/sources.py @@ -0,0 +1,69 @@ +"""BAGEL-local ``ConversationItem.meta`` routing tags. + +Items carry no module ownership (see :mod:`...utils.conversation`), so BAGEL +routes its rows with two BAGEL-private meta keys: + +* ``meta[BAGEL_CONTEXT_KEY]`` — which image context copy a row belongs to: + ``BAGEL_SIGLIP_CONTEXT`` (understanding features, read by SigLIP-NaViT) or + ``BAGEL_VAE_CONTEXT`` (latent context/target, read by the VAE and flow + connector). The VAE preprocessor expands each raw image by its data-layer + ``_IMG_TAG_KEY`` into one or both copies — ``_img_tag`` alone cannot tell the + two copies of one ``edit`` image apart. The chat template stamps the same tag + on the vision start/end marker rows around each copy, and MoT reads it to + pick attention mode and expert routing. +* ``meta[BAGEL_PHASE_KEY]`` — the inference/flow step an ``output`` row is in: + the understanding start token, then per denoise step flow query -> hidden -> + velocity, and finally the generated latent handed to VAE decode. + +An FSDP placeholder is flagged ``is_dummy=True`` and keeps the context tag of +the rows it stands in for, so each encoder selects real and dummy rows with +one filter. +""" + +from __future__ import annotations + +from ...utils.conversation import ConversationItem, iter_desired_items + + +BAGEL_CONTEXT_KEY = "bagel_context" +BAGEL_SIGLIP_CONTEXT = "siglip" +BAGEL_VAE_CONTEXT = "vae" + +BAGEL_PHASE_KEY = "bagel_phase" +BAGEL_START_TOKEN = "start_token" +BAGEL_FLOW_QUERY = "flow_query" +BAGEL_FLOW_HIDDEN = "flow_hidden" +BAGEL_FLOW_VELOCITY = "flow_velocity" +BAGEL_GENERATED_LATENT = "generated_latent" + + +def bagel_context(item: ConversationItem) -> str | None: + return item.meta.get(BAGEL_CONTEXT_KEY) + + +def bagel_phase(item: ConversationItem) -> str | None: + return item.meta.get(BAGEL_PHASE_KEY) + + +def get_tail_phase_item(conversation: list[ConversationItem], phase: str) -> ConversationItem | None: + """Return the latest ``output`` row in ``phase`` of a single conversation.""" + return next( + iter_desired_items([conversation], types=["output"], reverse_item=True, meta={BAGEL_PHASE_KEY: [phase]}), + None, + ) + + +__all__ = [ + "BAGEL_CONTEXT_KEY", + "BAGEL_FLOW_HIDDEN", + "BAGEL_FLOW_QUERY", + "BAGEL_FLOW_VELOCITY", + "BAGEL_GENERATED_LATENT", + "BAGEL_PHASE_KEY", + "BAGEL_SIGLIP_CONTEXT", + "BAGEL_START_TOKEN", + "BAGEL_VAE_CONTEXT", + "bagel_context", + "bagel_phase", + "get_tail_phase_item", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py new file mode 100644 index 0000000000..f780d497c9 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/__init__.py @@ -0,0 +1,24 @@ +"""BAGEL token embedding and LM head module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_config(): + from .configuration import BagelTextEncoderConfig + + return BagelTextEncoderConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_model(): + from .modeling import BagelTextEncoder + + return BagelTextEncoder + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_text_encoder") +def register_bagel_text_encoder_accelerated_model(): + from .accelerated import BagelTextEncoderAccelerated + + return BagelTextEncoderAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py new file mode 100644 index 0000000000..1a106ff116 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/__init__.py @@ -0,0 +1,10 @@ +"""Accelerated runtime for the bagel/text_encoder OmniModule (conversation graph).""" + +from .accelerated import ( + BagelTextEncoderAccelerated, +) + + +__all__ = [ + "BagelTextEncoderAccelerated", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py new file mode 100644 index 0000000000..48ec08ae65 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/accelerated/accelerated.py @@ -0,0 +1,176 @@ +"""VeOmni-accelerated BagelTextEncoder — training graph hooks. + +FSM ``generate`` (and ``encode_image_markers``) now live natively on +:class:`~.modeling.BagelTextEncoder`; this file only owns the training +pre/forward/post hooks (packed-token embedding + MoT CE loss). +""" + +from typing import Any, Dict, List, Optional, Tuple + +import torch +from transformers import PreTrainedTokenizerBase + +from .....mixins.training_module_mixin import post_forward, pre_forward +from .....utils.conversation import ConversationItem, iter_desired_items +from ....text_encoder.accelerated import ( + TrainingMixin as BaseTrainingMixin, +) +from ....text_encoder.accelerated import ( + VeOmniMixin as BaseVeOmniMixin, +) +from ..chat_template import BagelChatTemplate +from ..configuration import BagelTextEncoderConfig +from ..modeling import BagelTextEncoder, prepare_bagel_encode_input_ids + + +class TrainingMixin(BaseTrainingMixin): + config: BagelTextEncoderConfig + device: torch.device + dtype: torch.dtype + _chat_template: BagelChatTemplate | None + _encode_batch_shape: torch.LongTensor | None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._chat_template: Optional[BagelChatTemplate] = None + + def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]: + """Delegate to native impl — overrides ``TrainingModuleMixin.dummy_inputs``'s + unrelated ``(*, batch_size, device, dtype)`` signature, which otherwise wins + this name collision by sitting ahead of :class:`BagelTextEncoder` in MRO. + """ + return BagelTextEncoder.dummy_inputs(self, kind=kind) + + @pre_forward("encode") + def encode_pre( + self, + conversation_list: Optional[List[List[ConversationItem]]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + return super().encode_pre(conversation_list, **kwargs) + + @post_forward("encode") + def encode_post(self, **outputs: Any) -> Dict[str, Any]: + return super().encode_post(**outputs) + + @pre_forward("decode") + def decode_pre( + self, + conversation_list: Optional[List[List[ConversationItem]]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + return super().decode_pre(conversation_list, **kwargs) + + @post_forward("decode") + def decode_post(self, **outputs: Any) -> Dict[str, Any]: + return super().decode_post(**outputs) + + def _anchor_dummy_decode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + dummy: Dict[str, torch.Tensor], + ) -> Dict[str, torch.Tensor]: + """Tie dummy CE loss to MoT hidden states without changing its value.""" + if conversation_list is None: + return dummy + + anchor = None + for item in iter_desired_items( + conversation_list, types=["text", "image", "output"], roles=["user", "assistant"] + ): + if item.is_dummy: + continue + value = item.value + if not torch.is_tensor(value): + continue + if value.dim() == 3 and value.shape[0] == 1: + value = value.squeeze(0) + if value.dim() == 2 and int(value.shape[-1]) == int(self.config.hidden_size): + anchor = value.to(device=self.device, dtype=self.dtype).sum() * 0.0 + break + if anchor is None: + return dummy + + return { + "hidden_states": dummy["hidden_states"] + anchor, + "labels": dummy["labels"], + } + + def _prepare_encode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + ) -> torch.Tensor: + """Pack non-dummy text token IDs into a flat tensor and record segment lengths.""" + if conversation_list is None: + raise ValueError("BagelTextEncoder._prepare_encode_inputs requires conversation_list.") + + input_ids, self._encode_batch_shape = prepare_bagel_encode_input_ids( + conversation_list, + device=self.device, + fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"], + ) + return input_ids + + def _prepare_decode_inputs( + self, + conversation_list: Optional[List[List[ConversationItem]]], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Collect text hidden states and pre-shifted causal LM labels across ``conversation_list``.""" + if conversation_list is None: + raise ValueError("BagelTextEncoder._prepare_decode_inputs requires conversation_list.") + + hidden_parts: List[torch.Tensor] = [] + shift_label_parts: List[torch.Tensor] = [] + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + + hidden_states = item.value + labels = item.meta["labels"] + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + labels = labels.reshape(-1) + if hidden_states.shape[0] != labels.shape[0]: + raise ValueError( + "BAGEL text decode requires hidden-state and label lengths to match: " + f"got {hidden_states.shape[0]} and {labels.shape[0]}." + ) + + shift_labels = torch.full_like(labels, -100, dtype=torch.long) + shift_labels[:-1] = labels[1:] + hidden_parts.append(hidden_states.to(device=self.device, dtype=self.dtype)) + shift_label_parts.append(shift_labels) + + if hidden_parts: + hidden_states = torch.cat(hidden_parts, dim=0) + shift_labels = torch.cat(shift_label_parts, dim=0).to(device=hidden_states.device, non_blocking=True) + if torch.any(shift_labels != -100): + return hidden_states, shift_labels + + dummy = self._anchor_dummy_decode_inputs(conversation_list, self.dummy_inputs(kind="decode")) + return dummy["hidden_states"], dummy["labels"] + + +class VeOmniMixin(TrainingMixin, BaseVeOmniMixin): + """BAGEL text-embedding accelerated wrapper — chat-template binding + training hooks. + + The T2I generation FSM (``generate`` / ``encode_image_markers``) lives on + the native :class:`~.modeling.BagelTextEncoder`; only the chat template + binding and CE-loss training hooks are accelerated-specific. + """ + + @property + def tokenizer(self) -> PreTrainedTokenizerBase: + return self._tokenizer + + @tokenizer.setter + def tokenizer(self, tokenizer: PreTrainedTokenizerBase) -> None: + self._tokenizer = tokenizer + self._chat_template = BagelChatTemplate(tokenizer) + + +class BagelTextEncoderAccelerated(VeOmniMixin, BagelTextEncoder): + pass + + +__all__ = ["BagelTextEncoderAccelerated"] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py new file mode 100644 index 0000000000..4fe4b089f2 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py @@ -0,0 +1,195 @@ +"""BAGEL chat template for text/materialized image prompt rows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +from transformers import PreTrainedTokenizerBase + +from ....utils.conversation import _OMNI_TOKEN_LEN, ConversationItem +from ...text_encoder.chat_template import ChatMarkers, TextEncoderChatTemplate +from ..sources import BAGEL_CONTEXT_KEY, bagel_context +from .processing import is_bagel_vision_marker + + +_OMNI_TOKENIZED = "_omni_tokenized" + + +@dataclass(frozen=True) +class BagelChatMarkers(ChatMarkers): + vision_start_token: str + vision_end_token: str + + +class BagelChatTemplate(TextEncoderChatTemplate): + """BAGEL-specific template over SeedOmni conversation rows. + + The template is intentionally row-preserving: text rows are tokenized, and + image rows are bracketed by marker text rows, but marker-image-marker packing + remains a MoT responsibility. + """ + + chat_markers: BagelChatMarkers + + def __init__(self, tokenizer: PreTrainedTokenizerBase) -> None: + super().__init__(tokenizer) + + bos_token = "<|im_start|>" + eos_token = "<|im_end|>" + vision_start_token = "<|vision_start|>" + vision_end_token = "<|vision_end|>" + + self.bos_token_id = self._resolve_token_id_from_token(tokenizer, bos_token) + self.eos_token_id = self._resolve_token_id_from_token(tokenizer, eos_token) + self.vision_start_token_id = self._resolve_token_id_from_token(tokenizer, vision_start_token) + self.vision_end_token_id = self._resolve_token_id_from_token(tokenizer, vision_end_token) + + self.chat_markers = BagelChatMarkers( + bos_token=bos_token, + eos_token=eos_token, + vision_start_token=vision_start_token, + vision_end_token=vision_end_token, + ) + + @staticmethod + def _resolve_token_id_from_token(tokenizer: PreTrainedTokenizerBase, token: str) -> int: + """Look up ``token`` in ``tokenizer`` and raise if it is missing or maps to ``unk_token_id``.""" + token_id = tokenizer.convert_tokens_to_ids(token) + if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None): + raise ValueError(f"BAGEL tokenizer is missing required token: {token!r}.") + return int(token_id) + + def tokenize_conversation( + self, + sample: list[ConversationItem], + *, + add_generation_prompt: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Apply the BAGEL chat template, wrap image rows with vision markers, and tokenize text rows.""" + parts = self.apply_chat_template( + sample, + inference=add_generation_prompt, + generation_kwargs=generation_kwargs, + ) + if add_generation_prompt: + parts = self.apply_generation_prompt(parts) + self.tokenize(parts) + return self.merge_text_embeds(parts) + + def apply_chat_template( + self, + sample: list[ConversationItem], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Wrap untokenized text items with ``<|im_start|>...<|im_end|>`` and insert image marker items.""" + del inference, generation_kwargs + + for item in sample: + if item.type == "text" and not is_bagel_vision_marker(item): + if not item.meta.get(_OMNI_TOKENIZED) and not torch.is_tensor(item.value): + item.value = f"{self.chat_markers.bos_token}{item.value}{self.chat_markers.eos_token}" + item.meta["loss_mask"] = int(item.role == "assistant") + + return self._insert_prompt_context_image_markers(sample) + + def apply_generation_prompt(self, sample: list[ConversationItem]) -> list[ConversationItem]: + return sample + + def tokenize(self, parts: list[ConversationItem]) -> None: + """Tokenize untokenized text items in place and populate ``input_ids``, ``labels``, and ``attention_mask``.""" + for item in parts: + if item.type != "text" or item.is_dummy or item.meta.get(_OMNI_TOKENIZED): + continue + + token_ids = self._token_ids_for_text_item(item) + loss_mask = int(item.meta.pop("loss_mask", int(item.role == "assistant"))) + item.value = token_ids + item.meta["input_ids"] = token_ids.detach() + item.meta["attention_mask"] = torch.ones_like(token_ids, dtype=torch.long) + item.meta["labels"] = ( + token_ids.detach().clone() if loss_mask else torch.full_like(token_ids, -100, dtype=torch.long) + ) + item.meta[_OMNI_TOKEN_LEN] = int(token_ids.numel()) + item.meta[_OMNI_TOKENIZED] = True + + if ( + item.value in {self.chat_markers.bos_token, self.chat_markers.eos_token} + and int(token_ids.numel()) != 1 + ): + raise ValueError("BAGEL start/end tokens must tokenize to exactly one token.") + + @staticmethod + def merge_text_embeds(parts: list[ConversationItem]) -> list[ConversationItem]: + return parts + + def _token_ids_for_text_item(self, item: ConversationItem) -> torch.Tensor: + """Convert ``item.value`` (tensor or string) into a 1-D int64 token-id tensor.""" + value = item.value + if torch.is_tensor(value): + token_ids = value.detach().reshape(-1).to(dtype=torch.long) + else: + token_ids = torch.tensor( + self.tokenizer(str(value), add_special_tokens=False)["input_ids"], + dtype=torch.long, + ) + return token_ids + + def _insert_prompt_context_image_markers(self, sample: list[ConversationItem]) -> list[ConversationItem]: + """Bracket each non-dummy image item with ``<|vision_start|>`` and ``<|vision_end|>`` marker items.""" + out: list[ConversationItem] = [] + + index = 0 + while index < len(sample): + item = sample[index] + + if item.type != "image" or item.is_dummy: + out.append(item) + index += 1 + continue + + # Add vision marker for image items: + image + . + # Markers carry the image's context tag so MoT can regroup the triplet. + context = bagel_context(item) + marker_meta = {"loss_mask": 0} if context is None else {"loss_mask": 0, BAGEL_CONTEXT_KEY: context} + if not (out and bagel_context(out[-1]) == context and is_bagel_vision_marker(out[-1], context=context)): + out.append( + ConversationItem( + type="text", + value=self.chat_markers.vision_start_token, + role=item.role, + meta=dict(marker_meta), + ) + ) + + out.append(item) + + if ( + index + 1 < len(sample) + and bagel_context(sample[index + 1]) == context + and is_bagel_vision_marker(sample[index + 1], context=context) + ): + out.append(sample[index + 1]) + index += 2 + else: + out.append( + ConversationItem( + type="text", + value=self.chat_markers.vision_end_token, + role=item.role, + meta=dict(marker_meta), + ) + ) + index += 1 + + return out + + +__all__ = [ + "BagelChatMarkers", + "BagelChatTemplate", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py new file mode 100644 index 0000000000..da13aff996 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/configuration.py @@ -0,0 +1,12 @@ +"""Configuration for :class:`BagelTextEncoder`.""" + +from ...text_encoder.configuration import TextEncoderConfig + + +class BagelTextEncoderConfig(TextEncoderConfig): + """BAGEL text encoder config.""" + + model_type = "bagel_text_encoder" + + def __init__(self, *args, tie_word_embeddings: bool = True, lm_head_bias: bool = False, **kwargs): + super().__init__(*args, tie_word_embeddings=tie_word_embeddings, lm_head_bias=lm_head_bias, **kwargs) diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py new file mode 100644 index 0000000000..2af5cbf15a --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py @@ -0,0 +1,193 @@ +"""BAGEL token embedding and LM head module.""" + +from typing import Any, Dict, List, Optional + +import torch + +from veomni.utils.tensor_utils import naflatten, unflatten + +from ....graphs.generation_graph import FSM_SIGNAL_KEY +from ....utils.conversation import ConversationItem, iter_desired_items, maybe_merge_outputs +from ...text_encoder.modeling import TextEncoder +from ..sources import BAGEL_FLOW_QUERY, BAGEL_PHASE_KEY, BAGEL_START_TOKEN +from .configuration import BagelTextEncoderConfig +from .processing import BagelTextEncoderPreprocessor, apply_image_marker + + +SIGNAL_START_IMAGE_GEN = "start_image_gen" +SIGNAL_TEXT_DONE = "text_done" + +# Sentinel written by BagelTextEncoderPreprocessor onto every text item so +# encode_pre can skip tokenizer work already completed by a DataLoader worker. +_OMNI_TOKENIZED = "_omni_tokenized" + + +def prepare_bagel_encode_input_ids( + conversation_list: Optional[List[List[ConversationItem]]], + *, + device: torch.device, + fallback_input_ids: torch.Tensor, +) -> tuple[torch.Tensor, torch.LongTensor | None]: + """Pack CPU-preprocessed BAGEL text token ids (training + inference).""" + if conversation_list is None: + raise ValueError("BagelTextEncoder.prepare_bagel_encode_input_ids requires conversation_list.") + + input_ids: List[torch.Tensor] = [] + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + if not item.meta.get(_OMNI_TOKENIZED): + raise ValueError("BAGEL text encoder expects CPU-preprocessed text items.") + token_ids = item.meta.get("input_ids") + input_ids.append(token_ids.reshape(-1)) + + if not input_ids: + return fallback_input_ids, None + + flat_ids, batch_shape = naflatten(input_ids) + return flat_ids.to(device, non_blocking=True), batch_shape + + +def scatter_bagel_text_embeds( + conversation_list: List[List[ConversationItem]], + segment_embeds: List[torch.Tensor], + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Write BAGEL text embeds back onto conversation text items.""" + segment_embeds_iterator = iter(segment_embeds) + for item in iter_desired_items(conversation_list, types=["text"]): + if item.is_dummy: + continue + item.value = next(segment_embeds_iterator).to(device=device, dtype=dtype) + if next(segment_embeds_iterator, None) is not None: + raise RuntimeError("BAGEL text segment count mismatch during embed scatter.") + + +class BagelTextEncoder(TextEncoder): + """BAGEL token embedding + LM head, plus the T2I generation FSM. + + Only the T2I generation FSM (image-gen signal, marker embedding) is + genuinely BAGEL-specific; accelerated-only behavior lives in + ``accelerated.py``. + """ + + config_class = BagelTextEncoderConfig + preprocessor_class = BagelTextEncoderPreprocessor + + def generate( + self, + conversation_list: Optional[List[ConversationItem]] = None, + generation_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Embed prompt text tokens, sample AR decode tokens, or emit ``start_image_gen``.""" + tail = conversation_list[-1] + batched = [conversation_list] + + if tail.type == "text" and tail.role == "user": + input_ids, batch_shape = prepare_bagel_encode_input_ids( + batched, + device=self.device, + fallback_input_ids=self.dummy_inputs(kind="encode")["input_ids"], + ) + inputs_embeds = self.encode(input_ids)["inputs_embeds"] + scatter_bagel_text_embeds( + batched, + unflatten(inputs_embeds, batch_shape), + device=self.device, + dtype=self.dtype, + ) + infer_type = str((generation_kwargs or {}).get("infer_type", "")) + if infer_type == "infer_und" and not self._bos_injected: + input_ids = self._token_id_tensor(self._chat_template.bos_token_id) + start_embed = self.encode(input_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype) + conversation_list.append( + ConversationItem( + type="output", + value=start_embed, + role="assistant", + meta={"input_ids": input_ids.reshape(-1).detach(), BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, + ) + ) + self._bos_injected = True + return {"conversation_list": batched[0]} + + if tail.type == "output": + outputs: Dict[str, Any] = {"conversation_list": batched[0]} + hidden_states = tail.value + if not torch.is_tensor(hidden_states): + raise TypeError("BAGEL text generate expects the tail output value to be a hidden-state tensor.") + if hidden_states.dim() == 2: + hidden_states = hidden_states.unsqueeze(0) + sampling = self._extract_sampling_kwargs(generation_kwargs, 1.0, 1.0, kwargs) + + output_token_id = self._sample_token(hidden_states, **sampling) + self._text_token_cache.append(output_token_id) + + input_ids = self._token_id_tensor(output_token_id) + inputs_embeds = self.encode(input_ids)["inputs_embeds"] + tail.value = inputs_embeds.to(device=self.device, dtype=self.dtype) + tail.meta["input_ids"] = input_ids.reshape(-1).detach() + maybe_merge_outputs(batched[0]) + if output_token_id == self._chat_template.eos_token_id: + outputs[FSM_SIGNAL_KEY] = SIGNAL_TEXT_DONE + outputs["generated"] = self._flush_text_generated(batched[0]) + return outputs + + outputs: Dict[str, Any] = {"conversation_list": batched[0]} + infer_type = str((generation_kwargs or {}).get("infer_type", "")) + if infer_type in {"infer_gen", "infer_edit"}: + outputs[FSM_SIGNAL_KEY] = SIGNAL_START_IMAGE_GEN + return outputs + + raise ValueError(f"Invalid type: {tail.type}") + + def encode_image_markers( + self, + conversation_list: Optional[List[ConversationItem]] = None, + generation_kwargs: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Bracket ``flow_query`` output items with embedded ``<|vision_start|>`` and ``<|vision_end|>`` tokens.""" + del generation_kwargs, kwargs + marker_embeds: Optional[torch.Tensor] = None + for item in iter_desired_items( + [conversation_list], + types=["output"], + meta={BAGEL_PHASE_KEY: [BAGEL_FLOW_QUERY]}, + ): + if item.is_dummy: + continue + + if marker_embeds is None: + marker_ids = torch.tensor( + [[self._chat_template.vision_start_token_id, self._chat_template.vision_end_token_id]], + dtype=torch.long, + device=self.device, + ) + marker_embeds = self.encode(marker_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype) + marker_embeds = marker_embeds.squeeze(0) + apply_image_marker(item, marker_embeds, device=self.device, dtype=self.dtype) + + return {"conversation_list": conversation_list} + + def dummy_inputs(self, kind: str = "encode") -> Dict[str, torch.Tensor]: + """Return minimal dummy ``encode`` or ``decode`` inputs for FSDP anchor execution.""" + if kind == "encode": + return {"input_ids": torch.zeros(1, device=self.device, dtype=torch.long)} + return { + "hidden_states": torch.zeros(1, int(self.config.hidden_size), device=self.device, dtype=self.dtype), + "labels": torch.full((1,), -100, device=self.device, dtype=torch.long), + } + + +__all__ = [ + "BagelTextEncoder", + "BagelTextEncoderConfig", + "SIGNAL_START_IMAGE_GEN", + "SIGNAL_TEXT_DONE", + "prepare_bagel_encode_input_ids", + "scatter_bagel_text_embeds", +] diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py new file mode 100644 index 0000000000..9fbda4b240 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/processing.py @@ -0,0 +1,110 @@ +"""Stateless text carrier helpers + worker-side preprocessor for BAGEL text encoder. + +:class:`BagelTextEncoderPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`). +Its :meth:`from_pretrained` loads the tokenizer and builds the +:class:`~veomni.models.seed_omni.modules.bagel.text_encoder.chat_template.BagelChatTemplate` +straight off the checkpoint dir — no model instance involved. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import torch + +from .....auto import build_tokenizer +from ....utils.conversation import ConversationItem +from ...text_encoder.processing import TextEncoderPreprocessor +from ..sources import BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context + + +if TYPE_CHECKING: + # chat_template.py imports is_bagel_vision_marker from this module, so the + # reverse import must stay deferred (see from_pretrained) to avoid a cycle. + from .chat_template import BagelChatTemplate + + +def is_bagel_vision_marker(item: ConversationItem, *, context: str | None = None) -> bool: + """Return whether ``item`` is a single-token text vision marker tagged with a BAGEL context.""" + if item.type != "text": + return False + if context is not None and bagel_context(item) != context: + return False + if bagel_context(item) not in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + return False + return _text_item_length(item) == 1 + + +def _text_item_length(item: ConversationItem) -> int | None: + """Return the token count of a text ``ConversationItem``, or ``None`` if untokenized.""" + value = item.value + if torch.is_tensor(value): + if value.dim() == 0: + return 1 + if value.dim() == 1: + return int(value.shape[0]) + if value.dim() == 2: + return int(value.shape[0]) + if value.dim() == 3 and int(value.shape[0]) == 1: + return int(value.shape[1]) + return None + input_ids = item.meta.get("input_ids") + if torch.is_tensor(input_ids): + return int(input_ids.reshape(-1).shape[0]) + return None + + +def apply_image_marker( + item: ConversationItem, + marker_embeds: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, +) -> None: + """Prepend and append ``marker_embeds`` to ``item.value`` if not already bracketed.""" + image_embeds = item.value + if not torch.is_tensor(image_embeds): + return + if image_embeds.dim() == 3 and image_embeds.shape[0] == 1: + image_embeds = image_embeds.squeeze(0) + if image_embeds.dim() != 2: + return + + image_embeds = image_embeds.to(device=device, dtype=dtype) + if ( + image_embeds.shape[0] >= 2 + and torch.equal(image_embeds[:1], marker_embeds[:1]) + and torch.equal(image_embeds[-1:], marker_embeds[1:]) + ): + return + item.value = torch.cat([marker_embeds[:1], image_embeds, marker_embeds[1:]], dim=0) + + +class BagelTextEncoderPreprocessor(TextEncoderPreprocessor): + """Worker-side chat-template + tokenize for BAGEL text encoder inputs.""" + + @classmethod + def build_chat_template( + cls, + module_path: str, + *, + config_overrides: dict[str, Any] | None = None, + ) -> BagelChatTemplate: + del config_overrides + from .chat_template import BagelChatTemplate # deferred: see module-level note + + return BagelChatTemplate(build_tokenizer(module_path)) + + def _tokenize_conversation_kwargs(self, inference: bool, **kwargs: Any) -> dict[str, Any]: + return { + "add_generation_prompt": inference, + "generation_kwargs": kwargs.get("generation_kwargs"), + } + + +__all__ = [ + "apply_image_marker", + "is_bagel_vision_marker", + "BagelTextEncoderPreprocessor", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/__init__.py new file mode 100644 index 0000000000..6ac7773a52 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/__init__.py @@ -0,0 +1,31 @@ +"""BAGEL latent autoencoder module.""" + +from ... import OMNI_ACCELERATED_MODEL_REGISTRY, OMNI_CONFIG_REGISTRY, OMNI_MODEL_REGISTRY, OMNI_PROCESSOR_REGISTRY + + +@OMNI_CONFIG_REGISTRY.register("bagel_vae") +def register_bagel_vae_config(): + from .configuration import BagelVAEConfig + + return BagelVAEConfig + + +@OMNI_MODEL_REGISTRY.register("bagel_vae") +def register_bagel_vae_model(): + from .modeling import BagelVAE + + return BagelVAE + + +@OMNI_ACCELERATED_MODEL_REGISTRY.register("bagel_vae") +def register_bagel_vae_accelerated_model(): + from .accelerated import BagelVAEAccelerated + + return BagelVAEAccelerated + + +@OMNI_PROCESSOR_REGISTRY.register("bagel_vae") +def register_bagel_vae_processor(): + from .processing import BagelVAEProcessor + + return BagelVAEProcessor diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py new file mode 100644 index 0000000000..907701452f --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/__init__.py @@ -0,0 +1,14 @@ +"""Accelerated runtime for the bagel/vae OmniModule (conversation graph).""" + +from .accelerated import ( + BAGEL_VAE_PIXEL_SHAPE, + BagelVAEAccelerated, + BagelVAEOfflineMixin, +) + + +__all__ = [ + "BAGEL_VAE_PIXEL_SHAPE", + "BagelVAEAccelerated", + "BagelVAEOfflineMixin", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py new file mode 100644 index 0000000000..ec2e0a8534 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -0,0 +1,332 @@ +"""SeedOmni carrier hooks for BAGEL's latent VAE module — training-graph hooks only. + +``encode_context()`` and ``decode_generated()`` (the FSM generation nodes) live +natively on :class:`~.modeling.BagelVAE` — this file only carries the training +pre/forward/post hooks and offline-encoding-cache endpoints. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from veomni.utils.device import get_device_id, get_device_type + +from .......distributed.parallel_state import get_parallel_state +from .......distributed.sequence_parallel import gather_outputs, slice_input_tensor +from .....mixins.base_mixin import BaseMixin +from .....mixins.metric_meter_mixin import MetricMeterMixin +from .....mixins.offline_encoding_mixin import OfflineEncodingMixin +from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward +from .....utils.conversation import ConversationItem +from ...sources import BAGEL_CONTEXT_KEY, BAGEL_VAE_CONTEXT +from ..configuration import BagelVAEConfig +from ..modeling import BagelVAE, select_bagel_vae_context_items +from ..processing import BAGEL_VAE_PIXEL_SHAPE, BAGEL_VAE_POSTERIOR, crop_latent_to_image_shape + + +def _posterior_from_cache( + cache: torch.Tensor, + *, + z_channels: int, +) -> tuple[tuple[torch.Tensor, torch.Tensor], bool]: + """Unpack ``(mean, logvar)`` from a cached ``(2, C, H, W)`` or ``(1, 2, C, H, W)`` posterior tensor.""" + if cache.dim() == 5 and int(cache.shape[0]) == 1: + cache = cache.squeeze(0) + if cache.dim() == 4 and int(cache.shape[0]) == 2 and int(cache.shape[1]) == z_channels: + return (cache[0].unsqueeze(0), cache[1].unsqueeze(0)), True + raise ValueError( + "BAGEL VAE posterior cache tensor must be shaped (2, C, H, W) " + f"or singleton-batched (1, 2, C, H, W); got {tuple(cache.shape)}." + ) + + +class BagelVAEOfflineMixin(OfflineEncodingMixin): + """Offline-cache tensor endpoints and training-graph hooks for VAE encode / process.""" + + config: BagelVAEConfig + device: torch.device + dtype: torch.dtype + _conversation_carrier: list[list[ConversationItem]] | None + _encode_items: list[ConversationItem] + _sp_encode_count: int | None + + def offline_encode( + self, + pixel_values: torch.Tensor, + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values`` into a stacked ``(mean, logvar)`` posterior tensor for offline caching.""" + del kwargs + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) + with self._runtime_context(pixel_values): + posterior = self._encode_posterior(pixel_values) + return {"encoded_cache": torch.cat(posterior, dim=1).to(dtype=self.dtype)} + + def online_process( + self, + encoded_cache: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Sample scaled latent grids from cached ``(mean, logvar)`` posteriors without running the encoder.""" + del kwargs + if not isinstance(encoded_cache, list): + encoded_cache = [encoded_cache] + + latents = [] + for cache in encoded_cache: + posterior, is_item_cache = _posterior_from_cache(cache, z_channels=int(self.config.z_channels)) + latent = self._sample_scaled_latents(posterior) + if is_item_cache: + latent = latent.squeeze(0) + latents.append(latent.to(dtype=cache.dtype)) + return {"latents": latents} + + @post_forward("offline_encode") + def offline_encode_post(self, encoded_cache: torch.Tensor) -> dict[str, Any]: + """Gather SP shards, crop cached posteriors to each image's shape, and store on carrier items.""" + if get_parallel_state().sp_size > 1: + # Keep the CNN's spatial outputs intact through gather; crop each + # cache item from its original pixel shape only after batch unpadding. + encoded_cache = self._gather_encode_output(encoded_cache) + + conversation = self._conversation_carrier + encode_items = self._encode_items + self._conversation_carrier = None + self._encode_items = [] + self._sp_encode_count = None + + for item, cache_tensor in zip(encode_items, encoded_cache, strict=True): + item.type = "image" + cache_tensor = crop_latent_to_image_shape( + cache_tensor, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + z_channels = int(self.config.z_channels) + if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels: + cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:]) + item.value = cache_tensor.detach().to(device=self.device, dtype=self.dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_VAE_POSTERIOR] = True + return {"conversation_list": conversation} + + @pre_forward("online_process") + def online_process_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Collect cached posterior tensors from VAE context items for ``online_process``.""" + del kwargs + self._conversation_carrier = conversation_list + self._encode_items = [] + self._sp_encode_count = None + + self._encode_items = select_bagel_vae_context_items(conversation_list) + encoded_cache: list[torch.Tensor] = [] + for item in self._encode_items: + cache = item.value + if not torch.is_tensor(cache): + raise ValueError("BAGEL VAE online_process requires tensor posterior cache.") + encoded_cache.append(cache.detach().to(device=self._online_process_device())) + return {"encoded_cache": encoded_cache} + + def _online_process_dtype(self) -> torch.dtype: + """Resolve the target dtype for ``online_process`` when the VAE is built on meta.""" + config_dtype = getattr(self.config, "dtype", None) or getattr(self.config, "torch_dtype", None) + if isinstance(config_dtype, torch.dtype): + return config_dtype + if isinstance(config_dtype, str) and hasattr(torch, config_dtype): + dtype = getattr(torch, config_dtype) + if isinstance(dtype, torch.dtype): + return dtype + return torch.get_default_dtype() + + def _online_process_device(self) -> torch.device: + """Resolve the target device for ``online_process`` when the VAE is built on meta.""" + device_type = get_device_type() + if device_type == "cpu": + return torch.device("cpu") + return torch.device(device_type, get_device_id()) + + +class TrainingMixin(TrainingModuleMixin): + """Training-graph hooks — depends on :class:`BagelVAE` modeling APIs.""" + + config: BagelVAEConfig + device: torch.device + dtype: torch.dtype + _conversation_carrier: list[list[ConversationItem]] | None + _encode_items: list[ConversationItem] + _sp_encode_count: int | None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._conversation_carrier: list[list[ConversationItem]] | None = None + self._encode_items: list[ConversationItem] = [] + self._sp_encode_count: int | None = None + + @pre_forward("encode", "offline_encode") + def encode_pre( + self, + conversation_list: list[list[ConversationItem]] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """Stack VAE context images into a batch tensor and slice across Ulysses SP ranks if enabled.""" + del kwargs + self._conversation_carrier = conversation_list + + self._encode_items = select_bagel_vae_context_items(conversation_list) + + self._metric_meter_stash_latent_tokens(self._vae_latent_token_lengths()) + + pixel_values = torch.stack([item.value for item in self._encode_items], dim=0).to( + device=self.device, dtype=self.dtype, non_blocking=True + ) + ps = get_parallel_state() + if ps.sp_size == 1: + self._sp_encode_count = None + return {"pixel_values": pixel_values} + + if ps.cp_size != 1: + raise ValueError(f"BAGEL VAE training supports Ulysses groups only; got cp_size={ps.cp_size}.") + + self._sp_encode_count = int(pixel_values.shape[0]) + if self._sp_encode_count == 0: + raise ValueError("BAGEL VAE SP requires at least one real or dummy image.") + # Slice only the image-batch dimension. Processor-added spatial zero + # padding is part of each image canvas and must pass through the CNN. + pixel_values = slice_input_tensor(pixel_values, dim=0, padding=True, group=ps.sp_group) + return {"pixel_values": pixel_values} + + @post_forward("encode", "online_process") + def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: + """Gather SP latent shards, crop spatial padding, and write latents onto conversation items.""" + if not isinstance(latents, list) and get_parallel_state().sp_size > 1: + # Remove SP's batch-padding images first. Per-image spatial latent + # padding is cropped below from BAGEL_VAE_PIXEL_SHAPE after gather. + latents = self._gather_encode_output(latents) + + conversation = self._conversation_carrier + encode_items = self._encode_items + self._conversation_carrier = None + self._encode_items = [] + self._sp_encode_count = None + + if isinstance(latents, list): + for item, latent in zip(encode_items, latents, strict=True): + item.type = "image" + item.value = latent + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + else: + for item, latent in zip(encode_items, latents, strict=True): + item.type = "image" + latent = crop_latent_to_image_shape( + latent, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + item.value = latent + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + return {"conversation_list": conversation} + + def _gather_encode_output(self, output: torch.Tensor) -> torch.Tensor: + """All-gather batch-sharded VAE outputs across the SP group and trim SP batch padding.""" + if self._sp_encode_count is None: + raise RuntimeError("BAGEL VAE SP image count was not initialized.") + output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group) + return output.narrow(0, 0, self._sp_encode_count) + + def _vae_latent_token_lengths(self) -> list[int]: + """Compute the unpadded latent grid token count per encoded image for FLOPs metering.""" + # Latent tokens per image = (H // ds) * (W // ds), using the real pixel shape + ds = max(int(self.config.downsample), 1) + lengths: list[int] = [] + for item in self._encode_items: + shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + if torch.is_tensor(shape): + dims = shape.detach().reshape(-1).tolist() + height, width = int(dims[0]), int(dims[1]) + else: + height, width = int(item.value.shape[-2]), int(item.value.shape[-1]) + lengths.append((height // ds) * (width // ds)) + return lengths + + def _metric_meter_stash_latent_tokens(self, lengths: list[int]) -> None: + # currently use the same counts for encode and offline_encode + self.metric_meter_set_seqlens("encode", lengths) + self.metric_meter_set_seqlens("offline_encode", lengths) + + +class MeterMixin(MetricMeterMixin): + """Per-module training meter for BAGEL's latent VAE codec (FLUX-style conv AE).""" + + config: BagelVAEConfig + + def estimate_flops(self, seqlens: list[int]) -> float: + """Estimate TFLOPs for the FLUX-style convolutional VAE encoder over ``seqlens``.""" + cfg = self.config + + if getattr(cfg, "freeze", True): + return 0.0 + + ch = int(cfg.ch) + ch_mult = [int(m) for m in cfg.ch_mult] + in_ch_mult = [1, *ch_mult] + num_res = len(ch_mult) + num_res_blocks = int(cfg.num_res_blocks) + in_channels = int(cfg.in_channels) + z_channels = int(cfg.z_channels) + + # Conv MACs per latent token (linear term): area(level i) = N · 4**(num_res-1-i). + def area_factor(level: int) -> int: + return 4 ** (num_res - 1 - level) + + lin_macs_per_token = in_channels * ch * 9 * area_factor(0) # conv_in (3×3) + + for i in range(num_res): + af = area_factor(i) + block_in = ch * in_ch_mult[i] + block_out = ch * ch_mult[i] + for j in range(num_res_blocks): + cin = block_in if j == 0 else block_out + lin_macs_per_token += cin * block_out * 9 * af # ResnetBlock.conv1 + lin_macs_per_token += block_out * block_out * 9 * af # ResnetBlock.conv2 + if j == 0 and block_in != block_out: + lin_macs_per_token += block_in * block_out * af # 1×1 nin_shortcut + if i != num_res - 1: + # Strided Downsample (3×3) emits the next level's spatial area. + lin_macs_per_token += block_out * block_out * 9 * area_factor(i + 1) + + # Mid block at the latent resolution (area factor 1): two ResnetBlocks + AttnBlock. + bc = ch * ch_mult[-1] + lin_macs_per_token += 4 * (bc * bc * 9) # block_1 + block_2 (conv1 + conv2 each) + lin_macs_per_token += 4 * (bc * bc) # attn q/k/v/proj_out are 1×1 convs + lin_macs_per_token += bc * (2 * z_channels) * 9 # conv_out (3×3) + + tokens = sum(seqlens) + # Mid self-attention (single head, dim = bc): QK^T + attn·V ⇒ 2·N**2·bc MACs. + attn_macs = sum(2 * (n * n) * bc for n in seqlens) + + total_macs = lin_macs_per_token * tokens + attn_macs + return 6 * total_macs / 1e12 + + +class VeOmniMixin(BaseMixin, BagelVAEOfflineMixin, TrainingMixin, MeterMixin): + """Carrier hooks for raw-image VAE encode and latent decode. + + ``encode_context()`` / ``decode_generated()`` already live on the native + :class:`~.modeling.BagelVAE` (via its own :class:`~.modeling.InferenceMixin`), + so no ``InferenceMixin`` is needed here. + """ + + config: BagelVAEConfig + + +class BagelVAEAccelerated(VeOmniMixin, BagelVAE): + pass + + +__all__ = ["BAGEL_VAE_PIXEL_SHAPE", "BagelVAEAccelerated", "BagelVAEOfflineMixin"] diff --git a/veomni/models/seed_omni/modules/bagel/vae/configuration.py b/veomni/models/seed_omni/modules/bagel/vae/configuration.py new file mode 100644 index 0000000000..c7cc266ec2 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/configuration.py @@ -0,0 +1,52 @@ +"""Configuration for BAGEL's latent VAE module.""" + +from ...module_configuration_base import OmniModuleConfig + + +class BagelVAEConfig(OmniModuleConfig): + """BAGEL FLUX-style latent autoencoder config.""" + + model_type = "bagel_vae" + + def __init__( + self, + resolution: int = 256, + in_channels: int = 3, + downsample: int = 8, + ch: int = 128, + out_ch: int = 3, + ch_mult: list[int] | None = None, + num_res_blocks: int = 2, + z_channels: int = 16, + scale_factor: float = 0.3611, + shift_factor: float = 0.1159, + max_image_size: int = 1024, + min_image_size: int = 512, + image_stride: int = 16, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + freeze: bool = True, + **kwargs, + ) -> None: + self.resolution = resolution + self.in_channels = in_channels + self.downsample = downsample + self.ch = ch + self.out_ch = out_ch + self.ch_mult = [1, 2, 4, 4] if ch_mult is None else ch_mult + self.num_res_blocks = num_res_blocks + self.z_channels = z_channels + self.scale_factor = scale_factor + self.shift_factor = shift_factor + self.max_image_size = max_image_size + self.min_image_size = min_image_size + self.image_stride = image_stride + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + self.freeze = freeze + super().__init__(**kwargs) + + +__all__ = ["BagelVAEConfig"] diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py new file mode 100644 index 0000000000..aef2f1da5c --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -0,0 +1,501 @@ +"""BAGEL latent VAE module. + +Tensor call-sites: +* ``bagel_vae.encode`` maps normalized image tensors to scaled latent grids. +* ``bagel_vae.decode`` maps generated latent grids to decoded image tensors. + +The VAE module is a codec boundary only. Flow timestep/noise sampling, latent +patchification, packed MoT indexes, and conversation-carrier mutation belong to +the Bagel VAE module mixin and downstream Bagel nodes. + +``encode_context()`` (image-understanding VAE encode) and ``decode_generated()`` +(final generated-latent decode) are the FSM generation-node methods, native here +so pure HF inference (no accelerated mixins) can run BAGEL's generation graphs. +""" + +from __future__ import annotations + +from contextlib import contextmanager, nullcontext +from typing import Any + +import torch +import torch.nn as nn +from einops import rearrange +from torch import Tensor + +from ....utils.conversation import ConversationItem, iter_desired_items +from ...module_modeling_base import PretrainedOmniModule +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_VAE_CONTEXT +from .configuration import BagelVAEConfig +from .processing import BAGEL_VAE_PIXEL_SHAPE, BagelVAEPreprocessor, BagelVAEProcessor, crop_latent_to_image_shape + + +def select_bagel_vae_context_items( + conversation_list: list[list[ConversationItem]] | None, + *, + exclude_dummy: bool = False, +) -> list[ConversationItem]: + """Select VAE context image items (training includes dummies; inference skips them).""" + if conversation_list is None: + raise ValueError("BagelVAE requires conversation_list to select VAE context items.") + + items = list(iter_desired_items(conversation_list, types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]})) + if exclude_dummy: + return [item for item in items if not item.is_dummy] + return items + + +class InferenceMixin: + """FSM ``encode_context`` / ``decode_generated`` — HF ``GenerationMixin`` analog. + + Listed *before* :class:`~...module_modeling_base.PretrainedOmniModule` in + :class:`BagelVAE`'s bases for consistency with every other module's + native / accelerated split (see ``janus/llama/modeling.py`` for the full + MRO rationale where a competing no-op default exists — this module + doesn't override ``reset_local_inference_state`` / ``finalize`` so there + is nothing to shadow here). + """ + + def encode_context( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """FSM node: encode image-understanding context images into VAE latents.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + image_items = select_bagel_vae_context_items(batched, exclude_dummy=True) + if not image_items: + return {"conversation_list": conversation_list} + + pixel_values = torch.stack([item.value for item in image_items], dim=0).to( + device=self.device, dtype=self.dtype, non_blocking=True + ) + outputs = self.encode(pixel_values=pixel_values) + for image_item, latent in zip(image_items, outputs["latents"], strict=True): + latent = crop_latent_to_image_shape( + latent, + image_item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + image_item.value = latent.to(device=self.device, dtype=self.dtype) + image_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + return {"conversation_list": conversation_list} + + def decode_generated( + self, + conversation_list: list[ConversationItem] | None = None, + generation_kwargs: dict[str, Any] | None = None, + **kwargs: Any, + ) -> dict[str, Any]: + """FSM node: decode the flow connector's completed latent into a PIL image.""" + del generation_kwargs, kwargs + if conversation_list is None: + return {"conversation_list": conversation_list} + + batched = [conversation_list] + decode_items = self._select_vae_decode_items(batched) + if not decode_items: + return {"conversation_list": conversation_list} + + latents = [] + for item in decode_items: + latents.append( + item.value.detach().squeeze(0) + if item.value.dim() == 4 and item.value.shape[0] == 1 + else item.value.detach() + ) + latents = torch.stack(latents, dim=0).to(device=self.device, dtype=self.dtype) + + outputs = self.decode(latents=latents) + pixel_values = outputs["pixel_values"] + for item, image in zip(decode_items, pixel_values, strict=True): + item.type = "image" + item.value = image.to(device=self.device, dtype=self.dtype) + if self._image_processor is None: + raise RuntimeError( + "BagelVAE: cannot postprocess decoded image — no image processor was " + "loaded. Ensure `preprocessor_config.json` ships next to the weights." + ) + return { + "conversation_list": conversation_list, + "generated": {"type": "image", "value": self._image_processor.postprocess(pixel_values[-1])[0]}, + } + + def _select_vae_decode_items( + self, conversation_list: list[list[ConversationItem]] | None + ) -> list[ConversationItem]: + """Select non-dummy ``generated_latent`` output items for VAE image decoding.""" + if conversation_list is None: + raise ValueError("BagelVAE decode requires conversation_list to select latent items.") + + # Final image decode consumes the completed latent emitted by the flow connector. + decode_items: list[ConversationItem] = [] + for item in iter_desired_items( + conversation_list, types=["output"], meta={BAGEL_PHASE_KEY: [BAGEL_GENERATED_LATENT]} + ): + if not item.is_dummy: + decode_items.append(item) + return decode_items + + +class BagelVAE(InferenceMixin, PretrainedOmniModule): + config_class = BagelVAEConfig + image_processor_class = BagelVAEProcessor + preprocessor_class = BagelVAEPreprocessor + base_model_prefix = "bagel_vae" + main_input_name = "pixel_values" + _no_split_modules: list[str] = ["ResnetBlock", "AttnBlock"] + supports_gradient_checkpointing = False + + def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): + # The VAE uses fixed local torch SDPA in its mid-block, with no HF backend dispatch. + return attn_implementation + + def __init__(self, config: BagelVAEConfig, **kwargs: Any) -> None: + super().__init__(config, **kwargs) + self.encoder = Encoder( + resolution=config.resolution, + in_channels=config.in_channels, + ch=config.ch, + ch_mult=config.ch_mult, + num_res_blocks=config.num_res_blocks, + z_channels=config.z_channels, + ) + self.decoder = Decoder( + resolution=config.resolution, + in_channels=config.in_channels, + ch=config.ch, + out_ch=config.out_ch, + ch_mult=config.ch_mult, + num_res_blocks=config.num_res_blocks, + z_channels=config.z_channels, + ) + self._image_processor: BagelVAEProcessor | None = None + self.post_init() + + def freeze_model(self) -> None: + """Set the VAE to eval mode and disable gradients when ``config.freeze`` is true.""" + if self.config.freeze: + self.eval() + self.requires_grad_(False) + + @property + def _encoder_device(self) -> torch.device: + return self.encoder.conv_in.weight.device + + @property + def _decoder_device(self) -> torch.device: + return self.decoder.conv_in.weight.device + + @contextmanager + def _runtime_context(self, tensor: torch.Tensor): + """Enter ``no_grad`` (when frozen) and autocast for VAE encoder/decoder execution.""" + grad_context = torch.enable_grad() + if self.config.freeze: + grad_context = torch.no_grad() + + autocast_context = nullcontext() + if self.dtype != torch.float32: + autocast_context = torch.amp.autocast(tensor.device.type, enabled=True, dtype=self.dtype) + + with grad_context, autocast_context: + yield + + def encode( + self, + pixel_values: torch.Tensor, + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values`` into scaled VAE latent grids.""" + del kwargs + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) + with self._runtime_context(pixel_values): + posterior = self._encode_posterior(pixel_values) + latents = self._sample_scaled_latents(posterior) + return {"latents": latents.to(dtype=self.dtype)} + + def decode( + self, + latents: torch.Tensor, + **kwargs: object, + ) -> dict[str, Any]: + """Unscale ``latents`` and decode them back to normalized pixel tensors.""" + del kwargs + latents = latents.to(device=self._decoder_device, dtype=self.dtype) + latents = latents / self.config.scale_factor + self.config.shift_factor + with self._runtime_context(latents): + pixel_values = self.decoder(latents) + return {"pixel_values": pixel_values.to(dtype=self.dtype)} + + def _encode_posterior(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return torch.chunk(self.encoder(pixel_values), 2, dim=1) + + def _sample_scaled_latents(self, posterior: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: + mean, logvar = posterior + latents = mean + torch.exp(0.5 * logvar) * torch.randn_like(mean) + return self.config.scale_factor * (latents - self.config.shift_factor) + + +def swish(x: Tensor) -> Tensor: + return x * torch.sigmoid(x) + + +class AttnBlock(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.in_channels = in_channels + self.norm = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.q = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.k = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.v = nn.Conv2d(in_channels, in_channels, kernel_size=1) + self.proj_out = nn.Conv2d(in_channels, in_channels, kernel_size=1) + + def attention(self, hidden_states: Tensor) -> Tensor: + hidden_states = self.norm(hidden_states) + query = self.q(hidden_states) + key = self.k(hidden_states) + value = self.v(hidden_states) + + batch, channels, height, width = query.shape + query = rearrange(query, "b c h w -> b 1 (h w) c").contiguous() + key = rearrange(key, "b c h w -> b 1 (h w) c").contiguous() + value = rearrange(value, "b c h w -> b 1 (h w) c").contiguous() + hidden_states = nn.functional.scaled_dot_product_attention(query, key, value) + return rearrange(hidden_states, "b 1 (h w) c -> b c h w", h=height, w=width, c=channels, b=batch) + + def forward(self, x: Tensor) -> Tensor: + return x + self.proj_out(self.attention(x)) + + +class ResnetBlock(nn.Module): + def __init__(self, in_channels: int, out_channels: int) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.norm1 = nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True) + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + self.norm2 = nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=1e-6, affine=True) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1) + if self.in_channels != self.out_channels: + self.nin_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, x: Tensor) -> Tensor: + hidden_states = self.norm1(x) + hidden_states = swish(hidden_states) + hidden_states = self.conv1(hidden_states) + hidden_states = self.norm2(hidden_states) + hidden_states = swish(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return x + hidden_states + + +class Downsample(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0) + + def forward(self, x: Tensor) -> Tensor: + x = nn.functional.pad(x, (0, 1, 0, 1), mode="constant", value=0) + return self.conv(x) + + +class Upsample(nn.Module): + def __init__(self, in_channels: int) -> None: + super().__init__() + self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor) -> Tensor: + x = nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + return self.conv(x) + + +class Encoder(nn.Module): + def __init__( + self, + resolution: int, + in_channels: int, + ch: int, + ch_mult: list[int], + num_res_blocks: int, + z_channels: int, + ) -> None: + super().__init__() + self.gradient_checkpointing = False + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.conv_in = nn.Conv2d(in_channels, self.ch, kernel_size=3, stride=1, padding=1) + + in_ch_mult = (1,) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + block_in = self.ch + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in) + self.down.append(down) + + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, 2 * z_channels, kernel_size=3, stride=1, padding=1) + + def forward(self, x: Tensor) -> Tensor: + """Encode input image tensor ``x`` into ``2 * z_channels`` mean/logvar parameters.""" + hidden_stack = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + block = self.down[i_level].block[i_block] + hidden_states = ( + self._gradient_checkpointing_func(block.__call__, hidden_stack[-1]) + if self.gradient_checkpointing and self.training + else block(hidden_stack[-1]) + ) + if len(self.down[i_level].attn) > 0: + attn = self.down[i_level].attn[i_block] + hidden_states = ( + self._gradient_checkpointing_func(attn.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else attn(hidden_states) + ) + hidden_stack.append(hidden_states) + if i_level != self.num_resolutions - 1: + downsample = self.down[i_level].downsample + hidden_stack.append( + self._gradient_checkpointing_func(downsample.__call__, hidden_stack[-1]) + if self.gradient_checkpointing and self.training + else downsample(hidden_stack[-1]) + ) + + hidden_states = hidden_stack[-1] + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states) + else: + hidden_states = self.mid.block_1(hidden_states) + hidden_states = self.mid.attn_1(hidden_states) + hidden_states = self.mid.block_2(hidden_states) + hidden_states = self.norm_out(hidden_states) + hidden_states = swish(hidden_states) + return self.conv_out(hidden_states) + + +class Decoder(nn.Module): + def __init__( + self, + ch: int, + out_ch: int, + ch_mult: list[int], + num_res_blocks: int, + in_channels: int, + resolution: int, + z_channels: int, + ) -> None: + super().__init__() + self.gradient_checkpointing = False + self.ch = ch + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.ffactor = 2 ** (self.num_resolutions - 1) + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + + self.conv_in = nn.Conv2d(z_channels, block_in, kernel_size=3, stride=1, padding=1) + + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in) + self.mid.attn_1 = AttnBlock(block_in) + self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in) + + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for _ in range(self.num_res_blocks + 1): + block.append(ResnetBlock(in_channels=block_in, out_channels=block_out)) + block_in = block_out + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in) + self.up.insert(0, up) + + self.norm_out = nn.GroupNorm(num_groups=32, num_channels=block_in, eps=1e-6, affine=True) + self.conv_out = nn.Conv2d(block_in, out_ch, kernel_size=3, stride=1, padding=1) + + def forward(self, z: Tensor) -> Tensor: + """Decode latent tensor ``z`` into reconstructed RGB image space.""" + hidden_states = self.conv_in(z) + + if self.gradient_checkpointing and self.training: + hidden_states = self._gradient_checkpointing_func(self.mid.block_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.attn_1.__call__, hidden_states) + hidden_states = self._gradient_checkpointing_func(self.mid.block_2.__call__, hidden_states) + else: + hidden_states = self.mid.block_1(hidden_states) + hidden_states = self.mid.attn_1(hidden_states) + hidden_states = self.mid.block_2(hidden_states) + + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + block = self.up[i_level].block[i_block] + hidden_states = ( + self._gradient_checkpointing_func(block.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else block(hidden_states) + ) + if len(self.up[i_level].attn) > 0: + attn = self.up[i_level].attn[i_block] + hidden_states = ( + self._gradient_checkpointing_func(attn.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else attn(hidden_states) + ) + if i_level != 0: + upsample = self.up[i_level].upsample + hidden_states = ( + self._gradient_checkpointing_func(upsample.__call__, hidden_states) + if self.gradient_checkpointing and self.training + else upsample(hidden_states) + ) + + hidden_states = self.norm_out(hidden_states) + hidden_states = swish(hidden_states) + return self.conv_out(hidden_states) + + +__all__ = [ + "BagelVAE", + "BagelVAEConfig", + "InferenceMixin", + "select_bagel_vae_context_items", +] diff --git a/veomni/models/seed_omni/modules/bagel/vae/processing.py b/veomni/models/seed_omni/modules/bagel/vae/processing.py new file mode 100644 index 0000000000..7fe71a0d0d --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/vae/processing.py @@ -0,0 +1,563 @@ +"""Image processor + worker-side preprocessor for BAGEL VAE. + +:class:`BagelVAEPreprocessor` is the picklable, weight-free CPU worker +counterpart (see :class:`~veomni.models.seed_omni.modules.module_processing_base.ModulePreprocessorBase`) — +built straight off the checkpoint dir, with no model instance involved. Like +:mod:`..siglip_navit.processing`, BAGEL ships no separate +``preprocessor_config.json``: the image processor is derived from the +module's own ``config.json`` via :meth:`BagelVAEProcessor.from_config`. +""" + +from __future__ import annotations + +import copy +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torchvision.transforms import InterpolationMode +from torchvision.transforms import functional as TVF +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature + +from ....utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem, iter_desired_items +from ...module_processing_base import ModulePreprocessorBase +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT, bagel_context +from .configuration import BagelVAEConfig + + +_VALID_IMG_TAGS = frozenset({"und", "gen", "edit"}) + + +class BagelVAEProcessor(BaseImageProcessor): + """BAGEL VAE image processor. + + Owns raw-image resize and normalization for VAE encode. Carrier selection, + context tagging, and latent scatter stay in the module mixin. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + image_stride: int = 16, + max_image_size: int = 1024, + min_image_size: int = 512, + max_pixels: int = 14 * 14 * 9 * 1024, + image_mean: list[float] | None = None, + image_std: list[float] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.image_stride = image_stride + self.max_image_size = max_image_size + self.min_image_size = min_image_size + self.max_pixels = max_pixels + self.image_mean = [0.5, 0.5, 0.5] if image_mean is None else image_mean + self.image_std = [0.5, 0.5, 0.5] if image_std is None else image_std + + @classmethod + def from_config(cls, config: Any) -> BagelVAEProcessor: + return cls( + image_stride=int(config.image_stride), + max_image_size=int(config.max_image_size), + min_image_size=int(config.min_image_size), + max_pixels=int(config.max_pixels), + image_mean=list(config.image_mean), + image_std=list(config.image_std), + ) + + def preprocess( + self, + images: Any, + *, + return_tensors: str | None = None, + device: torch.device | None = None, + dtype: torch.dtype | None = None, + **kwargs: Any, + ) -> BatchFeature: + """Resize, normalize, and batch-pad ``images`` into ``pixel_values`` and ``pixel_shapes``.""" + del return_tensors, kwargs + image_list = images if isinstance(images, list) else [images] + pixel_values = [ + _preprocess_image( + image, + image_stride=self.image_stride, + max_image_size=self.max_image_size, + min_image_size=self.min_image_size, + max_pixels=self.max_pixels, + image_mean=self.image_mean, + image_std=self.image_std, + ) + for image in image_list + ] + pixel_shapes = torch.tensor( + [[int(pixel.shape[-2]), int(pixel.shape[-1])] for pixel in pixel_values], + dtype=torch.long, + ) + pixel_values = torch.stack(_pad_to_batch_size(pixel_values), dim=0) + + if dtype is not None: + pixel_values = pixel_values.to(dtype=dtype) + if device is not None: + pixel_values = pixel_values.to(device=device) + pixel_shapes = pixel_shapes.to(device=device) + return BatchFeature(data={"pixel_values": pixel_values, "pixel_shapes": pixel_shapes}) + + def postprocess(self, images: torch.Tensor | list[torch.Tensor], **kwargs: Any) -> list[Image.Image]: + """Convert decoded VAE image tensors in ``[-1, 1]`` back to RGB PIL images.""" + del kwargs + image_list = images if isinstance(images, list) else [images] + out: list[Image.Image] = [] + for image in image_list: + if image.dim() == 4: + out.extend(_decoded_tensor_to_pil(item) for item in image) + else: + out.append(_decoded_tensor_to_pil(image)) + return out + + +def _preprocess_image( + image: Any, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, + image_mean: list[float], + image_std: list[float], +) -> torch.Tensor: + """Resize and normalize ``image`` into a stride-aligned ``[3, H, W]`` float32 tensor.""" + if isinstance(image, (Image.Image, np.ndarray)): + pil_image = _to_rgb_pil(image if isinstance(image, Image.Image) else Image.fromarray(image)) + pil_image = _resize_pil( + pil_image, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + tensor = _pil_to_rgb_tensor(pil_image) + else: + tensor = _to_rgb_tensor(image) + tensor = _resize_tensor( + tensor, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + mean = torch.tensor(image_mean, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + std = torch.tensor(image_std, dtype=torch.float32, device=tensor.device).view(-1, 1, 1) + return tensor.sub(mean).div(std) + + +def _to_rgb_tensor(image: Any) -> torch.Tensor: + """Convert a PIL image, NumPy array, or tensor into a ``[3, H, W]`` RGB tensor in ``[0, 1]``.""" + if isinstance(image, Image.Image): + pil_image = _to_rgb_pil(image) + array = np.array(pil_image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + if isinstance(image, np.ndarray): + return _to_rgb_tensor(Image.fromarray(image)) + if torch.is_tensor(image): + tensor = image.detach().to(dtype=torch.float32) + if tensor.dim() != 3: + raise TypeError(f"BAGEL VAE image tensor must be 3-D, got shape {tuple(tensor.shape)}.") + if tensor.shape[0] in (1, 3, 4): + pass + elif tensor.shape[-1] in (1, 3, 4): + tensor = tensor.permute(2, 0, 1).contiguous() + else: + raise TypeError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(tensor.shape)}.") + if tensor.shape[0] == 1: + tensor = tensor.repeat(3, 1, 1) + elif tensor.shape[0] == 4: + alpha = tensor[3:4] + if alpha.max() > 1: + alpha = alpha / 255.0 + rgb = tensor[:3] + if rgb.max() > 1: + rgb = rgb / 255.0 + tensor = rgb * alpha + (1.0 - alpha) + if tensor.max() > 1: + tensor = tensor / 255.0 + return tensor[:3].clamp(0.0, 1.0) + raise TypeError(f"BAGEL VAE image item value must be PIL, numpy, or tensor, got {type(image).__name__}.") + + +def _resize_tensor( + tensor: torch.Tensor, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, +) -> torch.Tensor: + """Resize a ``[3, H, W]`` image tensor with bicubic interpolation to stride-aligned bounds.""" + height, width = tensor.shape[-2:] + new_width, new_height = _target_size( + width, + height, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + if new_width == width and new_height == height: + return tensor + resized = F.interpolate( + tensor.unsqueeze(0), + size=(new_height, new_width), + mode="bicubic", + align_corners=False, + antialias=True, + ) + return resized.squeeze(0).clamp(0.0, 1.0) + + +def _resize_pil( + image: Image.Image, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, + img_num: int = 1, +) -> Image.Image: + """Resize a PIL image with bicubic interpolation to stride-aligned bounds.""" + width, height = image.size + new_width, new_height = _target_size( + width, + height, + image_stride=image_stride, + max_image_size=max_image_size, + min_image_size=min_image_size, + max_pixels=max_pixels, + ) + if new_width == width and new_height == height: + return image + return TVF.resize( + image, + (new_height, new_width), + interpolation=InterpolationMode.BICUBIC, + antialias=True, + ) + + +def _pad_to_batch_size(images: list[torch.Tensor]) -> list[torch.Tensor]: + """Zero-pad variable-size ``[C, H, W]`` image tensors on the bottom/right to a common canvas.""" + if not images: + return images + max_height = max(int(image.shape[-2]) for image in images) + max_width = max(int(image.shape[-1]) for image in images) + padded = [] + for image in images: + pad_height = max_height - int(image.shape[-2]) + pad_width = max_width - int(image.shape[-1]) + if pad_height or pad_width: + image = F.pad(image, (0, pad_width, 0, pad_height), value=0.0) + padded.append(image) + return padded + + +def crop_latent_to_image_shape( + latent: torch.Tensor, + pixel_shape: torch.Tensor | None, + *, + downsample: int, +) -> torch.Tensor: + """Crop spatial batch padding off ``latent`` using the pre-pad ``pixel_shape`` metadata.""" + if not torch.is_tensor(pixel_shape): + return latent + + shape = pixel_shape.detach().reshape(-1).to(device="cpu", dtype=torch.long) + if int(shape.numel()) != 2: + raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].") + + downsample = max(int(downsample), 1) + latent_height = int(shape[0].item()) // downsample + latent_width = int(shape[1].item()) // downsample + if latent_height <= 0 or latent_width <= 0: + raise ValueError( + "BAGEL VAE pixel shape metadata produces an empty latent crop: " + f"pixel_shape={tuple(int(v.item()) for v in shape)}, downsample={downsample}." + ) + if latent_height > int(latent.shape[-2]) or latent_width > int(latent.shape[-1]): + raise ValueError( + "BAGEL VAE latent crop exceeds encoded latent shape: " + f"crop=({latent_height}, {latent_width}), latent_shape={tuple(latent.shape)}." + ) + return latent[..., :latent_height, :latent_width] + + +def _target_size( + width: int, + height: int, + *, + image_stride: int, + max_image_size: int, + min_image_size: int, + max_pixels: int, +) -> tuple[int, int]: + """Compute stride-aligned ``(new_width, new_height)`` within pixel and side-length bounds.""" + scale = min(max_image_size / max(width, height), 1.0) + scale = max(scale, min_image_size / min(width, height)) + new_width, new_height = _apply_scale(width, height, scale, image_stride) + if new_width * new_height > max_pixels: + scale = max_pixels / (new_width * new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride) + if max(new_width, new_height) > max_image_size: + scale = max_image_size / max(new_width, new_height) + new_width, new_height = _apply_scale(new_width, new_height, scale, image_stride) + return new_width, new_height + + +def _to_rgb_pil(image: Image.Image) -> Image.Image: + """Convert ``image`` to RGB, compositing transparency over a white background.""" + if image.mode == "RGBA" or image.info.get("transparency", None) is not None: + rgba = image.convert("RGBA") + white = Image.new(mode="RGB", size=rgba.size, color=(255, 255, 255)) + white.paste(rgba, mask=rgba.split()[3]) + return white + return image.convert("RGB") + + +def _pil_to_rgb_tensor(image: Image.Image) -> torch.Tensor: + array = np.array(image, copy=True) + return torch.from_numpy(array).permute(2, 0, 1).contiguous().to(dtype=torch.float32).div_(255.0) + + +def _apply_scale(width: int, height: int, scale: float, stride: int) -> tuple[int, int]: + new_width = round(width * scale) + new_height = round(height * scale) + return ( + max(stride, int(round(new_width / stride) * stride)), + max(stride, int(round(new_height / stride) * stride)), + ) + + +def _decoded_tensor_to_pil(image: torch.Tensor) -> Image.Image: + """Convert a single decoded ``[-1, 1]`` tensor into an RGB ``PIL.Image``.""" + if image.dim() == 4 and image.shape[0] == 1: + image = image.squeeze(0) + if image.dim() != 3: + raise ValueError(f"BAGEL VAE generated image must be rank 3, got shape {tuple(image.shape)}.") + image = (image.detach().to(dtype=torch.float32).clamp(-1.0, 1.0) * 0.5 + 0.5).clamp(0.0, 1.0) + if image.shape[0] in (1, 3, 4): + image = image[:3].permute(1, 2, 0) + elif image.shape[-1] not in (1, 3, 4): + raise ValueError(f"Unable to infer channel dimension for BAGEL VAE image tensor {tuple(image.shape)}.") + image = (image[..., :3] * 255.0).round().to(dtype=torch.uint8).cpu().numpy() + return Image.fromarray(image) + + +def copy_image_item(item: ConversationItem) -> ConversationItem: + """Deep-copy an image ``ConversationItem`` so VAE and SigLIP context copies do not alias.""" + value = item.value + if torch.is_tensor(value): + value = value.clone() + elif isinstance(value, Image.Image): + value = value.copy() + else: + value = copy.deepcopy(value) + return ConversationItem( + type=item.type, + value=value, + role=item.role, + is_dummy=item.is_dummy, + meta=copy.deepcopy(item.meta), + ) + + +def route_image_contexts( + conversation_list: list[list[ConversationItem]], + *, + inference: bool, + infer_type: object, +) -> None: + """Expand raw images into their SigLIP and/or VAE context copies (``BAGEL_CONTEXT_KEY``).""" + for sample in conversation_list: + routed: list[ConversationItem] = [] + for item in sample: + if item.type != "image" or item.is_dummy: + routed.append(item) + continue + + if bagel_context(item) in {BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT}: + routed.append(item) + continue + + # Interactive inference has no dataset preprocessor, so raw user + # request images use ``infer_type`` in generation_kwargs for routing. + if inference and item.role == "user": + if infer_type == "infer_edit": + vae_item = copy_image_item(item) + vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.extend([vae_item, item]) + else: + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.append(item) + continue + + # Training data declares each raw image's role via ``_img_tag``. + tag = item.meta.get(_IMG_TAG_KEY) + if tag is None or tag not in _VALID_IMG_TAGS: + raise ValueError(f"BAGEL route_image_contexts received image with invalid {_IMG_TAG_KEY}: {tag!r}.") + if tag == "und": + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.append(item) + elif tag == "gen": + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + routed.append(item) + else: + # tag == "edit": the same physical image is both VAE context and + # SigLIP context. Preserve the infer_edit ordering (VAE before + # SigLIP) so downstream marker wrapping stays stable. + vae_item = copy_image_item(item) + vae_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_SIGLIP_CONTEXT + routed.extend([vae_item, item]) + + sample[:] = routed + + +BAGEL_VAE_PIXEL_SHAPE = "bagel_vae_pixel_shape" +# Set by ``offline_encode``: the item already holds its cached posterior, not pixels. +BAGEL_VAE_POSTERIOR = "bagel_vae_posterior" + + +class BagelVAEPreprocessor(ModulePreprocessorBase): + """Worker-side image normalize for BAGEL VAE context/target images.""" + + def __init__( + self, + image_processor: Any, + dtype: torch.dtype | None = None, + dummy_pixel_values: torch.Tensor | None = None, + dummy_pixel_shape: torch.Tensor | None = None, + ) -> None: + self._image_processor = image_processor + self._dtype = dtype + self._dummy_pixel_values = dummy_pixel_values + self._dummy_pixel_shape = dummy_pixel_shape + + @classmethod + def from_pretrained( + cls, + module_path: str, + *, + config_overrides: dict[str, Any] | None = None, + **kwargs: Any, + ) -> BagelVAEPreprocessor: + """Build straight from the checkpoint dir — no model instance needed. + + BAGEL ships no standalone ``preprocessor_config.json``; the image + processor is derived from the module's own ``config.json``, with + ``config_overrides`` (the module's YAML ``model_config:`` block) on top. + """ + del kwargs + # ``model_config=`` (not ``**kwargs``): HF drops kwargs the class does not declare. + config = BagelVAEConfig.from_pretrained(module_path, model_config=config_overrides) + return cls(BagelVAEProcessor.from_config(config)) + + def bind_dummy_inputs(self, config: BagelVAEConfig, dtype: torch.dtype | None = None) -> None: + """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model.""" + size = max(int(config.image_stride), int(config.downsample)) + self._dtype = dtype + self._dummy_pixel_values = torch.zeros(int(config.in_channels), size, size, dtype=dtype) + self._dummy_pixel_shape = torch.tensor([size, size], dtype=torch.long) + + def preprocess_conversations( + self, + conversation_list: list[list[ConversationItem]], + *, + inference: bool = False, + generation_kwargs: dict[str, Any] | None = None, + ) -> None: + """Route image contexts, normalize VAE pixel tensors, and append per-sample dummy anchors.""" + infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type") + route_image_contexts(conversation_list, inference=inference, infer_type=infer_type) + + image_items: list[ConversationItem] = [] + missing_samples: list[list[ConversationItem]] = [] + for sample in conversation_list: + sample_image_items = list( + iter_desired_items([sample], types=["image"], meta={BAGEL_CONTEXT_KEY: [BAGEL_VAE_CONTEXT]}) + ) + if sample_image_items: + for item in sample_image_items: + if item.meta.get(BAGEL_VAE_POSTERIOR) and _OMNI_TOKEN_LEN not in item.meta: + pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1) + if torch.is_tensor(pixel_shape) and int(pixel_shape.numel()) == 2: + dims = pixel_shape.detach().reshape(-1).tolist() + item.meta[_OMNI_TOKEN_LEN] = (int(dims[0]) // stride) * (int(dims[1]) // stride) + image_items.extend( + item for item in sample_image_items if not item.is_dummy and not item.meta.get(BAGEL_VAE_POSTERIOR) + ) + elif not inference: + missing_samples.append(sample) + + stride = max(int(getattr(self._image_processor, "image_stride", 16)), 1) + if image_items: + inputs = self._image_processor( + images=[item.value for item in image_items], return_tensors="pt", dtype=self._dtype + ) + for item, pixels, pixel_shape in zip( + image_items, inputs["pixel_values"], inputs["pixel_shapes"], strict=True + ): + item.value = pixels.to(dtype=self._dtype) + item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT + item.meta[BAGEL_VAE_PIXEL_SHAPE] = pixel_shape.to(dtype=torch.long) + dims = pixel_shape.detach().reshape(-1).tolist() + item.meta[_OMNI_TOKEN_LEN] = max(1, (int(dims[0]) // stride) * (int(dims[1]) // stride)) + + if not missing_samples: + return + + if image_items: + # VAE encode stacks every context item in the micro-batch, so dummies must + # match a real processed image shape whenever the batch has one. + reference_item = image_items[0] + dummy_pixel_values = torch.zeros_like(reference_item.value, dtype=self._dtype) + dummy_pixel_shape = reference_item.meta[BAGEL_VAE_PIXEL_SHAPE].to(dtype=torch.long) + else: + if self._dummy_pixel_values is None: + raise RuntimeError( + f"{type(self).__name__}: dummy inputs not bound — call bind_dummy_inputs() " + "before training use (pure inference never reaches this branch)." + ) + dummy_pixel_values = self._dummy_pixel_values.to(dtype=self._dtype) + dummy_pixel_shape = self._dummy_pixel_shape.to(dtype=torch.long) + + dummy_dims = dummy_pixel_shape.detach().reshape(-1).tolist() + dummy_token_len = max(1, (int(dummy_dims[0]) // stride) * (int(dummy_dims[1]) // stride)) + + # The placeholder stands in for a generation target, the primary VAE row. + for sample in missing_samples: + sample.append( + ConversationItem( + type="image", + value=dummy_pixel_values.clone(), + role="assistant", + is_dummy=True, + meta={ + _IMG_TAG_KEY: "gen", + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + BAGEL_VAE_PIXEL_SHAPE: dummy_pixel_shape.clone(), + _OMNI_TOKEN_LEN: dummy_token_len, + }, + ) + ) + + +__all__ = [ + "BagelVAEProcessor", + "BagelVAEPreprocessor", + "copy_image_item", + "crop_latent_to_image_shape", + "route_image_contexts", +] diff --git a/veomni/models/seed_omni/modules/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/text_encoder/chat_template.py index 7a261bc608..2f941a0bfc 100644 --- a/veomni/models/seed_omni/modules/text_encoder/chat_template.py +++ b/veomni/models/seed_omni/modules/text_encoder/chat_template.py @@ -4,7 +4,7 @@ import torch -from ...utils.conversation import ConversationItem, ItemRole, ItemType, ItemValue +from ...utils.conversation import _OMNI_TOKEN_LEN, ConversationItem, ItemRole, ItemType, ItemValue @dataclass(frozen=True) @@ -117,6 +117,7 @@ def tokenize(self, parts: list[ConversationItem]) -> None: part.value = torch.tensor(input_ids, dtype=torch.long) part.meta["labels"] = torch.tensor(labels, dtype=torch.long) part.meta["attention_mask"] = torch.ones(len(input_ids), dtype=torch.long) + part.meta[_OMNI_TOKEN_LEN] = len(input_ids) @staticmethod def merge_text_embeds(parts: list[ConversationItem]) -> list[ConversationItem]: @@ -135,6 +136,7 @@ def merge_text_embeds(parts: list[ConversationItem]) -> list[ConversationItem]: prev.value = torch.cat([prev.value, part.value]) prev.meta["labels"] = torch.cat([prev.meta["labels"], part.meta["labels"]]) prev.meta["attention_mask"] = torch.cat([prev.meta["attention_mask"], part.meta["attention_mask"]]) + prev.meta[_OMNI_TOKEN_LEN] = int(prev.value.numel()) continue merged.append(part) return merged diff --git a/veomni/models/seed_omni/modules/text_encoder/modeling.py b/veomni/models/seed_omni/modules/text_encoder/modeling.py index 0af86eeeb8..b4f1fa6165 100644 --- a/veomni/models/seed_omni/modules/text_encoder/modeling.py +++ b/veomni/models/seed_omni/modules/text_encoder/modeling.py @@ -97,7 +97,7 @@ def _encode_prompt(self, conversation_list: List[ConversationItem]) -> Dict[str, def generate( self, conversation_list: Optional[List[ConversationItem]] = None, - generation_kwargs: Dict[str, Any] = dict, + generation_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> Dict[str, Any]: """One FSM inference step — implemented per module (see class docstring).""" diff --git a/veomni/utils/save_safetensor_utils.py b/veomni/utils/save_safetensor_utils.py index 405391dd92..b7292128b3 100644 --- a/veomni/utils/save_safetensor_utils.py +++ b/veomni/utils/save_safetensor_utils.py @@ -33,6 +33,42 @@ logger = helper.create_logger(__name__) +def _attach_weight_conversions_from_mro(model: torch.nn.Module) -> None: + """Recover class-name converters after FSDP2 wraps the module class. + + ``fully_shard`` builds a subclass whose ``__name__`` no longer matches the + registered mapping. ``model_type`` fallback is wrong for Bagel: eager and + accelerated share ``bagel_qwen2_mot``, but only accelerated exports fused + QKV through converters. Walk the MRO until a class-name mapping hits. + """ + if getattr(model, "_weight_conversions", None) is not None: + return + + from transformers.conversion_mapping import get_checkpoint_conversion_mapping + + for cls in type(model).__mro__: + conversions = get_checkpoint_conversion_mapping(cls.__name__) + if conversions is not None: + model._weight_conversions = list(conversions) + return + + +def _revert_model_weight_conversions_for_hf( + model: Optional[torch.nn.Module], + state_dict: Dict[str, torch.Tensor], +) -> Dict[str, torch.Tensor]: + """Revert runtime weight conversions when exporting an HF checkpoint.""" + while isinstance(getattr(model, "module", None), torch.nn.Module): + model = model.module + if model is None or not getattr(model, "_export_hf_checkpoint_with_weight_conversions", False): + return state_dict + + from transformers.core_model_loading import revert_weight_conversion + + _attach_weight_conversions_from_mro(model) + return revert_weight_conversion(model, state_dict) + + @torch.no_grad() def get_model_save_state( model: torch.nn.Module, @@ -127,6 +163,7 @@ def _should_quantize(origin: str) -> bool: # Use flat state dict so DCP FQNs match the original HF weight_map keys # (e.g. "model.embed_tokens.weight" instead of "model.model.embed_tokens.weight") save_state = ModelState(model, parallel_state=parallel_state).state_dict() + save_state = _revert_model_weight_conversions_for_hf(model, save_state) # Convert float32 tensors to bfloat16 on a copy of the state dict, # so the original model parameters remain unchanged. @@ -238,6 +275,7 @@ def _save_hf_safetensor_legacy( model_assets: Optional[Sequence], ckpt_manager: str, output_dir: Optional[str], + model: Optional[torch.nn.Module] = None, ): """Legacy HuggingFace safetensors save via checkpoint conversion (rank-0 only).""" model_state_dict = ckpt_to_state_dict( @@ -245,6 +283,7 @@ def _save_hf_safetensor_legacy( ckpt_manager=ckpt_manager, output_dir=output_dir, ) + model_state_dict = _revert_model_weight_conversions_for_hf(model, model_state_dict) save_model_weights(save_hf_safetensor_path, model_state_dict, model_assets=model_assets) logger.info_rank0(f"HuggingFace checkpoint saved at {save_hf_safetensor_path} successfully!") @@ -330,6 +369,7 @@ def save_hf_safetensor( model_assets, ckpt_manager, output_dir, + model, ) # Ensure all ranks finish saving before anyone proceeds From cd620b30f0d70d6a245aae90c9025bbb8cae150b Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 15:22:17 +0800 Subject: [PATCH 02/22] [ops, omni, model, config, docs, test] feat: add BAGEL MoT async Ulysses variant Co-authored-by: Cursor --- .../bagel_7b_mot/train/modules_train.yaml | 2 + .../with_cache/modules_train.yaml | 2 + docs/seed_omni/models/bagel.md | 26 + tests/ops/async_ulysses/test_async_ulysses.py | 196 ++++++- tests/ops/base/test_builtin_entries.py | 2 + .../modules/bagel/test_async_ulysses.py | 529 ++++++++++++++++++ .../modules/bagel/test_graph_config.py | 2 + .../seed_omni/modules/bagel/test_qwen2_mot.py | 3 + .../qwen2_mot/accelerated/accelerated.py | 95 +++- veomni/ops/README.md | 2 +- veomni/ops/kernels/async_ulysses/__init__.py | 22 +- .../async_ulysses/o_proj/bagel/__init__.py | 15 + .../async_ulysses/o_proj/bagel/eager.py | 158 ++++++ .../async_ulysses/qkv_proj/bagel/__init__.py | 15 + .../async_ulysses/qkv_proj/bagel/eager.py | 424 ++++++++++++++ 15 files changed, 1488 insertions(+), 5 deletions(-) create mode 100644 tests/seed_omni/modules/bagel/test_async_ulysses.py create mode 100644 veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py create mode 100644 veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py create mode 100644 veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py create mode 100644 veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml index 48b58617b1..c03909f5bd 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/modules_train.yaml @@ -19,3 +19,5 @@ bagel_qwen2_mot: model_path: bagel_qwen2_mot ops_implementation: attn_implementation: flex_attention + accelerator: + enable_async: true diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml index 48b58617b1..c03909f5bd 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/modules_train.yaml @@ -19,3 +19,5 @@ bagel_qwen2_mot: model_path: bagel_qwen2_mot ops_implementation: attn_implementation: flex_attention + accelerator: + enable_async: true diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md index 57dc34d2f3..d05ce8b4d3 100644 --- a/docs/seed_omni/models/bagel.md +++ b/docs/seed_omni/models/bagel.md @@ -125,6 +125,32 @@ code skip it via `item.is_dummy`. Training uses the accelerated Qwen2-MoT class from `train/modules_train.yaml`. The default packed attention backend is FlexAttention. +On H20 (SM90), eligible BF16/FP16 calls automatically use Flex's FlashAttention-4 +backend from the standard `gpu` extra. The MoT training module enables async +Ulysses when sequence parallelism is active. A single-node H20 baseline is: + +```bash +bash train.sh tasks/omni/train_omni.py \ + configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml \ + --model.accelerator.ulysses_size 4 \ + --train.micro_batch_size 1 +``` + +The online and cached-VAE training module configs both set +`bagel_qwen2_mot.accelerator.enable_async: true`. For a matched synchronous +control, override +`--model.model_config.modules.bagel_qwen2_mot.accelerator.enable_async false`. +With SP disabled, the fused synchronous QKV path is used regardless of this +flag. Both paths retain the same split-QKV external checkpoint schema. + +The async path dispatches through the `("async_ulysses_qkv", "bagel")` and +`("async_ulysses_o", "bagel")` compound ops, overlapping Q/K exchanges with +later routed projections and inverse exchanges with backward projection work. +It forwards the module's configured `rms_norm` and `rope` handles into the +compound ops and passes `skip_ulysses=True` to attention because Q/K/V have +already been redistributed. Inference continues to use its existing packed +prefill and FlashAttention-2 denoise/decode paths. + ```bash bash train.sh tasks/omni/train_omni.py \ configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml diff --git a/tests/ops/async_ulysses/test_async_ulysses.py b/tests/ops/async_ulysses/test_async_ulysses.py index e3d2931caf..0e1ca4a86c 100644 --- a/tests/ops/async_ulysses/test_async_ulysses.py +++ b/tests/ops/async_ulysses/test_async_ulysses.py @@ -42,6 +42,10 @@ ("async_ulysses_o", "standard"), ("async_ulysses_o", "dit"), ) +_BAGEL_EAGER_ROWS = ( + ("async_ulysses_qkv", "bagel"), + ("async_ulysses_o", "bagel"), +) def test_linear_backward_matches_autograd_under_cpu_bf16_autocast(): @@ -134,7 +138,7 @@ def fake_pad(tensor: Tensor, dim, **kwargs): def fake_unpad(tensor: Tensor, dim, size, **kwargs): return tensor - for op, variant in _EAGER_ROWS: + for op, variant in (*_EAGER_ROWS, *_BAGEL_EAGER_ROWS): module = _eager_module(op, variant) monkeypatch.setattr(module, "all_to_all_tensor", fake_all_to_all) monkeypatch.setattr(module, "padding_tensor_for_seqeunce_parallel", fake_pad) @@ -722,3 +726,193 @@ def test_dit_parity_world_size_matches_skip_gate(): assert DIT_SP_WORLD_SIZE == 4 assert AsyncUlyssesDiTSequenceParallelTest.world_size.fget(object()) == DIT_SP_WORLD_SIZE assert callable(AsyncUlyssesDiTSequenceParallelTest._configure_repro) + + +@pytest.mark.parametrize("empty_route", [None, 0, 1]) +def test_bagel_qkv_and_o_match_sequential_and_use_nested_handles( + monkeypatch: pytest.MonkeyPatch, + empty_route: int | None, +) -> None: + """Verify two-route BAGEL async Ulysses QKV and O projections match sequential execution.""" + _mock_identity_comm(monkeypatch) + torch.manual_seed(6201) + seq, hidden, num_q_heads, num_kv_heads, head_dim = 6, 16, 4, 2, 4 + query_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + eps = 1e-6 + + if empty_route == 0: + route_0_indices = torch.empty(0, dtype=torch.long) + route_1_indices = torch.arange(seq, dtype=torch.long) + elif empty_route == 1: + route_0_indices = torch.arange(seq, dtype=torch.long) + route_1_indices = torch.empty(0, dtype=torch.long) + else: + route_0_indices = torch.tensor([0, 2, 4], dtype=torch.long) + route_1_indices = torch.tensor([1, 3, 5], dtype=torch.long) + + angles = torch.randn(seq, head_dim // 2, dtype=torch.float64) + cos = torch.cat((angles.cos(), angles.cos()), dim=-1) + sin = torch.cat((angles.sin(), angles.sin()), dim=-1) + + hidden_states = torch.randn(seq, hidden, dtype=torch.float64, requires_grad=True) + weights_0 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64) + weights_1 = _qkv_weights(hidden, query_size, kv_size, dtype=torch.float64) + norm_q_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_k_0 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_q_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + norm_k_1 = torch.randn(head_dim, dtype=torch.float64, requires_grad=True) + o_weight_0 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True) + o_bias_0 = torch.randn(hidden, dtype=torch.float64, requires_grad=True) + o_weight_1 = torch.randn(hidden, query_size, dtype=torch.float64, requires_grad=True) + o_bias_1 = torch.randn(hidden, dtype=torch.float64, requires_grad=True) + + diff_tensors = ( + hidden_states, + *weights_0, + norm_q_0, + norm_k_0, + *weights_1, + norm_q_1, + norm_k_1, + o_weight_0, + o_bias_0, + o_weight_1, + o_bias_1, + ) + actual_leaves = _leaf(*diff_tensors) + expected_leaves = _leaf(*diff_tensors) + + ( + h_a, + qw0_a, + qb0_a, + kw0_a, + kb0_a, + vw0_a, + vb0_a, + nq0_a, + nk0_a, + qw1_a, + qb1_a, + kw1_a, + kb1_a, + vw1_a, + vb1_a, + nq1_a, + nk1_a, + ow0_a, + ob0_a, + ow1_a, + ob1_a, + ) = actual_leaves + q_a, k_a, v_a = VeomniOp("async_ulysses_qkv", "bagel")( + h_a, + route_0_indices, + route_1_indices, + cos, + sin, + qw0_a, + qb0_a, + kw0_a, + kb0_a, + vw0_a, + vb0_a, + nq0_a, + nk0_a, + qw1_a, + qb1_a, + kw1_a, + kb1_a, + vw1_a, + vb1_a, + nq1_a, + nk1_a, + head_dim=head_dim, + rms_norm_eps=eps, + unpadded_dim_size=seq, + group=object(), + rms_norm=VeomniOp("rms_norm", "standard", "eager"), + rope=VeomniOp("rope", "full", "eager"), + ) + attn_a = ( + q_a + + k_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + + v_a.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + ) + out_a = VeomniOp("async_ulysses_o", "bagel")( + attn_a, + route_0_indices, + route_1_indices, + ow0_a, + ob0_a, + ow1_a, + ob1_a, + group=object(), + ) + + ( + h_e, + qw0_e, + qb0_e, + kw0_e, + kb0_e, + vw0_e, + vb0_e, + nq0_e, + nk0_e, + qw1_e, + qb1_e, + kw1_e, + kb1_e, + vw1_e, + vb1_e, + nq1_e, + nk1_e, + ow0_e, + ob0_e, + ow1_e, + ob1_e, + ) = expected_leaves + rms = VeomniOp("rms_norm", "standard", "eager") + rope = VeomniOp("rope", "full", "eager") + + def routed_qkv(w0, b0, w1, b1, heads, n0=None, n1=None): + p0 = F.linear(h_e.index_select(0, route_0_indices), w0, b0).reshape(-1, heads, head_dim) + p1 = F.linear(h_e.index_select(0, route_1_indices), w1, b1).reshape(-1, heads, head_dim) + if n0 is not None and n1 is not None: + p0 = rms(p0, n0, eps=eps) + p1 = rms(p1, n1, eps=eps) + combined = p0.new_zeros((seq, heads, head_dim)) + combined = combined.index_copy(0, route_0_indices, p0) + combined = combined.index_copy(0, route_1_indices, p1) + return combined + + q_e = routed_qkv(qw0_e, qb0_e, qw1_e, qb1_e, num_q_heads, nq0_e, nq1_e) + k_e = routed_qkv(kw0_e, kb0_e, kw1_e, kb1_e, num_kv_heads, nk0_e, nk1_e) + v_e = routed_qkv(vw0_e, vb0_e, vw1_e, vb1_e, num_kv_heads) + q_e, k_e = rope( + q_e.transpose(0, 1).unsqueeze(0), + k_e.transpose(0, 1).unsqueeze(0), + cos.unsqueeze(0), + sin.unsqueeze(0), + unsqueeze_dim=1, + ) + q_e = q_e.squeeze(0).transpose(0, 1).contiguous() + k_e = k_e.squeeze(0).transpose(0, 1).contiguous() + attn_e = ( + q_e + + k_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + + v_e.repeat_interleave(num_q_heads // num_kv_heads, dim=1) + ) + flat_e = attn_e.reshape(seq, query_size) + out_e = flat_e.new_zeros((seq, hidden)) + out_e = out_e.index_copy(0, route_0_indices, F.linear(flat_e.index_select(0, route_0_indices), ow0_e, ob0_e)) + out_e = out_e.index_copy(0, route_1_indices, F.linear(flat_e.index_select(0, route_1_indices), ow1_e, ob1_e)) + + torch.testing.assert_close(out_a, out_e) + grad_out = torch.randn_like(out_a) + out_a.backward(grad_out) + out_e.backward(grad_out) + for actual, expected in zip(actual_leaves, expected_leaves, strict=True): + torch.testing.assert_close(actual.grad, expected.grad) diff --git a/tests/ops/base/test_builtin_entries.py b/tests/ops/base/test_builtin_entries.py index c981ab5e4e..c849cf76f4 100644 --- a/tests/ops/base/test_builtin_entries.py +++ b/tests/ops/base/test_builtin_entries.py @@ -53,8 +53,10 @@ ("moe_unpermute", "standard", "triton", "cuda", _GPU_NVIDIA_SM70, ("triton",)), ("async_ulysses_qkv", "standard", "eager", "any", _ANY, ()), ("async_ulysses_qkv", "dit", "eager", "any", _ANY, ()), + ("async_ulysses_qkv", "bagel", "eager", "any", _ANY, ()), ("async_ulysses_o", "standard", "eager", "any", _ANY, ()), ("async_ulysses_o", "dit", "eager", "any", _ANY, ()), + ("async_ulysses_o", "bagel", "eager", "any", _ANY, ()), ("attention", "standard", "eager", "any", _ANY, ()), ("attention", "standard", "sdpa", "any", _ANY, ()), ("attention", "standard", "flash_attention_2", "cuda", _GPU_SM80, ("flash_attn",)), diff --git a/tests/seed_omni/modules/bagel/test_async_ulysses.py b/tests/seed_omni/modules/bagel/test_async_ulysses.py new file mode 100644 index 0000000000..4ae61a9172 --- /dev/null +++ b/tests/seed_omni/modules/bagel/test_async_ulysses.py @@ -0,0 +1,529 @@ +"""Distributed numerical guards for BAGEL MoT async Ulysses and Flex-FA4.""" + +from __future__ import annotations + +import copy +from dataclasses import replace + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard + +from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.tools.launch_utils import torchrun +from veomni.distributed.parallel_state import _init_parallel_state, use_parallel_state +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + pad_mot_attention_metadata, +) +from veomni.ops import VeomniOp +from veomni.ops.kernels.attention.standard import flex as flex_backend +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type, get_torch_device +from veomni.utils.helper import enable_high_precision_for_bf16 + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(nproc=4)) + + +def _rms_norm_forward( + input_tensor: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Compute reference FP32-accumulated RMSNorm.""" + invvar = torch.rsqrt(input_tensor.float().square().mean(dim=-1, keepdim=True) + eps) + output = (input_tensor.float() * invvar * weight.float()).to(input_tensor.dtype) + return output, invvar + + +def _apply_rotary( + input_tensor: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> torch.Tensor: + """Compute reference rotate-half RoPE on ``[tokens, heads, head_dim]``.""" + first, second = input_tensor.chunk(2, dim=-1) + rotated = torch.cat((-second, first), dim=-1) + return input_tensor * cos.unsqueeze(1) + rotated * sin.unsqueeze(1) + + +def _reference_qkv_projection( + hidden_states: torch.Tensor, + route_0_indices: torch.Tensor, + route_1_indices: torch.Tensor, + position_cos: torch.Tensor, + position_sin: torch.Tensor, + parameters: dict[str, torch.Tensor], + *, + num_query_heads: int, + num_key_value_heads: int, + head_dim: int, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute full-sequence sequential two-route Q/K/V projection, RMSNorm, and RoPE.""" + + def project(name: str, route: int, indices: torch.Tensor, num_heads: int) -> torch.Tensor: + projected = F.linear( + hidden_states.index_select(0, indices), + parameters[f"{name}_weight_{route}"], + parameters[f"{name}_bias_{route}"], + ) + return projected.reshape(-1, num_heads, head_dim) + + def combine(name: str, num_heads: int, *, normalize: bool) -> torch.Tensor: + output = hidden_states.new_zeros(hidden_states.shape[0], num_heads, head_dim) + for route, indices in ((0, route_0_indices), (1, route_1_indices)): + projected = project(name, route, indices, num_heads) + if normalize: + projected, _ = _rms_norm_forward(projected, parameters[f"{name}_norm_weight_{route}"], eps) + output = output.index_copy(0, indices, projected) + return output + + query = _apply_rotary(combine("q", num_query_heads, normalize=True), position_cos, position_sin) + key = _apply_rotary(combine("k", num_key_value_heads, normalize=True), position_cos, position_sin) + value = combine("v", num_key_value_heads, normalize=False) + return query, key, value + + +def _reference_output_projection( + hidden_states: torch.Tensor, + route_0_indices: torch.Tensor, + route_1_indices: torch.Tensor, + parameters: dict[str, torch.Tensor], +) -> torch.Tensor: + """Compute full-sequence sequential two-route output projection.""" + flattened_states = hidden_states.flatten(1) + output = flattened_states.new_zeros(flattened_states.shape) + for route, indices in ((0, route_0_indices), (1, route_1_indices)): + projected = F.linear( + flattened_states.index_select(0, indices), + parameters[f"proj_weight_{route}"], + parameters[f"proj_bias_{route}"], + ) + output = output.index_copy(0, indices, projected) + return output + + +def _make_parameters( + device: torch.device, + *, + hidden_size: int, + query_size: int, + key_value_size: int, + head_dim: int, +) -> dict[str, torch.Tensor]: + """Allocate random two-route QKV, QK-norm, and output projection parameters.""" + shapes = { + "q_weight_0": (query_size, hidden_size), + "q_bias_0": (query_size,), + "k_weight_0": (key_value_size, hidden_size), + "k_bias_0": (key_value_size,), + "v_weight_0": (key_value_size, hidden_size), + "v_bias_0": (key_value_size,), + "q_norm_weight_0": (head_dim,), + "k_norm_weight_0": (head_dim,), + "q_weight_1": (query_size, hidden_size), + "q_bias_1": (query_size,), + "k_weight_1": (key_value_size, hidden_size), + "k_bias_1": (key_value_size,), + "v_weight_1": (key_value_size, hidden_size), + "v_bias_1": (key_value_size,), + "q_norm_weight_1": (head_dim,), + "k_norm_weight_1": (head_dim,), + "proj_weight_0": (hidden_size, query_size), + "proj_bias_0": (hidden_size,), + "proj_weight_1": (hidden_size, query_size), + "proj_bias_1": (hidden_size,), + } + parameters = {} + for name, shape in shapes.items(): + value = torch.randn(shape, device=device, dtype=torch.float32) + if "norm_weight" in name: + value = value * 0.1 + 1 + parameters[name] = value.requires_grad_() + return parameters + + +def _clone_parameters(parameters: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Detach and clone ``parameters`` into independent leaf tensors.""" + return {name: parameter.detach().clone().requires_grad_() for name, parameter in parameters.items()} + + +def _async_ulysses_mot_worker(num_key_value_heads: int, route_case: str) -> None: + """Compare 4-rank ``async_ulysses_qkv`` / ``async_ulysses_o`` ``bagel`` ops against sequential reference.""" + rank = dist.get_rank() + world_size = dist.get_world_size() + assert world_size == 4 + device = torch.device(f"{get_device_type()}:{rank}") + group = dist.group.WORLD + enable_high_precision_for_bf16() + torch.manual_seed(4201) + + sequence_length = 8 + local_sequence_length = sequence_length // world_size + hidden_size = 32 + num_query_heads = 8 + head_dim = hidden_size // num_query_heads + query_size = num_query_heads * head_dim + key_value_size = num_key_value_heads * head_dim + eps = 1e-6 + + full_input = torch.randn(sequence_length, hidden_size, device=device, dtype=torch.float32) + dist.broadcast(full_input, src=0) + angles = torch.randn(sequence_length, head_dim // 2, device=device, dtype=torch.float32) + dist.broadcast(angles, src=0) + position_cos = torch.cat((angles.cos(), angles.cos()), dim=-1) + position_sin = torch.cat((angles.sin(), angles.sin()), dim=-1) + full_output_gradient = torch.randn_like(full_input) + dist.broadcast(full_output_gradient, src=0) + + parameter_seed = _make_parameters( + device, + hidden_size=hidden_size, + query_size=query_size, + key_value_size=key_value_size, + head_dim=head_dim, + ) + for parameter in parameter_seed.values(): + dist.broadcast(parameter, src=0) + async_parameters = _clone_parameters(parameter_seed) + reference_parameters = _clone_parameters(parameter_seed) + + start = rank * local_sequence_length + stop = start + local_sequence_length + local_input = full_input[start:stop].detach().clone().requires_grad_() + reference_input = full_input.detach().clone().requires_grad_() + token_types = torch.arange(sequence_length, device=device) % 2 + if route_case == "empty_gen": + token_types.zero_() + elif route_case == "rank_routes": + token_types = torch.arange(sequence_length, device=device) // local_sequence_length % 2 + elif route_case == "padding": + token_types[-1] = -1 + full_route_0_indices = torch.where(token_types == 0)[0] + full_route_1_indices = torch.where(token_types == 1)[0] + local_route_0_indices = torch.where(token_types[start:stop] == 0)[0] + local_route_1_indices = torch.where(token_types[start:stop] == 1)[0] + + qkv_op = VeomniOp("async_ulysses_qkv", "bagel") + o_op = VeomniOp("async_ulysses_o", "bagel") + rms_op = VeomniOp("rms_norm", "standard", "eager") + rope_op = VeomniOp("rope", "full", "eager") + + async_query, async_key, async_value = qkv_op( + hidden_states=local_input, + route_0_indices=local_route_0_indices, + route_1_indices=local_route_1_indices, + position_cos=position_cos[start:stop], + position_sin=position_sin[start:stop], + q_weight_0=async_parameters["q_weight_0"], + q_bias_0=async_parameters["q_bias_0"], + k_weight_0=async_parameters["k_weight_0"], + k_bias_0=async_parameters["k_bias_0"], + v_weight_0=async_parameters["v_weight_0"], + v_bias_0=async_parameters["v_bias_0"], + q_norm_weight_0=async_parameters["q_norm_weight_0"], + k_norm_weight_0=async_parameters["k_norm_weight_0"], + q_weight_1=async_parameters["q_weight_1"], + q_bias_1=async_parameters["q_bias_1"], + k_weight_1=async_parameters["k_weight_1"], + k_bias_1=async_parameters["k_bias_1"], + v_weight_1=async_parameters["v_weight_1"], + v_bias_1=async_parameters["v_bias_1"], + q_norm_weight_1=async_parameters["q_norm_weight_1"], + k_norm_weight_1=async_parameters["k_norm_weight_1"], + head_dim=head_dim, + rms_norm_eps=eps, + unpadded_dim_size=sequence_length, + group=group, + rms_norm=rms_op, + rope=rope_op, + ) + + reference_query, reference_key, reference_value = _reference_qkv_projection( + reference_input, + full_route_0_indices, + full_route_1_indices, + position_cos, + position_sin, + reference_parameters, + num_query_heads=num_query_heads, + num_key_value_heads=num_key_value_heads, + head_dim=head_dim, + eps=eps, + ) + local_query_heads = num_query_heads // world_size + exchanged_key_value_heads = max(num_key_value_heads, world_size) + local_key_value_heads = exchanged_key_value_heads // world_size + torch.testing.assert_close( + async_query, + reference_query[:, rank * local_query_heads : (rank + 1) * local_query_heads], + ) + repeated_reference_key = torch.repeat_interleave( + reference_key, + repeats=exchanged_key_value_heads // num_key_value_heads, + dim=1, + ) + repeated_reference_value = torch.repeat_interleave( + reference_value, + repeats=exchanged_key_value_heads // num_key_value_heads, + dim=1, + ) + torch.testing.assert_close( + async_key, + repeated_reference_key[:, rank * local_key_value_heads : (rank + 1) * local_key_value_heads], + ) + torch.testing.assert_close( + async_value, + repeated_reference_value[:, rank * local_key_value_heads : (rank + 1) * local_key_value_heads], + ) + + local_gqa_repeats = local_query_heads // local_key_value_heads + async_attention_output = async_query + torch.repeat_interleave( + async_key + async_value, + repeats=local_gqa_repeats, + dim=1, + ) + reference_attention_output = reference_query + torch.repeat_interleave( + reference_key + reference_value, + repeats=num_query_heads // num_key_value_heads, + dim=1, + ) + async_output = o_op( + hidden_states=async_attention_output, + route_0_indices=local_route_0_indices, + route_1_indices=local_route_1_indices, + proj_weight_0=async_parameters["proj_weight_0"], + proj_bias_0=async_parameters["proj_bias_0"], + proj_weight_1=async_parameters["proj_weight_1"], + proj_bias_1=async_parameters["proj_bias_1"], + group=group, + ) + reference_output = _reference_output_projection( + reference_attention_output, + full_route_0_indices, + full_route_1_indices, + reference_parameters, + ) + torch.testing.assert_close(async_output, reference_output[start:stop], atol=2e-5, rtol=2e-5) + + async_output.backward(full_output_gradient[start:stop]) + reference_output.backward(full_output_gradient) + + gathered_input_grads = [torch.empty_like(local_input.grad) for _ in range(world_size)] + dist.all_gather(gathered_input_grads, local_input.grad) + torch.testing.assert_close(torch.cat(gathered_input_grads), reference_input.grad, atol=3e-5, rtol=3e-5) + for name, async_parameter in async_parameters.items(): + assert async_parameter.grad is not None, f"missing async gradient for {name}" + assert reference_parameters[name].grad is not None, f"missing reference gradient for {name}" + async_gradient = async_parameter.grad.detach().clone() + dist.all_reduce(async_gradient, group=group) + torch.testing.assert_close( + async_gradient, + reference_parameters[name].grad, + atol=5e-5, + rtol=5e-5, + msg=f"gradient mismatch for {name}", + ) + + +@pytest.mark.skipif(get_torch_device().device_count() < 4, reason="device_count should be >= 4") +@pytest.mark.parametrize( + ("num_key_value_heads", "route_case"), + [(2, "mixed"), (4, "empty_gen"), (4, "rank_routes"), (4, "padding")], +) +def test_async_ulysses_mot_matches_eager_two_route_projection_forward_backward( + num_key_value_heads: int, + route_case: str, +) -> None: + """Verify two-route BAGEL async Ulysses forward and backward across 4 ranks.""" + torchrun(_async_ulysses_mot_worker, world_size=4, num_key_value_heads=num_key_value_heads, route_case=route_case) + + +def _bagel_fa4_async_worker(ops_impl: str, checkpointing: bool, fsdp: bool, backend: str, dtype: torch.dtype) -> None: + """Compare synchronous and asynchronous Ulysses in ``BagelQwen2MoTAccelerated`` over 2 steps.""" + rank = dist.get_rank() + device = torch.device(f"{get_device_type()}:{rank}") + torch.manual_seed(6321) + enable_high_precision_for_bf16() + state = _init_parallel_state(dp_size=1, ulysses_size=4, name="bagel_sync_test") + async_state = replace(state, async_enabled=True) + + # SP4 leaves seven Q heads and one KV head per rank, matching BAGEL's + # non-power-of-two GQA geometry. Keep weights small while exercising FA4. + config = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=896, + intermediate_size=1024, + num_hidden_layers=1, + num_attention_heads=28, + num_key_value_heads=4, + max_position_embeddings=1024, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "flex_attention", + "rms_norm_implementation": ops_impl, + "rotary_pos_emb_implementation": ops_impl, + "swiglu_mlp_implementation": "eager", + }, + ) + reference = accelerated.BagelQwen2MoTAccelerated(config).to(device=device, dtype=dtype).train() + candidate = copy.deepcopy(reference) + if checkpointing: + for model in (reference, candidate): + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + if fsdp: + # FP32 parameter/gradient storage with BF16 forward matches the mixed + # precision contract used by the production FSDP2 wrapper. + for model in (reference, candidate): + model.float() + policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32) + for layer in model.model.layers: + fully_shard(layer, mesh=state.fsdp_mesh, mp_policy=policy) + fully_shard(model.model, mesh=state.fsdp_mesh, mp_policy=policy) + + selected_backends = [] + original_resolve = flex_backend.resolve_flex_attention + + def capture_backend(*args, **kwargs): + if backend == "TRITON": + args[1]["BACKEND"] = "TRITON" + interface = original_resolve(*args, **kwargs) + selected_backends.append(interface) + return interface + + flex_backend.resolve_flex_attention = capture_backend + # Two independent documents, a full span, and two mutually invisible noise + # spans exercise real MoT mask rules. The final SP shard includes padding. + sequence_length, padded_length = 259, 260 + metadata = build_mot_attention_metadata( + [[65, 64, 64], [33, 33]], + [["causal", "noise", "noise"], ["full", "causal"]], + device=device, + ) + metadata = pad_mot_attention_metadata(metadata, padded_length=padded_length) + hidden = torch.randn(padded_length, config.hidden_size, device=device, dtype=dtype) + token_types = torch.zeros(padded_length, device=device, dtype=torch.long) + token_types[65:193] = 1 + token_types[sequence_length:] = -1 + hidden[sequence_length:] = 0 + output_gradient = torch.randn_like(hidden) + output_gradient[sequence_length:] = 0 + positions = torch.arange(padded_length, device=device) + start, stop = rank * 65, (rank + 1) * 65 + local_und = torch.where(token_types[start:stop] == 0)[0] + local_gen = torch.where(token_types[start:stop] == 1)[0] + inputs = { + "packed_position_ids": positions[start:stop], + "packed_attention_metadata": metadata, + "packed_und_token_indexes": local_und, + "packed_gen_token_indexes": local_gen, + } + optimizers = [torch.optim.SGD(model.parameters(), lr=1e-3) for model in (reference, candidate)] + try: + # A second update catches weight-view and saved-tensor lifetime errors + # that may be hidden by the first FSDP unshard/recompute pass. + for _ in range(2): + outputs, input_gradients = [], [] + for model, model_state in ((reference, state), (candidate, async_state)): + local_input = hidden[start:stop].detach().clone().requires_grad_() + with use_parallel_state(model_state): + output = model.model(packed_sequence=local_input, **inputs).packed_query_sequence + output.backward(output_gradient[start:stop]) + outputs.append(output.detach()) + input_gradients.append(local_input.grad) + tolerance = 2e-4 if dtype == torch.float32 else 3e-2 + torch.testing.assert_close(outputs[1], outputs[0], atol=tolerance, rtol=tolerance) + torch.testing.assert_close(input_gradients[1], input_gradients[0], atol=tolerance, rtol=tolerance) + for (name, expected), (actual_name, actual) in zip( + reference.named_parameters(), candidate.named_parameters(), strict=True + ): + assert name == actual_name + assert expected.grad is not None and actual.grad is not None, name + expected_grad, actual_grad = expected.grad, actual.grad + if fsdp: + expected_grad, actual_grad = expected_grad.to_local(), actual_grad.to_local() + assert torch.isfinite(actual_grad).all(), name + if dtype == torch.float32: + torch.testing.assert_close( + actual_grad, + expected_grad, + atol=2e-4, + rtol=2e-4, + msg=lambda message, name=name: f"{name}: {message}", + ) + else: + gradient_pairs = [(name, actual_grad, expected_grad)] + if "qkv_proj" in name and not fsdp: + split_sizes = reference.model.layers[0].self_attn.qkv_split_sizes + gradient_pairs = [ + (f"{name}.{part}", actual_part, expected_part) + for part, actual_part, expected_part in zip( + ("q", "k", "v"), + actual_grad.split(split_sizes), + expected_grad.split(split_sizes), + strict=True, + ) + ] + # Split and fused schedules can differ at BF16 precision. + # Check each Q/K/V block separately so a large Q gradient + # cannot hide an incorrect K/V gradient. The FP32 control + # above retains strict pointwise derivative checks. + for gradient_name, actual_part, expected_part in gradient_pairs: + error = (actual_part.float() - expected_part.float()).norm() + scale = expected_part.float().norm() + assert error <= 0.01 * scale + 1e-7, ( + f"{gradient_name}: relative gradient L2 error {error / scale.clamp_min(1e-12)}" + ) + + for optimizer in optimizers: + optimizer.step() + optimizer.zero_grad(set_to_none=True) + expected_backend = ( + flex_backend._flex_attention_fa4 if backend == "FLASH" else flex_backend._flex_attention_triton + ) + assert selected_backends and all(selected is expected_backend for selected in selected_backends) + finally: + flex_backend.resolve_flex_attention = original_resolve + + +@pytest.mark.skipif( + not IS_CUDA_AVAILABLE or get_torch_device().device_count() < 4, + reason="BAGEL FA4 + async Ulysses integration requires four SM90+ GPUs", +) +@pytest.mark.parametrize( + ("ops_impl", "checkpointing", "fsdp", "backend", "dtype"), + [ + pytest.param("eager", False, False, "TRITON", torch.float32, id="fp32-triton-control"), + pytest.param("eager", False, False, "FLASH", torch.bfloat16, id="bf16-fa4-eager"), + pytest.param("liger_kernel", True, False, "FLASH", torch.bfloat16, id="bf16-fa4-liger-checkpoint"), + pytest.param("liger_kernel", True, True, "FLASH", torch.bfloat16, id="bf16-fa4-liger-checkpoint-fsdp2"), + ], +) +def test_bagel_fa4_async_matches_sync_forward_backward_and_updates( + ops_impl: str, + checkpointing: bool, + fsdp: bool, + backend: str, + dtype: torch.dtype, +) -> None: + """Verify BAGEL async Ulysses matches synchronous Ulysses under Flex-FA4, checkpointing, and FSDP2.""" + device = torch.device(get_device_type()) + if backend == "FLASH" and ( + flex_backend.get_gpu_compute_capability(device) < 90 or not flex_backend._fa4._flash_attn_cute_available() + ): + pytest.skip("Flex-FA4 integration requires SM90+ and flash-attn-4") + torchrun( + _bagel_fa4_async_worker, + world_size=4, + ops_impl=ops_impl, + checkpointing=checkpointing, + fsdp=fsdp, + backend=backend, + dtype=dtype, + ) diff --git a/tests/seed_omni/modules/bagel/test_graph_config.py b/tests/seed_omni/modules/bagel/test_graph_config.py index 9247e0c37d..d50c50cd9b 100644 --- a/tests/seed_omni/modules/bagel/test_graph_config.py +++ b/tests/seed_omni/modules/bagel/test_graph_config.py @@ -37,6 +37,7 @@ def test_bagel_train_yaml_loads_with_v2_module_names(): assert "end" in endpoints runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") + assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True def test_bagel_train_graph_fan_in_execution_order(): @@ -112,6 +113,7 @@ def test_bagel_train_plus_infer_merges_generation_graph(infer_graph: str): assert cfg.generation_graph is not None runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") + assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True assert cfg.generation_graph["initial"] == "prompt_encode" assert "done" not in cfg.generation_graph["states"] assert any( diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 7e32069fb4..0b1adcfdec 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -200,7 +200,10 @@ def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config() -> layer = model.model.layers[0] assert layer.input_layernorm.veomni_rms_norm.impl == "eager" assert layer.mlp.veomni_swiglu_mlp.impl == "eager" + assert layer.self_attn.veomni_rms_norm.impl == "eager" assert layer.self_attn.veomni_rope.impl == "eager" + assert layer.self_attn.veomni_async_ulysses_qkv.variant == "bagel" + assert layer.self_attn.veomni_async_ulysses_o.variant == "bagel" assert layer.self_attn.veomni_attn is not None assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention" assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2" diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index 5c7cecb203..453bb1e1e1 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -1,7 +1,7 @@ """VeOmni-accelerated BAGEL Qwen2-MoT — instance-local VeomniOp ops plus training-graph hooks. Native weights and packed ``forward`` live on :class:`BagelQwen2MoTCore`. -This module adds packed CFG denoise, fused RMSNorm / SwiGLU / RoPE, +This module adds packed CFG denoise, async Ulysses, fused RMSNorm / SwiGLU / RoPE, attention ``VeomniOp`` dispatch, and the training pre/forward/post hooks. """ @@ -469,7 +469,10 @@ def __init__( rms_norm_cls: type[Qwen2RMSNorm] = Qwen2RMSNorm, ): super().__init__(config, layer_idx, rms_norm_cls=rms_norm_cls) + self.veomni_rms_norm = VeomniOp("rms_norm", "standard", config.ops_config.rms_norm_implementation) self.veomni_rope = VeomniOp("rope", "full", config.ops_config.rotary_pos_emb_implementation) + self.veomni_async_ulysses_qkv = VeomniOp("async_ulysses_qkv", "bagel") + self.veomni_async_ulysses_o = VeomniOp("async_ulysses_o", "bagel") attn_impl = _normalize_packed_attn_impl(config.ops_config.attn_implementation) if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl) @@ -609,6 +612,96 @@ def attend_inference( ) return packed_attn_output.squeeze(0) + def forward_packed_train( + self, + packed_sequence: torch.Tensor, + attention_mask: BlockMask | MagiAttentionMask, + packed_position_cos: torch.Tensor, + packed_position_sin: torch.Tensor, + packed_und_token_indexes: torch.Tensor, + packed_gen_token_indexes: torch.Tensor, + ) -> torch.Tensor: + """Run packed training attention, using async Ulysses when enabled on the active parallel state.""" + parallel_state = get_parallel_state() + use_async_ulysses = parallel_state.ulysses_enabled and bool(parallel_state.async_enabled) + if not use_async_ulysses: + return super().forward_packed_train( + packed_sequence, + attention_mask, + packed_position_cos, + packed_position_sin, + packed_und_token_indexes, + packed_gen_token_indexes, + ) + attn_op = self._resolve_packed_attn_op(context="training") + + q_weight_und, k_weight_und, v_weight_und = self.qkv_proj_und.weight.split(self.qkv_split_sizes, dim=0) + q_bias_und, k_bias_und, v_bias_und = ( + (None, None, None) + if self.qkv_proj_und.bias is None + else self.qkv_proj_und.bias.split(self.qkv_split_sizes, dim=0) + ) + q_weight_gen, k_weight_gen, v_weight_gen = self.qkv_proj_gen.weight.split(self.qkv_split_sizes, dim=0) + q_bias_gen, k_bias_gen, v_bias_gen = ( + (None, None, None) + if self.qkv_proj_gen.bias is None + else self.qkv_proj_gen.bias.split(self.qkv_split_sizes, dim=0) + ) + rms_norm_op = getattr(self.q_norm, "veomni_rms_norm", self.veomni_rms_norm) + packed_query_states, packed_key_states, packed_value_states = self.veomni_async_ulysses_qkv( + hidden_states=packed_sequence, + route_0_indices=packed_und_token_indexes, + route_1_indices=packed_gen_token_indexes, + position_cos=packed_position_cos, + position_sin=packed_position_sin, + q_weight_0=q_weight_und, + q_bias_0=q_bias_und, + k_weight_0=k_weight_und, + k_bias_0=k_bias_und, + v_weight_0=v_weight_und, + v_bias_0=v_bias_und, + q_norm_weight_0=self.q_norm.weight, + k_norm_weight_0=self.k_norm.weight, + q_weight_1=q_weight_gen, + q_bias_1=q_bias_gen, + k_weight_1=k_weight_gen, + k_bias_1=k_bias_gen, + v_weight_1=v_weight_gen, + v_bias_1=v_bias_gen, + q_norm_weight_1=self.q_norm_moe_gen.weight, + k_norm_weight_1=self.k_norm_moe_gen.weight, + head_dim=self.head_dim, + rms_norm_eps=self.config.rms_norm_eps, + # The SP pre-hook already pads the packed sequence and builds a + # BlockMask for that full padded length. Keep those padding rows + # through attention; the module post-hook removes them after the + # local output shards have been gathered. + unpadded_dim_size=packed_sequence.shape[0] * parallel_state.ulysses_size, + group=parallel_state.ulysses_group, + rms_norm=rms_norm_op, + rope=self.veomni_rope, + ) + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return self.veomni_async_ulysses_o( + hidden_states=packed_attn_output.squeeze(0), + route_0_indices=packed_und_token_indexes, + route_1_indices=packed_gen_token_indexes, + proj_weight_0=self.o_proj.weight, + proj_bias_0=self.o_proj.bias, + proj_weight_1=self.o_proj_moe_gen.weight, + proj_bias_1=self.o_proj_moe_gen.bias, + group=parallel_state.ulysses_group, + ) + class InferenceMixinAccelerated(InferenceMixin): """Packed CFG denoise for the FSDP/runtime MoT class. diff --git a/veomni/ops/README.md b/veomni/ops/README.md index 431534cbe0..6b4313fa4a 100644 --- a/veomni/ops/README.md +++ b/veomni/ops/README.md @@ -43,7 +43,7 @@ when the selected implementation runs. | Op | Variants | Implementations | |---|---|---| | `attention` | `standard` | `eager`, `sdpa`, and FlashAttention/AITER/FlexAttention/MagiAttention/SageAttention names | -| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit` | `eager` orchestration | +| `async_ulysses_qkv`, `async_ulysses_o` | `standard`, `dit`, `bagel` | `eager` orchestration | | `rms_norm` | `standard`, `deepseek_v4`, `unweighted`, `offset` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; all accept optional `group_size` (fused weighted rows delegate grouped calls to eager) | | `layer_norm` | `standard` | `eager`, `apex` (`fused_layer_norm_cuda`); nested by async Ulysses QKV when `norm_type="layernorm"` | | `rope` | `full`, `partial`, `interleave`, `mrope`, `deepseek_v4`, `wan` | `eager`, `liger_kernel`, `torch_npu`, and `triton` where supported; `full` also accepts rank-3 vision layout | diff --git a/veomni/ops/kernels/async_ulysses/__init__.py b/veomni/ops/kernels/async_ulysses/__init__.py index 114f4093ef..9b065a579e 100644 --- a/veomni/ops/kernels/async_ulysses/__init__.py +++ b/veomni/ops/kernels/async_ulysses/__init__.py @@ -14,13 +14,15 @@ """Async Ulysses compound kernels. -``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard`` and ``dit`` -eager rows. Nested RMSNorm and LayerNorm use raw kernel pairs. +``async_ulysses_qkv`` / ``async_ulysses_o`` each have ``standard``, ``dit``, +and ``bagel`` eager rows. Nested RMSNorm, LayerNorm, and RoPE use raw kernel pairs. """ from ...registry import register_op +from .o_proj.bagel import eager as o_bagel from .o_proj.dit import eager as o_dit from .o_proj.standard import eager as o_standard +from .qkv_proj.bagel import eager as qkv_bagel from .qkv_proj.dit import eager as qkv_dit from .qkv_proj.standard import eager as qkv_standard @@ -41,6 +43,14 @@ qkv_dit.backward, description="VeOmni asynchronous Ulysses QKV projection for diffusion transformers", ) +register_op( + "async_ulysses_qkv", + "bagel", + "eager", + qkv_bagel.forward, + qkv_bagel.backward, + description="VeOmni asynchronous Ulysses QKV projection for BAGEL two-route MoT attention", +) register_op( "async_ulysses_o", "standard", @@ -57,3 +67,11 @@ o_dit.backward, description="VeOmni asynchronous Ulysses output projection for diffusion transformers", ) +register_op( + "async_ulysses_o", + "bagel", + "eager", + o_bagel.forward, + o_bagel.backward, + description="VeOmni asynchronous Ulysses output projection for BAGEL two-route MoT attention", +) diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py new file mode 100644 index 0000000000..bc1d3afdf2 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""async_ulysses_o bagel implementations.""" diff --git a/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py new file mode 100644 index 0000000000..41cdabfdc3 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/o_proj/bagel/eager.py @@ -0,0 +1,158 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BAGEL two-route MoT async Ulysses output projection: inverse A2A then routed linear.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn.functional as F +from torch import Tensor + +from ......distributed.sequence_parallel.comm import get_ulysses_sequence_parallel_group +from ......distributed.sequence_parallel.ulysses import all_to_all_tensor +from ......distributed.sequence_parallel.utils import ( + padding_tensor_for_seqeunce_parallel, + unpadding_tensor_for_seqeunce_parallel, +) +from .....registry import SavedState +from ...shared.backward import linear_input_backward, linear_parameter_backward + + +@dataclass +class _BagelOMeta: + """Saved all-to-all group, input sequence length, head geometry, and bias flags.""" + + sp_group: Any + input_sequence_length: int + num_heads: int + head_dim: int + has_proj_bias_0: bool + has_proj_bias_1: bool + + +def forward( + hidden_states: Tensor, + route_0_indices: Tensor, + route_1_indices: Tensor, + proj_weight_0: Tensor, + proj_bias_0: Tensor | None, + proj_weight_1: Tensor, + proj_bias_1: Tensor | None, + *, + group: Any = None, +) -> tuple[Tensor, SavedState]: + """Inverse Ulysses exchange followed by a two-route MoT output projection.""" + sp_group = get_ulysses_sequence_parallel_group() if group is None else group + if sp_group is None: + raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.") + input_sequence_length = hidden_states.shape[0] + hidden_states = padding_tensor_for_seqeunce_parallel(hidden_states, 0, group=sp_group) + hidden_states = all_to_all_tensor( + hidden_states, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=False, + ) + num_heads = hidden_states.shape[1] + head_dim = hidden_states.shape[2] + flattened_states = hidden_states.reshape(hidden_states.shape[0], num_heads * head_dim) + + output_0 = F.linear(flattened_states.index_select(0, route_0_indices), proj_weight_0, proj_bias_0) + output_1 = F.linear(flattened_states.index_select(0, route_1_indices), proj_weight_1, proj_bias_1) + output = output_0.new_zeros((flattened_states.shape[0], proj_weight_0.shape[0])) + output.index_copy_(0, route_0_indices, output_0) + output.index_copy_(0, route_1_indices, output_1) + + return output, SavedState( + ( + flattened_states, + route_0_indices, + route_1_indices, + proj_weight_0, + proj_weight_1, + ), + _BagelOMeta( + sp_group=sp_group, + input_sequence_length=input_sequence_length, + num_heads=num_heads, + head_dim=head_dim, + has_proj_bias_0=proj_bias_0 is not None, + has_proj_bias_1=proj_bias_1 is not None, + ), + ) + + +def backward(grad_output: Tensor, saved: SavedState) -> tuple[Tensor | None, ...]: + """Compute routed input gradients, launch reverse all-to-all, and overlap parameter gradients.""" + meta = saved.metadata + assert isinstance(meta, _BagelOMeta) + flattened_states, route_0_indices, route_1_indices, proj_weight_0, proj_weight_1 = saved.tensors + grad_output_0 = grad_output.index_select(0, route_0_indices) + grad_output_1 = grad_output.index_select(0, route_1_indices) + + flattened_states_0 = flattened_states.index_select(0, route_0_indices) + flattened_states_1 = flattened_states.index_select(0, route_1_indices) + grad_flattened_states = torch.zeros_like(flattened_states) + grad_flattened_states.index_copy_( + 0, + route_0_indices, + linear_input_backward(grad_output_0, flattened_states_0, proj_weight_0), + ) + grad_flattened_states.index_copy_( + 0, + route_1_indices, + linear_input_backward(grad_output_1, flattened_states_1, proj_weight_1), + ) + grad_hidden_states = grad_flattened_states.reshape(flattened_states.shape[0], meta.num_heads, meta.head_dim) + grad_hidden_wait = all_to_all_tensor( + grad_hidden_states, + scatter_dim=1, + gather_dim=0, + group=meta.sp_group, + async_op=True, + ) + + grad_proj_weight_0, grad_proj_bias_0 = linear_parameter_backward( + grad_output_0, + flattened_states_0, + proj_weight_0, + has_bias=meta.has_proj_bias_0, + ) + grad_proj_weight_1, grad_proj_bias_1 = linear_parameter_backward( + grad_output_1, + flattened_states_1, + proj_weight_1, + has_bias=meta.has_proj_bias_1, + ) + + grad_hidden_states = unpadding_tensor_for_seqeunce_parallel( + grad_hidden_wait(), + 0, + meta.input_sequence_length, + group=meta.sp_group, + ) + return ( + grad_hidden_states, + None, + None, + grad_proj_weight_0, + grad_proj_bias_0, + grad_proj_weight_1, + grad_proj_bias_1, + ) diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py new file mode 100644 index 0000000000..693dc5493e --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2025 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""async_ulysses_qkv bagel implementations.""" diff --git a/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py new file mode 100644 index 0000000000..1f0e0111c9 --- /dev/null +++ b/veomni/ops/kernels/async_ulysses/qkv_proj/bagel/eager.py @@ -0,0 +1,424 @@ +# Copyright 2026 Bytedance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BAGEL two-route MoT async Ulysses QKV: routed project, QK norm, RoPE, then A2A.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import Tensor + +from ......distributed.sequence_parallel.comm import ( + get_ulysses_sequence_parallel_group, + get_ulysses_sequence_parallel_world_size, +) +from ......distributed.sequence_parallel.ulysses import all_to_all_tensor +from ......distributed.sequence_parallel.utils import ( + padding_tensor_for_seqeunce_parallel, + unpadding_tensor_for_seqeunce_parallel, +) +from .....compound import InnerHandle, append_inner, resolve_inner_op, take_inner +from .....registry import OpEntry, SavedState +from ...shared.backward import linear_backward, reduce_repeated_kv_gradient + + +@dataclass +class _BagelQKVMeta: + """Non-tensor metadata and nested-op specs for two-route MoT QKV backward.""" + + sp_group: Any + head_dim: int + num_q_heads: int + num_kv_heads: int + repeat_kv: int + has_q_bias_0: bool + has_k_bias_0: bool + has_v_bias_0: bool + has_q_bias_1: bool + has_k_bias_1: bool + has_v_bias_1: bool + rms: OpEntry + rope: OpEntry + rms_q_0: tuple[Any, int] + rms_q_1: tuple[Any, int] + rms_k_0: tuple[Any, int] + rms_k_1: tuple[Any, int] + rope_q: tuple[Any, int] + rope_k: tuple[Any, int] + + +def _resolve_sp_group_and_size(group: Any) -> tuple[Any, int]: + """Return the active Ulysses process group and its world size.""" + sp_group = get_ulysses_sequence_parallel_group() if group is None else group + if sp_group is None: + raise ValueError("Async MoT Ulysses requires an initialized Ulysses process group.") + ulysses_size = ( + dist.get_world_size(sp_group) + if isinstance(sp_group, dist.ProcessGroup) + else get_ulysses_sequence_parallel_world_size() + ) + return sp_group, ulysses_size + + +def _project_route( + hidden_states: Tensor, + indices: Tensor, + weight: Tensor, + bias: Tensor | None, + num_heads: int, + head_dim: int, +) -> Tensor: + """Project rows selected by ``indices`` and reshape to ``[tokens, heads, head_dim]``.""" + projected = F.linear(hidden_states.index_select(0, indices), weight, bias) + return projected.reshape(-1, num_heads, head_dim) + + +def _rotate_single_forward( + rope: OpEntry, + x: Tensor, + cos: Tensor, + sin: Tensor, +) -> tuple[Tensor, SavedState]: + """Apply full RoPE to one ``[tokens, heads, head_dim]`` tensor before all-to-all.""" + q_in = x.transpose(0, 1).unsqueeze(0) + dummy_k = q_in.new_zeros((1, 1, x.shape[0], x.shape[2])) + (q_out, _), saved = rope.forward(q_in, dummy_k, cos.unsqueeze(0), sin.unsqueeze(0), None, 1) + return q_out.squeeze(0).transpose(0, 1).contiguous(), saved + + +def _rotate_single_backward( + rope: OpEntry, + grad_output: Tensor, + saved: SavedState, +) -> Tensor: + """Invert ``_rotate_single_forward`` on a ``[tokens, heads, head_dim]`` gradient.""" + grad_q = grad_output.transpose(0, 1).unsqueeze(0) + dummy_grad_k = grad_q.new_zeros((1, 1, grad_output.shape[0], grad_output.shape[2])) + dq, *_ = rope.backward((grad_q, dummy_grad_k), saved) + return dq.squeeze(0).transpose(0, 1).contiguous() + + +def forward( + hidden_states: Tensor, + route_0_indices: Tensor, + route_1_indices: Tensor, + position_cos: Tensor, + position_sin: Tensor, + q_weight_0: Tensor, + q_bias_0: Tensor | None, + k_weight_0: Tensor, + k_bias_0: Tensor | None, + v_weight_0: Tensor, + v_bias_0: Tensor | None, + q_norm_weight_0: Tensor, + k_norm_weight_0: Tensor, + q_weight_1: Tensor, + q_bias_1: Tensor | None, + k_weight_1: Tensor, + k_bias_1: Tensor | None, + v_weight_1: Tensor, + v_bias_1: Tensor | None, + q_norm_weight_1: Tensor, + k_norm_weight_1: Tensor, + *, + head_dim: int, + rms_norm_eps: float, + unpadded_dim_size: int, + group: Any = None, + rms_norm: InnerHandle = None, + rope: InnerHandle = None, +) -> tuple[tuple[Tensor, Tensor, Tensor], SavedState]: + """Two-route Q/K/V projection, per-route RMSNorm, RoPE, and async Ulysses exchange. + + Inputs are token-major ``[local_sequence, hidden_size]`` with local route + index tensors ``route_0_indices`` and ``route_1_indices``. Outputs are + ``[global_sequence, local_heads, head_dim]`` for Q, K, and V. + """ + num_q_heads = q_weight_0.shape[0] // head_dim + num_kv_heads = k_weight_0.shape[0] // head_dim + + sp_group, ulysses_size = _resolve_sp_group_and_size(group) + if num_q_heads % ulysses_size != 0: + raise ValueError(f"num_query_heads ({num_q_heads}) must be divisible by ulysses_size ({ulysses_size}).") + if ulysses_size > num_kv_heads: + if ulysses_size % num_kv_heads != 0: + raise ValueError( + f"ulysses_size ({ulysses_size}) must be divisible by num_key_value_heads ({num_kv_heads})." + ) + repeat_kv = ulysses_size // num_kv_heads + else: + if num_kv_heads % ulysses_size != 0: + raise ValueError( + f"num_key_value_heads ({num_kv_heads}) must be divisible by ulysses_size ({ulysses_size})." + ) + repeat_kv = 1 + + sequence_length = hidden_states.shape[0] + rms_entry = resolve_inner_op(rms_norm, op="rms_norm", variant="standard") + rope_entry = resolve_inner_op(rope, op="rope", variant="full") + + # Q projection + per-route RMSNorm + RoPE, followed by the first async + # exchange. K/V projection executes while earlier communication is in flight. + q_0 = _project_route(hidden_states, route_0_indices, q_weight_0, q_bias_0, num_q_heads, head_dim) + q_1 = _project_route(hidden_states, route_1_indices, q_weight_1, q_bias_1, num_q_heads, head_dim) + normalized_q_0, saved_rms_q_0 = rms_entry.forward(q_0, q_norm_weight_0, eps=rms_norm_eps) + normalized_q_1, saved_rms_q_1 = rms_entry.forward(q_1, q_norm_weight_1, eps=rms_norm_eps) + query = normalized_q_0.new_zeros((sequence_length, num_q_heads, head_dim)) + query.index_copy_(0, route_0_indices, normalized_q_0) + query.index_copy_(0, route_1_indices, normalized_q_1) + query, saved_rope_q = _rotate_single_forward(rope_entry, query, position_cos, position_sin) + query_wait = all_to_all_tensor( + query, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + k_0 = _project_route(hidden_states, route_0_indices, k_weight_0, k_bias_0, num_kv_heads, head_dim) + k_1 = _project_route(hidden_states, route_1_indices, k_weight_1, k_bias_1, num_kv_heads, head_dim) + normalized_k_0, saved_rms_k_0 = rms_entry.forward(k_0, k_norm_weight_0, eps=rms_norm_eps) + normalized_k_1, saved_rms_k_1 = rms_entry.forward(k_1, k_norm_weight_1, eps=rms_norm_eps) + key = normalized_k_0.new_zeros((sequence_length, num_kv_heads, head_dim)) + key.index_copy_(0, route_0_indices, normalized_k_0) + key.index_copy_(0, route_1_indices, normalized_k_1) + key, saved_rope_k = _rotate_single_forward(rope_entry, key, position_cos, position_sin) + if repeat_kv > 1: + key = torch.repeat_interleave(key, dim=1, repeats=repeat_kv) + key_wait = all_to_all_tensor( + key, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + v_0 = _project_route(hidden_states, route_0_indices, v_weight_0, v_bias_0, num_kv_heads, head_dim) + v_1 = _project_route(hidden_states, route_1_indices, v_weight_1, v_bias_1, num_kv_heads, head_dim) + value = v_0.new_zeros((sequence_length, num_kv_heads, head_dim)) + value.index_copy_(0, route_0_indices, v_0) + value.index_copy_(0, route_1_indices, v_1) + if repeat_kv > 1: + value = torch.repeat_interleave(value, dim=1, repeats=repeat_kv) + value_wait = all_to_all_tensor( + value, + scatter_dim=1, + gather_dim=0, + group=sp_group, + async_op=True, + ) + + query = unpadding_tensor_for_seqeunce_parallel(query_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + key = unpadding_tensor_for_seqeunce_parallel(key_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + value = unpadding_tensor_for_seqeunce_parallel(value_wait(), 0, unpadded_dim_size, group=sp_group).contiguous() + + saved: list[Tensor | None] = [ + hidden_states, + route_0_indices, + route_1_indices, + q_weight_0, + k_weight_0, + v_weight_0, + q_weight_1, + k_weight_1, + v_weight_1, + ] + meta = _BagelQKVMeta( + sp_group=sp_group, + head_dim=head_dim, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + repeat_kv=repeat_kv, + has_q_bias_0=q_bias_0 is not None, + has_k_bias_0=k_bias_0 is not None, + has_v_bias_0=v_bias_0 is not None, + has_q_bias_1=q_bias_1 is not None, + has_k_bias_1=k_bias_1 is not None, + has_v_bias_1=v_bias_1 is not None, + rms=rms_entry, + rope=rope_entry, + rms_q_0=append_inner(saved, saved_rms_q_0), + rms_q_1=append_inner(saved, saved_rms_q_1), + rms_k_0=append_inner(saved, saved_rms_k_0), + rms_k_1=append_inner(saved, saved_rms_k_1), + rope_q=append_inner(saved, saved_rope_q), + rope_k=append_inner(saved, saved_rope_k), + ) + return (query, key, value), SavedState(tuple(saved), meta) + + +def backward(grad_output: tuple[Tensor, Tensor, Tensor], saved: SavedState) -> tuple[Tensor | None, ...]: + """Overlap inverse Ulysses all-to-all exchanges with routed RoPE, RMSNorm, and linear gradients.""" + meta = saved.metadata + assert isinstance(meta, _BagelQKVMeta) + ( + hidden_states, + route_0_indices, + route_1_indices, + q_weight_0, + k_weight_0, + v_weight_0, + q_weight_1, + k_weight_1, + v_weight_1, + *rest_list, + ) = saved.tensors + rest = tuple(rest_list) + saved_rms_q_0, rest = take_inner(rest, meta.rms_q_0) + saved_rms_q_1, rest = take_inner(rest, meta.rms_q_1) + saved_rms_k_0, rest = take_inner(rest, meta.rms_k_0) + saved_rms_k_1, rest = take_inner(rest, meta.rms_k_1) + saved_rope_q, rest = take_inner(rest, meta.rope_q) + saved_rope_k, rest = take_inner(rest, meta.rope_k) + + grad_query, grad_key, grad_value = grad_output + sp_group = meta.sp_group + repeat_kv = meta.repeat_kv + num_kv_heads = meta.num_kv_heads + hidden_0 = hidden_states.index_select(0, route_0_indices) + hidden_1 = hidden_states.index_select(0, route_1_indices) + + # V inverse exchange has no independent projection work before it, so + # collect it first. K then overlaps V projection gradients, and Q + # overlaps K normalization/projection gradients. + grad_value = padding_tensor_for_seqeunce_parallel(grad_value.contiguous(), 0, group=sp_group) + grad_value_wait = all_to_all_tensor( + grad_value, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + grad_value = grad_value_wait() + if repeat_kv > 1: + grad_value = reduce_repeated_kv_gradient( + grad_value, + num_kv_heads, + repeat_kv, + head_dimension=1, + ) + + grad_key = padding_tensor_for_seqeunce_parallel(grad_key.contiguous(), 0, group=sp_group) + grad_key_wait = all_to_all_tensor( + grad_key, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + + grad_hidden_v_0, grad_v_weight_0, grad_v_bias_0 = linear_backward( + grad_value.index_select(0, route_0_indices), + hidden_0, + v_weight_0, + has_bias=meta.has_v_bias_0, + ) + grad_hidden_v_1, grad_v_weight_1, grad_v_bias_1 = linear_backward( + grad_value.index_select(0, route_1_indices), + hidden_1, + v_weight_1, + has_bias=meta.has_v_bias_1, + ) + + grad_key = grad_key_wait() + if repeat_kv > 1: + grad_key = reduce_repeated_kv_gradient( + grad_key, + num_kv_heads, + repeat_kv, + head_dimension=1, + ) + + grad_query = padding_tensor_for_seqeunce_parallel(grad_query.contiguous(), 0, group=sp_group) + grad_query_wait = all_to_all_tensor( + grad_query, + scatter_dim=0, + gather_dim=1, + group=sp_group, + async_op=True, + ) + + grad_key = _rotate_single_backward(meta.rope, grad_key, saved_rope_k) + grad_k_0, grad_k_norm_weight_0 = meta.rms.backward( + grad_key.index_select(0, route_0_indices), + saved_rms_k_0, + ) + grad_k_1, grad_k_norm_weight_1 = meta.rms.backward( + grad_key.index_select(0, route_1_indices), + saved_rms_k_1, + ) + grad_hidden_k_0, grad_k_weight_0, grad_k_bias_0 = linear_backward( + grad_k_0, hidden_0, k_weight_0, has_bias=meta.has_k_bias_0 + ) + grad_hidden_k_1, grad_k_weight_1, grad_k_bias_1 = linear_backward( + grad_k_1, hidden_1, k_weight_1, has_bias=meta.has_k_bias_1 + ) + + grad_query = grad_query_wait() + grad_query = _rotate_single_backward(meta.rope, grad_query, saved_rope_q) + grad_q_0, grad_q_norm_weight_0 = meta.rms.backward( + grad_query.index_select(0, route_0_indices), + saved_rms_q_0, + ) + grad_q_1, grad_q_norm_weight_1 = meta.rms.backward( + grad_query.index_select(0, route_1_indices), + saved_rms_q_1, + ) + grad_hidden_q_0, grad_q_weight_0, grad_q_bias_0 = linear_backward( + grad_q_0, hidden_0, q_weight_0, has_bias=meta.has_q_bias_0 + ) + grad_hidden_q_1, grad_q_weight_1, grad_q_bias_1 = linear_backward( + grad_q_1, hidden_1, q_weight_1, has_bias=meta.has_q_bias_1 + ) + + grad_hidden_states = torch.zeros_like(hidden_states) + grad_hidden_states.index_add_( + 0, + route_0_indices, + grad_hidden_q_0 + grad_hidden_k_0 + grad_hidden_v_0, + ) + grad_hidden_states.index_add_( + 0, + route_1_indices, + grad_hidden_q_1 + grad_hidden_k_1 + grad_hidden_v_1, + ) + + return ( + grad_hidden_states, + None, + None, + None, + None, + grad_q_weight_0, + grad_q_bias_0, + grad_k_weight_0, + grad_k_bias_0, + grad_v_weight_0, + grad_v_bias_0, + grad_q_norm_weight_0, + grad_k_norm_weight_0, + grad_q_weight_1, + grad_q_bias_1, + grad_k_weight_1, + grad_k_bias_1, + grad_v_weight_1, + grad_v_bias_1, + grad_q_norm_weight_1, + grad_k_norm_weight_1, + ) From eaf90da371a48b8a06e76dabbb35a281ca948a08 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 17:02:28 +0800 Subject: [PATCH 03/22] [omni, model, trainer, config, docs, test] fix: enable FlexAttention in Bagel eager inference and fix und/edit inference flow Co-authored-by: Cursor --- .../infer/modules_infer_eager.yaml | 2 +- docs/seed_omni/models/bagel.md | 6 +- .../modules/bagel/test_bagel_eager.py | 8 ++- .../modules/bagel/test_generation.py | 17 +++++- .../modules/bagel/test_graph_config.py | 3 + .../seed_omni/trainer/test_omni_inferencer.py | 9 ++- .../modules/bagel/qwen2_mot/modeling.py | 55 ++++++++++++++++--- veomni/trainer/omni/omni_inferencer.py | 1 + 8 files changed, 84 insertions(+), 17 deletions(-) diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml index 956e0e5d89..8c062c522a 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/infer/modules_infer_eager.yaml @@ -36,7 +36,7 @@ bagel_flow_connector: bagel_qwen2_mot: model_path: bagel_qwen2_mot ops_implementation: - attn_implementation: sdpa + attn_implementation: flex_attention accelerator: fsdp_config: fsdp_mode: eager diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md index d05ce8b4d3..981e720226 100644 --- a/docs/seed_omni/models/bagel.md +++ b/docs/seed_omni/models/bagel.md @@ -27,7 +27,7 @@ inference block maps each scenario to a separate generation graph. |------|------| | `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, train, and `infer` block. | | `train/modules_train.yaml` | Per-module training paths. `bagel_qwen2_mot` is the accelerated class with `flex_attention`. | -| `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses SDPA. | +| `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses FlexAttention. | | `infer/modules_infer_fsdp.yaml` | Distributed inference: every module uses FSDP2; MoT uses FlexAttention. | | `train/graph_train.yaml` | Training DAG. | | `infer/graph_infer_und.yaml` | Image/text understanding to text. | @@ -193,9 +193,9 @@ training does not use FlashAttention-2. | Backend | Class | Config | Use | |---------|-------|--------|-----| -| FlexAttention | accelerated | `train/modules_train.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training / FSDP inference | +| FlexAttention | accelerated / eager | `train/modules_train.yaml`, `infer/modules_infer_eager.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training and inference | | MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM90+ fused training; see nfunc below | -| SDPA | eager | `infer/modules_infer_eager.yaml` | Single-process inference and the dense-mask oracle | +| SDPA | eager | CLI override onto `bagel_qwen2_mot` | Dense-mask oracle | Leave the YAML on `flex_attention` unless you are opting into Magi. Enable Magi from the CLI (the launcher rewrites it to `veomni_magi_attention`): diff --git a/tests/seed_omni/modules/bagel/test_bagel_eager.py b/tests/seed_omni/modules/bagel/test_bagel_eager.py index 551c203e81..dba9303004 100644 --- a/tests/seed_omni/modules/bagel/test_bagel_eager.py +++ b/tests/seed_omni/modules/bagel/test_bagel_eager.py @@ -60,7 +60,7 @@ def _eager_cuda_model(): @pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") def test_eager_und_generate_runs() -> None: - """Verify eager understanding generation populates the KV cache and emits a finite output embedding.""" + """Verify eager understanding generation populates the KV cache on prefill and emits a finite output embedding on decode.""" model = _eager_cuda_model() prompt = ConversationItem( type="text", @@ -75,13 +75,17 @@ def test_eager_und_generate_runs() -> None: ) outputs = model.generate([prompt, bos], generation_kwargs={"infer_type": "infer_und"}) + assert model._generation_state.main.cache is not None + assert outputs["conversation_list"][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + + outputs = model.generate(outputs["conversation_list"], generation_kwargs={"infer_type": "infer_und"}) conversation = outputs["conversation_list"] tail = conversation[-1] assert tail.type == "output" + assert BAGEL_PHASE_KEY not in tail.meta assert torch.is_tensor(tail.value) assert tail.value.shape == (1, int(model.config.hidden_size)) assert torch.isfinite(tail.value).all() - assert model._generation_state.main.cache is not None @pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py index 3850c4f5c2..7c05a84d8e 100644 --- a/tests/seed_omni/modules/bagel/test_generation.py +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -74,7 +74,7 @@ def test_bagel_text_encoder_injects_raw_bos_for_understanding() -> None: def test_bagel_qwen_first_understanding_step_prefills_prompt_then_decodes_bos(monkeypatch) -> None: - """Verify ``BagelQwen2MoT.generate`` prefills the prompt before decoding the tail BOS token.""" + """Verify ``BagelQwen2MoT.generate`` prefills the prompt on ``prompt_encode`` and decodes the tail BOS token on ``text_ar``.""" BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).eval() @@ -89,8 +89,14 @@ def test_bagel_qwen_first_understanding_step_prefills_prompt_then_decodes_bos(mo calls: list[tuple[str, list[ConversationItem]]] = [] def _prefill(self, conversation_list, generation_kwargs): - del self, generation_kwargs + del generation_kwargs calls.append(("prefill", list(conversation_list))) + self._generation_state.main.install_cache( + cache=self._new_empty_cache(), + cache_len=2, + next_position_id=torch.tensor(2), + device=self.device, + ) return torch.full((2, hidden_size), 3.0) def _decode(self, conversation_list): @@ -105,7 +111,14 @@ def _decode(self, conversation_list): [prompt, bos], generation_kwargs={"infer_type": "infer_und"}, )["conversation_list"] + assert calls == [("prefill", [prompt])] + assert conversation[-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 2.0)) + conversation = model.generate( + conversation, + generation_kwargs={"infer_type": "infer_und"}, + )["conversation_list"] assert calls == [("prefill", [prompt]), ("decode", [prompt, bos])] assert conversation == [prompt, bos] assert BAGEL_PHASE_KEY not in bos.meta diff --git a/tests/seed_omni/modules/bagel/test_graph_config.py b/tests/seed_omni/modules/bagel/test_graph_config.py index d50c50cd9b..090eec0f90 100644 --- a/tests/seed_omni/modules/bagel/test_graph_config.py +++ b/tests/seed_omni/modules/bagel/test_graph_config.py @@ -38,6 +38,9 @@ def test_bagel_train_yaml_loads_with_v2_module_names(): runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True + for infer_modules in ("infer/modules_infer_eager.yaml", "infer/modules_infer_fsdp.yaml"): + infer_runtime = load_module_runtime_args(modules_path=bagel_cfg_dir() / infer_modules, for_inference=True) + assert infer_runtime["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") def test_bagel_train_graph_fan_in_execution_order(): diff --git a/tests/seed_omni/trainer/test_omni_inferencer.py b/tests/seed_omni/trainer/test_omni_inferencer.py index 258ced4713..d9a3b10f96 100644 --- a/tests/seed_omni/trainer/test_omni_inferencer.py +++ b/tests/seed_omni/trainer/test_omni_inferencer.py @@ -88,15 +88,20 @@ def generate(request, generation_kwargs=None): seen["input_ids"] = request["input_ids"] return [] + def processor(**kwargs): + seen["processor_kwargs"] = kwargs + return {"input_ids": torch.ones(2, dtype=torch.long)} + inferencer = _inferencer_with("infer_gen", {}) inferencer._distributed = distributed inferencer.device = torch.device("meta") - inferencer.processor = lambda **kwargs: {"input_ids": torch.ones(2, dtype=torch.long)} + inferencer.processor = processor inferencer.model = SimpleNamespace(reset=lambda: None, generate=generate) - inferencer._run(InferenceRequest(prompt="hi")) + inferencer._run(InferenceRequest(prompt="hi", generation_kwargs={"infer_type": "infer_edit"})) assert seen["input_ids"].device.type == expected_device + assert seen["processor_kwargs"]["infer_type"] == "infer_edit" def test_extract_generated_text_keeps_only_filled_text_items(): diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index 870ca1e7e0..bb8585c651 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -7,10 +7,12 @@ import torch import torch.nn as nn from torch.nn.attention import SDPBackend, sdpa_kernel +from torch.nn.attention.flex_attention import BlockMask from torch.nn.functional import scaled_dot_product_attention from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm from transformers.utils import ModelOutput +from veomni.ops import VeomniOp from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type from ....utils.conversation import ConversationItem @@ -26,10 +28,13 @@ ) from .configuration import BagelQwen2MoTConfig from .generation_state import MotGenerationState -from .masking import build_mot_sdpa_mask +from .masking import build_mot_block_mask, build_mot_sdpa_mask from .processing import PackedConversation, preprocess_mot_inputs +_FLEX_ATTN_IMPLEMENTATIONS = frozenset({"flex_attention", "veomni_flex_attention"}) + + class InferenceMixin: """FSM ``generate`` / serial denoise / velocity-collection — HF ``GenerationMixin`` analog. @@ -107,7 +112,7 @@ def generate( if tail.type != "output" or bagel_phase(tail) != BAGEL_START_TOKEN: raise ValueError("BAGEL understanding prefill requires a tail assistant BOS embedding.") self._prefill_prompt(conversation_list[:-1], generation_kwargs) - hidden_states = self._decode_next_token(conversation_list) + return {"conversation_list": conversation_list} else: hidden_states = self._decode_next_token(conversation_list) @@ -439,7 +444,7 @@ class BagelQwen2MoTCore(PretrainedOmniModule): _no_split_modules = ["BagelQwen2MoTDecoderLayer"] supports_gradient_checkpointing = True _supports_sdpa = True - _supports_flex_attn = False + _supports_flex_attn = True _export_hf_checkpoint_with_weight_conversions = False def __init__(self, config: BagelQwen2MoTConfig): @@ -784,6 +789,12 @@ def __init__( self.o_proj_moe_gen = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) self.q_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) self.k_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) + attn_impl = config.ops_config.attn_implementation + self.veomni_attn: VeomniOp | None = ( + VeomniOp("attention", "standard", "veomni_flex_attention") + if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS + else None + ) def _build_qkv_proj(self) -> None: self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True) @@ -818,8 +829,9 @@ def build_attention_mask( packed_attention_metadata: torch.Tensor, *, attn_implementation: str | None = None, - ) -> torch.Tensor: - del attn_implementation + ) -> torch.Tensor | BlockMask: + if attn_implementation in _FLEX_ATTN_IMPLEMENTATIONS: + return build_mot_block_mask(packed_attention_metadata) return build_mot_sdpa_mask(packed_attention_metadata) def apply_rotary_pos_emb( @@ -882,8 +894,22 @@ def attend( packed_query_states: torch.Tensor, packed_key_states: torch.Tensor, packed_value_states: torch.Tensor, - attention_mask: torch.Tensor, + attention_mask: torch.Tensor | BlockMask, ) -> torch.Tensor: + if isinstance(attention_mask, BlockMask): + attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + packed_key_states.transpose(0, 1).unsqueeze(0), + packed_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0).reshape(-1, self.num_heads * self.head_dim) + packed_attn_output = _sdpa_packed_attention( packed_query_states, packed_key_states, @@ -1017,7 +1043,7 @@ def attend_inference( merged_key_states: torch.Tensor, merged_value_states: torch.Tensor, *, - attention_mask: Optional[torch.Tensor], + attention_mask: Optional[torch.Tensor | BlockMask], is_causal: bool, cu_seq_lens_q: torch.Tensor, cu_seq_lens_k: torch.Tensor, @@ -1025,6 +1051,21 @@ def attend_inference( max_length_k: int, ) -> torch.Tensor: del max_length_q, max_length_k + + if isinstance(attention_mask, BlockMask): + attn_op = self.veomni_attn or VeomniOp("attention", "standard", "veomni_flex_attention") + packed_attn_output, _ = attn_op( + self, + packed_query_states.transpose(0, 1).unsqueeze(0), + merged_key_states.transpose(0, 1).unsqueeze(0), + merged_value_states.transpose(0, 1).unsqueeze(0), + attention_mask, + dropout=0.0, + scaling=self.head_dim**-0.5, + skip_ulysses=True, + ) + return packed_attn_output.squeeze(0) + return _sdpa_packed_attention( packed_query_states, merged_key_states, diff --git a/veomni/trainer/omni/omni_inferencer.py b/veomni/trainer/omni/omni_inferencer.py index 15f31d47a8..b54c2d3620 100644 --- a/veomni/trainer/omni/omni_inferencer.py +++ b/veomni/trainer/omni/omni_inferencer.py @@ -242,6 +242,7 @@ def _run(self, req: InferenceRequest) -> dict[str, Any]: videos=req.videos or None, mm_configs=req.mm_configs or None, inference=True, + **req.generation_kwargs, ) if self._distributed: # Every module sits on this rank's device, as in training. An eager From e60a76eb911c0cd151b96499002ee9cfbc93e345 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 18:21:03 +0800 Subject: [PATCH 04/22] [omni, model, test] refactor: align Bagel VeomniOp dispatch and instance isolation Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 175 +++++++++++++++++- .../qwen2_mot/accelerated/accelerated.py | 150 +++++++-------- .../modules/bagel/qwen2_mot/modeling.py | 26 ++- .../modules/bagel/siglip_navit/modeling.py | 66 +++---- 4 files changed, 277 insertions(+), 140 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 0b1adcfdec..6ab8ab3309 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -8,6 +8,7 @@ from transformers.utils import SAFE_WEIGHTS_NAME from scripts.ci.accelerator.launch_plan import LaunchSpec +from tests.models.conftest import available_nvidia_ops # noqa: F401 from tests.seed_omni.modules.bagel.helpers import config_cls, tiny_bagel_qwen2_cfg from veomni.models.checkpoint.weights import load_model_weights from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated @@ -176,7 +177,7 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).bias, expected.bias) -def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config() -> None: +def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config(available_nvidia_ops) -> None: # noqa: F811 """Verify ``BagelQwen2MoTAccelerated`` binds ``VeomniOp`` handles from ``config.ops_config``.""" cfg = BagelQwen2MoTConfig( vocab_size=128, @@ -206,6 +207,7 @@ def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config() -> assert layer.self_attn.veomni_async_ulysses_o.variant == "bagel" assert layer.self_attn.veomni_attn is not None assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention" + assert layer.self_attn.veomni_denoise_attn is not None assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2" @@ -233,12 +235,167 @@ def test_bagel_siglip_navit_selects_module_level_ops_and_forwards() -> None: packed_patches = torch.randn(4, 3 * 14 * 14) position_ids = torch.arange(4, dtype=torch.long) - cu_seqlens = torch.tensor([0, 4], dtype=torch.int32) - out = model.vision_model( - patchified_pixel_values=packed_patches, - patchified_position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_seqlen=4, + for cu_seqlens, max_seqlen in ( + (torch.tensor([0, 4], dtype=torch.int32), 4), + (torch.tensor([0, 2, 4], dtype=torch.int32), 2), + ): + out = model.vision_model( + patchified_pixel_values=packed_patches, + patchified_position_ids=position_ids, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + ) + assert out.shape == (4, 32) + assert torch.isfinite(out).all() + + +class _CallCounter: + def __init__(self, inner): + self.inner = inner + self.calls = 0 + + @property + def impl(self): + return self.inner.impl + + def __call__(self, *args, **kwargs): + self.calls += 1 + return self.inner(*args, **kwargs) + + +@pytest.mark.parametrize("hidden_act", ["silu", "swish", "gelu"]) +def test_bagel_qwen2_mot_mlp_and_rms_norm_dispatch_through_veomni_ops(hidden_act: str) -> None: + """Verify ``Qwen2RMSNormAccelerated`` and ``Qwen2MLPAccelerated`` route through their ``VeomniOp`` handles.""" + from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP as HFQwen2MLP + from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm as HFQwen2RMSNorm + + from tests.models.compare import assert_module_forward_and_grads_match + + cfg = BagelQwen2MoTConfig( + **{ + **tiny_bagel_qwen2_cfg(), + "hidden_act": hidden_act, + "ops_implementation": { + "attn_implementation": "sdpa", + "rms_norm_implementation": "eager", + "swiglu_mlp_implementation": "eager", + "rotary_pos_emb_implementation": "eager", + }, + } + ) + + torch.manual_seed(0) + hf_norm = HFQwen2RMSNorm(cfg.hidden_size, eps=cfg.rms_norm_eps) + ours_norm = accelerated.Qwen2RMSNormAccelerated(cfg.hidden_size, eps=cfg.rms_norm_eps, config=cfg) + ours_norm.load_state_dict(hf_norm.state_dict()) + norm_counter = _CallCounter(ours_norm.veomni_rms_norm) + ours_norm.veomni_rms_norm = norm_counter + assert_module_forward_and_grads_match(hf_norm, ours_norm, torch.randn(3, cfg.hidden_size)) + assert norm_counter.calls == 1 + empty_norm_out = ours_norm(torch.empty(0, cfg.hidden_size, requires_grad=True)) + assert empty_norm_out.shape == (0, cfg.hidden_size) + assert norm_counter.calls == 2 + + hf_mlp = HFQwen2MLP(cfg) + ours_mlp = accelerated.Qwen2MLPAccelerated(cfg) + ours_mlp.load_state_dict(hf_mlp.state_dict()) + mlp_counter = _CallCounter(ours_mlp.veomni_swiglu_mlp) + ours_mlp.veomni_swiglu_mlp = mlp_counter + assert_module_forward_and_grads_match(hf_mlp, ours_mlp, torch.randn(3, cfg.hidden_size)) + if hidden_act in {"silu", "swish"}: + assert mlp_counter.calls == 1 + empty_mlp_out = ours_mlp(torch.empty(0, cfg.hidden_size, requires_grad=True)) + assert empty_mlp_out.shape == (0, cfg.hidden_size) + assert mlp_counter.calls == 2 + else: + assert mlp_counter.calls == 0 + + +def test_bagel_modules_keep_construction_ops_when_config_mutated() -> None: + """Verify ``BagelQwen2MoTAccelerated``, ``BagelSiglipNavit``, and ``BagelTextEncoderAccelerated`` keep construction-time ops after ``config.ops_implementation`` is mutated.""" + from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig + from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit + from veomni.models.seed_omni.modules.bagel.text_encoder.accelerated.accelerated import BagelTextEncoderAccelerated + from veomni.models.seed_omni.modules.bagel.text_encoder.configuration import BagelTextEncoderConfig + + poison_ops = { + "attn_implementation": "sdpa", + "rms_norm_implementation": "liger_kernel", + "swiglu_mlp_implementation": "liger_kernel", + "rotary_pos_emb_implementation": "liger_kernel", + "rotary_pos_emb_vision_implementation": "liger_kernel", + "cross_entropy_loss_implementation": "liger_kernel", + } + + # 1. BagelSiglipNavit forward isolation + siglip_cfg = BagelSiglipNavitConfig( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=4, + image_size=28, + patch_size=14, + rope=True, + ops_implementation={ + "attn_implementation": "eager", + "rotary_pos_emb_vision_implementation": "eager", + }, + ) + siglip = BagelSiglipNavit(siglip_cfg).eval() + siglip_attn = siglip.vision_model.encoder.layers[0].self_attn + assert siglip_attn.veomni_attn.impl == "eager" + assert siglip_attn.veomni_rope.impl == "eager" + + patches = torch.randn(4, 3 * 14 * 14) + pos_ids = torch.arange(4, dtype=torch.long) + cu_seqlens = torch.tensor([0, 2, 4], dtype=torch.int32) + with torch.no_grad(): + siglip_ref = siglip.vision_model(patches, pos_ids, cu_seqlens=cu_seqlens, max_seqlen=2) + siglip_cfg.ops_implementation = dict(poison_ops) + assert siglip_attn.veomni_attn.impl == "eager" + assert siglip_attn.veomni_rope.impl == "eager" + siglip_out = siglip.vision_model(patches, pos_ids, cu_seqlens=cu_seqlens, max_seqlen=2) + torch.testing.assert_close(siglip_out, siglip_ref) + + # 2. BagelTextEncoderAccelerated decode loss isolation + text_cfg = BagelTextEncoderConfig( + vocab_size=32, + hidden_size=16, + ops_implementation={"cross_entropy_loss_implementation": "eager"}, ) - assert out.shape == (4, 32) - assert torch.isfinite(out).all() + text_enc = BagelTextEncoderAccelerated(text_cfg).eval() + assert text_enc.veomni_ce.impl == "eager" + hidden = torch.randn(4, 16) + shift_labels = torch.tensor([1, 2, 3, -100], dtype=torch.long) + with torch.no_grad(): + loss_ref = text_enc.decode(hidden_states=hidden, shift_labels=shift_labels)["loss"] + text_cfg.ops_implementation = dict(poison_ops) + assert text_enc.veomni_ce.impl == "eager" + loss_out = text_enc.decode(hidden_states=hidden, shift_labels=shift_labels)["loss"] + torch.testing.assert_close(loss_out, loss_ref) + + # 3. BagelQwen2MoTAccelerated RoPE / RMSNorm / SwiGLU / Attention isolation + mot_cfg = _flex_config() + mot = accelerated.BagelQwen2MoTAccelerated(mot_cfg).eval() + layer = mot.model.layers[0] + rope_counter = _CallCounter(layer.self_attn.veomni_rope) + layer.self_attn.veomni_rope = rope_counter + q = torch.randn(3, layer.self_attn.num_heads, layer.self_attn.head_dim) + k = torch.randn(3, layer.self_attn.num_key_value_heads, layer.self_attn.head_dim) + cos = torch.ones(3, layer.self_attn.head_dim) + sin = torch.zeros(3, layer.self_attn.head_dim) + with torch.no_grad(): + q_ref, k_ref = layer.self_attn.apply_rotary_pos_emb(q, k, cos, sin) + mlp_ref = layer.mlp(torch.randn(3, mot_cfg.hidden_size)) + mot_cfg.ops_implementation = dict(poison_ops) + assert layer.input_layernorm.veomni_rms_norm.impl == "eager" + assert layer.mlp.veomni_swiglu_mlp.impl == "eager" + assert layer.self_attn.veomni_rms_norm.impl == "eager" + assert layer.self_attn.veomni_rope.impl == "eager" + assert layer.self_attn.veomni_attn_impl == "veomni_flex_attention" + assert mot.model._bound_attn_implementation() == "veomni_flex_attention" + q_out, k_out = layer.self_attn.apply_rotary_pos_emb(q, k, cos, sin) + torch.testing.assert_close(q_out, q_ref) + torch.testing.assert_close(k_out, k_ref) + assert rope_counter.calls == 2 + assert torch.isfinite(mlp_ref).all() diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index 453bb1e1e1..040c5be99d 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -48,9 +48,6 @@ BagelQwen2MoTCore, InferenceMixin, ) -from ..modeling import ( - _apply_rotary_pos_emb as _apply_rotary_pos_emb_eager, -) from ..processing import PackedConversation, preprocess_mot_inputs @@ -342,7 +339,7 @@ class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): class Qwen2RMSNormAccelerated(Qwen2RMSNorm): - """Qwen2 RMSNorm with instance-local VeomniOp fused dispatch.""" + """Qwen2 RMSNorm with instance-local VeomniOp dispatch.""" def __init__(self, hidden_size: int, eps: float = 1e-6, *, config: BagelQwen2MoTConfig | None = None) -> None: super().__init__(hidden_size, eps=eps) @@ -351,49 +348,36 @@ def __init__(self, hidden_size: int, eps: float = 1e-6, *, config: BagelQwen2MoT self.veomni_rms_norm = VeomniOp("rms_norm", "standard", rms_impl) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Apply RMSNorm, falling back to eager on empty modality branches.""" - # Empty modality branches retain zero-gradient FSDP anchors. Liger's - # Triton kernel cannot launch the resulting zero-row grid. - if hidden_states.numel() == 0: - return super().forward(hidden_states) - - if self.veomni_rms_norm.impl != "eager": - return self.veomni_rms_norm(hidden_states, self.weight, eps=self.variance_epsilon) - - return super().forward(hidden_states) + """Apply the instance-local ``rms_norm`` ``VeomniOp``.""" + return self.veomni_rms_norm(hidden_states, self.weight, eps=self.variance_epsilon) class Qwen2MLPAccelerated(Qwen2MLP): - """Qwen2 SwiGLU MLP with instance-local VeomniOp fused dispatch.""" + """Qwen2 SwiGLU MLP with instance-local VeomniOp dispatch.""" def __init__(self, config: BagelQwen2MoTConfig) -> None: super().__init__(config) self.veomni_swiglu_mlp = VeomniOp("swiglu_mlp", "standard", config.ops_config.swiglu_mlp_implementation) def forward(self, x: torch.Tensor) -> torch.Tensor: - """Apply SwiGLU MLP, falling back to eager on empty modality branches.""" - # Empty modality branches retain zero-gradient FSDP anchors. Liger's - # Triton kernel cannot launch the resulting zero-row grid. - if x.numel() == 0: - return super().forward(x) - - if self.veomni_swiglu_mlp.impl != "eager": - if self.config.hidden_act not in {"silu", "swish"}: - raise ValueError( - f"Liger SwiGLU requires hidden_act='silu' or 'swish', got {self.config.hidden_act!r}. " - "Set model.ops_implementation.swiglu_mlp_implementation='eager' " - "to use the Transformers reference implementation." - ) - if has_plain_linear_projections(self): - return self.veomni_swiglu_mlp( - x, - self.gate_proj.weight, - self.gate_proj.bias if self.gate_proj.bias is not None else self.gate_proj.weight.new_empty(0), - self.up_proj.weight, - self.up_proj.bias if self.up_proj.bias is not None else self.gate_proj.weight.new_empty(0), - self.down_proj.weight, - self.down_proj.bias if self.down_proj.bias is not None else self.down_proj.weight.new_empty(0), - ) + """Apply the instance-local ``swiglu_mlp`` ``VeomniOp`` for ``silu``/``swish`` on plain linear projections.""" + if self.config.hidden_act not in {"silu", "swish"} and self.veomni_swiglu_mlp.impl != "eager": + raise ValueError( + f"Liger SwiGLU requires hidden_act='silu' or 'swish', got {self.config.hidden_act!r}. " + "Set model.ops_implementation.swiglu_mlp_implementation='eager' " + "to use the Transformers reference implementation." + ) + + if self.config.hidden_act in {"silu", "swish"} and has_plain_linear_projections(self): + return self.veomni_swiglu_mlp( + x, + self.gate_proj.weight, + self.gate_proj.bias if self.gate_proj.bias is not None else self.gate_proj.weight.new_empty(0), + self.up_proj.weight, + self.up_proj.bias if self.up_proj.bias is not None else self.gate_proj.weight.new_empty(0), + self.down_proj.weight, + self.down_proj.bias if self.down_proj.bias is not None else self.down_proj.weight.new_empty(0), + ) return super().forward(x) @@ -407,38 +391,36 @@ def _apply_rotary_pos_emb( *, rope_op: VeomniOp | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Packed RoPE with optional instance-local VeomniOp dispatch.""" + """Packed RoPE dispatched through the instance-local ``VeomniOp``.""" if rope_op is None: rope_op = VeomniOp("rope", "full", "eager") - if q.numel() != 0 and k.numel() != 0 and rope_op.impl != "eager": - if unsqueeze_dim != 1: - raise NotImplementedError("BAGEL packed fused RoPE requires unsqueeze_dim=1.") - if q.ndim != 3 or k.ndim != 3 or cos.ndim != 2 or sin.ndim != 2: - raise NotImplementedError( - "BAGEL fused RoPE requires packed q/k tensors shaped [tokens, heads, head_dim] " - "and cos/sin tensors shaped [tokens, head_dim]." - ) - if not (q.shape[0] == k.shape[0] == cos.shape[0] == sin.shape[0]): - raise ValueError("BAGEL packed q/k/cos/sin tensors must share the token dimension.") - if not (q.shape[-1] == k.shape[-1] == cos.shape[-1] == sin.shape[-1]): - raise NotImplementedError("Liger full RoPE does not support partial rotary dimensions.") - - # The shared full-RoPE kernels consume [batch, heads, sequence, head_dim]. - # BAGEL stores packed Q/K as [tokens, heads, head_dim], so adapt through - # a synthetic batch dimension and restore the packed layout afterwards. - q_embed, k_embed = rope_op( - q.transpose(0, 1).unsqueeze(0), - k.transpose(0, 1).unsqueeze(0), - cos.unsqueeze(0), - sin.unsqueeze(0), - unsqueeze_dim=1, - ) - return ( - q_embed.squeeze(0).transpose(0, 1).contiguous(), - k_embed.squeeze(0).transpose(0, 1).contiguous(), + if unsqueeze_dim != 1: + raise NotImplementedError("BAGEL packed fused RoPE requires unsqueeze_dim=1.") + if q.ndim != 3 or k.ndim != 3 or cos.ndim != 2 or sin.ndim != 2: + raise NotImplementedError( + "BAGEL fused RoPE requires packed q/k tensors shaped [tokens, heads, head_dim] " + "and cos/sin tensors shaped [tokens, head_dim]." ) - - return _apply_rotary_pos_emb_eager(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim) + if not (q.shape[0] == k.shape[0] == cos.shape[0] == sin.shape[0]): + raise ValueError("BAGEL packed q/k/cos/sin tensors must share the token dimension.") + if not (q.shape[-1] == k.shape[-1] == cos.shape[-1] == sin.shape[-1]): + raise NotImplementedError("Liger full RoPE does not support partial rotary dimensions.") + + # The shared full-RoPE kernels consume [batch, heads, sequence, head_dim] + # for LLM-dtype arithmetic (whereas 3-D [S, H, D] triggers vision FP32 promotion). + # BAGEL stores packed Q/K as [tokens, heads, head_dim], so adapt through + # a synthetic batch dimension and restore the packed layout afterwards. + q_embed, k_embed = rope_op( + q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), + cos.unsqueeze(0), + sin.unsqueeze(0), + unsqueeze_dim=1, + ) + return ( + q_embed.squeeze(0).transpose(0, 1).contiguous(), + k_embed.squeeze(0).transpose(0, 1).contiguous(), + ) _PACKED_FUSED_ATTN_IMPLEMENTATIONS = ( @@ -473,17 +455,23 @@ def __init__( self.veomni_rope = VeomniOp("rope", "full", config.ops_config.rotary_pos_emb_implementation) self.veomni_async_ulysses_qkv = VeomniOp("async_ulysses_qkv", "bagel") self.veomni_async_ulysses_o = VeomniOp("async_ulysses_o", "bagel") - attn_impl = _normalize_packed_attn_impl(config.ops_config.attn_implementation) - if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: - self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl) - elif attn_impl in _MAGI_ATTN_IMPLEMENTATIONS: + attn_impl = _normalize_packed_attn_impl( + config.ops_config.attn_implementation + if "attn_implementation" in config.ops_implementation + else "veomni_flex_attention" + ) + self.veomni_attn_impl = attn_impl + if attn_impl in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: try: - self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + self.veomni_attn: VeomniOp | None = VeomniOp("attention", "standard", attn_impl) except RuntimeError: self.veomni_attn = None else: self.veomni_attn = None - self.veomni_denoise_attn = VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + try: + self.veomni_denoise_attn: VeomniOp | None = VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + except RuntimeError: + self.veomni_denoise_attn = None self.register_load_state_dict_pre_hook(combine_qkv_state_dict_pre_hook) self.register_state_dict_post_hook(split_qkv_state_dict_post_hook) @@ -532,17 +520,16 @@ def apply_rotary_pos_emb( return _apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=unsqueeze_dim, rope_op=self.veomni_rope) def _resolve_packed_attn_op(self, *, context: str) -> VeomniOp: - """Resolve the packed fused attention ``VeomniOp`` for ``context``.""" - attn_impl = _normalize_packed_attn_impl(self.config._attn_implementation) - if attn_impl not in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: + """Resolve the packed fused attention ``VeomniOp`` bound at initialization for ``context``.""" + if self.veomni_attn is not None: + return self.veomni_attn + if self.veomni_attn_impl not in _PACKED_FUSED_ATTN_IMPLEMENTATIONS: raise ValueError( f"BAGEL Qwen2-MoT {context} requires packed fused attention " f"({', '.join(_PACKED_FUSED_ATTN_IMPLEMENTATIONS)}), got " - f"{self.config._attn_implementation!r}." + f"{self.veomni_attn_impl!r}." ) - if self.veomni_attn is not None and self.veomni_attn.impl == attn_impl: - return self.veomni_attn - return VeomniOp("attention", "standard", attn_impl) + return VeomniOp("attention", "standard", self.veomni_attn_impl) def attend( self, @@ -594,7 +581,8 @@ def attend_inference( ) return packed_attn_output.squeeze(0) - packed_attn_output, _ = self.veomni_denoise_attn( + denoise_op = self.veomni_denoise_attn or VeomniOp("attention", "standard", _FLASH_ATTENTION_2) + packed_attn_output, _ = denoise_op( self, packed_query_states.transpose(0, 1).unsqueeze(0), merged_key_states.transpose(0, 1).unsqueeze(0), diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index bb8585c651..de15ae7672 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -789,12 +789,16 @@ def __init__( self.o_proj_moe_gen = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False) self.q_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) self.k_norm_moe_gen = _build_rms_norm(rms_norm_cls, self.head_dim, config) - attn_impl = config.ops_config.attn_implementation - self.veomni_attn: VeomniOp | None = ( - VeomniOp("attention", "standard", "veomni_flex_attention") - if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS - else None + attn_impl = ( + config.ops_config.attn_implementation if "attn_implementation" in config.ops_implementation else "sdpa" ) + self.veomni_attn_impl = attn_impl + self.veomni_attn: VeomniOp | None = None + if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: + try: + self.veomni_attn = VeomniOp("attention", "standard", "veomni_flex_attention") + except RuntimeError: + self.veomni_attn = None def _build_qkv_proj(self) -> None: self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True) @@ -1398,6 +1402,14 @@ def __init__( self.use_moe = "Mo" in config.layer_module self.attention_cls = attention_cls + def _bound_attn_implementation(self) -> str | None: + """Return the attention implementation bound onto the decoder layers at initialization.""" + if len(self.layers) > 0: + bound = getattr(self.layers[0].self_attn, "veomni_attn_impl", None) + if bound is not None: + return bound + return self.config._attn_implementation + def _build_inference_attention_mask( self, packed_query_sequence: torch.Tensor, @@ -1419,7 +1431,7 @@ def _build_inference_attention_mask( ) return self.attention_cls.build_attention_mask( packed_attention_metadata.to(device=packed_query_sequence.device), - attn_implementation=self.config._attn_implementation, + attn_implementation=self._bound_attn_implementation(), ) def _apply_inference_final_norm( @@ -1450,7 +1462,7 @@ def forward_packed_train( """Run packed training forward across all MoT decoder layers and final expert norms.""" attention_mask = self.attention_cls.build_attention_mask( packed_attention_metadata, - attn_implementation=self.config._attn_implementation, + attn_implementation=self._bound_attn_implementation(), ) cos, sin = self.rotary_emb(packed_sequence, packed_position_ids.unsqueeze(0)) packed_position_cos = cos.squeeze(0) diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py index a15c0fc55b..236e7564ee 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -320,11 +320,6 @@ def forward( return patch_embeds + position_embeds.to(device=patch_embeds.device, dtype=patch_embeds.dtype) -def _is_flash_attention_impl(impl: str | None) -> bool: - """Return whether ``impl`` names a FlashAttention backend.""" - return impl is not None and "flash_attention" in impl - - class BagelSiglipAttention(nn.Module): def __init__(self, config: BagelSiglipNavitConfig) -> None: """Initialize NaViT self-attention projections and instance-local ``VeomniOp`` handles.""" @@ -332,6 +327,7 @@ def __init__(self, config: BagelSiglipNavitConfig) -> None: self.config = config self.num_heads = config.num_attention_heads self.head_dim = config.hidden_size // config.num_attention_heads + self.num_key_value_groups = 1 self.scaling = self.head_dim**-0.5 self.attention_dropout = config.attention_dropout self.is_causal = False @@ -349,9 +345,13 @@ def __init__(self, config: BagelSiglipNavitConfig) -> None: self.veomni_rope = VeomniOp("rope", "full", rope_impl) attn_impl = config.ops_config.attn_implementation - if not attn_impl or attn_impl == "eager": - attn_impl = "veomni_sdpa" - self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + if "attn_implementation" not in config.ops_implementation: + try: + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) + except RuntimeError: + self.veomni_attn = VeomniOp("attention", "standard", "veomni_sdpa") + else: + self.veomni_attn = VeomniOp("attention", "standard", attn_impl) def forward( self, @@ -388,41 +388,21 @@ def forward( value_states = value_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) dropout = 0.0 if not self.training else self.attention_dropout - if _is_flash_attention_impl(self.veomni_attn.impl): - attn_output, _ = self.veomni_attn( - self, - query_states, - key_states, - value_states, - attention_mask=None, - scaling=self.scaling, - dropout=dropout, - cu_seq_lens_q=cu_seqlens, - cu_seq_lens_k=cu_seqlens, - max_length_q=max_seqlen, - max_length_k=max_seqlen, - is_causal=False, - skip_ulysses=True, - ) - else: - lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() - splits = [torch.split(tensor, lengths, dim=2) for tensor in (query_states, key_states, value_states)] - attn_outputs = [ - self.veomni_attn( - self, - q, - k, - v, - attention_mask=None, - scaling=self.scaling, - dropout=dropout, - is_causal=False, - skip_ulysses=True, - )[0] - for q, k, v in zip(*splits, strict=True) - ] - attn_output = torch.cat(attn_outputs, dim=1) - + attn_output, _ = self.veomni_attn( + self, + query_states, + key_states, + value_states, + attention_mask=None, + scaling=self.scaling, + dropout=dropout, + cu_seq_lens_q=cu_seqlens, + cu_seq_lens_k=cu_seqlens, + max_length_q=max_seqlen, + max_length_k=max_seqlen, + is_causal=False, + skip_ulysses=True, + ) return self.out_proj(attn_output.reshape(total_q_len, -1).to(hidden_states.dtype)) From d7f447ed201939b57a19af571908bf1ce8881ff1 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 20:12:54 +0800 Subject: [PATCH 05/22] [omni, model, ckpt, test] fix: pad micro-batch VAE inputs, sample per-image flow timesteps, and keep fused QKV state_dict keys Co-authored-by: Cursor --- .../modules/bagel/test_processing.py | 101 ++++++++++++++++++ .../seed_omni/modules/bagel/test_qwen2_mot.py | 46 +++++--- .../bagel/flow_connector/processing.py | 2 +- .../qwen2_mot/accelerated/accelerated.py | 2 - .../bagel/qwen2_mot/checkpoint_conversion.py | 29 ----- .../bagel/vae/accelerated/accelerated.py | 9 +- .../seed_omni/modules/bagel/vae/modeling.py | 10 +- 7 files changed, 151 insertions(+), 48 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index 4cf4896753..230d5c53f1 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -230,6 +230,50 @@ def test_bagel_flow_training_embed_treats_edit_context_as_clean_and_gen_as_targe assert "timestep" not in edit_context.meta assert gen_target.meta["flow_velocity_target"].shape == (2, 1) assert gen_target.meta["timestep"].shape == (2,) + assert torch.equal(gen_target.meta["timestep"][0], gen_target.meta["timestep"][1]) + assert torch.equal(inputs["timesteps"][2], inputs["timesteps"][3]) + + +def test_bagel_flow_preprocess_latent_embed_samples_one_shared_timestep_per_image() -> None: + """Verify ``preprocess_latent_embed`` samples one scalar timestep per image and expands it across patch tokens.""" + from veomni.models.seed_omni.modules.bagel.flow_connector.processing import preprocess_latent_embed + + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + cfg = BagelFlowConnectorConfig( + hidden_size=4, + z_channels=1, + latent_patch_size=1, + patch_latent_dim=1, + max_latent_size=4, + timestep_frequency_embedding_size=4, + timestep_shift=3.0, + ) + first = ConversationItem( + type="image", + value=torch.zeros(1, 2, 2), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + second = ConversationItem( + type="image", + value=torch.ones(1, 2, 3), + role="assistant", + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, + ) + torch.manual_seed(1234) + inputs, lengths = preprocess_latent_embed( + [first, second], + config=cfg, + device=torch.device("cpu"), + dtype=torch.float32, + timestep_shift=3.0, + ) + assert lengths == [4, 6] + assert torch.all(first.meta["timestep"] == first.meta["timestep"][0]) + assert torch.all(second.meta["timestep"] == second.meta["timestep"][0]) + assert not torch.equal(first.meta["timestep"][0], second.meta["timestep"][0]) + assert torch.equal(inputs["timesteps"][:4], first.meta["timestep"]) + assert torch.equal(inputs["timesteps"][4:], second.meta["timestep"]) def test_mot_forward_post_scatters_virtual_marker_triplet_hidden_states() -> None: @@ -960,3 +1004,60 @@ def test_bagel_siglip_preprocessor_from_pretrained_applies_config_overrides(tmp_ ) assert pre._image_processor.image_size == 56 assert pre._image_processor.min_image_size == 28 + + +def test_bagel_vae_encode_and_offline_encode_pad_per_sample_preprocessed_microbatch() -> None: + """Verify ``encode`` and ``offline_encode`` pad variable-shape images across per-sample-preprocessed micro-batches.""" + pre = BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=8, max_pixels=64), + dtype=torch.float32, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.float32), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ) + # Dataset-stage transform runs preprocessor on one sample at a time: + sample_rect_a = [ + ConversationItem( + type="image", + value=torch.full((3, 4, 8), 5, dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + ] + sample_rect_b = [ + ConversationItem( + type="image", + value=torch.full((3, 8, 4), 7, dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + ] + sample_text_only = [ConversationItem(type="text", value="hello", role="user")] + for sample in (sample_rect_a, sample_rect_b, sample_text_only): + pre(_b([sample])) + + assert sample_rect_a[0].value.shape == (3, 4, 8) + assert sample_rect_b[0].value.shape == (3, 8, 4) + assert sample_text_only[-1].is_dummy + assert sample_text_only[-1].value.shape == (3, 2, 2) + + vae = _tiny_vae(downsample=2) + + # 1. Online encode with ragged real + dummy micro-batch + online_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_text_only]) + pre_inputs = vae.pre_forward("encode", conversation_list=online_batch) + assert pre_inputs["pixel_values"].shape == (3, 3, 8, 8) + encoded_out = vae.encode(**pre_inputs) + vae.post_forward("encode", **encoded_out) + assert online_batch[0][0].value.shape == (2, 2, 4) + assert online_batch[1][0].value.shape == (2, 4, 2) + assert online_batch[2][-1].value.shape == (2, 1, 1) + + # 2. Offline encode with ragged real + dummy micro-batch + offline_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_text_only]) + off_pre = vae.pre_forward("offline_encode", conversation_list=offline_batch) + assert off_pre["pixel_values"].shape == (3, 3, 8, 8) + off_out = vae.offline_encode(**off_pre) + vae.post_forward("offline_encode", **off_out) + assert offline_batch[0][0].value.shape == (2, 2, 2, 4) + assert offline_batch[1][0].value.shape == (2, 2, 4, 2) + assert offline_batch[2][-1].value.shape == (2, 2, 1, 1) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 6ab8ab3309..82d99cfe87 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -133,31 +133,53 @@ def test_checkpoint_tensor_converter_combines_legacy_qkv_groups() -> None: def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> None: - """Verify fused runtime QKV parameters round-trip through split HF safetensors and ``load_model_weights``.""" + """Verify fused runtime QKV parameters keep fused keys in ``state_dict`` / DCP and split on HF safetensors export.""" + from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict + + from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT + from veomni.utils.save_safetensor_utils import _revert_model_weight_conversions_for_hf + torch.manual_seed(3014) + eager_cfg = BagelQwen2MoTConfig(**{**tiny_bagel_qwen2_cfg(), "num_key_value_heads": 1}) + eager_ref = BagelQwen2MoT(eager_cfg) + eager_split_state = eager_ref.state_dict() + assert "model.layers.0.self_attn.q_proj.weight" in eager_split_state + assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in eager_split_state + reference = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + reference.load_state_dict(eager_split_state) internal_parameter_names = set(dict(reference.named_parameters())) assert "model.layers.0.self_attn.qkv_proj_und.weight" in internal_parameter_names assert "model.layers.0.self_attn.qkv_proj_gen.weight" in internal_parameter_names assert "model.layers.0.self_attn.q_proj.weight" not in internal_parameter_names assert "model.layers.0.self_attn.q_proj_moe_gen.weight" not in internal_parameter_names - checkpoint_state = reference.state_dict() - assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_state - assert "model.layers.0.self_attn.k_proj.weight" in checkpoint_state - assert "model.layers.0.self_attn.v_proj.weight" in checkpoint_state - assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in checkpoint_state - assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_state - assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in checkpoint_state - - state_dict_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) - state_dict_loaded.load_state_dict(checkpoint_state) + # PyTorch state_dict() and DCP get_model_state_dict() keep fused parameter keys so + # FQN resolution works under DDP as well as FSDP2. + runtime_state = reference.state_dict() + dcp_state = get_model_state_dict(reference) + assert "model.layers.0.self_attn.qkv_proj_und.weight" in runtime_state + assert "model.layers.0.self_attn.qkv_proj_gen.weight" in runtime_state + assert "model.layers.0.self_attn.q_proj.weight" not in runtime_state + assert set(dcp_state) == set(runtime_state) + + dcp_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + set_model_state_dict(dcp_loaded, dcp_state) for name in ("qkv_proj_und", "qkv_proj_gen"): expected = getattr(reference.model.layers[0].self_attn, name) - actual = getattr(state_dict_loaded.model.layers[0].self_attn, name) + actual = getattr(dcp_loaded.model.layers[0].self_attn, name) torch.testing.assert_close(actual.weight, expected.weight) torch.testing.assert_close(actual.bias, expected.bias) + # HF export reverts fused QKV back to split q_proj/k_proj/v_proj keys. + reverted_hf_state = _revert_model_weight_conversions_for_hf(reference, dict(dcp_state)) + assert "model.layers.0.self_attn.q_proj.weight" in reverted_hf_state + assert "model.layers.0.self_attn.k_proj.weight" in reverted_hf_state + assert "model.layers.0.self_attn.v_proj.weight" in reverted_hf_state + assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in reverted_hf_state + assert "model.layers.0.self_attn.qkv_proj_und.weight" not in reverted_hf_state + assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in reverted_hf_state + reference.save_pretrained(tmp_path) with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint: checkpoint_keys = set(checkpoint.keys()) diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py index c19124e84b..dc9c0d0deb 100644 --- a/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/processing.py @@ -32,7 +32,7 @@ def preprocess_latent_embed( token_count = int(clean.shape[0]) noise = torch.randn_like(clean) - timestep = torch.sigmoid(torch.randn(token_count, device=device, dtype=torch.float32)) + timestep = torch.sigmoid(torch.randn(1, device=device, dtype=torch.float32)).expand(token_count) timestep = float(timestep_shift) * timestep / (1.0 + (float(timestep_shift) - 1.0) * timestep) timestep_values = timestep.reshape(-1, 1).to(device=clean.device, dtype=torch.float32) noised = (1.0 - timestep_values) * clean + timestep_values * noise diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index 040c5be99d..e1a316eff8 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -39,7 +39,6 @@ BagelQwen2MoTCheckpointTensorConverter, combine_qkv_state_dict_pre_hook, register_accelerated_qkv_conversion_mapping, - split_qkv_state_dict_post_hook, ) from ..configuration import BagelQwen2MoTConfig from ..masking import build_mot_block_mask, build_mot_magi_mask, pad_mot_attention_metadata @@ -473,7 +472,6 @@ def __init__( except RuntimeError: self.veomni_denoise_attn = None self.register_load_state_dict_pre_hook(combine_qkv_state_dict_pre_hook) - self.register_state_dict_post_hook(split_qkv_state_dict_post_hook) def _build_qkv_proj(self) -> None: qkv_size = sum(self.qkv_split_sizes) diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py index 44a8199290..092e3bf21d 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py @@ -61,35 +61,6 @@ def combine_qkv_state_dict_pre_hook( state_dict[combined_key] = torch.cat([state_dict.pop(key) for key in checkpoint_keys], dim=0) -def split_qkv_state_dict_post_hook( - module: torch.nn.Module, - state_dict: dict[str, torch.Tensor], - prefix: str, - local_metadata: dict[str, Any], -) -> None: - """Expose legacy split Q/K/V keys while keeping combined runtime parameters.""" - del local_metadata - split_sizes = module.qkv_split_sizes - for combined_name, checkpoint_names in _QKV_BRANCHES: - for kind in ("weight", "bias"): - combined_key = f"{prefix}{combined_name}.{kind}" - tensor = state_dict.get(combined_key) - if tensor is None or tensor.is_meta or hasattr(tensor, "device_mesh"): - # Transformers inspects a meta model's state dict to discover the - # runtime parameter schema before applying its weight converters. - # DCP likewise requires DTensor keys to remain resolvable to the - # live combined parameters before the HF export layer converts them. - continue - - state_dict.pop(combined_key) - for checkpoint_name, chunk in zip( - checkpoint_names, - tensor.split(split_sizes, dim=0), - strict=True, - ): - state_dict[f"{prefix}{checkpoint_name}.{kind}"] = chunk - - class _ConcatenateQKV(ConversionOps): """Concatenate checkpoint Q/K/V tensors along the projection dimension.""" diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py index ec2e0a8534..a314c6b956 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -23,7 +23,12 @@ from ...sources import BAGEL_CONTEXT_KEY, BAGEL_VAE_CONTEXT from ..configuration import BagelVAEConfig from ..modeling import BagelVAE, select_bagel_vae_context_items -from ..processing import BAGEL_VAE_PIXEL_SHAPE, BAGEL_VAE_POSTERIOR, crop_latent_to_image_shape +from ..processing import ( + BAGEL_VAE_PIXEL_SHAPE, + BAGEL_VAE_POSTERIOR, + _pad_to_batch_size, + crop_latent_to_image_shape, +) def _posterior_from_cache( @@ -182,7 +187,7 @@ def encode_pre( self._metric_meter_stash_latent_tokens(self._vae_latent_token_lengths()) - pixel_values = torch.stack([item.value for item in self._encode_items], dim=0).to( + pixel_values = torch.stack(_pad_to_batch_size([item.value for item in self._encode_items]), dim=0).to( device=self.device, dtype=self.dtype, non_blocking=True ) ps = get_parallel_state() diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py index aef2f1da5c..abe8456886 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -27,7 +27,13 @@ from ...module_modeling_base import PretrainedOmniModule from ..sources import BAGEL_CONTEXT_KEY, BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_VAE_CONTEXT from .configuration import BagelVAEConfig -from .processing import BAGEL_VAE_PIXEL_SHAPE, BagelVAEPreprocessor, BagelVAEProcessor, crop_latent_to_image_shape +from .processing import ( + BAGEL_VAE_PIXEL_SHAPE, + BagelVAEPreprocessor, + BagelVAEProcessor, + _pad_to_batch_size, + crop_latent_to_image_shape, +) def select_bagel_vae_context_items( @@ -72,7 +78,7 @@ def encode_context( if not image_items: return {"conversation_list": conversation_list} - pixel_values = torch.stack([item.value for item in image_items], dim=0).to( + pixel_values = torch.stack(_pad_to_batch_size([item.value for item in image_items]), dim=0).to( device=self.device, dtype=self.dtype, non_blocking=True ) outputs = self.encode(pixel_values=pixel_values) From f1a297018d109ec3c9e9d0de0f53c82afd59340c Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 21:40:13 +0800 Subject: [PATCH 06/22] [omni, model, test] fix: fall back to sync attention on LoRA projections and fix multi-image SigLIP FLOPs Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 101 ++++++++++++++++++ .../qwen2_mot/accelerated/accelerated.py | 17 ++- .../siglip_navit/accelerated/accelerated.py | 61 ++++++++++- 3 files changed, 175 insertions(+), 4 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 82d99cfe87..dae44ba665 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -421,3 +421,104 @@ def test_bagel_modules_keep_construction_ops_when_config_mutated() -> None: torch.testing.assert_close(k_out, k_ref) assert rope_counter.calls == 2 assert torch.isfinite(mlp_ref).all() + + +@pytest.mark.parametrize("target_proj", ("qkv_proj_und", "qkv_proj_gen", "o_proj", "o_proj_moe_gen")) +def test_bagel_qwen2_mot_async_ulysses_falls_back_to_sync_on_lora_projections( + monkeypatch: pytest.MonkeyPatch, + target_proj: str, +) -> None: + """Verify ``BagelQwen2MoTAttentionAccelerated.forward_packed_train`` falls back to synchronous projections when LoRA wraps any projection layer.""" + from types import SimpleNamespace + + from veomni.lora.layers import LoraLinear + + cfg = _flex_config() + attn = accelerated.BagelQwen2MoTAttentionAccelerated(cfg, layer_idx=0).eval() + base_linear = getattr(attn, target_proj) + lora_linear = LoraLinear(base_linear, adapter_name="default", r=2, lora_alpha=4) + with torch.no_grad(): + lora_linear.lora_B["default"].weight.fill_(0.25) + setattr(attn, target_proj, lora_linear) + + def _fake_attn_op(module, q, k, v, mask, **kwargs): + del module, k, mask, kwargs + v_expanded = v.repeat_interleave(q.shape[1] // v.shape[1], dim=1) + return (q + v_expanded).transpose(1, 2).contiguous(), None + + attn.veomni_attn = _fake_attn_op + async_qkv_counter = _CallCounter(lambda **kwargs: (_ for _ in ()).throw(AssertionError("async qkv called"))) + async_o_counter = _CallCounter(lambda **kwargs: (_ for _ in ()).throw(AssertionError("async o called"))) + attn.veomni_async_ulysses_qkv = async_qkv_counter + attn.veomni_async_ulysses_o = async_o_counter + + packed_sequence = torch.randn(4, cfg.hidden_size) + cos = torch.ones(4, attn.head_dim) + sin = torch.zeros(4, attn.head_dim) + und_idx = torch.tensor([0, 2], dtype=torch.long) + gen_idx = torch.tensor([1, 3], dtype=torch.long) + + monkeypatch.setattr( + accelerated, + "get_parallel_state", + lambda: SimpleNamespace(ulysses_enabled=False, async_enabled=False, ulysses_size=1, ulysses_group=None), + ) + with torch.no_grad(): + sync_ref = attn.forward_packed_train(packed_sequence, None, cos, sin, und_idx, gen_idx) + + monkeypatch.setattr( + accelerated, + "get_parallel_state", + lambda: SimpleNamespace(ulysses_enabled=True, async_enabled=True, ulysses_size=2, ulysses_group=None), + ) + with torch.no_grad(): + async_with_lora = attn.forward_packed_train(packed_sequence, None, cos, sin, und_idx, gen_idx) + + assert async_qkv_counter.calls == 0 + assert async_o_counter.calls == 0 + torch.testing.assert_close(async_with_lora, sync_ref) + + +def test_bagel_siglip_navit_multi_image_sample_flops_and_seqlens() -> None: + """Verify multi-image samples preserve per-sample token counts while using per-image lengths for attention FLOPs.""" + from veomni.models.seed_omni.modules.bagel.siglip_navit.accelerated.accelerated import BagelSiglipNavitAccelerated + from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig + from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN + from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT + from veomni.models.seed_omni.utils.conversation import ConversationItem + + cfg = BagelSiglipNavitConfig( + num_hidden_layers=1, + ops_implementation={ + "attn_implementation": "eager", + "rotary_pos_emb_vision_implementation": "eager", + }, + ) + model = BagelSiglipNavitAccelerated(cfg) + model.config.num_hidden_layers = 26 + + patch_dim = cfg.num_channels * cfg.patch_size * cfg.patch_size + items = [ + ConversationItem( + type="image", + role="user", + value=torch.zeros(1024, patch_dim, dtype=torch.bfloat16), + meta={ + BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, + _OMNI_TOKEN_LEN: 1024, + _OMNI_POSITION_IDS: torch.zeros(1024, dtype=torch.long), + }, + ) + for _ in range(2) + ] + model.forward_pre(conversation_list=[items]) + model.metric_meter_add("forward") + flops, seqlens = model.metric_meter_collect() + + assert seqlens == [2048] + expected_per_image_flops = model.estimate_flops([1024, 1024]) + overcounted_sample_flops = model.estimate_flops([2048]) + assert flops == pytest.approx(expected_per_image_flops) + assert flops == pytest.approx(5.834821, rel=1e-4) + assert overcounted_sample_flops == pytest.approx(6.588592, rel=1e-4) + assert flops < overcounted_sample_flops diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index e1a316eff8..af5fabe47f 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -598,6 +598,17 @@ def attend_inference( ) return packed_attn_output.squeeze(0) + def _has_plain_linear_projections(self) -> bool: + """Whether async Ulysses may read this layer's projection weights directly. + + A wrapped projection such as ``LoraLinear`` has no ``weight`` and must run + its own forward, so attention falls back to the synchronous projection path. + """ + return all( + isinstance(getattr(self, name), torch.nn.Linear) + for name in ("qkv_proj_und", "qkv_proj_gen", "o_proj", "o_proj_moe_gen") + ) + def forward_packed_train( self, packed_sequence: torch.Tensor, @@ -609,7 +620,11 @@ def forward_packed_train( ) -> torch.Tensor: """Run packed training attention, using async Ulysses when enabled on the active parallel state.""" parallel_state = get_parallel_state() - use_async_ulysses = parallel_state.ulysses_enabled and bool(parallel_state.async_enabled) + use_async_ulysses = ( + parallel_state.ulysses_enabled + and bool(parallel_state.async_enabled) + and self._has_plain_linear_projections() + ) if not use_async_ulysses: return super().forward_packed_train( packed_sequence, diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py index 9941e82d0a..209734a0a3 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py @@ -53,7 +53,11 @@ def forward_pre( self._image_items = select_siglip_image_items(conversation_list) out = siglip_inputs_from_preprocessed_items(self._image_items, device=self.device, dtype=self.dtype) - self.metric_meter_set_seqlens("forward", self._metric_sample_token_lens(conversation_list)) + self.metric_meter_set_seqlens( + "forward", + self._metric_sample_token_lens(conversation_list), + image_seqlens=[int(item.meta[_OMNI_TOKEN_LEN]) for item in self._image_items], + ) ps = get_parallel_state() if ps.sp_size == 1: @@ -162,7 +166,47 @@ class MeterMixin(MetricMeterMixin): config: BagelSiglipNavitConfig - def estimate_flops(self, seqlens: list[int]) -> float: + def metric_meter_set_seqlens( + self, + method: str, + seqlens: list[int], + *, + image_seqlens: list[int] | None = None, + ) -> None: + """Stash per-sample ``seqlens`` plus per-image ``image_seqlens`` for attention FLOPs.""" + super().metric_meter_set_seqlens(method, seqlens) + if not hasattr(self, "_metric_full_image_seqlens"): + self._metric_full_image_seqlens: dict[str, list[int]] = {} + resolved_image_seqlens = seqlens if image_seqlens is None else image_seqlens + self._metric_full_image_seqlens[method] = [int(s) for s in resolved_image_seqlens] + + def _metric_meter_image_seqlen_buffer(self) -> list[int]: + if not hasattr(self, "_metric_meter_image_seqlens"): + self._metric_meter_image_seqlens: list[int] = [] + return self._metric_meter_image_seqlens + + def metric_meter_add(self, method: str) -> None: + """Move stashed per-sample and per-image token lengths into the step buffers.""" + super().metric_meter_add(method) + image_stash = getattr(self, "_metric_full_image_seqlens", None) + if image_stash: + self._metric_meter_image_seqlen_buffer().extend(image_stash.get(method, [])) + image_stash.clear() + + def metric_meter_collect(self) -> tuple[float, list[int]]: + """Return ``(theoretical_flops, sample_seqlens)`` using per-image lengths for attention FLOPs.""" + seqlens = self._metric_meter_seqlen_buffer() + self._metric_meter_seqlens = [] + image_seqlens = self._metric_meter_image_seqlen_buffer() + self._metric_meter_image_seqlens = [] + return self.estimate_flops(seqlens, image_seqlens=image_seqlens), seqlens + + def estimate_flops( + self, + seqlens: list[int], + *, + image_seqlens: list[int] | None = None, + ) -> float: """Estimate TFLOPs for the SigLIP-NaViT encoder and MLP connector over ``seqlens``.""" cfg = self.config dim = cfg.hidden_size @@ -173,8 +217,19 @@ def estimate_flops(self, seqlens: list[int]) -> float: mlp_n = dim * cfg.intermediate_size * 2 connector_n = dim * cfg.output_size + cfg.output_size * cfg.output_size dense_n = patch_embed_n + (attn_linear_n + mlp_n) * cfg.num_hidden_layers + connector_n + if image_seqlens is None: + buffered = getattr(self, "_metric_meter_image_seqlens", None) + stashed = getattr(self, "_metric_full_image_seqlens", {}).get("forward") + if buffered and sum(buffered) == sum(seqlens): + attn_seqlens = buffered + elif stashed and sum(stashed) == sum(seqlens): + attn_seqlens = stashed + else: + attn_seqlens = seqlens + else: + attn_seqlens = image_seqlens tokens = sum(seqlens) - seqlen_sq = sum(length * length for length in seqlens) + seqlen_sq = sum(length * length for length in attn_seqlens) dense_flops = 6 * dense_n * tokens attn_flops = 12 * seqlen_sq * head_dim * heads * cfg.num_hidden_layers return (dense_flops + attn_flops) / 1e12 From b49a3290778efe9886dbbcc846419828f62be01d Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 2 Nov 2026 22:25:07 +0800 Subject: [PATCH 07/22] [omni, model, docs, test] fix: support LoRA-wrapped Bagel projections and align SM100+ Magi contract Co-authored-by: Cursor --- docs/seed_omni/models/bagel.md | 42 ++++-------- .../modules/bagel/test_bagel_accel_align.py | 42 +++++++----- .../seed_omni/modules/bagel/test_qwen2_mot.py | 67 +++++++++++++++++++ .../modules/bagel/flow_connector/modeling.py | 19 ++++-- .../modules/bagel/siglip_navit/modeling.py | 20 ++++-- 5 files changed, 130 insertions(+), 60 deletions(-) diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md index 981e720226..7ecfa2e642 100644 --- a/docs/seed_omni/models/bagel.md +++ b/docs/seed_omni/models/bagel.md @@ -193,12 +193,12 @@ training does not use FlashAttention-2. | Backend | Class | Config | Use | |---------|-------|--------|-----| -| FlexAttention | accelerated / eager | `train/modules_train.yaml`, `infer/modules_infer_eager.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training and inference | -| MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM90+ fused training; see nfunc below | +| FlexAttention | accelerated / eager | `train/modules_train.yaml`, `infer/modules_infer_eager.yaml`, `infer/modules_infer_fsdp.yaml` | Default packed training and inference (only supported packed backend on SM90) | +| MagiAttention | accelerated | CLI override onto `bagel_qwen2_mot` | SM100+ CUTE DSL/JIT fused training | | SDPA | eager | CLI override onto `bagel_qwen2_mot` | Dense-mask oracle | -Leave the YAML on `flex_attention` unless you are opting into Magi. Enable Magi -from the CLI (the launcher rewrites it to `veomni_magi_attention`): +Leave the YAML on `flex_attention` unless you are opting into Magi on SM100+. +Enable Magi from the CLI (the launcher rewrites it to `veomni_magi_attention`): ```bash bash train.sh tasks/omni/train_omni.py \ @@ -207,32 +207,14 @@ bash train.sh tasks/omni/train_omni.py \ --train.micro_batch_size 1 ``` -Magi requires physical batch size 1, `cp_size == 1`, and NVIDIA SM90 or newer. -Ulysses still works. On 8 ranks, `--model.accelerator.ulysses_size 4` gives -`dp_shard=2` and SP4. Packed Magi/Flex is the training (and FSDP prefill) path; -the denoise loop still uses FlashAttention-2. The Magi adapter contract is in +Magi requires physical batch size 1, `cp_size == 1`, and NVIDIA SM100 or newer +(with CUTE DSL/JIT installed via `uv sync --extra gpu --extra magi --dev`). +SM90 and older GPUs only support FlexAttention. Ulysses still works. On 8 ranks, +`--model.accelerator.ulysses_size 4` gives `dp_shard=2` and SP4. Packed +Magi/Flex is the training (and FSDP prefill) path; the denoise loop still uses +FlashAttention-2. The Magi adapter contract is in [`docs/transformers_v5/veomni_attention_interface.md`](../../transformers_v5/veomni_attention_interface.md). -#### SM90 nfunc vs dataset - -On SM90, Magi uses a precompiled CUTLASS overlay. `nfunc` is baked at install -time: it is the HSTU interval count for the worst query in a sample, not the -Magi range-list length. The installer default is `1,3,5`. A later exact -`uv sync` removes the overlay, so rerun the installer before SM90 Magi runs. - -Text-only and single-image gen samples are typically `nfunc=1`. Multi-turn -`seed_edit_p23_multi_turn` punches noise-key holes, so nfunc grows with the -number of assistant images (about `2G-1`). That mixture needs **nfunc ≥ 11**: - -```bash -bash scripts/kernel/install_magi_sm90.sh --nfunc 1,3,5,7,9,11 -``` - -If a sample's runtime nfunc is missing from the compiled matrix, the kernel -fails with `Compile-time kNFunc (...) must match runtime arbitrary_func_num (...)`. -SM100+ uses CUTE DSL/JIT and compiles unseen nfunc at runtime, so it does not -need this overlay matrix. - ### 3.2 Offline VAE posterior cache (two stages) Caching is not a special framework mode — it is a different `train_graph` plus a @@ -370,8 +352,8 @@ YAML. The Bagel module and graph contracts cover carrier context-tag routing, generation state transitions, packing/cache behavior, and graph config structure. Packed MoT also compares Flex and Magi against eager SDPA on toy CE / MSE / gradients -in `tests/seed_omni/modules/bagel/test_bagel_accel_align.py`. Magi cases skip unless the -SM90 CUTLASS overlay or SM100+ CUTE JIT backend is present. +in `tests/seed_omni/modules/bagel/test_bagel_accel_align.py`. Magi cases skip unless an +NVIDIA SM100+ GPU with the CUTE DSL/JIT backend is present. ```bash .venv/bin/python -m pytest -q tests/seed_omni/modules/bagel diff --git a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py index a531d7be6c..863eb3b6e2 100644 --- a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py +++ b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py @@ -8,6 +8,7 @@ import pytest import torch +from scripts.ci.accelerator.launch_plan import LaunchSpec from tests.seed_omni.modules.bagel.helpers import ( ALIGN_ATOL, ALIGN_GRAD_ATOL, @@ -25,7 +26,7 @@ tiny_align_qwen2_cfg, ) from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated -from veomni.ops.kernels.attention.standard.magi import _fa4_cuda as magi_fa4_backend +from veomni.ops.kernels.attention.standard.magi import _kernel as magi_kernel from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type @@ -36,30 +37,18 @@ def _is_magi_ffa_available() -> bool: - """Return whether a runnable MagiAttention CUTLASS or CUTE backend is installed on this GPU.""" + """Return whether a runnable MagiAttention CUTE DSL/JIT backend is installed on this GPU.""" if not IS_CUDA_AVAILABLE or importlib.util.find_spec("magi_attention") is None: return False - kernel_mode = magi_fa4_backend._get_magi_kernel_mode(torch.device(get_device_type())) - if kernel_mode == magi_fa4_backend._MAGI_KERNEL_CUTE_JIT: - return importlib.util.find_spec("flash_attn_cute") is not None - if kernel_mode != magi_fa4_backend._MAGI_KERNEL_CUTLASS: - return False - - try: - from flash_attn_cute.ffa_fa3 import flash_attn_interface - except (ImportError, OSError, RuntimeError): - return False - - return all( - callable(getattr(flash_attn_interface, name, None)) for name in ("_flash_attn_forward", "_flash_attn_backward") + device = torch.device(get_device_type()) + return magi_kernel.get_kernel_mode(device) == magi_kernel.KERNEL_CUTE_JIT and ( + importlib.util.find_spec("flash_attn_cute") is not None ) _MAGI_FFA_AVAILABLE = _is_magi_ffa_available() -_MAGI_FFA_REASON = ( - "MagiAttention alignment requires a supported NVIDIA GPU with its CUTLASS overlay or CUTE DSL/JIT backend" -) +_MAGI_FFA_REASON = "MagiAttention alignment requires an NVIDIA SM100+ GPU with the CUTE DSL/JIT backend" def _eager_config(): @@ -177,3 +166,20 @@ def test_accelerated_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: @pytest.mark.parametrize("case", _CASES) def test_magi_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: _assert_accelerated_matches_eager(case, _MAGI_IMPLEMENTATION) + + +@pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) +def test_is_magi_ffa_available_checks_current_kernel_interface(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify ``_is_magi_ffa_available`` queries ``magi_kernel.get_kernel_mode`` cleanly when Magi is installed.""" + monkeypatch.setattr("tests.seed_omni.modules.bagel.test_bagel_accel_align.IS_CUDA_AVAILABLE", True) + monkeypatch.setattr( + "tests.seed_omni.modules.bagel.test_bagel_accel_align.get_device_type", + lambda: magi_kernel.CUDA_DEVICE_TYPE, + ) + monkeypatch.setattr(importlib.util, "find_spec", lambda name: object()) + + monkeypatch.setattr(magi_kernel, "get_kernel_mode", lambda device: magi_kernel.KERNEL_UNSUPPORTED) + assert _is_magi_ffa_available() is False + + monkeypatch.setattr(magi_kernel, "get_kernel_mode", lambda device: magi_kernel.KERNEL_CUTE_JIT) + assert _is_magi_ffa_available() is True diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index dae44ba665..eaa900f84c 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -522,3 +522,70 @@ def test_bagel_siglip_navit_multi_image_sample_flops_and_seqlens() -> None: assert flops == pytest.approx(5.834821, rel=1e-4) assert overcounted_sample_flops == pytest.approx(6.588592, rel=1e-4) assert flops < overcounted_sample_flops + + +def test_bagel_flow_connector_and_siglip_navit_support_lora_wrapped_projections() -> None: + """Verify ``BagelFlowConnector`` and ``BagelSiglipNavit`` run with ``LoraLinear``-wrapped projections.""" + from veomni.lora.layers import LoraLinear + from veomni.models.seed_omni.modules.bagel.flow_connector.configuration import BagelFlowConnectorConfig + from veomni.models.seed_omni.modules.bagel.flow_connector.modeling import BagelFlowConnector + from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig + from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit + + flow_cfg = BagelFlowConnectorConfig( + hidden_size=16, + z_channels=4, + latent_patch_size=2, + max_latent_size=8, + timestep_frequency_embedding_size=8, + ) + flow = BagelFlowConnector(flow_cfg).eval() + flow.vae2llm = LoraLinear(flow.vae2llm, adapter_name="default", r=2, lora_alpha=4) + flow.llm2vae = LoraLinear(flow.llm2vae, adapter_name="default", r=2, lora_alpha=4) + flow.time_embedder.mlp[0] = LoraLinear(flow.time_embedder.mlp[0], adapter_name="default", r=2, lora_alpha=4) + flow.time_embedder.mlp[2] = LoraLinear(flow.time_embedder.mlp[2], adapter_name="default", r=2, lora_alpha=4) + with torch.no_grad(): + flow.vae2llm.lora_B["default"].weight.fill_(0.1) + flow.llm2vae.lora_B["default"].weight.fill_(0.1) + latent_out = flow.embed_latent( + latents=torch.randn(3, flow_cfg.patch_latent_dim), + position_ids=torch.tensor([0, 1, 2], dtype=torch.long), + timesteps=torch.tensor([0.2, 0.5, 0.8], dtype=torch.float32), + )["latent_embeds"] + vel_out = flow.decode_velocity(torch.randn(3, flow_cfg.hidden_size))["velocity"] + assert latent_out.shape == (3, flow_cfg.hidden_size) + assert vel_out.shape == (3, flow_cfg.patch_latent_dim) + assert torch.isfinite(latent_out).all() + assert torch.isfinite(vel_out).all() + + siglip_cfg = BagelSiglipNavitConfig( + hidden_size=16, + output_size=24, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=4, + image_size=28, + patch_size=14, + rope=False, + ops_implementation={ + "attn_implementation": "eager", + "rotary_pos_emb_vision_implementation": "eager", + }, + ) + siglip = BagelSiglipNavit(siglip_cfg).eval() + siglip.vision_model.embeddings.patch_embedding = LoraLinear( + siglip.vision_model.embeddings.patch_embedding, adapter_name="default", r=2, lora_alpha=4 + ) + siglip.connector.fc1 = LoraLinear(siglip.connector.fc1, adapter_name="default", r=2, lora_alpha=4) + siglip.connector.fc2 = LoraLinear(siglip.connector.fc2, adapter_name="default", r=2, lora_alpha=4) + patch_dim = siglip_cfg.num_channels * siglip_cfg.patch_size * siglip_cfg.patch_size + with torch.no_grad(): + siglip_out = siglip( + patchified_pixel_values=torch.randn(4, patch_dim), + patchified_position_ids=torch.arange(4, dtype=torch.long), + cu_seqlens=torch.tensor([0, 2, 4], dtype=torch.int32), + max_seqlen=2, + token_lens=torch.tensor([2, 2], dtype=torch.int32), + ) + assert siglip_out["image_embeds"].shape == (4, siglip_cfg.output_size) + assert torch.isfinite(siglip_out["image_embeds"]).all() diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py index fe3a47afe7..b01fe1a022 100644 --- a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py @@ -286,6 +286,11 @@ def _emit_final_latent( return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: SIGNAL_IMAGE_COMPLETE} +def _module_weight(module: nn.Module) -> torch.Tensor: + """Return the underlying weight parameter for plain or LoRA-wrapped linear modules.""" + return getattr(module, "base_layer", module).weight + + class BagelFlowConnector(InferenceMixin, PretrainedOmniModule): config_class = BagelFlowConnectorConfig base_model_prefix = "bagel_flow_connector" @@ -315,11 +320,11 @@ def _init_weights(self, module: nn.Module) -> None: @property def _vae2llm_device(self) -> torch.device: - return self.vae2llm.weight.device + return _module_weight(self.vae2llm).device @property def _llm2vae_device(self) -> torch.device: - return self.llm2vae.weight.device + return _module_weight(self.llm2vae).device @property def _pos_embed_device(self) -> torch.device: @@ -327,7 +332,7 @@ def _pos_embed_device(self) -> torch.device: @property def _time_embedder_device(self) -> torch.device: - return self.time_embedder.mlp[0].weight.device + return _module_weight(self.time_embedder.mlp[0]).device def embed_latent( self, @@ -336,7 +341,8 @@ def embed_latent( timesteps: torch.Tensor, ) -> dict[str, torch.Tensor]: """Project patchified latents and add timestep and 2D sinusoidal position embeddings.""" - latents = latents.to(device=self._vae2llm_device, dtype=self.vae2llm.weight.dtype) + vae2llm_weight = _module_weight(self.vae2llm) + latents = latents.to(device=vae2llm_weight.device, dtype=vae2llm_weight.dtype) position_ids = position_ids.to(device=self._pos_embed_device, dtype=torch.long).reshape(-1) timesteps = timesteps.to(device=self._time_embedder_device, dtype=torch.float32).reshape(-1) if position_ids.numel() != latents.shape[0]: @@ -352,7 +358,8 @@ def embed_latent( } def decode_velocity(self, hidden_states: torch.Tensor) -> dict[str, torch.Tensor]: - hidden_states = hidden_states.to(device=self._llm2vae_device) + llm2vae_weight = _module_weight(self.llm2vae) + hidden_states = hidden_states.to(device=llm2vae_weight.device, dtype=llm2vae_weight.dtype) return {"velocity": self.llm2vae(hidden_states)} @@ -379,7 +386,7 @@ def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> to def forward(self, t: torch.Tensor) -> torch.Tensor: t_freq = self.timestep_embedding(t, self.frequency_embedding_size) - first_weight = self.mlp[0].weight + first_weight = _module_weight(self.mlp[0]) return self.mlp(t_freq.to(device=first_weight.device, dtype=first_weight.dtype)) diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py index 236e7564ee..923276a7d4 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -121,6 +121,11 @@ def generate( return {"conversation_list": batched[0]} +def _module_weight(module: nn.Module) -> torch.Tensor: + """Return the underlying weight parameter for plain or LoRA-wrapped modules.""" + return getattr(module, "base_layer", module).weight + + class BagelSiglipNavit(InferenceMixin, PretrainedOmniModule): config_class = BagelSiglipNavitConfig image_processor_class = BagelSiglipNavitProcessor @@ -148,11 +153,11 @@ def _init_weights(self, module: nn.Module) -> None: @property def _vit_device(self) -> torch.device: - return self.vision_model.embeddings.patch_embedding.weight.device + return _module_weight(self.vision_model.embeddings.patch_embedding).device @property def _connector_device(self) -> torch.device: - return self.connector.fc1.weight.device + return _module_weight(self.connector.fc1).device @property def _pos_embed_device(self) -> torch.device: @@ -204,7 +209,8 @@ def _project_with_position( vit_hidden: torch.Tensor, patchified_position_ids: torch.LongTensor, ) -> torch.Tensor: - vit_hidden = vit_hidden.to(device=self._connector_device, dtype=self.connector.fc1.weight.dtype) + fc1_weight = _module_weight(self.connector.fc1) + vit_hidden = vit_hidden.to(device=fc1_weight.device, dtype=fc1_weight.dtype) patchified_position_ids = patchified_position_ids.to(device=self._pos_embed_device) image_embeds = self.connector(vit_hidden) return image_embeds + self.vit_pos_embed(patchified_position_ids).to( @@ -308,14 +314,16 @@ def forward( patchified_pixel_values: torch.Tensor, patchified_position_ids: torch.LongTensor, ) -> torch.Tensor: + patch_weight = _module_weight(self.patch_embedding) patchified_pixel_values = patchified_pixel_values.to( - device=self.patch_embedding.weight.device, - dtype=self.patch_embedding.weight.dtype, + device=patch_weight.device, + dtype=patch_weight.dtype, ) patch_embeds = self.patch_embedding(patchified_pixel_values) if self.position_embedding is None: return patch_embeds - position_ids = patchified_position_ids.to(device=self.position_embedding.weight.device, dtype=torch.long) + position_weight = _module_weight(self.position_embedding) + position_ids = patchified_position_ids.to(device=position_weight.device, dtype=torch.long) position_embeds = self.position_embedding(position_ids) return patch_embeds + position_embeds.to(device=patch_embeds.device, dtype=patch_embeds.dtype) From 094f45447f7fdd6c43d774cdf58b695771d1985a Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Tue, 3 Nov 2026 14:19:47 +0800 Subject: [PATCH 08/22] [omni, model, test] fix: reduce flow loss over fsdp_group, avoid VAE spatial padding, and fix meta rotary/marker checks Co-authored-by: Cursor --- .../modules/bagel/test_processing.py | 170 +++++++++++++++--- .../seed_omni/modules/bagel/test_qwen2_mot.py | 22 +++ .../flow_connector/accelerated/accelerated.py | 24 ++- .../modules/bagel/qwen2_mot/modeling.py | 15 +- .../bagel/text_encoder/chat_template.py | 4 +- .../bagel/vae/accelerated/accelerated.py | 108 ++++++++--- .../seed_omni/modules/bagel/vae/modeling.py | 56 ++++-- .../seed_omni/modules/bagel/vae/processing.py | 29 ++- 8 files changed, 348 insertions(+), 80 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index 230d5c53f1..5d160c4131 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -1006,19 +1006,19 @@ def test_bagel_siglip_preprocessor_from_pretrained_applies_config_overrides(tmp_ assert pre._image_processor.min_image_size == 28 -def test_bagel_vae_encode_and_offline_encode_pad_per_sample_preprocessed_microbatch() -> None: - """Verify ``encode`` and ``offline_encode`` pad variable-shape images across per-sample-preprocessed micro-batches.""" +def test_bagel_vae_encode_and_offline_encode_preserve_native_shapes_and_match_independent_posteriors() -> None: + """Verify ``encode`` and ``offline_encode`` group equal shapes without spatial padding and match independent encoding.""" pre = BagelVAEPreprocessor( BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=8, max_pixels=64), dtype=torch.float32, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.float32), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + dummy_pixel_values=torch.zeros(3, 4, 4, dtype=torch.float32), + dummy_pixel_shape=torch.tensor([4, 4], dtype=torch.long), ) - # Dataset-stage transform runs preprocessor on one sample at a time: + torch.manual_seed(42) sample_rect_a = [ ConversationItem( type="image", - value=torch.full((3, 4, 8), 5, dtype=torch.uint8), + value=torch.randint(0, 255, (3, 4, 8), dtype=torch.uint8), role="assistant", meta={_IMG_TAG_KEY: "gen"}, ) @@ -1026,38 +1026,162 @@ def test_bagel_vae_encode_and_offline_encode_pad_per_sample_preprocessed_microba sample_rect_b = [ ConversationItem( type="image", - value=torch.full((3, 8, 4), 7, dtype=torch.uint8), + value=torch.randint(0, 255, (3, 8, 4), dtype=torch.uint8), + role="assistant", + meta={_IMG_TAG_KEY: "gen"}, + ) + ] + sample_rect_c = [ + ConversationItem( + type="image", + value=torch.randint(0, 255, (3, 4, 8), dtype=torch.uint8), role="assistant", meta={_IMG_TAG_KEY: "gen"}, ) ] sample_text_only = [ConversationItem(type="text", value="hello", role="user")] - for sample in (sample_rect_a, sample_rect_b, sample_text_only): + for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): pre(_b([sample])) assert sample_rect_a[0].value.shape == (3, 4, 8) assert sample_rect_b[0].value.shape == (3, 8, 4) + assert sample_rect_c[0].value.shape == (3, 4, 8) assert sample_text_only[-1].is_dummy - assert sample_text_only[-1].value.shape == (3, 2, 2) + assert sample_text_only[-1].value.shape == (3, 4, 4) - vae = _tiny_vae(downsample=2) + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) - # 1. Online encode with ragged real + dummy micro-batch - online_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_text_only]) - pre_inputs = vae.pre_forward("encode", conversation_list=online_batch) - assert pre_inputs["pixel_values"].shape == (3, 3, 8, 8) - encoded_out = vae.encode(**pre_inputs) - vae.post_forward("encode", **encoded_out) - assert online_batch[0][0].value.shape == (2, 2, 4) - assert online_batch[1][0].value.shape == (2, 4, 2) - assert online_batch[2][-1].value.shape == (2, 1, 1) + # Independent single-image offline_encode oracle + independent_posteriors = [] + for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): + single = copy.deepcopy([sample]) + single_pre = vae.pre_forward("offline_encode", conversation_list=single) + single_out = vae.offline_encode(**single_pre) + vae.post_forward("offline_encode", **single_out) + independent_posteriors.append(single[0][-1].value) - # 2. Offline encode with ragged real + dummy micro-batch - offline_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_text_only]) + # 1. Offline encode with ragged real + dummy micro-batch + offline_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only]) off_pre = vae.pre_forward("offline_encode", conversation_list=offline_batch) - assert off_pre["pixel_values"].shape == (3, 3, 8, 8) + assert isinstance(off_pre["pixel_values"], list) + assert [tuple(t.shape) for t in off_pre["pixel_values"]] == [(3, 4, 8), (3, 8, 4), (3, 4, 8), (3, 4, 4)] off_out = vae.offline_encode(**off_pre) vae.post_forward("offline_encode", **off_out) assert offline_batch[0][0].value.shape == (2, 2, 2, 4) assert offline_batch[1][0].value.shape == (2, 2, 4, 2) - assert offline_batch[2][-1].value.shape == (2, 2, 1, 1) + assert offline_batch[2][0].value.shape == (2, 2, 2, 4) + assert offline_batch[3][-1].value.shape == (2, 2, 2, 2) + for idx, expected_posterior in enumerate(independent_posteriors): + torch.testing.assert_close(offline_batch[idx][-1].value, expected_posterior) + + # 2. Online encode with ragged real + dummy micro-batch (matching RNG sequence) + torch.manual_seed(777) + independent_latents = [] + for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): + single = copy.deepcopy([sample]) + single_pre = vae.pre_forward("encode", conversation_list=single) + single_out = vae.encode(**single_pre) + vae.post_forward("encode", **single_out) + independent_latents.append(single[0][-1].value) + + torch.manual_seed(777) + online_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only]) + pre_inputs = vae.pre_forward("encode", conversation_list=online_batch) + encoded_out = vae.encode(**pre_inputs) + vae.post_forward("encode", **encoded_out) + assert online_batch[0][0].value.shape == (2, 2, 4) + assert online_batch[1][0].value.shape == (2, 4, 2) + assert online_batch[2][0].value.shape == (2, 2, 4) + assert online_batch[3][-1].value.shape == (2, 2, 2) + for idx, expected_latent in enumerate(independent_latents): + torch.testing.assert_close(online_batch[idx][-1].value, expected_latent) + + +def test_bagel_flow_decode_velocity_reduces_loss_and_token_count_over_fsdp_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verify ``decode_velocity_post`` reduces flow loss and valid-token counts over ``fsdp_group`` including zero-target ranks.""" + import torch.distributed as dist + + from veomni.models.seed_omni.modules.bagel.flow_connector.accelerated import accelerated as flow_acc + + BagelFlowConnector = model_cls("bagel_flow_connector") + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + model = BagelFlowConnector( + BagelFlowConnectorConfig( + hidden_size=4, + z_channels=1, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + timestep_shift=1.0, + ) + ) + fake_fsdp_group = object() + monkeypatch.setattr( + flow_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=1, cp_size=1, sp_group=None, fsdp_group=fake_fsdp_group), + ) + monkeypatch.setattr(dist, "get_world_size", lambda group=None: 2) + + # Case 1: Generation rank (4 tokens) paired with a text-only rank (0 tokens on peer) + peer_loss_sum = 0.0 + peer_tokens = 0 + + def _fake_all_reduce_gen_vs_text(tensor: torch.Tensor, group=None): + assert group is fake_fsdp_group + if tensor.dtype == torch.long: + tensor.add_(peer_tokens) + else: + tensor.add_(peer_loss_sum) + + monkeypatch.setattr(dist, "all_reduce", _fake_all_reduce_gen_vs_text) + + gen_item = ConversationItem( + type="image", + value=torch.ones(4, 4), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: "gen", + "flow_velocity_target": torch.zeros(4, 2), + }, + ) + gen_conv = [[gen_item]] + model.decode_velocity_pre(conversation_list=gen_conv) + pred_vel = torch.full((4, 2), 2.0, requires_grad=True) + gen_out = model.decode_velocity_post(pred_vel) + assert gen_out["_loss"].item() == pytest.approx(4.0) + gen_out["_loss"].backward() + # Single-rank unreduced grad per element is 2 * (2 - 0) / (4 * 2) = 0.5; + # with world_size=2 and peer_tokens=0, ReduceLoss scales backward by 2.0 so FSDP's /2 preserves 0.5. + torch.testing.assert_close(pred_vel.grad, torch.full_like(pred_vel, 1.0)) + + # Zero-target rank paired with the 4-token generation rank on peer (peer_loss_sum = 16.0, peer_tokens = 4) + peer_loss_sum = 16.0 + peer_tokens = 4 + text_conv = [[ConversationItem(type="text", value=torch.ones(2, 4), role="user")]] + model.decode_velocity_pre(conversation_list=text_conv) + dummy_vel = torch.ones(1, 2, requires_grad=True) + zero_out = model.decode_velocity_post(dummy_vel) + assert zero_out["_loss"].item() == pytest.approx(4.0) + zero_out["_loss"].backward() + torch.testing.assert_close(dummy_vel.grad, torch.zeros_like(dummy_vel)) + + +def test_bagel_chat_template_rejects_multi_token_start_or_end_markers() -> None: + """Verify ``BagelChatTemplate.tokenize`` raises when ``<|im_start|>`` or ``<|im_end|>`` splits into multiple tokens.""" + + class _SplittingTokenizer(_FakeBagelTokenizer): + def __call__(self, text: str, add_special_tokens: bool = False) -> dict[str, list[int]]: + assert add_special_tokens is False + if text in {"<|im_start|>", "<|im_end|>"}: + return {"input_ids": [1, 2]} + return super().__call__(text, add_special_tokens=add_special_tokens) + + template = BagelChatTemplate(_SplittingTokenizer()) + for marker in ("<|im_start|>", "<|im_end|>"): + with pytest.raises(ValueError, match="BAGEL start/end tokens must tokenize to exactly one token"): + template.tokenize([ConversationItem(type="text", value=marker, role="user")]) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index eaa900f84c..38bbd61530 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -589,3 +589,25 @@ def test_bagel_flow_connector_and_siglip_navit_support_lora_wrapped_projections( ) assert siglip_out["image_embeds"].shape == (4, siglip_cfg.output_size) assert torch.isfinite(siglip_out["image_embeds"]).all() + + +def test_bagel_qwen2_rotary_embedding_to_empty_from_meta_rebuilds_fp32_table() -> None: + """Verify ``BagelQwen2RotaryEmbedding`` built on ``meta`` rebuilds valid fp32 buffers on ``to_empty``.""" + from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2RotaryEmbedding + + cfg = _flex_config() + ref_rotary = BagelQwen2RotaryEmbedding(cfg, device=torch.device("cpu")) + with torch.device("meta"): + meta_model = accelerated.BagelQwen2MoTAccelerated(cfg) + assert meta_model.model.rotary_emb.inv_freq.is_meta + assert meta_model.model.rotary_emb.original_inv_freq.is_meta + + materialized = meta_model.to_empty(device="cpu").to(dtype=torch.bfloat16) + rotary = materialized.model.rotary_emb + assert not rotary.inv_freq.is_meta + assert not rotary.original_inv_freq.is_meta + assert rotary.inv_freq.dtype == torch.float32 + assert rotary.original_inv_freq.dtype == torch.float32 + assert rotary.inv_freq.data_ptr() != rotary.original_inv_freq.data_ptr() + torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) + torch.testing.assert_close(rotary.original_inv_freq, ref_rotary.original_inv_freq) diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py index f9146848eb..25c9037283 100644 --- a/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/accelerated/accelerated.py @@ -14,7 +14,12 @@ import torch from .......distributed.parallel_state import get_parallel_state -from .......distributed.sequence_parallel import gather_outputs, slice_input_tensor, sp_pad +from .......distributed.sequence_parallel import ( + gather_outputs, + reduce_sequence_parallel_loss, + slice_input_tensor, + sp_pad, +) from .....mixins.base_mixin import BaseMixin from .....mixins.metric_meter_mixin import MetricMeterMixin from .....mixins.training_module_mixin import TrainingModuleMixin, post_forward, pre_forward @@ -236,12 +241,21 @@ def decode_velocity_post(self, velocity: torch.Tensor) -> dict[str, Any]: self._sp_decode_length = None real_velocity_parts = self._scatter_velocity(decode_target_groups, decode_lengths, velocity) - loss = velocity.sum() * 0.0 + loss = velocity.float().sum() * 0.0 if target is not None and real_velocity_parts: real_velocity = torch.cat(real_velocity_parts, dim=0) - mse = (real_velocity - target.to(device=velocity.device, dtype=velocity.dtype)).square() - token_count = torch.tensor(float(mse.shape[0]), device=mse.device, dtype=mse.dtype) - loss = loss + mse.mean(dim=-1).sum() / token_count + mse = (real_velocity - target.to(device=velocity.device, dtype=velocity.dtype)).float().square() + num_valid_tokens = torch.tensor(mse.shape[0], device=velocity.device, dtype=torch.long) + loss = loss + mse.mean(dim=-1).sum() / num_valid_tokens.to(dtype=loss.dtype) + else: + num_valid_tokens = torch.zeros((), device=velocity.device, dtype=torch.long) + + ps = get_parallel_state() + loss_reduction_group = ( + ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None) + ) + if loss_reduction_group is not None: + loss = reduce_sequence_parallel_loss(loss, num_valid_tokens, group=loss_reduction_group) return {"conversation_list": conversation, "_loss": loss} def _anchor_dummy_decode_velocity_inputs( diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index de15ae7672..871ee725aa 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -703,15 +703,16 @@ def __init__(self, config: BagelQwen2MoTConfig, device: Optional[torch.device] = self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) def _apply(self, fn: Any, recurse: bool = True) -> nn.Module: - # Snapshot the true fp32 table before ``fn`` can quantize it (``.to(bfloat16)``). - # Casting first and then ``.float()`` keeps bf16 rounding, which is not official - # mixed-precision train (buffers stay fp32) and is not bitwise with a fresh table. - inv_freq = self.inv_freq.detach().to(dtype=torch.float32).clone() - original_inv_freq = self.original_inv_freq.detach().to(dtype=torch.float32).clone() + # Rebuild the true fp32 table after ``fn`` so ``.to(bfloat16)`` does not + # quantize rotary buffers and ``to_empty(device=...)`` from ``meta`` + # materializes valid fp32 inverse frequencies on the target device. module = super()._apply(fn, recurse=recurse) device = self.inv_freq.device - self.inv_freq = inv_freq.to(device=device) - self.original_inv_freq = original_inv_freq.to(device=device) + if device.type == "meta": + return module + inv_freq, _ = self.compute_default_rope_parameters(self.config, device) + self.inv_freq = inv_freq + self.original_inv_freq = inv_freq.clone() return module @staticmethod diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py index 4fe4b089f2..5b3552b2b3 100644 --- a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py @@ -106,6 +106,7 @@ def tokenize(self, parts: list[ConversationItem]) -> None: if item.type != "text" or item.is_dummy or item.meta.get(_OMNI_TOKENIZED): continue + raw_value = item.value token_ids = self._token_ids_for_text_item(item) loss_mask = int(item.meta.pop("loss_mask", int(item.role == "assistant"))) item.value = token_ids @@ -118,7 +119,8 @@ def tokenize(self, parts: list[ConversationItem]) -> None: item.meta[_OMNI_TOKENIZED] = True if ( - item.value in {self.chat_markers.bos_token, self.chat_markers.eos_token} + isinstance(raw_value, str) + and raw_value in {self.chat_markers.bos_token, self.chat_markers.eos_token} and int(token_ids.numel()) != 1 ): raise ValueError("BAGEL start/end tokens must tokenize to exactly one token.") diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py index a314c6b956..3a88fffc3f 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -7,6 +7,7 @@ from __future__ import annotations +import math from typing import Any import torch @@ -14,7 +15,7 @@ from veomni.utils.device import get_device_id, get_device_type from .......distributed.parallel_state import get_parallel_state -from .......distributed.sequence_parallel import gather_outputs, slice_input_tensor +from .......distributed.sequence_parallel import gather_outputs from .....mixins.base_mixin import BaseMixin from .....mixins.metric_meter_mixin import MetricMeterMixin from .....mixins.offline_encoding_mixin import OfflineEncodingMixin @@ -26,8 +27,8 @@ from ..processing import ( BAGEL_VAE_PIXEL_SHAPE, BAGEL_VAE_POSTERIOR, - _pad_to_batch_size, crop_latent_to_image_shape, + unpad_vae_item_pixel_values, ) @@ -59,11 +60,15 @@ class BagelVAEOfflineMixin(OfflineEncodingMixin): def offline_encode( self, - pixel_values: torch.Tensor, + pixel_values: torch.Tensor | list[torch.Tensor], **kwargs: object, ) -> dict[str, Any]: - """Encode ``pixel_values`` into a stacked ``(mean, logvar)`` posterior tensor for offline caching.""" + """Encode ``pixel_values`` into ``(mean, logvar)`` posterior tensor(s) for offline caching.""" del kwargs + if isinstance(pixel_values, list): + posteriors = self._encode_posterior_list(pixel_values) + return {"encoded_cache": [torch.cat(posterior, dim=0).to(dtype=self.dtype) for posterior in posteriors]} + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) with self._runtime_context(pixel_values): posterior = self._encode_posterior(pixel_values) @@ -89,27 +94,30 @@ def online_process( return {"latents": latents} @post_forward("offline_encode") - def offline_encode_post(self, encoded_cache: torch.Tensor) -> dict[str, Any]: + def offline_encode_post(self, encoded_cache: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: """Gather SP shards, crop cached posteriors to each image's shape, and store on carrier items.""" if get_parallel_state().sp_size > 1: - # Keep the CNN's spatial outputs intact through gather; crop each - # cache item from its original pixel shape only after batch unpadding. - encoded_cache = self._gather_encode_output(encoded_cache) + if isinstance(encoded_cache, list): + encoded_cache = self._gather_encode_output_list(encoded_cache) + else: + encoded_cache = self._gather_encode_output(encoded_cache) conversation = self._conversation_carrier encode_items = self._encode_items self._conversation_carrier = None self._encode_items = [] self._sp_encode_count = None + self._sp_latent_shapes = None + z_channels = int(self.config.z_channels) for item, cache_tensor in zip(encode_items, encoded_cache, strict=True): item.type = "image" - cache_tensor = crop_latent_to_image_shape( - cache_tensor, - item.meta.get(BAGEL_VAE_PIXEL_SHAPE), - downsample=int(self.config.downsample), - ) - z_channels = int(self.config.z_channels) + if not isinstance(encoded_cache, list): + cache_tensor = crop_latent_to_image_shape( + cache_tensor, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels: cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:]) item.value = cache_tensor.detach().to(device=self.device, dtype=self.dtype) @@ -172,6 +180,7 @@ def __init__(self, *args, **kwargs) -> None: self._conversation_carrier: list[list[ConversationItem]] | None = None self._encode_items: list[ConversationItem] = [] self._sp_encode_count: int | None = None + self._sp_latent_shapes: list[tuple[int, int]] | None = None @pre_forward("encode", "offline_encode") def encode_pre( @@ -179,7 +188,7 @@ def encode_pre( conversation_list: list[list[ConversationItem]] | None = None, **kwargs: Any, ) -> dict[str, Any]: - """Stack VAE context images into a batch tensor and slice across Ulysses SP ranks if enabled.""" + """Collect unpadded VAE context images and slice across Ulysses SP ranks if enabled.""" del kwargs self._conversation_carrier = conversation_list @@ -187,38 +196,46 @@ def encode_pre( self._metric_meter_stash_latent_tokens(self._vae_latent_token_lengths()) - pixel_values = torch.stack(_pad_to_batch_size([item.value for item in self._encode_items]), dim=0).to( - device=self.device, dtype=self.dtype, non_blocking=True - ) + pixel_values = [ + unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True) + for item in self._encode_items + ] ps = get_parallel_state() if ps.sp_size == 1: self._sp_encode_count = None + self._sp_latent_shapes = None return {"pixel_values": pixel_values} if ps.cp_size != 1: raise ValueError(f"BAGEL VAE training supports Ulysses groups only; got cp_size={ps.cp_size}.") - self._sp_encode_count = int(pixel_values.shape[0]) + self._sp_encode_count = len(pixel_values) if self._sp_encode_count == 0: raise ValueError("BAGEL VAE SP requires at least one real or dummy image.") - # Slice only the image-batch dimension. Processor-added spatial zero - # padding is part of each image canvas and must pass through the CNN. - pixel_values = slice_input_tensor(pixel_values, dim=0, padding=True, group=ps.sp_group) - return {"pixel_values": pixel_values} + + pad_count = (-self._sp_encode_count) % ps.sp_size + padded_pixels = pixel_values + [pixel_values[-1]] * pad_count + ds = max(int(self.config.downsample), 1) + self._sp_latent_shapes = [(int(image.shape[-2]) // ds, int(image.shape[-1]) // ds) for image in padded_pixels] + per_rank = len(padded_pixels) // ps.sp_size + start = ps.sp_rank * per_rank + return {"pixel_values": padded_pixels[start : start + per_rank]} @post_forward("encode", "online_process") def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: - """Gather SP latent shards, crop spatial padding, and write latents onto conversation items.""" - if not isinstance(latents, list) and get_parallel_state().sp_size > 1: - # Remove SP's batch-padding images first. Per-image spatial latent - # padding is cropped below from BAGEL_VAE_PIXEL_SHAPE after gather. - latents = self._gather_encode_output(latents) + """Gather SP latent shards, crop spatial padding if needed, and write latents onto conversation items.""" + if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None: + if isinstance(latents, list): + latents = self._gather_encode_output_list(latents) + else: + latents = self._gather_encode_output(latents) conversation = self._conversation_carrier encode_items = self._encode_items self._conversation_carrier = None self._encode_items = [] self._sp_encode_count = None + self._sp_latent_shapes = None if isinstance(latents, list): for item, latent in zip(encode_items, latents, strict=True): @@ -244,6 +261,41 @@ def _gather_encode_output(self, output: torch.Tensor) -> torch.Tensor: output = gather_outputs(output, gather_dim=0, group=get_parallel_state().sp_group) return output.narrow(0, 0, self._sp_encode_count) + def _gather_encode_output_list(self, outputs: list[torch.Tensor]) -> list[torch.Tensor]: + """All-gather variable-shape per-image VAE outputs across the SP group and trim SP padding.""" + if self._sp_encode_count is None or self._sp_latent_shapes is None: + raise RuntimeError("BAGEL VAE SP image shapes were not initialized.") + ps = get_parallel_state() + per_rank = len(self._sp_latent_shapes) // ps.sp_size + if len(outputs) != per_rank: + raise RuntimeError(f"BAGEL VAE SP local output count mismatch: expected {per_rank}, got {len(outputs)}.") + leading_shape = tuple(int(dim) for dim in outputs[0].shape[:-2]) + leading_numel = math.prod(leading_shape) + rank_numels = [ + sum( + leading_numel * height * width + for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank] + ) + for rank in range(ps.sp_size) + ] + max_rank_numel = max(rank_numels) + flat_local = torch.cat([tensor.reshape(-1) for tensor in outputs], dim=0) + if int(flat_local.numel()) < max_rank_numel: + flat_local = torch.nn.functional.pad(flat_local, (0, max_rank_numel - int(flat_local.numel()))) + gathered = gather_outputs(flat_local.unsqueeze(0), gather_dim=0, group=ps.sp_group) + + restored: list[torch.Tensor] = [] + for rank in range(ps.sp_size): + offset = 0 + rank_flat = gathered[rank] + for height, width in self._sp_latent_shapes[rank * per_rank : (rank + 1) * per_rank]: + numel = leading_numel * height * width + restored.append(rank_flat[offset : offset + numel].reshape(*leading_shape, height, width)) + offset += numel + if len(restored) == self._sp_encode_count: + return restored + return restored + def _vae_latent_token_lengths(self) -> list[int]: """Compute the unpadded latent grid token count per encoded image for FLOPs metering.""" # Latent tokens per image = (H // ds) * (W // ds), using the real pixel shape diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py index abe8456886..ea183c1c63 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -28,11 +28,9 @@ from ..sources import BAGEL_CONTEXT_KEY, BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_VAE_CONTEXT from .configuration import BagelVAEConfig from .processing import ( - BAGEL_VAE_PIXEL_SHAPE, BagelVAEPreprocessor, BagelVAEProcessor, - _pad_to_batch_size, - crop_latent_to_image_shape, + unpad_vae_item_pixel_values, ) @@ -78,16 +76,12 @@ def encode_context( if not image_items: return {"conversation_list": conversation_list} - pixel_values = torch.stack(_pad_to_batch_size([item.value for item in image_items]), dim=0).to( - device=self.device, dtype=self.dtype, non_blocking=True - ) + pixel_values = [ + unpad_vae_item_pixel_values(item).to(device=self.device, dtype=self.dtype, non_blocking=True) + for item in image_items + ] outputs = self.encode(pixel_values=pixel_values) for image_item, latent in zip(image_items, outputs["latents"], strict=True): - latent = crop_latent_to_image_shape( - latent, - image_item.meta.get(BAGEL_VAE_PIXEL_SHAPE), - downsample=int(self.config.downsample), - ) image_item.value = latent.to(device=self.device, dtype=self.dtype) image_item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT return {"conversation_list": conversation_list} @@ -214,11 +208,19 @@ def _runtime_context(self, tensor: torch.Tensor): def encode( self, - pixel_values: torch.Tensor, + pixel_values: torch.Tensor | list[torch.Tensor], **kwargs: object, ) -> dict[str, Any]: """Encode ``pixel_values`` into scaled VAE latent grids.""" del kwargs + if isinstance(pixel_values, list): + posteriors = self._encode_posterior_list(pixel_values) + latents: list[torch.Tensor] = [] + for posterior in posteriors: + with self._runtime_context(posterior[0]): + latents.append(self._sample_scaled_latents(posterior).squeeze(0).to(dtype=self.dtype)) + return {"latents": latents} + pixel_values = pixel_values.to(device=self._encoder_device, dtype=self.dtype) with self._runtime_context(pixel_values): posterior = self._encode_posterior(pixel_values) @@ -241,6 +243,36 @@ def decode( def _encode_posterior(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: return torch.chunk(self.encoder(pixel_values), 2, dim=1) + def _encode_posterior_list( + self, + pixel_values: list[torch.Tensor], + ) -> list[tuple[torch.Tensor, torch.Tensor]]: + """Encode a list of ``[C, H, W]`` images at their native shapes, batching equal shapes.""" + if not pixel_values: + return [] + + shape_groups: dict[tuple[int, int], list[int]] = {} + normalized: list[torch.Tensor] = [] + for idx, image in enumerate(pixel_values): + if image.dim() == 4 and int(image.shape[0]) == 1: + image = image.squeeze(0) + if image.dim() != 3: + raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(image.shape)}.") + normalized.append(image) + shape_key = (int(image.shape[-2]), int(image.shape[-1])) + shape_groups.setdefault(shape_key, []).append(idx) + + posteriors: list[tuple[torch.Tensor, torch.Tensor] | None] = [None] * len(normalized) + for indices in shape_groups.values(): + batch = torch.stack([normalized[i] for i in indices], dim=0).to( + device=self._encoder_device, dtype=self.dtype + ) + with self._runtime_context(batch): + mean_batch, logvar_batch = self._encode_posterior(batch) + for idx, mean_i, logvar_i in zip(indices, mean_batch, logvar_batch, strict=True): + posteriors[idx] = (mean_i.unsqueeze(0), logvar_i.unsqueeze(0)) + return posteriors # type: ignore[return-value] + def _sample_scaled_latents(self, posterior: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: mean, logvar = posterior latents = mean + torch.exp(0.5 * logvar) * torch.randn_like(mean) diff --git a/veomni/models/seed_omni/modules/bagel/vae/processing.py b/veomni/models/seed_omni/modules/bagel/vae/processing.py index 7fe71a0d0d..879b019a94 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/processing.py +++ b/veomni/models/seed_omni/modules/bagel/vae/processing.py @@ -262,6 +262,25 @@ def _pad_to_batch_size(images: list[torch.Tensor]) -> list[torch.Tensor]: return padded +def unpad_vae_item_pixel_values(item: ConversationItem) -> torch.Tensor: + """Return ``item.value`` cropped to its ``BAGEL_VAE_PIXEL_SHAPE`` if present.""" + pixel_values = item.value + if not torch.is_tensor(pixel_values): + raise TypeError("BAGEL VAE item value must be a tensor.") + if pixel_values.dim() == 4 and int(pixel_values.shape[0]) == 1: + pixel_values = pixel_values.squeeze(0) + if pixel_values.dim() != 3: + raise ValueError(f"BAGEL VAE expects rank-3 [C, H, W] image tensor, got {tuple(pixel_values.shape)}.") + pixel_shape = item.meta.get(BAGEL_VAE_PIXEL_SHAPE) + if torch.is_tensor(pixel_shape): + dims = pixel_shape.detach().reshape(-1).tolist() + if len(dims) != 2: + raise ValueError("BAGEL VAE pixel shape metadata must contain [height, width].") + height, width = int(dims[0]), int(dims[1]) + pixel_values = pixel_values[..., :height, :width] + return pixel_values + + def crop_latent_to_image_shape( latent: torch.Tensor, pixel_shape: torch.Tensor | None, @@ -465,7 +484,7 @@ def from_pretrained( def bind_dummy_inputs(self, config: BagelVAEConfig, dtype: torch.dtype | None = None) -> None: """Training-only FSDP-anchor dummy — pure ``(config, dtype)``, no live model.""" - size = max(int(config.image_stride), int(config.downsample)) + size = max(int(config.image_stride), int(config.downsample) * 2) self._dtype = dtype self._dummy_pixel_values = torch.zeros(int(config.in_channels), size, size, dtype=dtype) self._dummy_pixel_shape = torch.tensor([size, size], dtype=torch.long) @@ -509,11 +528,12 @@ def preprocess_conversations( for item, pixels, pixel_shape in zip( image_items, inputs["pixel_values"], inputs["pixel_shapes"], strict=True ): - item.value = pixels.to(dtype=self._dtype) + dims = pixel_shape.detach().reshape(-1).tolist() + height, width = int(dims[0]), int(dims[1]) + item.value = pixels[..., :height, :width].contiguous().to(dtype=self._dtype) item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT item.meta[BAGEL_VAE_PIXEL_SHAPE] = pixel_shape.to(dtype=torch.long) - dims = pixel_shape.detach().reshape(-1).tolist() - item.meta[_OMNI_TOKEN_LEN] = max(1, (int(dims[0]) // stride) * (int(dims[1]) // stride)) + item.meta[_OMNI_TOKEN_LEN] = max(1, (height // stride) * (width // stride)) if not missing_samples: return @@ -560,4 +580,5 @@ def preprocess_conversations( "copy_image_item", "crop_latent_to_image_shape", "route_image_contexts", + "unpad_vae_item_pixel_values", ] From c41d2d16192e21895c10f65d21bf005c161db13c Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 4 Nov 2026 12:08:06 +0800 Subject: [PATCH 09/22] [omni, model, test] fix: align VAE encoder passes across shard group and preserve inference compute dtype Co-authored-by: Cursor --- .../modules/bagel/test_bagel_eager.py | 28 ++++++----- .../modules/bagel/test_processing.py | 48 +++++++++++++++++++ .../modules/bagel/qwen2_mot/modeling.py | 43 ++++++++--------- .../bagel/vae/accelerated/accelerated.py | 22 +++++++++ .../seed_omni/modules/bagel/vae/modeling.py | 17 ++++++- 5 files changed, 121 insertions(+), 37 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_bagel_eager.py b/tests/seed_omni/modules/bagel/test_bagel_eager.py index dba9303004..ef457fc75a 100644 --- a/tests/seed_omni/modules/bagel/test_bagel_eager.py +++ b/tests/seed_omni/modules/bagel/test_bagel_eager.py @@ -20,14 +20,15 @@ BAGEL_START_TOKEN, ) from veomni.models.seed_omni.utils.conversation import ConversationItem -from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type + + +pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) def _eager_config(**overrides): return config_cls("bagel_qwen2_mot")(**{**tiny_bagel_qwen2_cfg(), **overrides, "attn_implementation": "sdpa"}) -@pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) def test_eager_packed_train_forward_runs() -> None: """Verify eager packed MoT forward and backward populate both und and gen expert gradients.""" model = native_model_cls("bagel_qwen2_mot")(_eager_config()).train() @@ -53,15 +54,14 @@ def test_eager_packed_train_forward_runs() -> None: assert gen_query_grad.abs().max() > 0 -def _eager_cuda_model(): - device = torch.device(get_device_type()) - return native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device=device, dtype=torch.bfloat16).eval() +def _eager_model(dtype: torch.dtype): + return native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device="cpu", dtype=dtype).eval() -@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") -def test_eager_und_generate_runs() -> None: - """Verify eager understanding generation populates the KV cache on prefill and emits a finite output embedding on decode.""" - model = _eager_cuda_model() +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_eager_und_generate_runs(dtype: torch.dtype) -> None: + """Verify eager understanding generation preserves the configured dtype across FP32, FP16, and BF16.""" + model = _eager_model(dtype) prompt = ConversationItem( type="text", value=torch.randn(4, int(model.config.hidden_size), device=model.device, dtype=model.dtype), @@ -84,14 +84,15 @@ def test_eager_und_generate_runs() -> None: assert tail.type == "output" assert BAGEL_PHASE_KEY not in tail.meta assert torch.is_tensor(tail.value) + assert tail.value.dtype == dtype assert tail.value.shape == (1, int(model.config.hidden_size)) assert torch.isfinite(tail.value).all() -@pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="eager sequential infer smoke uses CUDA bf16") -def test_eager_gen_denoise_runs() -> None: - """Verify eager ``denoise_branch`` transitions ``flow_query`` into ``flow_hidden``.""" - model = _eager_cuda_model() +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_eager_gen_denoise_runs(dtype: torch.dtype) -> None: + """Verify eager ``denoise_branch`` preserves the configured dtype across FP32, FP16, and BF16.""" + model = _eager_model(dtype) hidden_size = int(model.config.hidden_size) prompt = ConversationItem( type="text", @@ -113,6 +114,7 @@ def test_eager_gen_denoise_runs() -> None: tail = outputs["conversation_list"][-1] assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN assert torch.is_tensor(tail.value) + assert tail.value.dtype == dtype assert tail.value.shape[0] == 5 assert tail.value.shape[-1] == hidden_size assert torch.isfinite(tail.value).all() diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index 5d160c4131..de6fe1a371 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -1185,3 +1185,51 @@ def __call__(self, text: str, add_special_tokens: bool = False) -> dict[str, lis for marker in ("<|im_start|>", "<|im_end|>"): with pytest.raises(ValueError, match="BAGEL start/end tokens must tokenize to exactly one token"): template.tokenize([ConversationItem(type="text", value=marker, role="user")]) + + +def test_bagel_vae_aligns_encoder_passes_across_shard_group_and_keeps_blocks_unsplit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verify ``BagelVAE`` keeps ``_no_split_modules`` empty and aligns ``_encode_posterior_list`` pass counts across ``fsdp_group``.""" + import torch.distributed as dist + + from veomni.models.seed_omni.modules.bagel.vae.accelerated import accelerated as vae_acc + + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) + assert not vae._no_split_modules + + # Reference single-rank posterior for two equal-shape images (1 shape group natively) + img_a = torch.randn(3, 4, 8) + img_b = torch.randn(3, 4, 8) + ref_out = vae.offline_encode([img_a, img_b])["encoded_cache"] + + fake_fsdp_group = object() + monkeypatch.setattr(dist, "is_initialized", lambda: True) + monkeypatch.setattr( + vae_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=2, cp_size=1, sp_group=object(), fsdp_group=fake_fsdp_group), + ) + + def _fake_all_reduce_max(tensor: torch.Tensor, op=None, group=None): + assert group is fake_fsdp_group + assert op == dist.ReduceOp.MAX + # Peer rank has 3 distinct image shapes, while this rank has only 1 shape group. + tensor.fill_(3) + + monkeypatch.setattr(dist, "all_reduce", _fake_all_reduce_max) + + encoder_calls = 0 + orig_encoder_forward = vae.encoder.forward + + def _counting_encoder_forward(x: torch.Tensor) -> torch.Tensor: + nonlocal encoder_calls + encoder_calls += 1 + return orig_encoder_forward(x) + + monkeypatch.setattr(vae.encoder, "forward", _counting_encoder_forward) + + aligned_out = vae.offline_encode([img_a, img_b])["encoded_cache"] + assert encoder_calls == 3 + for actual, expected in zip(aligned_out, ref_out, strict=True): + torch.testing.assert_close(actual, expected) diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index 871ee725aa..97879af674 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -987,7 +987,6 @@ def project_qkv_inference( packed_key_states = self.k_norm(packed_key_states) return packed_query_states, packed_key_states, packed_value_states - packed_query_sequence = packed_query_sequence.to(torch.bfloat16) packed_qkv_states = packed_query_sequence.new_zeros( (packed_query_sequence.shape[0], sum(self.qkv_split_sizes)) ) @@ -1000,16 +999,18 @@ def project_qkv_inference( packed_query_states, packed_key_states, packed_value_states = packed_qkv_states.split( self.qkv_split_sizes, dim=-1 ) - packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim).to(torch.float32) - packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim).to(torch.float32) + packed_query_states = packed_query_states.view(-1, self.num_heads, self.head_dim) + packed_key_states = packed_key_states.view(-1, self.num_key_value_heads, self.head_dim) packed_value_states = packed_value_states.view(-1, self.num_key_value_heads, self.head_dim) - packed_query_states[packed_text_indexes] = self.q_norm(packed_query_states[packed_text_indexes]) - packed_query_states[packed_vae_token_indexes] = self.q_norm_moe_gen( + normed_query_states = packed_query_states.new_zeros(packed_query_states.shape) + normed_query_states[packed_text_indexes] = self.q_norm(packed_query_states[packed_text_indexes]) + normed_query_states[packed_vae_token_indexes] = self.q_norm_moe_gen( packed_query_states[packed_vae_token_indexes] ) - packed_key_states[packed_text_indexes] = self.k_norm(packed_key_states[packed_text_indexes]) - packed_key_states[packed_vae_token_indexes] = self.k_norm_moe_gen(packed_key_states[packed_vae_token_indexes]) - return packed_query_states, packed_key_states, packed_value_states + normed_key_states = packed_key_states.new_zeros(packed_key_states.shape) + normed_key_states[packed_text_indexes] = self.k_norm(packed_key_states[packed_text_indexes]) + normed_key_states[packed_vae_token_indexes] = self.k_norm_moe_gen(packed_key_states[packed_vae_token_indexes]) + return normed_query_states, normed_key_states, packed_value_states def merge_kv_cache( self, @@ -1096,11 +1097,10 @@ def project_o_inference( if not is_gen: return self.o_proj(packed_attn_output) - packed_attn_output[packed_text_indexes] = self.o_proj(packed_attn_output[packed_text_indexes]) - packed_attn_output[packed_vae_token_indexes] = self.o_proj_moe_gen( - packed_attn_output[packed_vae_token_indexes] - ) - return packed_attn_output + output = packed_attn_output.new_zeros(packed_attn_output.shape) + output[packed_text_indexes] = self.o_proj(packed_attn_output[packed_text_indexes]) + output[packed_vae_token_indexes] = self.o_proj_moe_gen(packed_attn_output[packed_vae_token_indexes]) + return output def forward_packed_inference( self, @@ -1125,6 +1125,7 @@ def forward_packed_inference( total_key_value_tokens: int, ) -> tuple[torch.Tensor, Optional[NaiveCache]]: is_gen = _check_packed_inference_mode(mode) + compute_dtype = packed_query_sequence.dtype packed_query_states, packed_key_states, packed_value_states = self.project_qkv_inference( packed_query_sequence, is_gen=is_gen, @@ -1141,9 +1142,9 @@ def forward_packed_inference( unsqueeze_dim=1, ) - packed_query_states = packed_query_states.to(torch.bfloat16) - packed_key_states = packed_key_states.to(torch.bfloat16) - packed_value_states = packed_value_states.to(torch.bfloat16) + packed_query_states = packed_query_states.to(compute_dtype) + packed_key_states = packed_key_states.to(compute_dtype) + packed_value_states = packed_value_states.to(compute_dtype) merged_key_states, merged_value_states = self.merge_kv_cache( packed_key_states, @@ -1166,7 +1167,7 @@ def forward_packed_inference( max_length_k=max_length_k, ) packed_attn_output = self.project_o_inference( - packed_attn_output, + packed_attn_output.to(compute_dtype), is_gen=is_gen, packed_text_indexes=packed_text_indexes, packed_vae_token_indexes=packed_vae_token_indexes, @@ -1293,13 +1294,11 @@ def _apply_inference_mlp( if not is_gen: return self.mlp(self.post_attention_layernorm(packed_query_sequence)) - packed_text_query_sequence = self.post_attention_layernorm(packed_query_sequence[packed_text_indexes]).to( - torch.bfloat16 - ) + packed_text_query_sequence = self.post_attention_layernorm(packed_query_sequence[packed_text_indexes]) packed_vae_query_sequence = self.post_attention_layernorm_moe_gen( packed_query_sequence[packed_vae_token_indexes] - ).to(torch.bfloat16) - mlp_output = torch.zeros_like(packed_query_sequence).to(torch.bfloat16) + ) + mlp_output = torch.zeros_like(packed_query_sequence) mlp_output[packed_text_indexes] = self.mlp(packed_text_query_sequence) mlp_output[packed_vae_token_indexes] = self.mlp_moe_gen(packed_vae_query_sequence) return mlp_output diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py index 3a88fffc3f..4ade802260 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -11,6 +11,7 @@ from typing import Any import torch +import torch.distributed as dist from veomni.utils.device import get_device_id, get_device_type @@ -254,6 +255,27 @@ def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, A item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT return {"conversation_list": conversation} + def _encode_posterior_list( + self, + pixel_values: list[torch.Tensor], + *, + min_passes: int = 0, + ) -> list[tuple[torch.Tensor, torch.Tensor]]: + """Align the number of encoder passes across ``fsdp_group`` / ``sp_group`` so FSDP unshards stay in lockstep.""" + if pixel_values and dist.is_initialized(): + ps = get_parallel_state() + shard_group = ps.fsdp_group if ps.fsdp_group is not None else (ps.sp_group if ps.sp_size > 1 else None) + if shard_group is not None: + local_shapes = {(int(image.shape[-2]), int(image.shape[-1])) for image in pixel_values} + passes_tensor = torch.tensor( + max(len(local_shapes), min_passes), + device=self._encoder_device, + dtype=torch.int64, + ) + dist.all_reduce(passes_tensor, op=dist.ReduceOp.MAX, group=shard_group) + min_passes = int(passes_tensor.item()) + return super()._encode_posterior_list(pixel_values, min_passes=min_passes) + def _gather_encode_output(self, output: torch.Tensor) -> torch.Tensor: """All-gather batch-sharded VAE outputs across the SP group and trim SP batch padding.""" if self._sp_encode_count is None: diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py index ea183c1c63..5c909c62d7 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -149,7 +149,7 @@ class BagelVAE(InferenceMixin, PretrainedOmniModule): preprocessor_class = BagelVAEPreprocessor base_model_prefix = "bagel_vae" main_input_name = "pixel_values" - _no_split_modules: list[str] = ["ResnetBlock", "AttnBlock"] + _no_split_modules: list[str] = [] supports_gradient_checkpointing = False def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): @@ -246,6 +246,8 @@ def _encode_posterior(self, pixel_values: torch.Tensor) -> tuple[torch.Tensor, t def _encode_posterior_list( self, pixel_values: list[torch.Tensor], + *, + min_passes: int = 0, ) -> list[tuple[torch.Tensor, torch.Tensor]]: """Encode a list of ``[C, H, W]`` images at their native shapes, batching equal shapes.""" if not pixel_values: @@ -262,8 +264,9 @@ def _encode_posterior_list( shape_key = (int(image.shape[-2]), int(image.shape[-1])) shape_groups.setdefault(shape_key, []).append(idx) + group_indices = list(shape_groups.values()) posteriors: list[tuple[torch.Tensor, torch.Tensor] | None] = [None] * len(normalized) - for indices in shape_groups.values(): + for indices in group_indices: batch = torch.stack([normalized[i] for i in indices], dim=0).to( device=self._encoder_device, dtype=self.dtype ) @@ -271,6 +274,16 @@ def _encode_posterior_list( mean_batch, logvar_batch = self._encode_posterior(batch) for idx, mean_i, logvar_i in zip(indices, mean_batch, logvar_batch, strict=True): posteriors[idx] = (mean_i.unsqueeze(0), logvar_i.unsqueeze(0)) + + if min_passes > len(group_indices): + dummy_batch = normalized[0].unsqueeze(0).to(device=self._encoder_device, dtype=self.dtype) + for _ in range(min_passes - len(group_indices)): + with self._runtime_context(dummy_batch): + mean_dummy, logvar_dummy = self._encode_posterior(dummy_batch) + assert posteriors[0] is not None + anchor = (mean_dummy.sum() + logvar_dummy.sum()) * 0.0 + posteriors[0] = (posteriors[0][0] + anchor, posteriors[0][1]) + return posteriors # type: ignore[return-value] def _sample_scaled_latents(self, posterior: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor: From 63a5c2ae9e8ac517a91833694c4999fff4bd481e Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 4 Nov 2026 14:25:22 +0800 Subject: [PATCH 10/22] [omni, model, test] fix: preserve native BagelVAE _no_split_modules and clear only on BagelVAEAccelerated Co-authored-by: Cursor --- tests/seed_omni/modules/bagel/test_processing.py | 5 +++++ .../seed_omni/modules/bagel/vae/accelerated/accelerated.py | 2 +- veomni/models/seed_omni/modules/bagel/vae/modeling.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index de6fe1a371..8fe7c394d9 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -1195,6 +1195,11 @@ def test_bagel_vae_aligns_encoder_passes_across_shard_group_and_keeps_blocks_uns from veomni.models.seed_omni.modules.bagel.vae.accelerated import accelerated as vae_acc + native_vae = native_model_cls("bagel_vae")( + config_cls("bagel_vae")(resolution=8, ch=32, ch_mult=[1, 1], num_res_blocks=1, z_channels=2, downsample=2) + ) + assert set(native_vae._no_split_modules) == {"ResnetBlock", "AttnBlock"} + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) assert not vae._no_split_modules diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py index 4ade802260..680751343d 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -405,7 +405,7 @@ class VeOmniMixin(BaseMixin, BagelVAEOfflineMixin, TrainingMixin, MeterMixin): class BagelVAEAccelerated(VeOmniMixin, BagelVAE): - pass + _no_split_modules: list[str] = [] __all__ = ["BAGEL_VAE_PIXEL_SHAPE", "BagelVAEAccelerated", "BagelVAEOfflineMixin"] diff --git a/veomni/models/seed_omni/modules/bagel/vae/modeling.py b/veomni/models/seed_omni/modules/bagel/vae/modeling.py index 5c909c62d7..fa518bd977 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/vae/modeling.py @@ -149,7 +149,7 @@ class BagelVAE(InferenceMixin, PretrainedOmniModule): preprocessor_class = BagelVAEPreprocessor base_model_prefix = "bagel_vae" main_input_name = "pixel_values" - _no_split_modules: list[str] = [] + _no_split_modules: list[str] = ["ResnetBlock", "AttnBlock"] supports_gradient_checkpointing = False def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): From a2c8857d96ef74c1439dbda69c41386dd2c9b05b Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 4 Nov 2026 15:44:54 +0800 Subject: [PATCH 11/22] [omni, model, test] fix: sample SP VAE posteriors after gather and isolate SigLIP SP RNG/dtype Co-authored-by: Cursor --- .../modules/bagel/test_processing.py | 148 ++++++++++++++++++ .../siglip_navit/accelerated/accelerated.py | 12 ++ .../modules/bagel/siglip_navit/modeling.py | 11 +- .../bagel/vae/accelerated/accelerated.py | 52 +++++- 4 files changed, 217 insertions(+), 6 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index 8fe7c394d9..1728ed0a49 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -1238,3 +1238,151 @@ def _counting_encoder_forward(x: torch.Tensor) -> torch.Tensor: assert encoder_calls == 3 for actual, expected in zip(aligned_out, ref_out, strict=True): torch.testing.assert_close(actual, expected) + + +def test_bagel_vae_sp_samples_posteriors_after_gather_and_keeps_flow_rng_aligned( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Verify SP-sharded VAE encoding defers posterior sampling until after gather so uneven image sizes keep flow RNG aligned across SP ranks.""" + from veomni.models.seed_omni.modules.bagel.flow_connector.accelerated import accelerated as flow_acc + from veomni.models.seed_omni.modules.bagel.vae.accelerated import accelerated as vae_acc + + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) + BagelFlowConnector = model_cls("bagel_flow_connector") + BagelFlowConnectorConfig = config_cls("bagel_flow_connector") + flow = BagelFlowConnector( + BagelFlowConnectorConfig( + hidden_size=4, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=8, + timestep_frequency_embedding_size=4, + timestep_shift=3.0, + ) + ) + + torch.manual_seed(99) + small_img = torch.randn(3, 4, 4) + large_img = torch.randn(3, 8, 8) + + def _make_batch() -> list[list[ConversationItem]]: + return [ + [ + ConversationItem( + type="image", + value=small_img.clone(), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: "gen", + BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 4], dtype=torch.long), + }, + ), + ConversationItem( + type="image", + value=large_img.clone(), + role="assistant", + meta={ + BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, + _IMG_TAG_KEY: "gen", + BAGEL_VAE_PIXEL_SHAPE: torch.tensor([8, 8], dtype=torch.long), + }, + ), + ] + ] + + # 1. Baseline sp_size=1 run + monkeypatch.setattr( + vae_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=1, sp_rank=0, cp_size=1, sp_group=None, fsdp_group=None), + ) + monkeypatch.setattr( + flow_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=1, sp_rank=0, cp_size=1, sp_group=None, fsdp_group=None), + ) + ref_batch = _make_batch() + torch.manual_seed(2026) + ref_pre = vae.pre_forward("encode", conversation_list=ref_batch) + ref_enc = vae.encode(**ref_pre) + vae.post_forward("encode", **ref_enc) + ref_rng_after_vae = torch.get_rng_state() + ref_flow_inputs = flow.pre_forward("embed_latent", conversation_list=ref_batch) + + # 2. Simulate 2-rank SP where rank 0 encodes small_img (4x4) and rank 1 encodes large_img (8x8) + rank_caches: dict[int, list[torch.Tensor]] = {} + for sp_rank in (0, 1): + monkeypatch.setattr( + vae_acc, + "get_parallel_state", + lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), + ) + batch_r = _make_batch() + torch.manual_seed(2026) + pre_r = vae.pre_forward("encode", conversation_list=batch_r) + enc_r = vae.encode(**pre_r) + # No RNG must be consumed during the SP-sharded encode step before gather + assert torch.equal(torch.get_rng_state(), torch.manual_seed(2026).get_state()) + rank_caches[sp_rank] = enc_r["latents"] + + def _fake_gather_outputs(x: torch.Tensor, gather_dim: int = 0, group=None) -> torch.Tensor: + del group + assert gather_dim == 0 + # Pad rank 0's flat posterior tensor to match max_rank_numel (rank 1's 2*2*4*4 = 64 elements) + flat_0 = rank_caches[0][0].reshape(-1) + flat_1 = rank_caches[1][0].reshape(-1) + max_numel = max(int(flat_0.numel()), int(flat_1.numel())) + pad_0 = torch.nn.functional.pad(flat_0, (0, max_numel - int(flat_0.numel()))).unsqueeze(0) + pad_1 = torch.nn.functional.pad(flat_1, (0, max_numel - int(flat_1.numel()))).unsqueeze(0) + return torch.cat([pad_0, pad_1], dim=0) + + monkeypatch.setattr(vae_acc, "gather_outputs", _fake_gather_outputs) + monkeypatch.setattr( + flow_acc, + "slice_sp_token_inputs", + lambda method, **inputs: (int(inputs["latents"].shape[0]), inputs), + ) + + rank_batches: dict[int, list[list[ConversationItem]]] = {} + rank_flow_inputs: dict[int, dict[str, torch.Tensor]] = {} + for sp_rank in (0, 1): + monkeypatch.setattr( + vae_acc, + "get_parallel_state", + lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), + ) + monkeypatch.setattr( + flow_acc, + "get_parallel_state", + lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), + ) + batch_r = _make_batch() + torch.manual_seed(2026) + pre_r = vae.pre_forward("encode", conversation_list=batch_r) + enc_r = vae.encode(**pre_r) + vae.post_forward("encode", **enc_r) + assert torch.equal(torch.get_rng_state(), ref_rng_after_vae) + flow_in_r = flow.pre_forward("embed_latent", conversation_list=batch_r) + rank_batches[sp_rank] = batch_r + rank_flow_inputs[sp_rank] = flow_in_r + + for idx in (0, 1): + torch.testing.assert_close(rank_batches[0][0][idx].value, ref_batch[0][idx].value) + torch.testing.assert_close(rank_batches[1][0][idx].value, ref_batch[0][idx].value) + torch.testing.assert_close( + rank_batches[0][0][idx].meta["flow_velocity_target"], + rank_batches[1][0][idx].meta["flow_velocity_target"], + ) + torch.testing.assert_close( + rank_batches[0][0][idx].meta["flow_velocity_target"], + ref_batch[0][idx].meta["flow_velocity_target"], + ) + torch.testing.assert_close( + rank_batches[0][0][idx].meta["timestep"], + rank_batches[1][0][idx].meta["timestep"], + ) + for key in ("latents", "position_ids", "timesteps"): + torch.testing.assert_close(rank_flow_inputs[0][key], rank_flow_inputs[1][key]) + torch.testing.assert_close(rank_flow_inputs[0][key], ref_flow_inputs[key]) diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py index 209734a0a3..496a4b9ea0 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py @@ -108,6 +108,18 @@ def forward_pre( "token_lens": local_token_lens, } + def forward(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """Isolate SP-sharded attention dropout RNG from downstream modules when enabled.""" + if ( + get_parallel_state().sp_size > 1 + and self.training + and float(getattr(self.config, "attention_dropout", 0.0)) > 0.0 + ): + devices = [self.device] if self.device.type != "cpu" else [] + with torch.random.fork_rng(devices=devices): + return super().forward(*args, **kwargs) + return super().forward(*args, **kwargs) + @post_forward("forward") def forward_post( self, diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py index 923276a7d4..9775c2a52d 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -391,9 +391,14 @@ def forward( query_states = torch.cat([qh, qw], dim=-1) key_states = torch.cat([kh, kw], dim=-1) - query_states = query_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) - key_states = key_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) - value_states = value_states.to(torch.bfloat16).transpose(0, 1).unsqueeze(0) + attn_dtype = ( + torch.bfloat16 + if value_states.dtype == torch.float32 and "flash_attention" in self.veomni_attn.impl + else value_states.dtype + ) + query_states = query_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) + key_states = key_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) + value_states = value_states.to(attn_dtype).transpose(0, 1).unsqueeze(0) dropout = 0.0 if not self.training else self.attention_dropout attn_output, _ = self.veomni_attn( diff --git a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py index 680751343d..cb0542b568 100644 --- a/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/vae/accelerated/accelerated.py @@ -222,14 +222,26 @@ def encode_pre( start = ps.sp_rank * per_rank return {"pixel_values": padded_pixels[start : start + per_rank]} + def encode( + self, + pixel_values: torch.Tensor | list[torch.Tensor], + **kwargs: object, + ) -> dict[str, Any]: + """Encode ``pixel_values``; under SP, defer posterior sampling until ``encode_post`` gathers all shards.""" + if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None: + return {"latents": self.offline_encode(pixel_values=pixel_values, **kwargs)["encoded_cache"]} + return super().encode(pixel_values=pixel_values, **kwargs) + @post_forward("encode", "online_process") def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, Any]: - """Gather SP latent shards, crop spatial padding if needed, and write latents onto conversation items.""" + """Gather SP posterior shards, sample scaled latents after gather, and write latents onto conversation items.""" if get_parallel_state().sp_size > 1 and self._sp_encode_count is not None: if isinstance(latents, list): - latents = self._gather_encode_output_list(latents) + gathered_cache = self._gather_encode_output_list(latents) + latents = self._sample_posteriors_list(gathered_cache) else: - latents = self._gather_encode_output(latents) + gathered_cache = self._gather_encode_output(latents) + latents = self._sample_posterior_tensor(gathered_cache, self._encode_items) conversation = self._conversation_carrier encode_items = self._encode_items @@ -255,6 +267,40 @@ def encode_post(self, latents: torch.Tensor | list[torch.Tensor]) -> dict[str, A item.meta[BAGEL_CONTEXT_KEY] = BAGEL_VAE_CONTEXT return {"conversation_list": conversation} + def _sample_posteriors_list(self, encoded_cache: list[torch.Tensor]) -> list[torch.Tensor]: + """Sample scaled latents in batch order from gathered ``(mean, logvar)`` posteriors.""" + z_channels = int(self.config.z_channels) + latents: list[torch.Tensor] = [] + for cache in encoded_cache: + if cache.dim() == 3 and int(cache.shape[0]) == 2 * z_channels: + cache = cache.reshape(2, z_channels, *cache.shape[-2:]) + posterior, is_item_cache = _posterior_from_cache(cache, z_channels=z_channels) + with self._runtime_context(posterior[0]): + latent = self._sample_scaled_latents(posterior) + if is_item_cache: + latent = latent.squeeze(0) + latents.append(latent.to(dtype=cache.dtype)) + return latents + + def _sample_posterior_tensor( + self, + encoded_cache: torch.Tensor, + encode_items: list[ConversationItem], + ) -> list[torch.Tensor]: + """Crop and sample scaled latents in batch order from a gathered 4-D posterior tensor.""" + z_channels = int(self.config.z_channels) + cropped_caches: list[torch.Tensor] = [] + for item, cache_tensor in zip(encode_items, encoded_cache, strict=True): + cache_tensor = crop_latent_to_image_shape( + cache_tensor, + item.meta.get(BAGEL_VAE_PIXEL_SHAPE), + downsample=int(self.config.downsample), + ) + if cache_tensor.dim() == 3 and int(cache_tensor.shape[0]) == 2 * z_channels: + cache_tensor = cache_tensor.reshape(2, z_channels, *cache_tensor.shape[-2:]) + cropped_caches.append(cache_tensor) + return self._sample_posteriors_list(cropped_caches) + def _encode_posterior_list( self, pixel_values: list[torch.Tensor], From a816fc9d5bf79aed0f716a8dc8f84c9ebaf0dee5 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 4 Nov 2026 15:59:09 +0800 Subject: [PATCH 12/22] [omni, model] fix: pass explicit device_type to fork_rng in BagelSiglipNavitAccelerated Co-authored-by: Cursor --- .../modules/bagel/siglip_navit/accelerated/accelerated.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py index 496a4b9ea0..607c4481b0 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/accelerated/accelerated.py @@ -115,8 +115,12 @@ def forward(self, *args: Any, **kwargs: Any) -> dict[str, Any]: and self.training and float(getattr(self.config, "attention_dropout", 0.0)) > 0.0 ): - devices = [self.device] if self.device.type != "cpu" else [] - with torch.random.fork_rng(devices=devices): + fork_kwargs: dict[str, Any] = ( + {"devices": []} + if self.device.type == "cpu" + else {"devices": [self.device], "device_type": self.device.type} + ) + with torch.random.fork_rng(**fork_kwargs): return super().forward(*args, **kwargs) return super().forward(*args, **kwargs) From 274c3bd0152cc7c817925f7eebe400733cf0448f Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Thu, 5 Nov 2026 14:19:07 +0800 Subject: [PATCH 13/22] [omni, model, task, test] fix: propagate resolved infer_type in native OmniModel and OmniProcessor Co-authored-by: Cursor --- tasks/omni/infer_omni_native.py | 2 +- tests/seed_omni/model/test_processor.py | 1 + tests/seed_omni/model/test_save_load.py | 10 ++ tests/seed_omni/modules/bagel/helpers.py | 4 +- .../modules/bagel/test_generation.py | 131 ++++++++++++++++++ .../modules/bagel/test_processing.py | 5 + .../omni_model/omni_model_runtime.py | 2 + veomni/models/seed_omni/modeling_omni.py | 4 + .../modules/bagel/text_encoder/modeling.py | 2 +- veomni/models/seed_omni/processing_omni.py | 17 ++- 10 files changed, 173 insertions(+), 5 deletions(-) diff --git a/tasks/omni/infer_omni_native.py b/tasks/omni/infer_omni_native.py index ef43d4d1f2..776cc04b00 100644 --- a/tasks/omni/infer_omni_native.py +++ b/tasks/omni/infer_omni_native.py @@ -77,7 +77,7 @@ def main() -> None: torch_dtype="auto", **scenario, ).eval() - processor = OmniProcessor.from_pretrained(args.model_path) + processor = OmniProcessor.from_pretrained(args.model_path, **scenario) model_input = processor( text=args.prompt, images=args.image or None, diff --git a/tests/seed_omni/model/test_processor.py b/tests/seed_omni/model/test_processor.py index d14ec82a2c..8b69afe393 100644 --- a/tests/seed_omni/model/test_processor.py +++ b/tests/seed_omni/model/test_processor.py @@ -155,6 +155,7 @@ def test_omni_processor_from_pretrained_collects_module_preprocessors( mock_from_pretrained.assert_called_once_with(tmp_path, infer_type="infer_gen") fake_mod_cls.preprocessor_class.from_pretrained.assert_called_once() assert len(processor._preprocessors) == 1 + assert processor.infer_type == "infer_gen" @patch("veomni.models.seed_omni.processing_omni.OMNI_MODEL_REGISTRY") diff --git a/tests/seed_omni/model/test_save_load.py b/tests/seed_omni/model/test_save_load.py index 0a798adb74..9d96e5db37 100644 --- a/tests/seed_omni/model/test_save_load.py +++ b/tests/seed_omni/model/test_save_load.py @@ -975,6 +975,16 @@ def test_omni_model_resolve_generation_kwargs_uses_config_defaults(): } assert model.resolve_generation_kwargs({"max_new_tokens": 8}) == {"max_new_tokens": 8} + model.config.infer_type = "infer_gen" + assert model.resolve_generation_kwargs(None) == { + "max_new_tokens": 64, + "infer_type": "infer_gen", + } + assert model.resolve_generation_kwargs({"infer_type": "infer_und"}) == { + "max_new_tokens": 64, + "infer_type": "infer_und", + } + def test_omni_model_post_init_unions_child_no_split_modules(): """HF ``post_init`` (re-run after children are attached) is the aggregator.""" diff --git a/tests/seed_omni/modules/bagel/helpers.py b/tests/seed_omni/modules/bagel/helpers.py index 3b8378380c..01d16df657 100644 --- a/tests/seed_omni/modules/bagel/helpers.py +++ b/tests/seed_omni/modules/bagel/helpers.py @@ -87,7 +87,9 @@ def load_omni_config( if train_graph_path is not None: model_config["train_graph"] = str(train_graph_path) if infer_graph_path is not None: - model_config["infer_graph"] = str(infer_graph_path) + scenario = infer_graph_path.stem.removeprefix("graph_") + model_config["infer_graph"] = {scenario: str(infer_graph_path)} + model_config["infer_type"] = scenario args = OmniArguments( model=OmniModelRuntimeArguments( model_path=model_path, diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py index 7c05a84d8e..afb2726545 100644 --- a/tests/seed_omni/modules/bagel/test_generation.py +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -553,3 +553,134 @@ def embed_context_latents(self, conversation_list: list[ConversationItem] | None item.value = torch.zeros(16, 8) item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} return {"conversation_list": conversation_list} + + +def test_bagel_native_omni_model_and_processor_propagate_resolved_infer_type_without_duplication(tmp_path): + """Verify native ``OmniProcessor`` and ``OmniModel.generate`` propagate ``config.infer_type`` to preprocessing and module calls without caller duplication.""" + from tests.seed_omni.modules.bagel.test_processing import _bagel_template, _FakeBagelTokenizer + from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + BagelSiglipNavitPreprocessor, + BagelSiglipNavitProcessor, + ) + from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor + from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEPreprocessor, BagelVAEProcessor + from veomni.models.seed_omni.processing_omni import OmniProcessor + + image_path = tmp_path / "sample.png" + Image.new("RGB", (8, 8), color=(12, 34, 56)).save(str(image_path)) + + def _build_preprocessors(): + return { + "bagel_vae": BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.float32, + ), + "bagel_siglip_navit": BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), + dtype=torch.float32, + ), + "bagel_text_encoder": BagelTextEncoderPreprocessor(_bagel_template()), + } + + # 1. Default converted checkpoint scenario (infer_und): real BagelTextEncoder + BagelQwen2MoT + und_cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_und.yaml", + ) + und_cfg.training_graphs = {} + assert und_cfg.infer_type == "infer_und" + + BagelTextEncoder = native_model_cls("bagel_text_encoder") + BagelTextEncoderConfig = config_cls("bagel_text_encoder") + BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") + BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") + + qwen_kwargs = { + **tiny_bagel_qwen2_cfg(), + "hidden_size": 8, + "intermediate_size": 16, + "num_attention_heads": 2, + "num_key_value_heads": 2, + } + hidden_size = int(qwen_kwargs["hidden_size"]) + + def _make_real_text_and_mot(): + text_enc = BagelTextEncoder( + BagelTextEncoderConfig(vocab_size=int(qwen_kwargs["vocab_size"]), hidden_size=hidden_size) + ).eval() + text_enc._tokenizer = _FakeBagelTokenizer() + text_enc._chat_template = _bagel_template() + text_enc._sample_token = lambda hidden_states, **kwargs: text_enc._chat_template.eos_token_id + mot = BagelQwen2MoT(BagelQwen2MoTConfig(**qwen_kwargs)).eval() + return text_enc, mot + + class _DimAlignedSiglip(_NoopBagelSiglip): + def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): + del kwargs + assert conversation_list is not None + for item in conversation_list: + if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT: + item.value = torch.zeros(4, hidden_size) + return {"conversation_list": conversation_list} + + und_text_encoder, und_qwen_mot = _make_real_text_and_mot() + und_model = OmniModel( + und_cfg, + { + "bagel_text_encoder": und_text_encoder, + "bagel_siglip_navit": _DimAlignedSiglip(), + "bagel_qwen2_mot": und_qwen_mot, + "bagel_flow_connector": _InferGenBagelFlow(), + "bagel_vae": _InferGenBagelVAE(), + }, + ).eval() + und_processor = OmniProcessor(_build_preprocessors(), infer_type=und_model.config.infer_type) + + # Caller passes neither infer_type to processor(...) nor infer_type in generation_kwargs + und_request = und_processor(text="hi", images=str(image_path)) + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_request["conversation_list"] if item.type == "image"] == [ + BAGEL_SIGLIP_CONTEXT + ] + und_model.reset() + und_generated = und_model.generate(und_request, generation_kwargs={"max_new_tokens": 4}) + assert any(item["type"] == "text" for item in und_generated) + assert und_qwen_mot._generation_state.main.cache is not None + + # 2. Image edit scenario (infer_edit): processor routes both VAE + SigLIP and real text_encoder + qwen_mot run edit FSM + edit_cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_edit.yaml", + ) + edit_cfg.training_graphs = {} + assert edit_cfg.infer_type == "infer_edit" + + edit_text_encoder, edit_qwen_mot = _make_real_text_and_mot() + edit_model = OmniModel( + edit_cfg, + { + "bagel_text_encoder": edit_text_encoder, + "bagel_siglip_navit": _DimAlignedSiglip(), + "bagel_qwen2_mot": edit_qwen_mot, + "bagel_flow_connector": _InferEditBagelFlow(), + "bagel_vae": _InferEditBagelVAE(), + }, + ).eval() + edit_processor = OmniProcessor(_build_preprocessors(), infer_type=edit_model.config.infer_type) + + # Caller passes neither infer_type to processor(...) nor infer_type in generation_kwargs + edit_request = edit_processor(text="edit", images=str(image_path)) + assert [ + item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_request["conversation_list"] if item.type == "image" + ] == [ + BAGEL_VAE_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + ] + edit_model.reset() + edit_generated = edit_model.generate( + edit_request, + generation_kwargs={"max_new_tokens": 8, "do_sample": False, "image_height": 64, "image_width": 64}, + ) + assert any(item["type"] == "image" for item in edit_generated) + assert edit_qwen_mot._generation_state.main.cache is not None diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index 1728ed0a49..bc70219522 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -502,6 +502,11 @@ class _FakeBagelTokenizer: def convert_tokens_to_ids(self, token): return self.special_tokens.get(token, self.unk_token_id) + def decode(self, token_ids, skip_special_tokens=True): + del skip_special_tokens + special_by_id = {v: k for k, v in self.special_tokens.items()} + return "".join(special_by_id.get(int(t), chr(max(32, int(t)))) for t in token_ids) + def __call__(self, text, add_special_tokens=False): del add_special_tokens input_ids = [] diff --git a/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py b/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py index a66b86cba8..45ed63bab5 100644 --- a/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py +++ b/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py @@ -468,6 +468,8 @@ def generate( modules = {name: self.get_module(name) for name in model._module_names} model.generation_graph.validate_modules({name: model.get_module(name) for name in model._module_names}) generation_kwargs = model.resolve_generation_kwargs(generation_kwargs) + if "infer_type" not in generation_kwargs and model.config.generation_graphs: + generation_kwargs["infer_type"] = next(iter(model.config.generation_graphs)) max_new_tokens = generation_kwargs.get("max_new_tokens", 2048) max_steps = max_new_tokens total_steps = 0 diff --git a/veomni/models/seed_omni/modeling_omni.py b/veomni/models/seed_omni/modeling_omni.py index 308d1ec42b..7cc51b3821 100644 --- a/veomni/models/seed_omni/modeling_omni.py +++ b/veomni/models/seed_omni/modeling_omni.py @@ -664,6 +664,8 @@ def generate( self.generation_graph.validate_modules({name: self.get_module(name) for name in self._module_names}) ctx: dict[str, Any] = request generation_kwargs = self.resolve_generation_kwargs(generation_kwargs) + if "infer_type" not in generation_kwargs and self.config.generation_graphs: + generation_kwargs["infer_type"] = next(iter(self.config.generation_graphs)) max_new_tokens = generation_kwargs.get("max_new_tokens", 2048) max_steps = max_new_tokens @@ -715,6 +717,8 @@ def resolve_generation_kwargs( ) -> dict[str, Any]: """Merge :attr:`~OmniConfig.generation_kwargs` defaults with per-request overrides.""" resolved = dict(self.config.generation_kwargs or {}) + if self.config.infer_type is not None: + resolved.setdefault("infer_type", self.config.infer_type) resolved.update(generation_kwargs or {}) return resolved diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py index 2af5cbf15a..e573c66188 100644 --- a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py @@ -99,7 +99,7 @@ def generate( device=self.device, dtype=self.dtype, ) - infer_type = str((generation_kwargs or {}).get("infer_type", "")) + infer_type = str((generation_kwargs or {}).get("infer_type") or "infer_und") if infer_type == "infer_und" and not self._bos_injected: input_ids = self._token_id_tensor(self._chat_template.bos_token_id) start_embed = self.encode(input_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype) diff --git a/veomni/models/seed_omni/processing_omni.py b/veomni/models/seed_omni/processing_omni.py index 65f2853837..a5600b26f6 100644 --- a/veomni/models/seed_omni/processing_omni.py +++ b/veomni/models/seed_omni/processing_omni.py @@ -78,8 +78,14 @@ def _as_list(media: MediaInput | None) -> list[Any]: class OmniProcessor: """Composed SeedOmni request preprocessor (HF ``AutoProcessor``-style API).""" - def __init__(self, preprocessors: dict[str, ModulePreprocessorBase]) -> None: + def __init__( + self, + preprocessors: dict[str, ModulePreprocessorBase], + *, + infer_type: str | None = None, + ) -> None: self._preprocessors: dict[str, ModulePreprocessorBase] = dict(preprocessors) + self.infer_type: str | None = infer_type def __len__(self) -> int: """Number of worker-side preprocessors contributed by active graph modules.""" @@ -148,7 +154,12 @@ def from_config( if preprocessor is not None: preprocessors[name] = preprocessor logger.info_rank0(f"OmniProcessor: module '{name}' contributes {type(preprocessor).__name__}.") - return cls(preprocessors) + infer_type = ( + config.infer_type + if config.infer_type is not None + else (next(iter(config.generation_graphs)) if config.generation_graphs else None) + ) + return cls(preprocessors, infer_type=infer_type) @classmethod def from_pretrained( @@ -242,6 +253,8 @@ def preprocess_batch( Training uses ``inference=False`` (default); single-request inference uses :meth:`__call__` with ``inference=True``. """ + if inference and self.infer_type is not None: + generation_kwargs.setdefault("infer_type", self.infer_type) for preprocessor in self._preprocessors.values(): preprocessor( batch, From a9124bc1579ba327b34ab056ec4c513518e08005 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Fri, 6 Nov 2026 15:38:24 +0800 Subject: [PATCH 14/22] [omni, model, test] fix: default BagelQwen2MoTConfig attn_implementation to flex_attention Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 19 +++++++++++++++++++ .../modules/bagel/qwen2_mot/configuration.py | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 38bbd61530..bd20efd4a8 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -611,3 +611,22 @@ def test_bagel_qwen2_rotary_embedding_to_empty_from_meta_rebuilds_fp32_table() - assert rotary.inv_freq.data_ptr() != rotary.original_inv_freq.data_ptr() torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) torch.testing.assert_close(rotary.original_inv_freq, ref_rotary.original_inv_freq) + + +def test_bagel_qwen2_mot_config_defaults_to_flex_attention() -> None: + """Verify ``BagelQwen2MoTConfig`` defaults to ``veomni_flex_attention`` when no attention backend is specified.""" + from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT + + cfg = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=64, + intermediate_size=128, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=1, + max_position_embeddings=64, + ) + assert cfg._attn_implementation == "veomni_flex_attention" + model = BagelQwen2MoT(cfg).eval() + assert model.model.layers[0].self_attn.veomni_attn is not None + assert model.model.layers[0].self_attn.veomni_attn.impl == "veomni_flex_attention" diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py index bf2905f746..5a330813fc 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/configuration.py @@ -35,6 +35,11 @@ def __init__( freeze_und: bool = False, **kwargs, ): + ops_impl = kwargs.get("ops_implementation") + if "attn_implementation" not in kwargs and not ( + isinstance(ops_impl, dict) and ops_impl.get("attn_implementation") is not None + ): + kwargs["attn_implementation"] = "flex_attention" super().__init__( vocab_size=vocab_size, hidden_size=hidden_size, From 9f186e138729debfba9be5feeb0b03f8035a2e6b Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Fri, 6 Nov 2026 16:01:33 +0800 Subject: [PATCH 15/22] [omni, config, docs, test] refactor: normalize OmniConfig graph types and inherit checkpoint generation_kwargs Co-authored-by: Cursor --- .../bagel_7b_mot/offline_cache/base.yaml | 18 ---------- .../Bagel/bagel_7b_mot/train/base.yaml | 18 ---------- .../Bagel/bagel_7b_mot/with_cache/base.yaml | 18 ---------- docs/seed_omni/models/bagel.md | 2 +- .../seed_omni/usage/training_and_inference.md | 4 ++- tests/seed_omni/arguments/test_module_args.py | 33 +++++++++++++++++++ tests/seed_omni/model/test_save_load.py | 18 ++++++---- veomni/arguments/omni_arguments_types.py | 5 ++- .../omni_model/omni_model_runtime.py | 2 -- veomni/models/seed_omni/configuration_omni.py | 18 ++++++++-- veomni/models/seed_omni/modeling_omni.py | 6 ++-- veomni/models/seed_omni/processing_omni.py | 11 ++----- veomni/models/seed_omni/utils/hf_layout.py | 3 +- 13 files changed, 74 insertions(+), 82 deletions(-) diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml index 6b8cfdef07..78b9c32181 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/offline_cache/base.yaml @@ -81,21 +81,3 @@ train: enable: false project: VeOmni_omni name: bagel_7b_mot_offline_cache - -infer: - generation_kwargs: - max_new_tokens: 2048 - temperature: 0.0 - top_p: 1.0 - do_sample: false - image_height: 1024 - image_width: 1024 - latent_downsample: 16 - num_timesteps: 50 - timestep_shift: 3.0 - cfg_text_scale: 1.0 - cfg_img_scale: 1.0 - cfg_interval: [0.0, 1.0] - cfg_renorm_min: 0.0 - cfg_renorm_type: global - enable_taylorseer: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml index 9bc7a9417e..41ba85608c 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/train/base.yaml @@ -81,21 +81,3 @@ train: enable: true project: VeOmni_omni name: bagel_7b_mot_omni_sft - -infer: - generation_kwargs: - max_new_tokens: 2048 - temperature: 0.0 - top_p: 1.0 - do_sample: false - image_height: 1024 - image_width: 1024 - latent_downsample: 16 - num_timesteps: 50 - timestep_shift: 3.0 - cfg_text_scale: 1.0 - cfg_img_scale: 1.0 - cfg_interval: [0.0, 1.0] - cfg_renorm_min: 0.0 - cfg_renorm_type: global - enable_taylorseer: false diff --git a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml index 198c0feead..517489071e 100644 --- a/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml +++ b/configs/seed_omni/Bagel/bagel_7b_mot/with_cache/base.yaml @@ -79,21 +79,3 @@ train: enable: true project: VeOmni_omni name: bagel_7b_mot_omni_sft_from_cache - -infer: - generation_kwargs: - max_new_tokens: 2048 - temperature: 0.0 - top_p: 1.0 - do_sample: false - image_height: 1024 - image_width: 1024 - latent_downsample: 16 - num_timesteps: 50 - timestep_shift: 3.0 - cfg_text_scale: 1.0 - cfg_img_scale: 1.0 - cfg_interval: [0.0, 1.0] - cfg_renorm_min: 0.0 - cfg_renorm_type: global - enable_taylorseer: false diff --git a/docs/seed_omni/models/bagel.md b/docs/seed_omni/models/bagel.md index 7ecfa2e642..eb249318d0 100644 --- a/docs/seed_omni/models/bagel.md +++ b/docs/seed_omni/models/bagel.md @@ -25,7 +25,7 @@ inference block maps each scenario to a separate generation graph. | File | Role | |------|------| -| `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, train, and `infer` block. | +| `train/base.yaml` | Launcher: model paths, `model.accelerator`, data, and train blocks. | | `train/modules_train.yaml` | Per-module training paths. `bagel_qwen2_mot` is the accelerated class with `flex_attention`. | | `infer/modules_infer_eager.yaml` | Single-process inference: every module loads eager; MoT uses FlexAttention. | | `infer/modules_infer_fsdp.yaml` | Distributed inference: every module uses FSDP2; MoT uses FlexAttention. | diff --git a/docs/seed_omni/usage/training_and_inference.md b/docs/seed_omni/usage/training_and_inference.md index a6bc5be706..9cb8957ad3 100644 --- a/docs/seed_omni/usage/training_and_inference.md +++ b/docs/seed_omni/usage/training_and_inference.md @@ -205,7 +205,9 @@ flowchart TB `generate()` always lives on the native `modeling.py`, so an all-eager Inferencer run builds the same object a native user gets from `OmniModel.from_pretrained`; -the launcher only projects the YAML onto `OmniConfig` first. +the launcher only projects the YAML onto `OmniConfig` first. When `infer_graph` +comes from the checkpoint, `infer.generation_kwargs` overrides the checkpoint's +`generation_kwargs` defaults per key rather than replacing the bag. **Native HF:** diff --git a/tests/seed_omni/arguments/test_module_args.py b/tests/seed_omni/arguments/test_module_args.py index 9de23dd8a4..f6d5e97012 100644 --- a/tests/seed_omni/arguments/test_module_args.py +++ b/tests/seed_omni/arguments/test_module_args.py @@ -509,6 +509,39 @@ def test_a_launcher_infer_graph_does_not_inherit_the_checkpoint_infer_type(tmp_p assert build_omni_model_runtime_args(args, for_inference=True).infer_type == "understanding" +def test_build_model_runtime_args_merges_checkpoint_generation_kwargs_when_infer_graph_from_checkpoint(tmp_path): + """Checkpoint ``generation_kwargs`` supply the defaults and ``infer.generation_kwargs`` overrides per key; + replacing ``infer_graph`` in the launcher drops the checkpoint's ``generation_kwargs``.""" + infer_dir = _cfg_dir() / "infer" + runtime_cfg = _model_runtime( + model_path=str(tmp_path), + infer_graph={"infer_und": str(infer_dir / "graph_infer_und.yaml")}, + infer_type="infer_und", + ) + runtime_cfg.generation_kwargs = {"num_timesteps": 50, "timestep_shift": 3.0} + export_root = tmp_path / "exported" + _with_module_configs(runtime_cfg.to_hf_config()).save_pretrained(export_root) + + args = OmniArguments( + model=OmniModelRuntimeArguments(model_path=str(export_root)), + data=OmniDataArguments(train_path=""), + infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}), + ) + resolved = build_omni_model_runtime_args(args, for_inference=True) + assert resolved.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0} + + args_custom_graph = OmniArguments( + model=OmniModelRuntimeArguments( + model_path=str(export_root), + model_config={"infer_graph": {"understanding": str(infer_dir / "graph_infer_und.yaml")}}, + ), + data=OmniDataArguments(train_path=""), + infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}), + ) + resolved_custom = build_omni_model_runtime_args(args_custom_graph, for_inference=True) + assert resolved_custom.generation_kwargs == {"num_timesteps": 10} + + def test_build_model_runtime_args_keeps_every_training_scenario_from_omni_checkpoint(tmp_path): """The checkpoint stores the whole ``training_graphs`` map and its ``train_type``; reading back only the active graph would rename it ``default`` and drop the rest.""" diff --git a/tests/seed_omni/model/test_save_load.py b/tests/seed_omni/model/test_save_load.py index 9d96e5db37..e545e3651e 100644 --- a/tests/seed_omni/model/test_save_load.py +++ b/tests/seed_omni/model/test_save_load.py @@ -968,22 +968,28 @@ def test_omni_model_resolve_generation_kwargs_uses_config_defaults(): model = _build_omni_model() model.config.generation_kwargs = {"max_new_tokens": 64} - assert model.resolve_generation_kwargs(None) == {"max_new_tokens": 64} + assert model.config.infer_type == "infer_gen" + assert model.resolve_generation_kwargs(None) == { + "max_new_tokens": 64, + "infer_type": "infer_gen", + } assert model.resolve_generation_kwargs({"temperature": 0.2}) == { "max_new_tokens": 64, "temperature": 0.2, + "infer_type": "infer_gen", } - assert model.resolve_generation_kwargs({"max_new_tokens": 8}) == {"max_new_tokens": 8} - - model.config.infer_type = "infer_gen" - assert model.resolve_generation_kwargs(None) == { - "max_new_tokens": 64, + assert model.resolve_generation_kwargs({"max_new_tokens": 8}) == { + "max_new_tokens": 8, "infer_type": "infer_gen", } assert model.resolve_generation_kwargs({"infer_type": "infer_und"}) == { "max_new_tokens": 64, "infer_type": "infer_und", } + assert model.resolve_generation_kwargs({"infer_type": None}) == { + "max_new_tokens": 64, + "infer_type": "infer_gen", + } def test_omni_model_post_init_unions_child_no_split_modules(): diff --git a/veomni/arguments/omni_arguments_types.py b/veomni/arguments/omni_arguments_types.py index f9eb8595d3..8c9dd433f5 100644 --- a/veomni/arguments/omni_arguments_types.py +++ b/veomni/arguments/omni_arguments_types.py @@ -290,7 +290,10 @@ def build_omni_model_runtime_args(args: "OmniArguments", *, for_inference: bool generation_graphs=generation_graphs, train_type=train_type, infer_type=infer_type, - generation_kwargs=dict(args.infer.generation_kwargs), + generation_kwargs={ + **(omni_cfg.generation_kwargs or {} if infer_graph_from_ckpt and omni_cfg is not None else {}), + **args.infer.generation_kwargs, + }, ) diff --git a/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py b/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py index 45ed63bab5..a66b86cba8 100644 --- a/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py +++ b/veomni/models/seed_omni/accelerated/omni_model/omni_model_runtime.py @@ -468,8 +468,6 @@ def generate( modules = {name: self.get_module(name) for name in model._module_names} model.generation_graph.validate_modules({name: model.get_module(name) for name in model._module_names}) generation_kwargs = model.resolve_generation_kwargs(generation_kwargs) - if "infer_type" not in generation_kwargs and model.config.generation_graphs: - generation_kwargs["infer_type"] = next(iter(model.config.generation_graphs)) max_new_tokens = generation_kwargs.get("max_new_tokens", 2048) max_steps = max_new_tokens total_steps = 0 diff --git a/veomni/models/seed_omni/configuration_omni.py b/veomni/models/seed_omni/configuration_omni.py index 323b8a30ca..a02d2f45b0 100644 --- a/veomni/models/seed_omni/configuration_omni.py +++ b/veomni/models/seed_omni/configuration_omni.py @@ -68,11 +68,10 @@ def select_graph( empty_hint: str = "", unknown_hint: str = "scenario", ) -> Any: - """Pick the active graph out of a scenario map; unset ``scenario`` takes the first.""" + """Pick the active graph out of a scenario map.""" if not graphs: raise ValueError(f"No graph scenarios are declared. {empty_hint}".strip()) - if scenario is None: - return next(iter(graphs.values())) + assert scenario is not None, f"`{unknown_hint}` must be normalized before calling `select_graph`." if scenario not in graphs: known = ", ".join(graphs) raise KeyError(f"Unknown {unknown_hint} {scenario!r}; expected one of: {known}.") @@ -220,6 +219,7 @@ def __init__( check_infer_type_names(generation_graphs or {}) self.train_type = train_type self.infer_type = infer_type + self._resolve_graph_types() self.generation_kwargs = generation_kwargs # Model-level kernels: this composed model's defaults for every module # it holds. Sparse — only what was explicitly selected. A loaded module @@ -454,6 +454,17 @@ def load_checkpoint_sidecars(self, checkpoint_root: Union[str, os.PathLike]) -> base_ops_implementation=self.ops_implementation, ) + def _resolve_graph_types(self) -> None: + """Bind the active scenario in each graph map once the map is final.""" + for graphs, attr in ((self.training_graphs, "train_type"), (self.generation_graphs, "infer_type")): + if not graphs: + continue + selected = getattr(self, attr) + if selected is None: + setattr(self, attr, next(iter(graphs))) + elif selected not in graphs: + raise KeyError(f"Unknown {attr} {selected!r}; expected one of: {', '.join(graphs)}.") + def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike]) -> None: """Load ``training_graphs`` and ``generation_graphs`` from YAML sidecars when present. @@ -469,6 +480,7 @@ def _load_graphs_from_pretrained(self, checkpoint_root: Union[str, os.PathLike]) if not self.generation_graphs: self.generation_graphs = self._read_graph_sidecar(root, DEFAULT_GENERATION_GRAPH_FILE, dict) check_infer_type_names(self.generation_graphs) + self._resolve_graph_types() @classmethod def _read_graph_sidecar(cls, root: str, filename: str, payload_type: type) -> Dict: diff --git a/veomni/models/seed_omni/modeling_omni.py b/veomni/models/seed_omni/modeling_omni.py index 7cc51b3821..305ea49060 100644 --- a/veomni/models/seed_omni/modeling_omni.py +++ b/veomni/models/seed_omni/modeling_omni.py @@ -664,8 +664,6 @@ def generate( self.generation_graph.validate_modules({name: self.get_module(name) for name in self._module_names}) ctx: dict[str, Any] = request generation_kwargs = self.resolve_generation_kwargs(generation_kwargs) - if "infer_type" not in generation_kwargs and self.config.generation_graphs: - generation_kwargs["infer_type"] = next(iter(self.config.generation_graphs)) max_new_tokens = generation_kwargs.get("max_new_tokens", 2048) max_steps = max_new_tokens @@ -717,9 +715,9 @@ def resolve_generation_kwargs( ) -> dict[str, Any]: """Merge :attr:`~OmniConfig.generation_kwargs` defaults with per-request overrides.""" resolved = dict(self.config.generation_kwargs or {}) - if self.config.infer_type is not None: - resolved.setdefault("infer_type", self.config.infer_type) resolved.update(generation_kwargs or {}) + if resolved.get("infer_type") is None and self.config.infer_type is not None: + resolved["infer_type"] = self.config.infer_type return resolved def collect_assets(self) -> list[Any]: diff --git a/veomni/models/seed_omni/processing_omni.py b/veomni/models/seed_omni/processing_omni.py index a5600b26f6..56a8b10cb2 100644 --- a/veomni/models/seed_omni/processing_omni.py +++ b/veomni/models/seed_omni/processing_omni.py @@ -154,12 +154,7 @@ def from_config( if preprocessor is not None: preprocessors[name] = preprocessor logger.info_rank0(f"OmniProcessor: module '{name}' contributes {type(preprocessor).__name__}.") - infer_type = ( - config.infer_type - if config.infer_type is not None - else (next(iter(config.generation_graphs)) if config.generation_graphs else None) - ) - return cls(preprocessors, infer_type=infer_type) + return cls(preprocessors, infer_type=config.infer_type) @classmethod def from_pretrained( @@ -253,8 +248,8 @@ def preprocess_batch( Training uses ``inference=False`` (default); single-request inference uses :meth:`__call__` with ``inference=True``. """ - if inference and self.infer_type is not None: - generation_kwargs.setdefault("infer_type", self.infer_type) + if inference and self.infer_type is not None and generation_kwargs.get("infer_type") is None: + generation_kwargs["infer_type"] = self.infer_type for preprocessor in self._preprocessors.values(): preprocessor( batch, diff --git a/veomni/models/seed_omni/utils/hf_layout.py b/veomni/models/seed_omni/utils/hf_layout.py index 2206705c35..885143eece 100644 --- a/veomni/models/seed_omni/utils/hf_layout.py +++ b/veomni/models/seed_omni/utils/hf_layout.py @@ -308,12 +308,11 @@ def _write_hf_view(source: HFSource) -> str: module_configs = layout.build_module_configs(source.path) training_graphs, generation_graphs = layout.load_graphs() - infer_type = layout.infer_type if layout.infer_type in generation_graphs else None config = OmniConfig( _module_entries={name: {"model_path": name} for name in module_configs}, training_graphs=training_graphs, generation_graphs=generation_graphs, - infer_type=infer_type, + infer_type=layout.infer_type, ) config._module_configs = dict(module_configs) config.save_pretrained(view) From b1e01e60d9c22f4a9f168dd0d799d92f93f0710a Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Fri, 6 Nov 2026 16:28:43 +0800 Subject: [PATCH 16/22] [omni, model, ckpt, test] refactor: move BAGEL fused QKV export into BagelQwen2MoTCheckpointTensorConverter.export_weights Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 14 ++++-- tests/utils/test_save_safetensor_utils.py | 22 ---------- veomni/models/checkpoint/convert.py | 2 + .../qwen2_mot/accelerated/accelerated.py | 1 - .../bagel/qwen2_mot/checkpoint_conversion.py | 44 ++++++++++++++++--- .../modules/bagel/qwen2_mot/modeling.py | 1 - veomni/utils/save_safetensor_utils.py | 40 ----------------- 7 files changed, 50 insertions(+), 74 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index bd20efd4a8..6406429ba4 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -137,7 +137,7 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT - from veomni.utils.save_safetensor_utils import _revert_model_weight_conversions_for_hf + from veomni.utils.save_safetensor_utils import get_model_save_state torch.manual_seed(3014) eager_cfg = BagelQwen2MoTConfig(**{**tiny_bagel_qwen2_cfg(), "num_key_value_heads": 1}) @@ -171,14 +171,22 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> torch.testing.assert_close(actual.weight, expected.weight) torch.testing.assert_close(actual.bias, expected.bias) - # HF export reverts fused QKV back to split q_proj/k_proj/v_proj keys. - reverted_hf_state = _revert_model_weight_conversions_for_hf(reference, dict(dcp_state)) + # Distributed HF export routes through BagelQwen2MoTCheckpointTensorConverter.export_weights + # and splits fused QKV back into q_proj/k_proj/v_proj keys. + reverted_hf_state = get_model_save_state(reference, None) assert "model.layers.0.self_attn.q_proj.weight" in reverted_hf_state assert "model.layers.0.self_attn.k_proj.weight" in reverted_hf_state assert "model.layers.0.self_attn.v_proj.weight" in reverted_hf_state assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in reverted_hf_state assert "model.layers.0.self_attn.qkv_proj_und.weight" not in reverted_hf_state assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in reverted_hf_state + for key in ( + "model.layers.0.self_attn.q_proj.weight", + "model.layers.0.self_attn.k_proj.weight", + "model.layers.0.self_attn.v_proj.weight", + "model.layers.0.self_attn.q_proj_moe_gen.weight", + ): + torch.testing.assert_close(reverted_hf_state[key], eager_split_state[key].to(torch.bfloat16)) reference.save_pretrained(tmp_path) with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint: diff --git a/tests/utils/test_save_safetensor_utils.py b/tests/utils/test_save_safetensor_utils.py index 01900e7865..6f2da449c8 100644 --- a/tests/utils/test_save_safetensor_utils.py +++ b/tests/utils/test_save_safetensor_utils.py @@ -443,28 +443,6 @@ def test_none_mapping_passed_through(self): ) self.assertIsNone(writer_cls.call_args.kwargs["fqn_to_index_mapping"]) - def test_revert_model_weight_conversions_unwraps_ddp_wrapper(self): - from veomni.utils.save_safetensor_utils import _revert_model_weight_conversions_for_hf - - class _Inner(nn.Module): - _export_hf_checkpoint_with_weight_conversions = True - - def __init__(self): - super().__init__() - self._weight_conversions = [] - - class _Wrapper(nn.Module): - def __init__(self, inner): - super().__init__() - self.module = inner - - inner = _Inner() - wrapped = _Wrapper(inner) - with patch("transformers.core_model_loading.revert_weight_conversion") as mock_revert: - mock_revert.side_effect = lambda m, sd: {"unwrapped_cls": type(m).__name__, **sd} - out = _revert_model_weight_conversions_for_hf(wrapped, {"w": torch.ones(1)}) - self.assertEqual(out["unwrapped_cls"], "_Inner") - def test_naflatten_and_unflatten_roundtrip_keeps_shape_on_cpu(self): from veomni.utils.tensor_utils import naflatten, unflatten diff --git a/veomni/models/checkpoint/convert.py b/veomni/models/checkpoint/convert.py index d4b4149478..f5bbffd78f 100644 --- a/veomni/models/checkpoint/convert.py +++ b/veomni/models/checkpoint/convert.py @@ -223,6 +223,8 @@ def get_checkpoint_tensor_converter( model: Union["nn.Module", "PreTrainedModel"], ) -> Optional["CheckpointTensorConverter"]: """Return the checkpoint tensor converter registered on *model*, or ``None``.""" + while isinstance(getattr(model, "module", None), nn.Module): + model = model.module factory = getattr(model, "_create_checkpoint_tensor_converter", None) if factory is None: return None diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index af5fabe47f..30ef5c38c5 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -743,7 +743,6 @@ def denoise_branch( class BagelQwen2MoTAccelerated(VeOmniMixin, InferenceMixinAccelerated, BagelQwen2MoTCore): _supports_flex_attn = True _supports_magi_attn = True - _export_hf_checkpoint_with_weight_conversions = True attention_cls = BagelQwen2MoTAttentionAccelerated mlp_cls = Qwen2MLPAccelerated rms_norm_cls = Qwen2RMSNormAccelerated diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py index 092e3bf21d..f11535eba3 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/checkpoint_conversion.py @@ -11,13 +11,14 @@ from __future__ import annotations import re +from collections.abc import Generator from typing import Any import torch from transformers.conversion_mapping import register_checkpoint_conversion_mapping from transformers.core_model_loading import ConversionOps, WeightConverter -from ......models.checkpoint.convert import ConvertedCheckpointTensor +from ......models.checkpoint.convert import ConvertedCheckpointTensor, export_weights _QKV_COMPONENTS = ("q", "k", "v") @@ -25,11 +26,17 @@ ("qkv_proj_und", ("q_proj", "k_proj", "v_proj")), ("qkv_proj_gen", ("q_proj_moe_gen", "k_proj_moe_gen", "v_proj_moe_gen")), ) +_QKV_BRANCH_TARGETS = dict(_QKV_BRANCHES) _QKV_CHECKPOINT_RE = re.compile( r"^(?P.*\.self_attn\.)" r"(?P[qkv]_proj(?:_moe_gen)?)\." r"(?Pweight|bias)$" ) +_QKV_FUSED_RE = re.compile( + r"^(?P.*\.self_attn\.)" + r"(?Pqkv_proj_(?:und|gen))\." + r"(?Pweight|bias)$" +) def combine_qkv_state_dict_pre_hook( @@ -106,11 +113,6 @@ def convert( values = next(iter(input_dict.values())) tensor = values[0] if isinstance(values, list) else values - # FSDP2 / DCP keeps a dim-0 DTensor on the fused parameter. Q/K/V row - # sizes do not line up with those shards, so gather before splitting. - full_tensor = getattr(tensor, "full_tensor", None) - if callable(full_tensor) and getattr(tensor, "device_mesh", None) is not None: - tensor = full_tensor() config = kwargs["config"] head_dim = config.hidden_size // config.num_attention_heads split_sizes = ( @@ -118,7 +120,10 @@ def convert( config.num_key_value_heads * head_dim, config.num_key_value_heads * head_dim, ) - return dict(zip(target_patterns, tensor.split(split_sizes, dim=0), strict=True)) + return { + target: chunk.clone() + for target, chunk in zip(target_patterns, tensor.split(split_sizes, dim=0), strict=True) + } @property def reverse_op(self) -> ConversionOps: @@ -201,3 +206,28 @@ def finalize(self) -> list[ConvertedCheckpointTensor]: ) raise ValueError(f"Incomplete BAGEL QKV checkpoint groups: {incomplete}.") return [] + + def export_weights(self, model: torch.nn.Module) -> Generator[tuple[str, torch.Tensor]]: + """Export model parameters in HF checkpoint format with fused ``qkv_proj_{und,gen}`` split into ``{q,k,v}_proj``.""" + base_model = model + while isinstance(getattr(base_model, "module", None), torch.nn.Module): + base_model = base_model.module + config = base_model.config + head_dim = config.hidden_size // config.num_attention_heads + split_sizes = ( + config.num_attention_heads * head_dim, + config.num_key_value_heads * head_dim, + config.num_key_value_heads * head_dim, + ) + for name, tensor in export_weights(base_model): + if tensor.dtype == torch.float32: + tensor = tensor.to(torch.bfloat16) + match = _QKV_FUSED_RE.fullmatch(name) + if match is None: + yield name, tensor + continue + prefix = match.group("prefix") + fused = match.group("fused") + kind = match.group("kind") + for target_proj, chunk in zip(_QKV_BRANCH_TARGETS[fused], tensor.split(split_sizes, dim=0), strict=True): + yield f"{prefix}{target_proj}.{kind}", chunk.clone() diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index 97879af674..7e8cb86e02 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -445,7 +445,6 @@ class BagelQwen2MoTCore(PretrainedOmniModule): supports_gradient_checkpointing = True _supports_sdpa = True _supports_flex_attn = True - _export_hf_checkpoint_with_weight_conversions = False def __init__(self, config: BagelQwen2MoTConfig): super().__init__(config) diff --git a/veomni/utils/save_safetensor_utils.py b/veomni/utils/save_safetensor_utils.py index b7292128b3..405391dd92 100644 --- a/veomni/utils/save_safetensor_utils.py +++ b/veomni/utils/save_safetensor_utils.py @@ -33,42 +33,6 @@ logger = helper.create_logger(__name__) -def _attach_weight_conversions_from_mro(model: torch.nn.Module) -> None: - """Recover class-name converters after FSDP2 wraps the module class. - - ``fully_shard`` builds a subclass whose ``__name__`` no longer matches the - registered mapping. ``model_type`` fallback is wrong for Bagel: eager and - accelerated share ``bagel_qwen2_mot``, but only accelerated exports fused - QKV through converters. Walk the MRO until a class-name mapping hits. - """ - if getattr(model, "_weight_conversions", None) is not None: - return - - from transformers.conversion_mapping import get_checkpoint_conversion_mapping - - for cls in type(model).__mro__: - conversions = get_checkpoint_conversion_mapping(cls.__name__) - if conversions is not None: - model._weight_conversions = list(conversions) - return - - -def _revert_model_weight_conversions_for_hf( - model: Optional[torch.nn.Module], - state_dict: Dict[str, torch.Tensor], -) -> Dict[str, torch.Tensor]: - """Revert runtime weight conversions when exporting an HF checkpoint.""" - while isinstance(getattr(model, "module", None), torch.nn.Module): - model = model.module - if model is None or not getattr(model, "_export_hf_checkpoint_with_weight_conversions", False): - return state_dict - - from transformers.core_model_loading import revert_weight_conversion - - _attach_weight_conversions_from_mro(model) - return revert_weight_conversion(model, state_dict) - - @torch.no_grad() def get_model_save_state( model: torch.nn.Module, @@ -163,7 +127,6 @@ def _should_quantize(origin: str) -> bool: # Use flat state dict so DCP FQNs match the original HF weight_map keys # (e.g. "model.embed_tokens.weight" instead of "model.model.embed_tokens.weight") save_state = ModelState(model, parallel_state=parallel_state).state_dict() - save_state = _revert_model_weight_conversions_for_hf(model, save_state) # Convert float32 tensors to bfloat16 on a copy of the state dict, # so the original model parameters remain unchanged. @@ -275,7 +238,6 @@ def _save_hf_safetensor_legacy( model_assets: Optional[Sequence], ckpt_manager: str, output_dir: Optional[str], - model: Optional[torch.nn.Module] = None, ): """Legacy HuggingFace safetensors save via checkpoint conversion (rank-0 only).""" model_state_dict = ckpt_to_state_dict( @@ -283,7 +245,6 @@ def _save_hf_safetensor_legacy( ckpt_manager=ckpt_manager, output_dir=output_dir, ) - model_state_dict = _revert_model_weight_conversions_for_hf(model, model_state_dict) save_model_weights(save_hf_safetensor_path, model_state_dict, model_assets=model_assets) logger.info_rank0(f"HuggingFace checkpoint saved at {save_hf_safetensor_path} successfully!") @@ -369,7 +330,6 @@ def save_hf_safetensor( model_assets, ckpt_manager, output_dir, - model, ) # Ensure all ranks finish saving before anyone proceeds From c0db513145eb0f4559e37374341f537ffc9ae79d Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Fri, 6 Nov 2026 17:14:34 +0800 Subject: [PATCH 17/22] [omni, model, test] fix: use available_nvidia_ops in BagelQwen2MoTConfig default flex_attention CPU test Co-authored-by: Cursor --- tests/seed_omni/modules/bagel/test_qwen2_mot.py | 4 +++- veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 6406429ba4..3834695d7a 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -621,7 +621,7 @@ def test_bagel_qwen2_rotary_embedding_to_empty_from_meta_rebuilds_fp32_table() - torch.testing.assert_close(rotary.original_inv_freq, ref_rotary.original_inv_freq) -def test_bagel_qwen2_mot_config_defaults_to_flex_attention() -> None: +def test_bagel_qwen2_mot_config_defaults_to_flex_attention(available_nvidia_ops) -> None: # noqa: F811 """Verify ``BagelQwen2MoTConfig`` defaults to ``veomni_flex_attention`` when no attention backend is specified.""" from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT @@ -633,6 +633,8 @@ def test_bagel_qwen2_mot_config_defaults_to_flex_attention() -> None: num_attention_heads=4, num_key_value_heads=1, max_position_embeddings=64, + bos_token_id=1, + eos_token_id=2, ) assert cfg._attn_implementation == "veomni_flex_attention" model = BagelQwen2MoT(cfg).eval() diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index 7e8cb86e02..f68ee0c9fa 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -792,13 +792,14 @@ def __init__( attn_impl = ( config.ops_config.attn_implementation if "attn_implementation" in config.ops_implementation else "sdpa" ) - self.veomni_attn_impl = attn_impl self.veomni_attn: VeomniOp | None = None if attn_impl in _FLEX_ATTN_IMPLEMENTATIONS: try: self.veomni_attn = VeomniOp("attention", "standard", "veomni_flex_attention") except RuntimeError: self.veomni_attn = None + attn_impl = "sdpa" + self.veomni_attn_impl = attn_impl def _build_qkv_proj(self) -> None: self.q_proj = nn.Linear(self.hidden_size, self.query_size, bias=True) From cafdccdcd569551bdfb3c38c856323743f5021c6 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 9 Nov 2026 12:19:35 +0800 Subject: [PATCH 18/22] [omni, model, config, docs, test] fix: inherit checkpoint generation_kwargs and unify BAGEL prefill dispatch Co-authored-by: Cursor --- .../seed_omni/usage/training_and_inference.md | 6 +- tests/seed_omni/arguments/test_module_args.py | 12 +-- tests/seed_omni/model/test_save_load.py | 20 +---- .../modules/bagel/test_bagel_eager.py | 22 ++++-- .../modules/bagel/test_generation.py | 73 ++++++++++--------- .../modules/bagel/test_processing.py | 8 +- veomni/arguments/omni_arguments_types.py | 2 +- veomni/models/seed_omni/modeling_omni.py | 2 - .../bagel/qwen2_mot/generation_state.py | 16 ---- .../modules/bagel/qwen2_mot/modeling.py | 46 ++++++------ .../bagel/text_encoder/chat_template.py | 26 ++++++- .../modules/bagel/text_encoder/modeling.py | 25 +------ 12 files changed, 122 insertions(+), 136 deletions(-) diff --git a/docs/seed_omni/usage/training_and_inference.md b/docs/seed_omni/usage/training_and_inference.md index 9cb8957ad3..17979d08af 100644 --- a/docs/seed_omni/usage/training_and_inference.md +++ b/docs/seed_omni/usage/training_and_inference.md @@ -205,9 +205,9 @@ flowchart TB `generate()` always lives on the native `modeling.py`, so an all-eager Inferencer run builds the same object a native user gets from `OmniModel.from_pretrained`; -the launcher only projects the YAML onto `OmniConfig` first. When `infer_graph` -comes from the checkpoint, `infer.generation_kwargs` overrides the checkpoint's -`generation_kwargs` defaults per key rather than replacing the bag. +the launcher only projects the YAML onto `OmniConfig` first. When the checkpoint +declares `generation_kwargs` defaults, `infer.generation_kwargs` overrides them +per key rather than replacing the bag. **Native HF:** diff --git a/tests/seed_omni/arguments/test_module_args.py b/tests/seed_omni/arguments/test_module_args.py index f6d5e97012..21c9c6546a 100644 --- a/tests/seed_omni/arguments/test_module_args.py +++ b/tests/seed_omni/arguments/test_module_args.py @@ -509,9 +509,9 @@ def test_a_launcher_infer_graph_does_not_inherit_the_checkpoint_infer_type(tmp_p assert build_omni_model_runtime_args(args, for_inference=True).infer_type == "understanding" -def test_build_model_runtime_args_merges_checkpoint_generation_kwargs_when_infer_graph_from_checkpoint(tmp_path): - """Checkpoint ``generation_kwargs`` supply the defaults and ``infer.generation_kwargs`` overrides per key; - replacing ``infer_graph`` in the launcher drops the checkpoint's ``generation_kwargs``.""" +def test_build_model_runtime_args_merges_checkpoint_generation_kwargs(tmp_path): + """Checkpoint ``generation_kwargs`` supply the defaults and ``infer.generation_kwargs`` overrides per key, + even when the launcher YAML also declares ``model.model_config.infer_graph``.""" infer_dir = _cfg_dir() / "infer" runtime_cfg = _model_runtime( model_path=str(tmp_path), @@ -530,7 +530,7 @@ def test_build_model_runtime_args_merges_checkpoint_generation_kwargs_when_infer resolved = build_omni_model_runtime_args(args, for_inference=True) assert resolved.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0} - args_custom_graph = OmniArguments( + args_with_launcher_graph = OmniArguments( model=OmniModelRuntimeArguments( model_path=str(export_root), model_config={"infer_graph": {"understanding": str(infer_dir / "graph_infer_und.yaml")}}, @@ -538,8 +538,8 @@ def test_build_model_runtime_args_merges_checkpoint_generation_kwargs_when_infer data=OmniDataArguments(train_path=""), infer=OmniInferArguments(generation_kwargs={"num_timesteps": 10}), ) - resolved_custom = build_omni_model_runtime_args(args_custom_graph, for_inference=True) - assert resolved_custom.generation_kwargs == {"num_timesteps": 10} + resolved_with_launcher_graph = build_omni_model_runtime_args(args_with_launcher_graph, for_inference=True) + assert resolved_with_launcher_graph.generation_kwargs == {"num_timesteps": 10, "timestep_shift": 3.0} def test_build_model_runtime_args_keeps_every_training_scenario_from_omni_checkpoint(tmp_path): diff --git a/tests/seed_omni/model/test_save_load.py b/tests/seed_omni/model/test_save_load.py index e545e3651e..0a798adb74 100644 --- a/tests/seed_omni/model/test_save_load.py +++ b/tests/seed_omni/model/test_save_load.py @@ -968,28 +968,12 @@ def test_omni_model_resolve_generation_kwargs_uses_config_defaults(): model = _build_omni_model() model.config.generation_kwargs = {"max_new_tokens": 64} - assert model.config.infer_type == "infer_gen" - assert model.resolve_generation_kwargs(None) == { - "max_new_tokens": 64, - "infer_type": "infer_gen", - } + assert model.resolve_generation_kwargs(None) == {"max_new_tokens": 64} assert model.resolve_generation_kwargs({"temperature": 0.2}) == { "max_new_tokens": 64, "temperature": 0.2, - "infer_type": "infer_gen", - } - assert model.resolve_generation_kwargs({"max_new_tokens": 8}) == { - "max_new_tokens": 8, - "infer_type": "infer_gen", - } - assert model.resolve_generation_kwargs({"infer_type": "infer_und"}) == { - "max_new_tokens": 64, - "infer_type": "infer_und", - } - assert model.resolve_generation_kwargs({"infer_type": None}) == { - "max_new_tokens": 64, - "infer_type": "infer_gen", } + assert model.resolve_generation_kwargs({"max_new_tokens": 8}) == {"max_new_tokens": 8} def test_omni_model_post_init_unions_child_no_split_modules(): diff --git a/tests/seed_omni/modules/bagel/test_bagel_eager.py b/tests/seed_omni/modules/bagel/test_bagel_eager.py index ef457fc75a..92d061c2c6 100644 --- a/tests/seed_omni/modules/bagel/test_bagel_eager.py +++ b/tests/seed_omni/modules/bagel/test_bagel_eager.py @@ -68,21 +68,31 @@ def test_eager_und_generate_runs(dtype: torch.dtype) -> None: role="user", ) bos = ConversationItem( - type="output", + type="text", value=torch.randn(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype), role="assistant", meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, ) - outputs = model.generate([prompt, bos], generation_kwargs={"infer_type": "infer_und"}) - assert model._generation_state.main.cache is not None - assert outputs["conversation_list"][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN - - outputs = model.generate(outputs["conversation_list"], generation_kwargs={"infer_type": "infer_und"}) + outputs = model.generate([prompt, bos]) conversation = outputs["conversation_list"] tail = conversation[-1] + assert model._generation_state.main.cache is not None + assert tail.type == "output" + assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + assert torch.is_tensor(tail.value) + assert tail.value.dtype == dtype + assert tail.value.shape == (1, int(model.config.hidden_size)) + assert torch.isfinite(tail.value).all() + + outputs = model.generate(conversation) + tail = outputs["conversation_list"][-1] assert tail.type == "output" assert BAGEL_PHASE_KEY not in tail.meta + + outputs = model.generate(outputs["conversation_list"]) + tail = outputs["conversation_list"][-1] + assert tail.type == "output" assert torch.is_tensor(tail.value) assert tail.value.dtype == dtype assert tail.value.shape == (1, int(model.config.hidden_size)) diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py index afb2726545..783f0d6d96 100644 --- a/tests/seed_omni/modules/bagel/test_generation.py +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -2,8 +2,6 @@ from __future__ import annotations -from types import SimpleNamespace - import pytest import torch import torch.nn as nn @@ -50,38 +48,43 @@ def _make_veomni_runtime(cfg, modules): def test_bagel_text_encoder_injects_raw_bos_for_understanding() -> None: - """Verify ``BagelTextEncoder.generate`` appends an embedded BOS output item on the first understanding step.""" + """Verify ``BagelTextEncoderPreprocessor`` appends a ``BAGEL_START_TOKEN`` BOS item for ``infer_und`` and ``BagelTextEncoder.generate`` embeds it.""" + from tests.seed_omni.modules.bagel.test_processing import _bagel_template + from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor + BagelTextEncoder = native_model_cls("bagel_text_encoder") BagelTextEncoderConfig = config_cls("bagel_text_encoder") - model = BagelTextEncoder(BagelTextEncoderConfig(vocab_size=8, hidden_size=4)).eval() - model._chat_template = SimpleNamespace(bos_token_id=3) + model = BagelTextEncoder(BagelTextEncoderConfig(vocab_size=256, hidden_size=4)).eval() + model._chat_template = _bagel_template() with torch.no_grad(): - model.embed_tokens.weight.copy_(torch.arange(32, dtype=torch.float32).reshape(8, 4)) + model.embed_tokens.weight.copy_(torch.arange(256 * 4, dtype=torch.float32).reshape(256, 4)) - prompt = ConversationItem( - type="text", - value=torch.tensor([1, 2]), - role="user", - meta={"_omni_tokenized": True, "input_ids": torch.tensor([1, 2])}, - ) - conversation = model.generate([prompt], generation_kwargs={"infer_type": "infer_und"})["conversation_list"] + preprocessor = BagelTextEncoderPreprocessor(model._chat_template) + sample = [ConversationItem(type="text", value="hi", role="user")] + preprocessor({"conversation_list": [sample]}, inference=True, generation_kwargs={"infer_type": "infer_und"}) + + assert len(sample) == 2 + assert sample[-1].type == "text" + assert sample[-1].role == "assistant" + assert sample[-1].meta[BAGEL_PHASE_KEY] == BAGEL_START_TOKEN + bos_id = model._chat_template.bos_token_id + assert sample[-1].meta["input_ids"].tolist() == [bos_id] + conversation = model.generate(sample)["conversation_list"] assert len(conversation) == 2 - assert conversation[-1].type == "output" assert conversation[-1].meta[BAGEL_PHASE_KEY] == BAGEL_START_TOKEN - assert conversation[-1].meta["input_ids"].tolist() == [3] - torch.testing.assert_close(conversation[-1].value, model.embed_tokens.weight[3].reshape(1, 4)) + torch.testing.assert_close(conversation[-1].value, model.embed_tokens.weight[bos_id].reshape(1, 4)) def test_bagel_qwen_first_understanding_step_prefills_prompt_then_decodes_bos(monkeypatch) -> None: - """Verify ``BagelQwen2MoT.generate`` prefills the prompt on ``prompt_encode`` and decodes the tail BOS token on ``text_ar``.""" + """Verify ``BagelQwen2MoT.generate`` prefills the full ``conversation_list`` (including ``BAGEL_START_TOKEN``) on ``prompt_encode`` and decodes the tail ``output`` item on ``text_ar``.""" BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).eval() hidden_size = int(model.config.hidden_size) prompt = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") bos = ConversationItem( - type="output", + type="text", value=torch.full((1, hidden_size), 2.0), role="assistant", meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, @@ -93,11 +96,11 @@ def _prefill(self, conversation_list, generation_kwargs): calls.append(("prefill", list(conversation_list))) self._generation_state.main.install_cache( cache=self._new_empty_cache(), - cache_len=2, - next_position_id=torch.tensor(2), + cache_len=3, + next_position_id=torch.tensor(3), device=self.device, ) - return torch.full((2, hidden_size), 3.0) + return torch.full((3, hidden_size), 3.0) def _decode(self, conversation_list): del self @@ -107,21 +110,21 @@ def _decode(self, conversation_list): monkeypatch.setattr(type(model), "_prefill_prompt", _prefill) monkeypatch.setattr(type(model), "_decode_next_token", _decode) - conversation = model.generate( - [prompt, bos], - generation_kwargs={"infer_type": "infer_und"}, - )["conversation_list"] - assert calls == [("prefill", [prompt])] - assert conversation[-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN - torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 2.0)) - - conversation = model.generate( - conversation, - generation_kwargs={"infer_type": "infer_und"}, - )["conversation_list"] - assert calls == [("prefill", [prompt]), ("decode", [prompt, bos])] - assert conversation == [prompt, bos] + conversation = model.generate([prompt, bos])["conversation_list"] + assert calls == [("prefill", [prompt, bos])] + assert bos.type == "output" + assert bos.meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 3.0)) + + conversation = model.generate(conversation)["conversation_list"] + assert calls == [("prefill", [prompt, bos])] assert BAGEL_PHASE_KEY not in bos.meta + torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 3.0)) + + conversation = model.generate(conversation)["conversation_list"] + assert calls == [("prefill", [prompt, bos]), ("decode", [prompt, bos])] + assert conversation == [prompt, bos] + assert bos.type == "output" torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 4.0)) diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index bc70219522..f6af9579bd 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -840,7 +840,9 @@ def test_bagel_preprocessors_route_tagged_edit_without_infer_type() -> None: def test_bagel_preprocessors_route_inference_und_user_image_to_siglip_only() -> None: - """Verify ``infer_und`` preprocessing routes user images to SigLIP context only.""" + """Verify ``infer_und`` preprocessing routes user images to SigLIP context only and appends the assistant start token.""" + from veomni.models.seed_omni.modules.bagel.sources import BAGEL_PHASE_KEY, BAGEL_START_TOKEN + vae_pre = BagelVAEPreprocessor( BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), dtype=torch.bfloat16, @@ -854,9 +856,11 @@ def test_bagel_preprocessors_route_inference_und_user_image_to_siglip_only() -> vae_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) text_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) - assert [item.type for item in batch[0]] == ["text", "image", "text"] + assert [item.type for item in batch[0]] == ["text", "image", "text", "text"] assert batch[0][1].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT assert torch.equal(batch[0][1].value, image) + assert batch[0][-1].role == "assistant" + assert batch[0][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN def test_bagel_vae_preprocessor_appends_per_sample_dummy_for_missing_context() -> None: diff --git a/veomni/arguments/omni_arguments_types.py b/veomni/arguments/omni_arguments_types.py index 8c9dd433f5..10b414e146 100644 --- a/veomni/arguments/omni_arguments_types.py +++ b/veomni/arguments/omni_arguments_types.py @@ -291,7 +291,7 @@ def build_omni_model_runtime_args(args: "OmniArguments", *, for_inference: bool train_type=train_type, infer_type=infer_type, generation_kwargs={ - **(omni_cfg.generation_kwargs or {} if infer_graph_from_ckpt and omni_cfg is not None else {}), + **(omni_cfg.generation_kwargs or {} if omni_cfg is not None else {}), **args.infer.generation_kwargs, }, ) diff --git a/veomni/models/seed_omni/modeling_omni.py b/veomni/models/seed_omni/modeling_omni.py index 305ea49060..308d1ec42b 100644 --- a/veomni/models/seed_omni/modeling_omni.py +++ b/veomni/models/seed_omni/modeling_omni.py @@ -716,8 +716,6 @@ def resolve_generation_kwargs( """Merge :attr:`~OmniConfig.generation_kwargs` defaults with per-request overrides.""" resolved = dict(self.config.generation_kwargs or {}) resolved.update(generation_kwargs or {}) - if resolved.get("infer_type") is None and self.config.infer_type is not None: - resolved["infer_type"] = self.config.infer_type return resolved def collect_assets(self) -> list[Any]: diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py index fd903c8950..530cbffc54 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/generation_state.py @@ -208,7 +208,6 @@ class MotGenerationState: also remembers the branch layout between denoise and velocity collection. """ - _infer_mode: str = "und" _velocity_buffer: dict[str, torch.Tensor] = field(default_factory=dict) _pending_denoise_branches: tuple[str, ...] | None = None _pending_denoise_query_len: int | None = None @@ -217,7 +216,6 @@ class MotGenerationState: _cfg_img: MotCacheContext = field(default_factory=lambda: MotCacheContext("image CFG")) def reset(self) -> None: - self._infer_mode = "und" self._velocity_buffer.clear() self._pending_denoise_branches = None self._pending_denoise_query_len = None @@ -225,20 +223,6 @@ def reset(self) -> None: self._cfg_text.reset() self._cfg_img.reset() - def update_infer_mode(self, generation_kwargs: dict[str, Any]) -> str: - """Update and return the sticky inference mode (``'und'`` or ``'gen'``) from ``generation_kwargs``.""" - # Later graph steps omit infer_type, so the mode is sticky until the - # graph resets this module's local inference state. - if "infer_type" in generation_kwargs: - infer_type = str(generation_kwargs["infer_type"]) - if infer_type not in {"infer_und", "infer_gen", "infer_edit"}: - raise ValueError(f"Unsupported BAGEL infer_type: {infer_type!r}.") - infer_mode = "und" if infer_type == "infer_und" else "gen" - else: - infer_mode = self._infer_mode or "und" - self._infer_mode = infer_mode - return self._infer_mode - # ── CFG cache contexts ────────────────────────────────── @property diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index f68ee0c9fa..44f78abf3a 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -102,30 +102,34 @@ def generate( raise ValueError("BAGEL Qwen2-MoT generate requires conversation_list.") generation_kwargs = generation_kwargs or {} - infer_mode = self._generation_state.update_infer_mode(generation_kwargs) - # Generation/edit rebuilds all CFG prompt caches. Understanding reuses - # the main cache and starts AR from the raw assistant BOS embedding. - if infer_mode == "gen": + tail = conversation_list[-1] + is_start_token = bagel_phase(tail) == BAGEL_START_TOKEN + # Prefill the full conversation_list (including the assistant start token + # on understanding requests). The first text_ar pass returns the prefilled + # start-token hidden state; subsequent AR steps decode the tail output item. + if self._generation_state.main.cache is None or tail.type != "output": hidden_states = self._prefill_prompt(conversation_list, generation_kwargs) - elif self._generation_state.main.cache is None: - tail = conversation_list[-1] - if tail.type != "output" or bagel_phase(tail) != BAGEL_START_TOKEN: - raise ValueError("BAGEL understanding prefill requires a tail assistant BOS embedding.") - self._prefill_prompt(conversation_list[:-1], generation_kwargs) + if is_start_token: + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + if hidden_states.dim() != 2: + raise ValueError( + f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}." + ) + tail.type = "output" + tail.value = hidden_states[-1:].contiguous() return {"conversation_list": conversation_list} - else: - hidden_states = self._decode_next_token(conversation_list) - - if infer_mode != "gen": - if hidden_states.dim() == 3 and hidden_states.size(0) == 1: - hidden_states = hidden_states.squeeze(0) - if hidden_states.dim() != 2: - raise ValueError(f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}.") - tail = conversation_list[-1] - if tail.type != "output": - raise ValueError(f"BAGEL understanding decode expects tail output item, got {tail.type!r}.") - tail.value = hidden_states[-1:].contiguous() + + if is_start_token: tail.meta.pop(BAGEL_PHASE_KEY, None) + return {"conversation_list": conversation_list} + + hidden_states = self._decode_next_token(conversation_list) + if hidden_states.dim() == 3 and hidden_states.size(0) == 1: + hidden_states = hidden_states.squeeze(0) + if hidden_states.dim() != 2: + raise ValueError(f"BAGEL Qwen2-MoT expected packed hidden states, got {tuple(hidden_states.shape)}.") + tail.value = hidden_states[-1:].contiguous() return {"conversation_list": conversation_list} def _prepare_denoise_query( diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py index 5b3552b2b3..8c2aba26a3 100644 --- a/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/chat_template.py @@ -10,7 +10,7 @@ from ....utils.conversation import _OMNI_TOKEN_LEN, ConversationItem from ...text_encoder.chat_template import ChatMarkers, TextEncoderChatTemplate -from ..sources import BAGEL_CONTEXT_KEY, bagel_context +from ..sources import BAGEL_CONTEXT_KEY, BAGEL_PHASE_KEY, BAGEL_START_TOKEN, bagel_context from .processing import is_bagel_vision_marker @@ -75,7 +75,7 @@ def tokenize_conversation( generation_kwargs=generation_kwargs, ) if add_generation_prompt: - parts = self.apply_generation_prompt(parts) + parts = self.apply_generation_prompt(parts, generation_kwargs=generation_kwargs) self.tokenize(parts) return self.merge_text_embeds(parts) @@ -97,8 +97,26 @@ def apply_chat_template( return self._insert_prompt_context_image_markers(sample) - def apply_generation_prompt(self, sample: list[ConversationItem]) -> list[ConversationItem]: - return sample + def apply_generation_prompt( + self, + sample: list[ConversationItem], + *, + generation_kwargs: dict[str, Any] | None = None, + ) -> list[ConversationItem]: + """Append the assistant ``<|im_start|>`` start token for understanding inference.""" + infer_type = None if generation_kwargs is None else generation_kwargs.get("infer_type") + if infer_type != "infer_und": + return sample + out = list(sample) + out.append( + ConversationItem( + type="text", + value=self.chat_markers.bos_token, + role="assistant", + meta={"loss_mask": 0, BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, + ) + ) + return out def tokenize(self, parts: list[ConversationItem]) -> None: """Tokenize untokenized text items in place and populate ``input_ids``, ``labels``, and ``attention_mask``.""" diff --git a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py index e573c66188..469caebc28 100644 --- a/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/text_encoder/modeling.py @@ -9,7 +9,7 @@ from ....graphs.generation_graph import FSM_SIGNAL_KEY from ....utils.conversation import ConversationItem, iter_desired_items, maybe_merge_outputs from ...text_encoder.modeling import TextEncoder -from ..sources import BAGEL_FLOW_QUERY, BAGEL_PHASE_KEY, BAGEL_START_TOKEN +from ..sources import BAGEL_FLOW_QUERY, BAGEL_PHASE_KEY from .configuration import BagelTextEncoderConfig from .processing import BagelTextEncoderPreprocessor, apply_image_marker @@ -82,11 +82,11 @@ def generate( generation_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> Dict[str, Any]: - """Embed prompt text tokens, sample AR decode tokens, or emit ``start_image_gen``.""" + """Embed prompt text tokens or sample the next AR decode token.""" tail = conversation_list[-1] batched = [conversation_list] - if tail.type == "text" and tail.role == "user": + if tail.type == "text": input_ids, batch_shape = prepare_bagel_encode_input_ids( batched, device=self.device, @@ -99,19 +99,6 @@ def generate( device=self.device, dtype=self.dtype, ) - infer_type = str((generation_kwargs or {}).get("infer_type") or "infer_und") - if infer_type == "infer_und" and not self._bos_injected: - input_ids = self._token_id_tensor(self._chat_template.bos_token_id) - start_embed = self.encode(input_ids)["inputs_embeds"].to(device=self.device, dtype=self.dtype) - conversation_list.append( - ConversationItem( - type="output", - value=start_embed, - role="assistant", - meta={"input_ids": input_ids.reshape(-1).detach(), BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, - ) - ) - self._bos_injected = True return {"conversation_list": batched[0]} if tail.type == "output": @@ -136,12 +123,6 @@ def generate( outputs["generated"] = self._flush_text_generated(batched[0]) return outputs - outputs: Dict[str, Any] = {"conversation_list": batched[0]} - infer_type = str((generation_kwargs or {}).get("infer_type", "")) - if infer_type in {"infer_gen", "infer_edit"}: - outputs[FSM_SIGNAL_KEY] = SIGNAL_START_IMAGE_GEN - return outputs - raise ValueError(f"Invalid type: {tail.type}") def encode_image_markers( From 1e74577ce74701fc640703a077cb2762cd644f00 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Mon, 9 Nov 2026 17:39:19 +0800 Subject: [PATCH 19/22] refactor: reuse patched Qwen2 primitives and deduplicate Bagel pos embed Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 8 +-- .../modules/bagel/flow_connector/modeling.py | 49 +--------------- .../modules/bagel/position_embedding.py | 58 +++++++++++++++++++ .../qwen2_mot/accelerated/accelerated.py | 57 +++--------------- .../modules/bagel/qwen2_mot/modeling.py | 52 +---------------- .../modules/bagel/siglip_navit/modeling.py | 45 +------------- 6 files changed, 76 insertions(+), 193 deletions(-) create mode 100644 veomni/models/seed_omni/modules/bagel/position_embedding.py diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 3834695d7a..50dc533c0b 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -295,7 +295,7 @@ def __call__(self, *args, **kwargs): @pytest.mark.parametrize("hidden_act", ["silu", "swish", "gelu"]) def test_bagel_qwen2_mot_mlp_and_rms_norm_dispatch_through_veomni_ops(hidden_act: str) -> None: - """Verify ``Qwen2RMSNormAccelerated`` and ``Qwen2MLPAccelerated`` route through their ``VeomniOp`` handles.""" + """Verify ``BagelQwen2MoTAccelerated`` reuses patched Qwen2 ``rms_norm_cls`` and ``mlp_cls`` ``VeomniOp`` handles.""" from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP as HFQwen2MLP from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm as HFQwen2RMSNorm @@ -316,7 +316,7 @@ def test_bagel_qwen2_mot_mlp_and_rms_norm_dispatch_through_veomni_ops(hidden_act torch.manual_seed(0) hf_norm = HFQwen2RMSNorm(cfg.hidden_size, eps=cfg.rms_norm_eps) - ours_norm = accelerated.Qwen2RMSNormAccelerated(cfg.hidden_size, eps=cfg.rms_norm_eps, config=cfg) + ours_norm = accelerated.BagelQwen2MoTAccelerated.rms_norm_cls(cfg.hidden_size, eps=cfg.rms_norm_eps, config=cfg) ours_norm.load_state_dict(hf_norm.state_dict()) norm_counter = _CallCounter(ours_norm.veomni_rms_norm) ours_norm.veomni_rms_norm = norm_counter @@ -327,7 +327,7 @@ def test_bagel_qwen2_mot_mlp_and_rms_norm_dispatch_through_veomni_ops(hidden_act assert norm_counter.calls == 2 hf_mlp = HFQwen2MLP(cfg) - ours_mlp = accelerated.Qwen2MLPAccelerated(cfg) + ours_mlp = accelerated.BagelQwen2MoTAccelerated.mlp_cls(cfg) ours_mlp.load_state_dict(hf_mlp.state_dict()) mlp_counter = _CallCounter(ours_mlp.veomni_swiglu_mlp) ours_mlp.veomni_swiglu_mlp = mlp_counter @@ -604,7 +604,7 @@ def test_bagel_qwen2_rotary_embedding_to_empty_from_meta_rebuilds_fp32_table() - from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2RotaryEmbedding cfg = _flex_config() - ref_rotary = BagelQwen2RotaryEmbedding(cfg, device=torch.device("cpu")) + ref_rotary = BagelQwen2RotaryEmbedding(cfg) with torch.device("meta"): meta_model = accelerated.BagelQwen2MoTAccelerated(cfg) assert meta_model.model.rotary_emb.inv_freq.is_meta diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py index b01fe1a022..3990bd78df 100644 --- a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py @@ -18,13 +18,13 @@ import math from typing import Any -import numpy as np import torch import torch.nn as nn from ....graphs.generation_graph import FSM_SIGNAL_KEY from ....utils.conversation import ConversationItem, iter_desired_items from ...module_modeling_base import PretrainedOmniModule +from ..position_embedding import PositionEmbedding from ..sources import ( BAGEL_CONTEXT_KEY, BAGEL_FLOW_HIDDEN, @@ -390,53 +390,6 @@ def forward(self, t: torch.Tensor) -> torch.Tensor: return self.mlp(t_freq.to(device=first_weight.device, dtype=first_weight.dtype)) -class PositionEmbedding(nn.Module): - def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: - super().__init__() - self.max_num_patch_per_side = max_num_patch_per_side - self.hidden_size = hidden_size - self.pos_embed = nn.Parameter( - torch.zeros(max_num_patch_per_side**2, hidden_size), - requires_grad=False, - ) - self.reset_parameters() - - def reset_parameters(self) -> None: - new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) - if hasattr(self.pos_embed.data, "to_local"): - self.pos_embed.data.to_local().copy_(new_data) - else: - self.pos_embed.data.copy_(new_data) - - def _init_weights(self) -> None: - self.reset_parameters() - - def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: - return self.pos_embed[position_ids.to(device=self.pos_embed.device)] - - -def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: - grid_h = np.arange(grid_size, dtype=np.float32) - grid_w = np.arange(grid_size, dtype=np.float32) - grid = np.meshgrid(grid_w, grid_h) - grid = np.stack(grid, axis=0).reshape(2, 1, grid_size, grid_size) - return torch.from_numpy(_get_2d_sincos_pos_embed_from_grid(embed_dim, grid)).float() - - -def _get_2d_sincos_pos_embed_from_grid(embed_dim: int, grid: np.ndarray) -> np.ndarray: - emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) - emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) - return np.concatenate([emb_h, emb_w], axis=1) - - -def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: - omega = np.arange(embed_dim // 2, dtype=np.float64) - omega /= embed_dim / 2.0 - omega = 1.0 / 10000**omega - out = np.einsum("m,d->md", pos.reshape(-1), omega) - return np.concatenate([np.sin(out), np.cos(out)], axis=1) - - __all__ = [ "BagelFlowConnector", "BagelFlowConnectorConfig", diff --git a/veomni/models/seed_omni/modules/bagel/position_embedding.py b/veomni/models/seed_omni/modules/bagel/position_embedding.py new file mode 100644 index 0000000000..7364939074 --- /dev/null +++ b/veomni/models/seed_omni/modules/bagel/position_embedding.py @@ -0,0 +1,58 @@ +"""Shared 2D sin-cos position embedding for BAGEL vision and latent patch grids.""" + +from __future__ import annotations + +import numpy as np +import torch +import torch.nn as nn + + +class PositionEmbedding(nn.Module): + """Fixed 2D sin-cos position embedding table indexed by flattened patch IDs.""" + + def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: + super().__init__() + self.max_num_patch_per_side = max_num_patch_per_side + self.hidden_size = hidden_size + self.pos_embed = nn.Parameter( + torch.zeros(max_num_patch_per_side**2, hidden_size), + requires_grad=False, + ) + self.reset_parameters() + + def reset_parameters(self) -> None: + """Populate ``pos_embed`` with deterministic 2D sin-cos values.""" + new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) + if hasattr(self.pos_embed.data, "to_local"): + self.pos_embed.data.to_local().copy_(new_data) + else: + self.pos_embed.data.copy_(new_data) + + def _init_weights(self) -> None: + self.reset_parameters() + + def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: + return self.pos_embed[position_ids.to(device=self.pos_embed.device)] + + +def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: + """Build a ``[grid_size**2, embed_dim]`` 2D sin-cos position embedding tensor.""" + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0).reshape(2, 1, grid_size, grid_size) + emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + return torch.from_numpy(np.concatenate([emb_h, emb_w], axis=1)).float() + + +def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: + """Build a 1D sin-cos position embedding array from flattened grid coordinates.""" + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + out = np.einsum("m,d->md", pos.reshape(-1), omega) + return np.concatenate([np.sin(out), np.cos(out)], axis=1) + + +__all__ = ["PositionEmbedding"] diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py index 30ef5c38c5..68671c8848 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/accelerated/accelerated.py @@ -12,7 +12,7 @@ import torch import torch.distributed as dist from torch.nn.attention.flex_attention import BlockMask -from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm +from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm from .......distributed.parallel_state import get_parallel_state from .......distributed.sequence_parallel import ( @@ -20,8 +20,13 @@ slice_input_tensor, sp_pad, ) +from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import ( + Qwen2MLP as PatchedQwen2MLP, +) +from .......models.transformers.qwen2.generated.patched_modeling_qwen2_gpu import ( + Qwen2RMSNorm as PatchedQwen2RMSNorm, +) from .......ops import VeomniOp -from .......ops.kernels.swiglu_mlp import has_plain_linear_projections from .......ops.mask import MagiAttentionMask from .....mixins.base_mixin import BaseMixin from .....mixins.metric_meter_mixin import MetricMeterMixin @@ -337,50 +342,6 @@ class VeOmniMixin(BaseMixin, TrainingMixin, MeterMixin): """ -class Qwen2RMSNormAccelerated(Qwen2RMSNorm): - """Qwen2 RMSNorm with instance-local VeomniOp dispatch.""" - - def __init__(self, hidden_size: int, eps: float = 1e-6, *, config: BagelQwen2MoTConfig | None = None) -> None: - super().__init__(hidden_size, eps=eps) - self.config = config - rms_impl = config.ops_config.rms_norm_implementation if config is not None else "eager" - self.veomni_rms_norm = VeomniOp("rms_norm", "standard", rms_impl) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Apply the instance-local ``rms_norm`` ``VeomniOp``.""" - return self.veomni_rms_norm(hidden_states, self.weight, eps=self.variance_epsilon) - - -class Qwen2MLPAccelerated(Qwen2MLP): - """Qwen2 SwiGLU MLP with instance-local VeomniOp dispatch.""" - - def __init__(self, config: BagelQwen2MoTConfig) -> None: - super().__init__(config) - self.veomni_swiglu_mlp = VeomniOp("swiglu_mlp", "standard", config.ops_config.swiglu_mlp_implementation) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - """Apply the instance-local ``swiglu_mlp`` ``VeomniOp`` for ``silu``/``swish`` on plain linear projections.""" - if self.config.hidden_act not in {"silu", "swish"} and self.veomni_swiglu_mlp.impl != "eager": - raise ValueError( - f"Liger SwiGLU requires hidden_act='silu' or 'swish', got {self.config.hidden_act!r}. " - "Set model.ops_implementation.swiglu_mlp_implementation='eager' " - "to use the Transformers reference implementation." - ) - - if self.config.hidden_act in {"silu", "swish"} and has_plain_linear_projections(self): - return self.veomni_swiglu_mlp( - x, - self.gate_proj.weight, - self.gate_proj.bias if self.gate_proj.bias is not None else self.gate_proj.weight.new_empty(0), - self.up_proj.weight, - self.up_proj.bias if self.up_proj.bias is not None else self.gate_proj.weight.new_empty(0), - self.down_proj.weight, - self.down_proj.bias if self.down_proj.bias is not None else self.down_proj.weight.new_empty(0), - ) - - return super().forward(x) - - def _apply_rotary_pos_emb( q: torch.Tensor, k: torch.Tensor, @@ -744,8 +705,8 @@ class BagelQwen2MoTAccelerated(VeOmniMixin, InferenceMixinAccelerated, BagelQwen _supports_flex_attn = True _supports_magi_attn = True attention_cls = BagelQwen2MoTAttentionAccelerated - mlp_cls = Qwen2MLPAccelerated - rms_norm_cls = Qwen2RMSNormAccelerated + mlp_cls = PatchedQwen2MLP + rms_norm_cls = PatchedQwen2RMSNorm @staticmethod def _create_checkpoint_tensor_converter( diff --git a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py index 44f78abf3a..e2b2975a5a 100644 --- a/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/qwen2_mot/modeling.py @@ -9,7 +9,7 @@ from torch.nn.attention import SDPBackend, sdpa_kernel from torch.nn.attention.flex_attention import BlockMask from torch.nn.functional import scaled_dot_product_attention -from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm +from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP, Qwen2RMSNorm, Qwen2RotaryEmbedding from transformers.utils import ModelOutput from veomni.ops import VeomniOp @@ -691,19 +691,8 @@ def _check_packed_inference_mode( raise ValueError(f"Unsupported BAGEL Qwen2 MoT inference mode: {mode!r}") -class BagelQwen2RotaryEmbedding(nn.Module): - """Official-compatible Qwen2 RoPE for BAGEL parity.""" - - def __init__(self, config: BagelQwen2MoTConfig, device: Optional[torch.device] = None): - super().__init__() - self.config = config - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - self.rope_type = "default" - self.attention_scaling = 1.0 - inv_freq, _ = self.compute_default_rope_parameters(config, device) - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) +class BagelQwen2RotaryEmbedding(Qwen2RotaryEmbedding): + """Qwen2 RoPE preserving fp32 inverse frequencies across dtype casts and meta materialization.""" def _apply(self, fn: Any, recurse: bool = True) -> nn.Module: # Rebuild the true fp32 table after ``fn`` so ``.to(bfloat16)`` does not @@ -718,41 +707,6 @@ def _apply(self, fn: Any, recurse: bool = True) -> nn.Module: self.original_inv_freq = inv_freq.clone() return module - @staticmethod - def compute_default_rope_parameters( - config: BagelQwen2MoTConfig, - device: Optional[torch.device] = None, - seq_len: Optional[int] = None, - ) -> tuple[torch.Tensor, float]: - """Compute default inverse frequencies and attention scaling factor for Qwen2 RoPE.""" - del seq_len - head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads - partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) - dim = int(head_dim * partial_rotary_factor) - base = getattr(config, "rope_theta", None) - if base is None: - base = config.rope_parameters["rope_theta"] - inv_freq = 1.0 / ( - base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim) - ) - return inv_freq, 1.0 - - @torch.no_grad() - def forward(self, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - inv_freq = self.inv_freq.to(device=x.device, dtype=torch.float32) - inv_freq_expanded = inv_freq[None, :, None].expand(position_ids.shape[0], -1, 1) - position_ids_expanded = position_ids[:, None, :].float() - device_type = x.device.type - device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos() - sin = emb.sin() - cos = cos * self.attention_scaling - sin = sin * self.attention_scaling - return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) - def _build_rms_norm( rms_norm_cls: type[Qwen2RMSNorm], diff --git a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py index 9775c2a52d..4147254847 100644 --- a/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/siglip_navit/modeling.py @@ -11,7 +11,6 @@ from typing import Any -import numpy as np import torch import torch.nn as nn from transformers.activations import ACT2FN @@ -20,6 +19,7 @@ from ....utils.conversation import ConversationItem, iter_desired_items from ...module_modeling_base import PretrainedOmniModule +from ..position_embedding import PositionEmbedding from ..sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT from .configuration import BagelSiglipNavitConfig from .processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN, BagelSiglipNavitPreprocessor, BagelSiglipNavitProcessor @@ -246,49 +246,6 @@ def _forward_one_side(grid: torch.Tensor, inv_freq: torch.Tensor) -> tuple[torch return emb.cos(), emb.sin() -class PositionEmbedding(nn.Module): - def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: - super().__init__() - self.max_num_patch_per_side = max_num_patch_per_side - self.hidden_size = hidden_size - self.pos_embed = nn.Parameter( - torch.zeros(max_num_patch_per_side**2, hidden_size), - requires_grad=False, - ) - self.reset_parameters() - - def reset_parameters(self) -> None: - new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) - if hasattr(self.pos_embed.data, "to_local"): - self.pos_embed.data.to_local().copy_(new_data) - else: - self.pos_embed.data.copy_(new_data) - - def _init_weights(self) -> None: - self.reset_parameters() - - def forward(self, position_ids: torch.LongTensor) -> torch.Tensor: - return self.pos_embed[position_ids.to(device=self.pos_embed.device)] - - -def _get_2d_sincos_pos_embed(embed_dim: int, grid_size: int) -> torch.Tensor: - grid_h = np.arange(grid_size, dtype=np.float32) - grid_w = np.arange(grid_size, dtype=np.float32) - grid = np.meshgrid(grid_w, grid_h) - grid = np.stack(grid, axis=0).reshape([2, 1, grid_size, grid_size]) - emb_h = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) - emb_w = _get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) - return torch.from_numpy(np.concatenate([emb_h, emb_w], axis=1)).float() - - -def _get_1d_sincos_pos_embed_from_grid(embed_dim: int, pos: np.ndarray) -> np.ndarray: - omega = np.arange(embed_dim // 2, dtype=np.float64) - omega /= embed_dim / 2.0 - omega = 1.0 / 10000**omega - out = np.einsum("m,d->md", pos.reshape(-1), omega) - return np.concatenate([np.sin(out), np.cos(out)], axis=1) - - class MLPConnector(nn.Module): def __init__(self, in_dim: int, out_dim: int, hidden_act: str) -> None: super().__init__() From 7fb58efa7f778a7ae8c01ff6c94e6b7fd6515b2d Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Tue, 10 Nov 2026 18:16:30 +0800 Subject: [PATCH 20/22] test: consolidate Bagel unit and E2E alignment tests Co-authored-by: Cursor --- tests/seed_omni/modules/bagel/helpers.py | 56 + .../modules/bagel/test_async_ulysses.py | 529 ------ .../modules/bagel/test_bagel_accel_align.py | 260 +-- .../modules/bagel/test_bagel_eager.py | 130 -- .../modules/bagel/test_generation.py | 774 ++------- .../modules/bagel/test_graph_config.py | 131 -- tests/seed_omni/modules/bagel/test_masking.py | 100 -- .../modules/bagel/test_processing.py | 1439 ++--------------- .../seed_omni/modules/bagel/test_qwen2_mot.py | 635 ++------ .../modules/bagel/test_source_routing.py | 171 -- 10 files changed, 683 insertions(+), 3542 deletions(-) delete mode 100644 tests/seed_omni/modules/bagel/test_async_ulysses.py delete mode 100644 tests/seed_omni/modules/bagel/test_bagel_eager.py delete mode 100644 tests/seed_omni/modules/bagel/test_graph_config.py delete mode 100644 tests/seed_omni/modules/bagel/test_masking.py delete mode 100644 tests/seed_omni/modules/bagel/test_source_routing.py diff --git a/tests/seed_omni/modules/bagel/helpers.py b/tests/seed_omni/modules/bagel/helpers.py index 01d16df657..59fcb3239d 100644 --- a/tests/seed_omni/modules/bagel/helpers.py +++ b/tests/seed_omni/modules/bagel/helpers.py @@ -22,6 +22,7 @@ from veomni.models.seed_omni.modules import OMNI_CONFIG_REGISTRY from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import PackedConversation, preprocess_mot_inputs from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT +from veomni.models.seed_omni.modules.bagel.text_encoder.chat_template import BagelChatTemplate from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem @@ -33,6 +34,61 @@ ALIGN_GRAD_RTOL = 5e-2 +class FakeBagelTokenizer: + """Char-ordinal tokenizer with the marker tokens/ids that ``BagelChatTemplate`` resolves.""" + + bos_token = "" + eos_token = "" + boi_token = "" + eoi_token = "" + bos_token_id = 1 + eos_token_id = 2 + boi_token_id = 3 + eoi_token_id = 4 + pad_token_id = 0 + unk_token_id = -1 + special_tokens = { + "": 1, + "": 2, + "": 3, + "": 4, + "<|im_start|>": 5, + "<|im_end|>": 2, + "<|vision_start|>": 7, + "<|vision_end|>": 8, + } + + def convert_tokens_to_ids(self, token): + return self.special_tokens.get(token, self.unk_token_id) + + def decode(self, token_ids, skip_special_tokens=True): + del skip_special_tokens + special_by_id = {v: k for k, v in self.special_tokens.items()} + return "".join(special_by_id.get(int(t), chr(max(32, int(t)))) for t in token_ids) + + def __call__(self, text, add_special_tokens=False): + del add_special_tokens + input_ids = [] + index = 0 + while index < len(text): + matched = False + for token, token_id in sorted(self.special_tokens.items(), key=lambda item: len(item[0]), reverse=True): + if text.startswith(token, index): + input_ids.append(token_id) + index += len(token) + matched = True + break + if matched: + continue + input_ids.append(ord(text[index])) + index += 1 + return {"input_ids": input_ids} + + +def bagel_template() -> BagelChatTemplate: + return BagelChatTemplate(FakeBagelTokenizer()) + + def bagel_cfg_dir() -> Path: return Path(__file__).resolve().parents[4] / "configs" / "seed_omni" / "Bagel" / "bagel_7b_mot" diff --git a/tests/seed_omni/modules/bagel/test_async_ulysses.py b/tests/seed_omni/modules/bagel/test_async_ulysses.py deleted file mode 100644 index 4ae61a9172..0000000000 --- a/tests/seed_omni/modules/bagel/test_async_ulysses.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Distributed numerical guards for BAGEL MoT async Ulysses and Flex-FA4.""" - -from __future__ import annotations - -import copy -from dataclasses import replace - -import pytest -import torch -import torch.distributed as dist -import torch.nn.functional as F -from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard - -from scripts.ci.accelerator.launch_plan import LaunchSpec -from tests.tools.launch_utils import torchrun -from veomni.distributed.parallel_state import _init_parallel_state, use_parallel_state -from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated -from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig -from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( - build_mot_attention_metadata, - pad_mot_attention_metadata, -) -from veomni.ops import VeomniOp -from veomni.ops.kernels.attention.standard import flex as flex_backend -from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type, get_torch_device -from veomni.utils.helper import enable_high_precision_for_bf16 - - -pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(nproc=4)) - - -def _rms_norm_forward( - input_tensor: torch.Tensor, - weight: torch.Tensor, - eps: float, -) -> tuple[torch.Tensor, torch.Tensor]: - """Compute reference FP32-accumulated RMSNorm.""" - invvar = torch.rsqrt(input_tensor.float().square().mean(dim=-1, keepdim=True) + eps) - output = (input_tensor.float() * invvar * weight.float()).to(input_tensor.dtype) - return output, invvar - - -def _apply_rotary( - input_tensor: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, -) -> torch.Tensor: - """Compute reference rotate-half RoPE on ``[tokens, heads, head_dim]``.""" - first, second = input_tensor.chunk(2, dim=-1) - rotated = torch.cat((-second, first), dim=-1) - return input_tensor * cos.unsqueeze(1) + rotated * sin.unsqueeze(1) - - -def _reference_qkv_projection( - hidden_states: torch.Tensor, - route_0_indices: torch.Tensor, - route_1_indices: torch.Tensor, - position_cos: torch.Tensor, - position_sin: torch.Tensor, - parameters: dict[str, torch.Tensor], - *, - num_query_heads: int, - num_key_value_heads: int, - head_dim: int, - eps: float, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Compute full-sequence sequential two-route Q/K/V projection, RMSNorm, and RoPE.""" - - def project(name: str, route: int, indices: torch.Tensor, num_heads: int) -> torch.Tensor: - projected = F.linear( - hidden_states.index_select(0, indices), - parameters[f"{name}_weight_{route}"], - parameters[f"{name}_bias_{route}"], - ) - return projected.reshape(-1, num_heads, head_dim) - - def combine(name: str, num_heads: int, *, normalize: bool) -> torch.Tensor: - output = hidden_states.new_zeros(hidden_states.shape[0], num_heads, head_dim) - for route, indices in ((0, route_0_indices), (1, route_1_indices)): - projected = project(name, route, indices, num_heads) - if normalize: - projected, _ = _rms_norm_forward(projected, parameters[f"{name}_norm_weight_{route}"], eps) - output = output.index_copy(0, indices, projected) - return output - - query = _apply_rotary(combine("q", num_query_heads, normalize=True), position_cos, position_sin) - key = _apply_rotary(combine("k", num_key_value_heads, normalize=True), position_cos, position_sin) - value = combine("v", num_key_value_heads, normalize=False) - return query, key, value - - -def _reference_output_projection( - hidden_states: torch.Tensor, - route_0_indices: torch.Tensor, - route_1_indices: torch.Tensor, - parameters: dict[str, torch.Tensor], -) -> torch.Tensor: - """Compute full-sequence sequential two-route output projection.""" - flattened_states = hidden_states.flatten(1) - output = flattened_states.new_zeros(flattened_states.shape) - for route, indices in ((0, route_0_indices), (1, route_1_indices)): - projected = F.linear( - flattened_states.index_select(0, indices), - parameters[f"proj_weight_{route}"], - parameters[f"proj_bias_{route}"], - ) - output = output.index_copy(0, indices, projected) - return output - - -def _make_parameters( - device: torch.device, - *, - hidden_size: int, - query_size: int, - key_value_size: int, - head_dim: int, -) -> dict[str, torch.Tensor]: - """Allocate random two-route QKV, QK-norm, and output projection parameters.""" - shapes = { - "q_weight_0": (query_size, hidden_size), - "q_bias_0": (query_size,), - "k_weight_0": (key_value_size, hidden_size), - "k_bias_0": (key_value_size,), - "v_weight_0": (key_value_size, hidden_size), - "v_bias_0": (key_value_size,), - "q_norm_weight_0": (head_dim,), - "k_norm_weight_0": (head_dim,), - "q_weight_1": (query_size, hidden_size), - "q_bias_1": (query_size,), - "k_weight_1": (key_value_size, hidden_size), - "k_bias_1": (key_value_size,), - "v_weight_1": (key_value_size, hidden_size), - "v_bias_1": (key_value_size,), - "q_norm_weight_1": (head_dim,), - "k_norm_weight_1": (head_dim,), - "proj_weight_0": (hidden_size, query_size), - "proj_bias_0": (hidden_size,), - "proj_weight_1": (hidden_size, query_size), - "proj_bias_1": (hidden_size,), - } - parameters = {} - for name, shape in shapes.items(): - value = torch.randn(shape, device=device, dtype=torch.float32) - if "norm_weight" in name: - value = value * 0.1 + 1 - parameters[name] = value.requires_grad_() - return parameters - - -def _clone_parameters(parameters: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: - """Detach and clone ``parameters`` into independent leaf tensors.""" - return {name: parameter.detach().clone().requires_grad_() for name, parameter in parameters.items()} - - -def _async_ulysses_mot_worker(num_key_value_heads: int, route_case: str) -> None: - """Compare 4-rank ``async_ulysses_qkv`` / ``async_ulysses_o`` ``bagel`` ops against sequential reference.""" - rank = dist.get_rank() - world_size = dist.get_world_size() - assert world_size == 4 - device = torch.device(f"{get_device_type()}:{rank}") - group = dist.group.WORLD - enable_high_precision_for_bf16() - torch.manual_seed(4201) - - sequence_length = 8 - local_sequence_length = sequence_length // world_size - hidden_size = 32 - num_query_heads = 8 - head_dim = hidden_size // num_query_heads - query_size = num_query_heads * head_dim - key_value_size = num_key_value_heads * head_dim - eps = 1e-6 - - full_input = torch.randn(sequence_length, hidden_size, device=device, dtype=torch.float32) - dist.broadcast(full_input, src=0) - angles = torch.randn(sequence_length, head_dim // 2, device=device, dtype=torch.float32) - dist.broadcast(angles, src=0) - position_cos = torch.cat((angles.cos(), angles.cos()), dim=-1) - position_sin = torch.cat((angles.sin(), angles.sin()), dim=-1) - full_output_gradient = torch.randn_like(full_input) - dist.broadcast(full_output_gradient, src=0) - - parameter_seed = _make_parameters( - device, - hidden_size=hidden_size, - query_size=query_size, - key_value_size=key_value_size, - head_dim=head_dim, - ) - for parameter in parameter_seed.values(): - dist.broadcast(parameter, src=0) - async_parameters = _clone_parameters(parameter_seed) - reference_parameters = _clone_parameters(parameter_seed) - - start = rank * local_sequence_length - stop = start + local_sequence_length - local_input = full_input[start:stop].detach().clone().requires_grad_() - reference_input = full_input.detach().clone().requires_grad_() - token_types = torch.arange(sequence_length, device=device) % 2 - if route_case == "empty_gen": - token_types.zero_() - elif route_case == "rank_routes": - token_types = torch.arange(sequence_length, device=device) // local_sequence_length % 2 - elif route_case == "padding": - token_types[-1] = -1 - full_route_0_indices = torch.where(token_types == 0)[0] - full_route_1_indices = torch.where(token_types == 1)[0] - local_route_0_indices = torch.where(token_types[start:stop] == 0)[0] - local_route_1_indices = torch.where(token_types[start:stop] == 1)[0] - - qkv_op = VeomniOp("async_ulysses_qkv", "bagel") - o_op = VeomniOp("async_ulysses_o", "bagel") - rms_op = VeomniOp("rms_norm", "standard", "eager") - rope_op = VeomniOp("rope", "full", "eager") - - async_query, async_key, async_value = qkv_op( - hidden_states=local_input, - route_0_indices=local_route_0_indices, - route_1_indices=local_route_1_indices, - position_cos=position_cos[start:stop], - position_sin=position_sin[start:stop], - q_weight_0=async_parameters["q_weight_0"], - q_bias_0=async_parameters["q_bias_0"], - k_weight_0=async_parameters["k_weight_0"], - k_bias_0=async_parameters["k_bias_0"], - v_weight_0=async_parameters["v_weight_0"], - v_bias_0=async_parameters["v_bias_0"], - q_norm_weight_0=async_parameters["q_norm_weight_0"], - k_norm_weight_0=async_parameters["k_norm_weight_0"], - q_weight_1=async_parameters["q_weight_1"], - q_bias_1=async_parameters["q_bias_1"], - k_weight_1=async_parameters["k_weight_1"], - k_bias_1=async_parameters["k_bias_1"], - v_weight_1=async_parameters["v_weight_1"], - v_bias_1=async_parameters["v_bias_1"], - q_norm_weight_1=async_parameters["q_norm_weight_1"], - k_norm_weight_1=async_parameters["k_norm_weight_1"], - head_dim=head_dim, - rms_norm_eps=eps, - unpadded_dim_size=sequence_length, - group=group, - rms_norm=rms_op, - rope=rope_op, - ) - - reference_query, reference_key, reference_value = _reference_qkv_projection( - reference_input, - full_route_0_indices, - full_route_1_indices, - position_cos, - position_sin, - reference_parameters, - num_query_heads=num_query_heads, - num_key_value_heads=num_key_value_heads, - head_dim=head_dim, - eps=eps, - ) - local_query_heads = num_query_heads // world_size - exchanged_key_value_heads = max(num_key_value_heads, world_size) - local_key_value_heads = exchanged_key_value_heads // world_size - torch.testing.assert_close( - async_query, - reference_query[:, rank * local_query_heads : (rank + 1) * local_query_heads], - ) - repeated_reference_key = torch.repeat_interleave( - reference_key, - repeats=exchanged_key_value_heads // num_key_value_heads, - dim=1, - ) - repeated_reference_value = torch.repeat_interleave( - reference_value, - repeats=exchanged_key_value_heads // num_key_value_heads, - dim=1, - ) - torch.testing.assert_close( - async_key, - repeated_reference_key[:, rank * local_key_value_heads : (rank + 1) * local_key_value_heads], - ) - torch.testing.assert_close( - async_value, - repeated_reference_value[:, rank * local_key_value_heads : (rank + 1) * local_key_value_heads], - ) - - local_gqa_repeats = local_query_heads // local_key_value_heads - async_attention_output = async_query + torch.repeat_interleave( - async_key + async_value, - repeats=local_gqa_repeats, - dim=1, - ) - reference_attention_output = reference_query + torch.repeat_interleave( - reference_key + reference_value, - repeats=num_query_heads // num_key_value_heads, - dim=1, - ) - async_output = o_op( - hidden_states=async_attention_output, - route_0_indices=local_route_0_indices, - route_1_indices=local_route_1_indices, - proj_weight_0=async_parameters["proj_weight_0"], - proj_bias_0=async_parameters["proj_bias_0"], - proj_weight_1=async_parameters["proj_weight_1"], - proj_bias_1=async_parameters["proj_bias_1"], - group=group, - ) - reference_output = _reference_output_projection( - reference_attention_output, - full_route_0_indices, - full_route_1_indices, - reference_parameters, - ) - torch.testing.assert_close(async_output, reference_output[start:stop], atol=2e-5, rtol=2e-5) - - async_output.backward(full_output_gradient[start:stop]) - reference_output.backward(full_output_gradient) - - gathered_input_grads = [torch.empty_like(local_input.grad) for _ in range(world_size)] - dist.all_gather(gathered_input_grads, local_input.grad) - torch.testing.assert_close(torch.cat(gathered_input_grads), reference_input.grad, atol=3e-5, rtol=3e-5) - for name, async_parameter in async_parameters.items(): - assert async_parameter.grad is not None, f"missing async gradient for {name}" - assert reference_parameters[name].grad is not None, f"missing reference gradient for {name}" - async_gradient = async_parameter.grad.detach().clone() - dist.all_reduce(async_gradient, group=group) - torch.testing.assert_close( - async_gradient, - reference_parameters[name].grad, - atol=5e-5, - rtol=5e-5, - msg=f"gradient mismatch for {name}", - ) - - -@pytest.mark.skipif(get_torch_device().device_count() < 4, reason="device_count should be >= 4") -@pytest.mark.parametrize( - ("num_key_value_heads", "route_case"), - [(2, "mixed"), (4, "empty_gen"), (4, "rank_routes"), (4, "padding")], -) -def test_async_ulysses_mot_matches_eager_two_route_projection_forward_backward( - num_key_value_heads: int, - route_case: str, -) -> None: - """Verify two-route BAGEL async Ulysses forward and backward across 4 ranks.""" - torchrun(_async_ulysses_mot_worker, world_size=4, num_key_value_heads=num_key_value_heads, route_case=route_case) - - -def _bagel_fa4_async_worker(ops_impl: str, checkpointing: bool, fsdp: bool, backend: str, dtype: torch.dtype) -> None: - """Compare synchronous and asynchronous Ulysses in ``BagelQwen2MoTAccelerated`` over 2 steps.""" - rank = dist.get_rank() - device = torch.device(f"{get_device_type()}:{rank}") - torch.manual_seed(6321) - enable_high_precision_for_bf16() - state = _init_parallel_state(dp_size=1, ulysses_size=4, name="bagel_sync_test") - async_state = replace(state, async_enabled=True) - - # SP4 leaves seven Q heads and one KV head per rank, matching BAGEL's - # non-power-of-two GQA geometry. Keep weights small while exercising FA4. - config = BagelQwen2MoTConfig( - vocab_size=128, - hidden_size=896, - intermediate_size=1024, - num_hidden_layers=1, - num_attention_heads=28, - num_key_value_heads=4, - max_position_embeddings=1024, - bos_token_id=1, - eos_token_id=2, - ops_implementation={ - "attn_implementation": "flex_attention", - "rms_norm_implementation": ops_impl, - "rotary_pos_emb_implementation": ops_impl, - "swiglu_mlp_implementation": "eager", - }, - ) - reference = accelerated.BagelQwen2MoTAccelerated(config).to(device=device, dtype=dtype).train() - candidate = copy.deepcopy(reference) - if checkpointing: - for model in (reference, candidate): - model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) - if fsdp: - # FP32 parameter/gradient storage with BF16 forward matches the mixed - # precision contract used by the production FSDP2 wrapper. - for model in (reference, candidate): - model.float() - policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32) - for layer in model.model.layers: - fully_shard(layer, mesh=state.fsdp_mesh, mp_policy=policy) - fully_shard(model.model, mesh=state.fsdp_mesh, mp_policy=policy) - - selected_backends = [] - original_resolve = flex_backend.resolve_flex_attention - - def capture_backend(*args, **kwargs): - if backend == "TRITON": - args[1]["BACKEND"] = "TRITON" - interface = original_resolve(*args, **kwargs) - selected_backends.append(interface) - return interface - - flex_backend.resolve_flex_attention = capture_backend - # Two independent documents, a full span, and two mutually invisible noise - # spans exercise real MoT mask rules. The final SP shard includes padding. - sequence_length, padded_length = 259, 260 - metadata = build_mot_attention_metadata( - [[65, 64, 64], [33, 33]], - [["causal", "noise", "noise"], ["full", "causal"]], - device=device, - ) - metadata = pad_mot_attention_metadata(metadata, padded_length=padded_length) - hidden = torch.randn(padded_length, config.hidden_size, device=device, dtype=dtype) - token_types = torch.zeros(padded_length, device=device, dtype=torch.long) - token_types[65:193] = 1 - token_types[sequence_length:] = -1 - hidden[sequence_length:] = 0 - output_gradient = torch.randn_like(hidden) - output_gradient[sequence_length:] = 0 - positions = torch.arange(padded_length, device=device) - start, stop = rank * 65, (rank + 1) * 65 - local_und = torch.where(token_types[start:stop] == 0)[0] - local_gen = torch.where(token_types[start:stop] == 1)[0] - inputs = { - "packed_position_ids": positions[start:stop], - "packed_attention_metadata": metadata, - "packed_und_token_indexes": local_und, - "packed_gen_token_indexes": local_gen, - } - optimizers = [torch.optim.SGD(model.parameters(), lr=1e-3) for model in (reference, candidate)] - try: - # A second update catches weight-view and saved-tensor lifetime errors - # that may be hidden by the first FSDP unshard/recompute pass. - for _ in range(2): - outputs, input_gradients = [], [] - for model, model_state in ((reference, state), (candidate, async_state)): - local_input = hidden[start:stop].detach().clone().requires_grad_() - with use_parallel_state(model_state): - output = model.model(packed_sequence=local_input, **inputs).packed_query_sequence - output.backward(output_gradient[start:stop]) - outputs.append(output.detach()) - input_gradients.append(local_input.grad) - tolerance = 2e-4 if dtype == torch.float32 else 3e-2 - torch.testing.assert_close(outputs[1], outputs[0], atol=tolerance, rtol=tolerance) - torch.testing.assert_close(input_gradients[1], input_gradients[0], atol=tolerance, rtol=tolerance) - for (name, expected), (actual_name, actual) in zip( - reference.named_parameters(), candidate.named_parameters(), strict=True - ): - assert name == actual_name - assert expected.grad is not None and actual.grad is not None, name - expected_grad, actual_grad = expected.grad, actual.grad - if fsdp: - expected_grad, actual_grad = expected_grad.to_local(), actual_grad.to_local() - assert torch.isfinite(actual_grad).all(), name - if dtype == torch.float32: - torch.testing.assert_close( - actual_grad, - expected_grad, - atol=2e-4, - rtol=2e-4, - msg=lambda message, name=name: f"{name}: {message}", - ) - else: - gradient_pairs = [(name, actual_grad, expected_grad)] - if "qkv_proj" in name and not fsdp: - split_sizes = reference.model.layers[0].self_attn.qkv_split_sizes - gradient_pairs = [ - (f"{name}.{part}", actual_part, expected_part) - for part, actual_part, expected_part in zip( - ("q", "k", "v"), - actual_grad.split(split_sizes), - expected_grad.split(split_sizes), - strict=True, - ) - ] - # Split and fused schedules can differ at BF16 precision. - # Check each Q/K/V block separately so a large Q gradient - # cannot hide an incorrect K/V gradient. The FP32 control - # above retains strict pointwise derivative checks. - for gradient_name, actual_part, expected_part in gradient_pairs: - error = (actual_part.float() - expected_part.float()).norm() - scale = expected_part.float().norm() - assert error <= 0.01 * scale + 1e-7, ( - f"{gradient_name}: relative gradient L2 error {error / scale.clamp_min(1e-12)}" - ) - - for optimizer in optimizers: - optimizer.step() - optimizer.zero_grad(set_to_none=True) - expected_backend = ( - flex_backend._flex_attention_fa4 if backend == "FLASH" else flex_backend._flex_attention_triton - ) - assert selected_backends and all(selected is expected_backend for selected in selected_backends) - finally: - flex_backend.resolve_flex_attention = original_resolve - - -@pytest.mark.skipif( - not IS_CUDA_AVAILABLE or get_torch_device().device_count() < 4, - reason="BAGEL FA4 + async Ulysses integration requires four SM90+ GPUs", -) -@pytest.mark.parametrize( - ("ops_impl", "checkpointing", "fsdp", "backend", "dtype"), - [ - pytest.param("eager", False, False, "TRITON", torch.float32, id="fp32-triton-control"), - pytest.param("eager", False, False, "FLASH", torch.bfloat16, id="bf16-fa4-eager"), - pytest.param("liger_kernel", True, False, "FLASH", torch.bfloat16, id="bf16-fa4-liger-checkpoint"), - pytest.param("liger_kernel", True, True, "FLASH", torch.bfloat16, id="bf16-fa4-liger-checkpoint-fsdp2"), - ], -) -def test_bagel_fa4_async_matches_sync_forward_backward_and_updates( - ops_impl: str, - checkpointing: bool, - fsdp: bool, - backend: str, - dtype: torch.dtype, -) -> None: - """Verify BAGEL async Ulysses matches synchronous Ulysses under Flex-FA4, checkpointing, and FSDP2.""" - device = torch.device(get_device_type()) - if backend == "FLASH" and ( - flex_backend.get_gpu_compute_capability(device) < 90 or not flex_backend._fa4._flash_attn_cute_available() - ): - pytest.skip("Flex-FA4 integration requires SM90+ and flash-attn-4") - torchrun( - _bagel_fa4_async_worker, - world_size=4, - ops_impl=ops_impl, - checkpointing=checkpointing, - fsdp=fsdp, - backend=backend, - dtype=dtype, - ) diff --git a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py index 863eb3b6e2..b3fe186308 100644 --- a/tests/seed_omni/modules/bagel/test_bagel_accel_align.py +++ b/tests/seed_omni/modules/bagel/test_bagel_accel_align.py @@ -1,12 +1,15 @@ -"""Accelerated Bagel Qwen2-MoT vs eager CE / MSE alignment on toy data.""" +"""Accelerated BAGEL Qwen2-MoT alignment: single-GPU eager parity and 4-GPU FSDP2 + Async Ulysses.""" from __future__ import annotations import copy import importlib.util +from dataclasses import replace import pytest import torch +import torch.distributed as dist +from torch.distributed.fsdp import MixedPrecisionPolicy, fully_shard from scripts.ci.accelerator.launch_plan import LaunchSpec from tests.seed_omni.modules.bagel.helpers import ( @@ -14,7 +17,6 @@ ALIGN_GRAD_ATOL, ALIGN_GRAD_RTOL, ALIGN_RTOL, - ToyCase, build_toy_conversation, clone_conversation, config_cls, @@ -25,13 +27,22 @@ run_eager_mot, tiny_align_qwen2_cfg, ) +from tests.tools.launch_utils import torchrun +from veomni.distributed.parallel_state import _init_parallel_state, use_parallel_state +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig +from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( + build_mot_attention_metadata, + pad_mot_attention_metadata, +) +from veomni.ops.kernels.attention.standard import flex as flex_backend from veomni.ops.kernels.attention.standard.magi import _kernel as magi_kernel -from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type +from veomni.utils.device import IS_CUDA_AVAILABLE, get_device_type, get_torch_device +from veomni.utils.helper import enable_high_precision_for_bf16 _PATCH_LATENT_DIM = 4 -_CASES: tuple[ToyCase, ...] = ("ce_only", "vit_ce", "mse_only", "mixed") _MAGI_IMPLEMENTATION = "veomni_magi_attention" _FLEX_IMPLEMENTATION = "veomni_flex_attention" @@ -51,46 +62,27 @@ def _is_magi_ffa_available() -> bool: _MAGI_FFA_REASON = "MagiAttention alignment requires an NVIDIA SM100+ GPU with the CUTE DSL/JIT backend" -def _eager_config(): - cfg = tiny_align_qwen2_cfg() - cfg.pop("ops_implementation", None) - return config_cls("bagel_qwen2_mot")(**cfg, attn_implementation="sdpa") - - -def _accelerated_config(attn_implementation: str): - cfg = tiny_align_qwen2_cfg() - cfg.pop("ops_implementation", None) - return config_cls("bagel_qwen2_mot")( - **cfg, - attn_implementation=attn_implementation, - ) - - -def _shared_heads(hidden_size: int, vocab_size: int, device: torch.device) -> tuple[torch.nn.Linear, torch.nn.Linear]: - lm_head = torch.nn.Linear(hidden_size, vocab_size, bias=False).to(device=device, dtype=torch.float32) - llm2vae = torch.nn.Linear(hidden_size, _PATCH_LATENT_DIM, bias=False).to(device=device, dtype=torch.float32) - return lm_head, llm2vae - - def _build_aligned_models(device: torch.device, dtype: torch.dtype, *, attn_implementation: str): torch.manual_seed(29) - eager = native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device=device, dtype=dtype).train() - accelerated = ( - BagelQwen2MoTAccelerated(_accelerated_config(attn_implementation)).to(device=device, dtype=dtype).train() - ) - accelerated.load_state_dict(copy.deepcopy(eager.state_dict())) - return eager, accelerated + cfg = tiny_align_qwen2_cfg() + cfg.pop("ops_implementation", None) + eager_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation="sdpa") + accel_cfg = config_cls("bagel_qwen2_mot")(**cfg, attn_implementation=attn_implementation) + eager = native_model_cls("bagel_qwen2_mot")(eager_cfg).to(device=device, dtype=dtype).train() + accel = BagelQwen2MoTAccelerated(accel_cfg).to(device=device, dtype=dtype).train() + accel.load_state_dict(copy.deepcopy(eager.state_dict())) + return eager, accel -def _assert_accelerated_matches_eager(case: ToyCase, attn_implementation: str) -> None: - """Verify that ``BagelQwen2MoTAccelerated`` matches eager SDPA outputs, losses, and gradients.""" +def _assert_accelerated_matches_eager(attn_implementation: str) -> None: + """Verify ``BagelQwen2MoTAccelerated`` matches eager SDPA outputs, CE/MSE losses, and gradients on mixed data.""" device = torch.device(get_device_type()) dtype = torch.bfloat16 - eager, accelerated = _build_aligned_models(device, dtype, attn_implementation=attn_implementation) + eager, accel = _build_aligned_models(device, dtype, attn_implementation=attn_implementation) hidden_size = int(eager.config.hidden_size) vocab_size = int(eager.config.vocab_size) conversation = build_toy_conversation( - case, + "mixed", hidden_size=hidden_size, vocab_size=vocab_size, patch_latent_dim=_PATCH_LATENT_DIM, @@ -98,45 +90,28 @@ def _assert_accelerated_matches_eager(case: ToyCase, attn_implementation: str) - dtype=dtype, ) eager_conversation = clone_conversation(conversation) - accelerated_conversation = clone_conversation(conversation) - lm_head, llm2vae = _shared_heads(hidden_size, vocab_size, device) + accel_conversation = clone_conversation(conversation) + lm_head = torch.nn.Linear(hidden_size, vocab_size, bias=False).to(device=device, dtype=torch.float32) + llm2vae = torch.nn.Linear(hidden_size, _PATCH_LATENT_DIM, bias=False).to(device=device, dtype=torch.float32) eager_hidden = run_eager_mot(eager, eager_conversation) - accelerated_hidden = run_accelerated_mot(accelerated, accelerated_conversation) - - assert torch.isfinite(eager_hidden).all() - assert torch.isfinite(accelerated_hidden).all() - assert eager_hidden.shape == accelerated_hidden.shape + accel_hidden = run_accelerated_mot(accel, accel_conversation) + assert torch.isfinite(eager_hidden).all() and torch.isfinite(accel_hidden).all() + assert eager_hidden.shape == accel_hidden.shape eager_ce = conversation_ce_loss(eager_conversation, lm_head) - accelerated_ce = conversation_ce_loss(accelerated_conversation, lm_head) + accel_ce = conversation_ce_loss(accel_conversation, lm_head) eager_mse = conversation_mse_loss(eager_conversation, llm2vae) - accelerated_mse = conversation_mse_loss(accelerated_conversation, llm2vae) - - if case in {"ce_only", "vit_ce", "mixed"}: - assert eager_ce is not None and accelerated_ce is not None - assert torch.isfinite(eager_ce) and torch.isfinite(accelerated_ce) - torch.testing.assert_close(accelerated_ce, eager_ce, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) - else: - assert eager_ce is None and accelerated_ce is None - - if case in {"mse_only", "mixed"}: - assert eager_mse is not None and accelerated_mse is not None - assert torch.isfinite(eager_mse) and torch.isfinite(accelerated_mse) - torch.testing.assert_close(accelerated_mse, eager_mse, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) - else: - assert eager_mse is None and accelerated_mse is None - - eager_loss = eager_hidden.float().square().mean() - accelerated_loss = accelerated_hidden.float().square().mean() - if eager_ce is not None: - eager_loss = eager_loss + eager_ce - accelerated_loss = accelerated_loss + accelerated_ce - if eager_mse is not None: - eager_loss = eager_loss + eager_mse - accelerated_loss = accelerated_loss + accelerated_mse + accel_mse = conversation_mse_loss(accel_conversation, llm2vae) + assert eager_ce is not None and accel_ce is not None + assert eager_mse is not None and accel_mse is not None + torch.testing.assert_close(accel_ce, eager_ce, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + torch.testing.assert_close(accel_mse, eager_mse, atol=ALIGN_ATOL, rtol=ALIGN_RTOL) + + eager_loss = eager_hidden.float().square().mean() + eager_ce + eager_mse + accel_loss = accel_hidden.float().square().mean() + accel_ce + accel_mse eager_loss.backward() - accelerated_loss.backward() + accel_loss.backward() for name in ( "model.layers.0.self_attn.o_proj.weight", @@ -144,11 +119,10 @@ def _assert_accelerated_matches_eager(case: ToyCase, attn_implementation: str) - "model.norm.weight", ): eager_grad = dict(eager.named_parameters())[name].grad - accelerated_grad = dict(accelerated.named_parameters())[name].grad - assert eager_grad is not None and accelerated_grad is not None - assert torch.isfinite(eager_grad).all() and torch.isfinite(accelerated_grad).all() + accel_grad = dict(accel.named_parameters())[name].grad + assert eager_grad is not None and accel_grad is not None torch.testing.assert_close( - accelerated_grad, + accel_grad, eager_grad, atol=ALIGN_GRAD_ATOL, rtol=ALIGN_GRAD_RTOL, @@ -156,30 +130,132 @@ def _assert_accelerated_matches_eager(case: ToyCase, attn_implementation: str) - ) +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1)) @pytest.mark.skipif(not IS_CUDA_AVAILABLE, reason="FlexAttention alignment requires CUDA") -@pytest.mark.parametrize("case", _CASES) -def test_accelerated_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: - _assert_accelerated_matches_eager(case, _FLEX_IMPLEMENTATION) +def test_accelerated_ce_mse_matches_eager_on_toy_data() -> None: + _assert_accelerated_matches_eager(_FLEX_IMPLEMENTATION) +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=1)) @pytest.mark.skipif(not _MAGI_FFA_AVAILABLE, reason=_MAGI_FFA_REASON) -@pytest.mark.parametrize("case", _CASES) -def test_magi_ce_mse_matches_eager_on_toy_data(case: ToyCase) -> None: - _assert_accelerated_matches_eager(case, _MAGI_IMPLEMENTATION) - - -@pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) -def test_is_magi_ffa_available_checks_current_kernel_interface(monkeypatch: pytest.MonkeyPatch) -> None: - """Verify ``_is_magi_ffa_available`` queries ``magi_kernel.get_kernel_mode`` cleanly when Magi is installed.""" - monkeypatch.setattr("tests.seed_omni.modules.bagel.test_bagel_accel_align.IS_CUDA_AVAILABLE", True) - monkeypatch.setattr( - "tests.seed_omni.modules.bagel.test_bagel_accel_align.get_device_type", - lambda: magi_kernel.CUDA_DEVICE_TYPE, +def test_magi_ce_mse_matches_eager_on_toy_data() -> None: + _assert_accelerated_matches_eager(_MAGI_IMPLEMENTATION) + + +def _bagel_fa4_async_worker() -> None: + """Compare synchronous and asynchronous Ulysses in ``BagelQwen2MoTAccelerated`` under FSDP2 + Flex-FA4 over 2 steps.""" + rank = dist.get_rank() + device = torch.device(f"{get_device_type()}:{rank}") + dtype = torch.bfloat16 + torch.manual_seed(6321) + enable_high_precision_for_bf16() + state = _init_parallel_state(dp_size=1, ulysses_size=4, name="bagel_sync_test") + async_state = replace(state, async_enabled=True) + + # SP4 leaves seven Q heads and one KV head per rank, matching BAGEL's + # non-power-of-two GQA geometry while exercising FA4 + Liger + checkpointing + FSDP2. + config = BagelQwen2MoTConfig( + vocab_size=128, + hidden_size=896, + intermediate_size=1024, + num_hidden_layers=1, + num_attention_heads=28, + num_key_value_heads=4, + max_position_embeddings=1024, + bos_token_id=1, + eos_token_id=2, + ops_implementation={ + "attn_implementation": "flex_attention", + "rms_norm_implementation": "liger_kernel", + "rotary_pos_emb_implementation": "liger_kernel", + "swiglu_mlp_implementation": "eager", + }, + ) + reference = accelerated.BagelQwen2MoTAccelerated(config).to(device=device, dtype=dtype).train() + candidate = copy.deepcopy(reference) + for model in (reference, candidate): + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + model.float() + policy = MixedPrecisionPolicy(param_dtype=torch.bfloat16, reduce_dtype=torch.float32) + for layer in model.model.layers: + fully_shard(layer, mesh=state.fsdp_mesh, mp_policy=policy) + fully_shard(model.model, mesh=state.fsdp_mesh, mp_policy=policy) + + selected_backends = [] + original_resolve = flex_backend.resolve_flex_attention + + def capture_backend(*args, **kwargs): + interface = original_resolve(*args, **kwargs) + selected_backends.append(interface) + return interface + + flex_backend.resolve_flex_attention = capture_backend + sequence_length, padded_length = 259, 260 + metadata = build_mot_attention_metadata( + [[65, 64, 64], [33, 33]], + [["causal", "noise", "noise"], ["full", "causal"]], + device=device, ) - monkeypatch.setattr(importlib.util, "find_spec", lambda name: object()) + metadata = pad_mot_attention_metadata(metadata, padded_length=padded_length) + hidden = torch.randn(padded_length, config.hidden_size, device=device, dtype=dtype) + token_types = torch.zeros(padded_length, device=device, dtype=torch.long) + token_types[65:193] = 1 + token_types[sequence_length:] = -1 + hidden[sequence_length:] = 0 + output_gradient = torch.randn_like(hidden) + output_gradient[sequence_length:] = 0 + positions = torch.arange(padded_length, device=device) + start, stop = rank * 65, (rank + 1) * 65 + inputs = { + "packed_position_ids": positions[start:stop], + "packed_attention_metadata": metadata, + "packed_und_token_indexes": torch.where(token_types[start:stop] == 0)[0], + "packed_gen_token_indexes": torch.where(token_types[start:stop] == 1)[0], + } + optimizers = [torch.optim.SGD(model.parameters(), lr=1e-3) for model in (reference, candidate)] + try: + for _ in range(2): + outputs, input_gradients = [], [] + for model, model_state in ((reference, state), (candidate, async_state)): + local_input = hidden[start:stop].detach().clone().requires_grad_() + with use_parallel_state(model_state): + output = model.model(packed_sequence=local_input, **inputs).packed_query_sequence + output.backward(output_gradient[start:stop]) + outputs.append(output.detach()) + input_gradients.append(local_input.grad) + torch.testing.assert_close(outputs[1], outputs[0], atol=3e-2, rtol=3e-2) + torch.testing.assert_close(input_gradients[1], input_gradients[0], atol=3e-2, rtol=3e-2) + for (name, expected), (actual_name, actual) in zip( + reference.named_parameters(), candidate.named_parameters(), strict=True + ): + assert name == actual_name + assert expected.grad is not None and actual.grad is not None, name + expected_grad, actual_grad = expected.grad.to_local(), actual.grad.to_local() + assert torch.isfinite(actual_grad).all(), name + error = (actual_grad.float() - expected_grad.float()).norm() + scale = expected_grad.float().norm() + assert error <= 0.01 * scale + 1e-7, ( + f"{name}: relative gradient L2 error {error / scale.clamp_min(1e-12)}" + ) + + for optimizer in optimizers: + optimizer.step() + optimizer.zero_grad(set_to_none=True) + assert selected_backends and all( + selected is flex_backend._flex_attention_fa4 for selected in selected_backends + ) + finally: + flex_backend.resolve_flex_attention = original_resolve - monkeypatch.setattr(magi_kernel, "get_kernel_mode", lambda device: magi_kernel.KERNEL_UNSUPPORTED) - assert _is_magi_ffa_available() is False - monkeypatch.setattr(magi_kernel, "get_kernel_mode", lambda device: magi_kernel.KERNEL_CUTE_JIT) - assert _is_magi_ffa_available() is True +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=4)) +@pytest.mark.skipif( + not IS_CUDA_AVAILABLE or get_torch_device().device_count() < 4, + reason="BAGEL FA4 + async Ulysses integration requires four SM90+ GPUs", +) +def test_bagel_fa4_async_matches_sync_forward_backward_and_updates() -> None: + """Verify BAGEL async Ulysses matches synchronous Ulysses under Flex-FA4, Liger, checkpointing, and FSDP2.""" + device = torch.device(get_device_type()) + if flex_backend.get_gpu_compute_capability(device) < 90 or not flex_backend._fa4._flash_attn_cute_available(): + pytest.skip("Flex-FA4 integration requires SM90+ and flash-attn-4") + torchrun(_bagel_fa4_async_worker, world_size=4) diff --git a/tests/seed_omni/modules/bagel/test_bagel_eager.py b/tests/seed_omni/modules/bagel/test_bagel_eager.py deleted file mode 100644 index 92d061c2c6..0000000000 --- a/tests/seed_omni/modules/bagel/test_bagel_eager.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Smoke coverage for native eager Bagel Qwen2-MoT train and sequential infer.""" - -from __future__ import annotations - -import pytest -import torch - -from scripts.ci.accelerator.launch_plan import LaunchSpec -from tests.seed_omni.modules.bagel.helpers import ( - build_toy_conversation, - config_cls, - native_model_cls, - run_eager_mot, - tiny_bagel_qwen2_cfg, -) -from veomni.models.seed_omni.modules.bagel.sources import ( - BAGEL_FLOW_HIDDEN, - BAGEL_FLOW_QUERY, - BAGEL_PHASE_KEY, - BAGEL_START_TOKEN, -) -from veomni.models.seed_omni.utils.conversation import ConversationItem - - -pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) - - -def _eager_config(**overrides): - return config_cls("bagel_qwen2_mot")(**{**tiny_bagel_qwen2_cfg(), **overrides, "attn_implementation": "sdpa"}) - - -def test_eager_packed_train_forward_runs() -> None: - """Verify eager packed MoT forward and backward populate both und and gen expert gradients.""" - model = native_model_cls("bagel_qwen2_mot")(_eager_config()).train() - conversation = build_toy_conversation( - "mixed", - hidden_size=int(model.config.hidden_size), - vocab_size=int(model.config.vocab_size), - patch_latent_dim=4, - device=model.device, - dtype=model.dtype, - ) - - hidden_states = run_eager_mot(model, conversation) - assert hidden_states.shape[-1] == int(model.config.hidden_size) - assert torch.isfinite(hidden_states).all() - - hidden_states.float().square().mean().backward() - query_grad = model.model.layers[0].self_attn.q_proj.weight.grad - gen_query_grad = model.model.layers[0].self_attn.q_proj_moe_gen.weight.grad - assert query_grad is not None and torch.isfinite(query_grad).all() - assert gen_query_grad is not None and torch.isfinite(gen_query_grad).all() - assert query_grad.abs().max() > 0 - assert gen_query_grad.abs().max() > 0 - - -def _eager_model(dtype: torch.dtype): - return native_model_cls("bagel_qwen2_mot")(_eager_config()).to(device="cpu", dtype=dtype).eval() - - -@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) -def test_eager_und_generate_runs(dtype: torch.dtype) -> None: - """Verify eager understanding generation preserves the configured dtype across FP32, FP16, and BF16.""" - model = _eager_model(dtype) - prompt = ConversationItem( - type="text", - value=torch.randn(4, int(model.config.hidden_size), device=model.device, dtype=model.dtype), - role="user", - ) - bos = ConversationItem( - type="text", - value=torch.randn(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype), - role="assistant", - meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, - ) - - outputs = model.generate([prompt, bos]) - conversation = outputs["conversation_list"] - tail = conversation[-1] - assert model._generation_state.main.cache is not None - assert tail.type == "output" - assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN - assert torch.is_tensor(tail.value) - assert tail.value.dtype == dtype - assert tail.value.shape == (1, int(model.config.hidden_size)) - assert torch.isfinite(tail.value).all() - - outputs = model.generate(conversation) - tail = outputs["conversation_list"][-1] - assert tail.type == "output" - assert BAGEL_PHASE_KEY not in tail.meta - - outputs = model.generate(outputs["conversation_list"]) - tail = outputs["conversation_list"][-1] - assert tail.type == "output" - assert torch.is_tensor(tail.value) - assert tail.value.dtype == dtype - assert tail.value.shape == (1, int(model.config.hidden_size)) - assert torch.isfinite(tail.value).all() - - -@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) -def test_eager_gen_denoise_runs(dtype: torch.dtype) -> None: - """Verify eager ``denoise_branch`` preserves the configured dtype across FP32, FP16, and BF16.""" - model = _eager_model(dtype) - hidden_size = int(model.config.hidden_size) - prompt = ConversationItem( - type="text", - value=torch.randn(3, hidden_size, device=model.device, dtype=model.dtype), - role="user", - ) - model.generate([prompt], generation_kwargs={"infer_type": "infer_gen"}) - - query = ConversationItem( - type="output", - value=torch.randn(5, hidden_size, device=model.device, dtype=model.dtype), - role="assistant", - meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, - ) - outputs = model.denoise_branch( - [query], - generation_kwargs={"infer_type": "infer_gen", "cfg_text_scale": 1.0, "cfg_img_scale": 1.0}, - ) - tail = outputs["conversation_list"][-1] - assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN - assert torch.is_tensor(tail.value) - assert tail.value.dtype == dtype - assert tail.value.shape[0] == 5 - assert tail.value.shape[-1] == hidden_size - assert torch.isfinite(tail.value).all() diff --git a/tests/seed_omni/modules/bagel/test_generation.py b/tests/seed_omni/modules/bagel/test_generation.py index 783f0d6d96..ff15caea08 100644 --- a/tests/seed_omni/modules/bagel/test_generation.py +++ b/tests/seed_omni/modules/bagel/test_generation.py @@ -1,340 +1,220 @@ -"""BAGEL generation graph smoke and eager/accelerated denoise structure.""" +"""End-to-end BAGEL generation tests across ``infer_und``, ``infer_gen``, and ``infer_edit``.""" from __future__ import annotations +from pathlib import Path + import pytest import torch -import torch.nn as nn from PIL import Image from scripts.ci.accelerator.launch_plan import LaunchSpec from tests.seed_omni.modules.bagel.helpers import ( + FakeBagelTokenizer, bagel_cfg_dir, + bagel_template, config_cls, load_omni_config, native_model_cls, tiny_bagel_qwen2_cfg, ) -from veomni.models.seed_omni.accelerated import OmniModelRuntime -from veomni.models.seed_omni.graphs.generation_graph import FSM_SIGNAL_KEY -from veomni.models.seed_omni.mixins.base_mixin import BaseMixin -from veomni.models.seed_omni.mixins.inference_module_mixin import InferenceModuleMixin from veomni.models.seed_omni.modeling_omni import OmniModel +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( + BagelSiglipNavitPreprocessor, + BagelSiglipNavitProcessor, +) from veomni.models.seed_omni.modules.bagel.sources import ( BAGEL_CONTEXT_KEY, BAGEL_FLOW_HIDDEN, BAGEL_FLOW_QUERY, - BAGEL_FLOW_VELOCITY, - BAGEL_GENERATED_LATENT, BAGEL_PHASE_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_START_TOKEN, BAGEL_VAE_CONTEXT, ) -from veomni.models.seed_omni.modules.module_configuration_base import OmniModuleConfig -from veomni.models.seed_omni.modules.module_modeling_base import PretrainedOmniModule +from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor +from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEPreprocessor, BagelVAEProcessor +from veomni.models.seed_omni.processing_omni import OmniProcessor from veomni.models.seed_omni.utils.conversation import ConversationItem -from veomni.models.seed_omni.utils.graph_profiler import GraphProfiler pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) -def _make_veomni_runtime(cfg, modules): - # The stubs implement generation endpoints only; OmniModel checks every graph's methods. - cfg.training_graphs = {} - model = OmniModel(cfg, modules).eval() - return OmniModelRuntime(model), model - - -def test_bagel_text_encoder_injects_raw_bos_for_understanding() -> None: - """Verify ``BagelTextEncoderPreprocessor`` appends a ``BAGEL_START_TOKEN`` BOS item for ``infer_und`` and ``BagelTextEncoder.generate`` embeds it.""" - from tests.seed_omni.modules.bagel.test_processing import _bagel_template - from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor - - BagelTextEncoder = native_model_cls("bagel_text_encoder") - BagelTextEncoderConfig = config_cls("bagel_text_encoder") - model = BagelTextEncoder(BagelTextEncoderConfig(vocab_size=256, hidden_size=4)).eval() - model._chat_template = _bagel_template() - with torch.no_grad(): - model.embed_tokens.weight.copy_(torch.arange(256 * 4, dtype=torch.float32).reshape(256, 4)) - - preprocessor = BagelTextEncoderPreprocessor(model._chat_template) - sample = [ConversationItem(type="text", value="hi", role="user")] - preprocessor({"conversation_list": [sample]}, inference=True, generation_kwargs={"infer_type": "infer_und"}) - - assert len(sample) == 2 - assert sample[-1].type == "text" - assert sample[-1].role == "assistant" - assert sample[-1].meta[BAGEL_PHASE_KEY] == BAGEL_START_TOKEN - bos_id = model._chat_template.bos_token_id - assert sample[-1].meta["input_ids"].tolist() == [bos_id] - - conversation = model.generate(sample)["conversation_list"] - assert len(conversation) == 2 - assert conversation[-1].meta[BAGEL_PHASE_KEY] == BAGEL_START_TOKEN - torch.testing.assert_close(conversation[-1].value, model.embed_tokens.weight[bos_id].reshape(1, 4)) - - -def test_bagel_qwen_first_understanding_step_prefills_prompt_then_decodes_bos(monkeypatch) -> None: - """Verify ``BagelQwen2MoT.generate`` prefills the full ``conversation_list`` (including ``BAGEL_START_TOKEN``) on ``prompt_encode`` and decodes the tail ``output`` item on ``text_ar``.""" - BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).eval() - hidden_size = int(model.config.hidden_size) - prompt = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") - bos = ConversationItem( - type="text", - value=torch.full((1, hidden_size), 2.0), - role="assistant", - meta={BAGEL_PHASE_KEY: BAGEL_START_TOKEN}, - ) - calls: list[tuple[str, list[ConversationItem]]] = [] - - def _prefill(self, conversation_list, generation_kwargs): - del generation_kwargs - calls.append(("prefill", list(conversation_list))) - self._generation_state.main.install_cache( - cache=self._new_empty_cache(), - cache_len=3, - next_position_id=torch.tensor(3), - device=self.device, - ) - return torch.full((3, hidden_size), 3.0) - - def _decode(self, conversation_list): - del self - calls.append(("decode", list(conversation_list))) - return torch.full((1, hidden_size), 4.0) - - monkeypatch.setattr(type(model), "_prefill_prompt", _prefill) - monkeypatch.setattr(type(model), "_decode_next_token", _decode) - - conversation = model.generate([prompt, bos])["conversation_list"] - assert calls == [("prefill", [prompt, bos])] - assert bos.type == "output" - assert bos.meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN - torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 3.0)) - - conversation = model.generate(conversation)["conversation_list"] - assert calls == [("prefill", [prompt, bos])] - assert BAGEL_PHASE_KEY not in bos.meta - torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 3.0)) - - conversation = model.generate(conversation)["conversation_list"] - assert calls == [("prefill", [prompt, bos]), ("decode", [prompt, bos])] - assert conversation == [prompt, bos] - assert bos.type == "output" - torch.testing.assert_close(bos.value, torch.full((1, hidden_size), 4.0)) - - -def test_bagel_infer_gen_denoise_signal_smoke(): - """Verify the ``infer_gen`` FSM transitions through ``prompt_encode``, ``query_denoise``, ``velocity_collect``, and ``image_decode``.""" +def _build_tiny_bagel_omni_model_and_processor(infer_graph_rel: str) -> tuple[OmniModel, OmniProcessor]: + """Build a tiny 5-module real BAGEL ``OmniModel`` and ``OmniProcessor`` for ``infer_graph_rel``.""" cfg = load_omni_config( modules_path=bagel_cfg_dir() / "train/modules_train.yaml", train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", - ) - runtime, _model = _make_veomni_runtime( - cfg, - { - "bagel_text_encoder": _InferGenTextEncoder(), - "bagel_siglip_navit": _NoopBagelSiglip(), - "bagel_qwen2_mot": _InferGenBagelQwen(), - "bagel_flow_connector": _InferGenBagelFlow(), - "bagel_vae": _InferGenBagelVAE(), - }, - ) - profiler = GraphProfiler() - request = {"conversation_list": [ConversationItem(type="text", value="prompt", role="user")]} - generated = runtime.generate( - request, - profiler=profiler, - generation_kwargs={ - "max_new_tokens": 8, - "do_sample": False, - "image_height": 64, - "image_width": 64, - }, + infer_graph_path=bagel_cfg_dir() / infer_graph_rel, ) + cfg.training_graphs = {} - trace = profiler.save_records() - assert any("transition: prompt_encode -> query_denoise" in entry for entry in trace) - assert any("transition: query_denoise -> velocity_collect" in entry for entry in trace) - assert any("transition: velocity_collect -> image_decode" in entry for entry in trace) - assert any("transition: image_decode -> done" in entry for entry in trace) - assert any(item["type"] == "image" for item in generated) - assert "timestep" not in request["conversation_list"][-1].meta + hidden_size = 8 + qwen_kwargs = { + **tiny_bagel_qwen2_cfg(), + "hidden_size": hidden_size, + "intermediate_size": 16, + "num_attention_heads": 2, + "num_key_value_heads": 2, + } + text_encoder = native_model_cls("bagel_text_encoder")( + config_cls("bagel_text_encoder")(vocab_size=int(qwen_kwargs["vocab_size"]), hidden_size=hidden_size) + ).eval() + text_encoder._tokenizer = FakeBagelTokenizer() + text_encoder._chat_template = bagel_template() + + siglip = native_model_cls("bagel_siglip_navit")( + config_cls("bagel_siglip_navit")( + hidden_size=hidden_size, + output_size=hidden_size, + intermediate_size=16, + num_hidden_layers=1, + num_attention_heads=2, + image_size=4, + patch_size=2, + rope=False, + ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"}, + ) + ).eval() + qwen_mot = native_model_cls("bagel_qwen2_mot")(config_cls("bagel_qwen2_mot")(**qwen_kwargs)).eval() + + flow = native_model_cls("bagel_flow_connector")( + config_cls("bagel_flow_connector")( + hidden_size=hidden_size, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + resolution=4, + ) + ).eval() -def test_bagel_infer_gen_user_image_runs_siglip_context_only(): - """Verify ``infer_gen`` encodes user images through SigLIP without creating VAE context items.""" - cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", - ) - siglip = _CountingInferGenBagelSiglip() - runtime, _model = _make_veomni_runtime( + vae_cfg = config_cls("bagel_vae")( + resolution=4, + ch=32, + ch_mult=[1], + num_res_blocks=1, + z_channels=2, + max_image_size=4, + min_image_size=4, + image_stride=2, + max_pixels=16, + downsample=1, + ) + vae = native_model_cls("bagel_vae")(vae_cfg).eval() + vae._image_processor = BagelVAEProcessor.from_config(vae_cfg) + + model = OmniModel( cfg, { - "bagel_text_encoder": _InferGenTextEncoder(), + "bagel_text_encoder": text_encoder, "bagel_siglip_navit": siglip, - "bagel_qwen2_mot": _InferGenBagelQwen(), - "bagel_flow_connector": _InferGenBagelFlow(), - "bagel_vae": _InferGenBagelVAE(), + "bagel_qwen2_mot": qwen_mot, + "bagel_flow_connector": flow, + "bagel_vae": vae, }, - ) - request = { - "conversation_list": [ - ConversationItem( - type="image", - value=Image.new("RGB", (1, 1)), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, + ).eval() + preprocessors = { + "bagel_vae": BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.float32, + ), + "bagel_siglip_navit": BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, ), - ConversationItem(type="text", value="prompt", role="user"), - ] + dtype=torch.float32, + ), + "bagel_text_encoder": BagelTextEncoderPreprocessor(bagel_template()), } - generated = runtime.generate( - request, - generation_kwargs={ - "max_new_tokens": 8, - "do_sample": False, - "image_height": 64, - "image_width": 64, - }, - ) + return model, OmniProcessor(preprocessors, infer_type=model.config.infer_type) - assert siglip.calls == 1 - assert all(item.meta.get(BAGEL_CONTEXT_KEY) != BAGEL_VAE_CONTEXT for item in request["conversation_list"]) - assert torch.equal(request["conversation_list"][0].value[0], torch.zeros(8)) - assert torch.equal(request["conversation_list"][0].value[1:], torch.ones(3, 8)) - assert any(item["type"] == "image" for item in generated) +def test_bagel_e2e_generation_across_und_gen_and_edit_scenarios(tmp_path: Path) -> None: + """Verify real 5-module BAGEL ``OmniProcessor`` + ``OmniModel.generate`` across ``infer_und``, ``infer_gen``, and ``infer_edit``.""" + image_path = tmp_path / "sample.png" + Image.new("RGB", (4, 4), color=(12, 34, 56)).save(str(image_path)) -def test_bagel_infer_edit_defaults_to_denoise_signal_smoke(): - """Verify ``infer_edit`` encodes both VAE and SigLIP context and runs the denoise loop to completion.""" - cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_edit.yaml", + # 1. infer_und: routes image to SigLIP only, appends BAGEL_START_TOKEN, prefills + decodes text + und_model, und_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_und.yaml") + assert und_model.config.infer_type == "infer_und" + und_request = und_processor(text="hi", images=str(image_path)) + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_request["conversation_list"] if item.type == "image"] == [ + BAGEL_SIGLIP_CONTEXT + ] + assert und_request["conversation_list"][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN + und_model.reset() + und_generated = und_model.generate( + und_request, + generation_kwargs={"max_new_tokens": 2, "do_sample": False}, ) - runtime, _model = _make_veomni_runtime( - cfg, - { - "bagel_text_encoder": _InferGenTextEncoder(), - "bagel_siglip_navit": _NoopBagelSiglip(), - "bagel_qwen2_mot": _InferGenBagelQwen(), - "bagel_flow_connector": _InferEditBagelFlow(), - "bagel_vae": _InferEditBagelVAE(), + assert any(item["type"] == "text" for item in und_generated) + assert und_model.modules_dict["bagel_qwen2_mot"]._generation_state.main.cache is not None + + # 2. infer_gen: text prompt -> CFG flow denoise loop -> decoded PIL image + gen_model, gen_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_gen.yaml") + assert gen_model.config.infer_type == "infer_gen" + gen_request = gen_processor(text="draw") + gen_model.reset() + gen_generated = gen_model.generate( + gen_request, + generation_kwargs={ + "num_timesteps": 2, + "image_height": 4, + "image_width": 4, + "latent_downsample": 2, + "cfg_text_scale": 2.0, + "cfg_img_scale": 1.0, }, ) - profiler = GraphProfiler() - request = { - "conversation_list": [ - ConversationItem( - type="image", - value=Image.new("RGB", (1, 1)), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, - ), - ConversationItem( - type="image", - value=Image.new("RGB", (1, 1)), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ), - ConversationItem(type="text", value="prompt", role="user"), - ] - } - generated = runtime.generate( - request, - profiler=profiler, + assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in gen_generated) + + # 3. infer_edit: routes user image to both VAE and SigLIP context -> 3-branch CFG denoise -> decoded PIL image + edit_model, edit_processor = _build_tiny_bagel_omni_model_and_processor("infer/graph_infer_edit.yaml") + assert edit_model.config.infer_type == "infer_edit" + edit_request = edit_processor(text="edit", images=str(image_path)) + assert [ + item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_request["conversation_list"] if item.type == "image" + ] == [BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT] + edit_model.reset() + edit_generated = edit_model.generate( + edit_request, generation_kwargs={ - "max_new_tokens": 8, - "do_sample": False, - "image_height": 64, - "image_width": 64, + "num_timesteps": 2, + "image_height": 4, + "image_width": 4, + "latent_downsample": 2, + "cfg_text_scale": 2.0, + "cfg_img_scale": 1.5, }, ) - - trace = profiler.save_records() - assert any("transition: prompt_encode -> query_denoise" in entry for entry in trace) - assert not any("transition: prompt_encode -> text_ar" in entry for entry in trace) - assert any("transition: image_decode -> done" in entry for entry in trace) - assert any(item["type"] == "image" for item in generated) - assert "timestep" not in request["conversation_list"][-1].meta + assert any(item["type"] == "image" and isinstance(item["value"], Image.Image) for item in edit_generated) -def _fake_cache(model: nn.Module, values: torch.Tensor): - cache = model._new_empty_cache() - cache.key_cache[0] = values.reshape(-1, 1, 1) - cache.value_cache[0] = (values + 100.0).reshape(-1, 1, 1) - return cache - - -def _install_three_branch_caches(model: nn.Module) -> None: +def test_bagel_qwen2_mot_accelerated_denoise_branch_packs_cfg_branches(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify accelerated ``denoise_branch`` stacks active CFG branches into one packed ``forward_inference`` call.""" + model = BagelQwen2MoTAccelerated(config_cls("bagel_qwen2_mot")(**tiny_bagel_qwen2_cfg())) state = model._generation_state - state.main.install_cache( - cache=_fake_cache(model, torch.tensor([10.0, 11.0])), - cache_len=2, - next_position_id=torch.tensor(3), - device=model.device, - ) - state.cfg_text.install_cache( - cache=model._new_empty_cache(), - cache_len=0, - next_position_id=torch.tensor(7), - device=model.device, - ) - state.cfg_img.install_cache( - cache=_fake_cache(model, torch.tensor([20.0, 21.0, 22.0])), - cache_len=3, - next_position_id=torch.tensor(11), - device=model.device, - ) - - -def test_bagel_qwen2_mot_eager_denoise_branch_runs_serial_forward_inference(monkeypatch): - """Verify eager ``denoise_branch`` runs one serial ``forward_inference`` call per active CFG branch.""" - BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) - _install_three_branch_caches(model) - query = torch.zeros(5, int(model.config.hidden_size)) - calls: list[dict[str, object]] = [] - - def _capture_forward_inference(self, **kwargs): - del self - calls.append(kwargs) - return {"hidden_states": kwargs["packed_query_sequence"]} - - monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference) - tail = ConversationItem( - type="output", - value=query, - role="assistant", - meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, - ) - model.denoise_branch([tail], generation_kwargs={"cfg_text_scale": 2.0, "cfg_img_scale": 1.5}) - - assert len(calls) == 3 - assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN - assert int(tail.value.shape[0]) == 15 - assert calls[0]["query_lens"].tolist() == [5] - assert calls[1]["query_lens"].tolist() == [5] - assert calls[2]["query_lens"].tolist() == [5] - - -def test_bagel_qwen2_mot_accelerated_denoise_branch_packs_cfg_branches(monkeypatch): - """Verify accelerated ``denoise_branch`` stacks all active CFG branches into one packed ``forward_inference`` call.""" - from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAccelerated + for branch, length, next_pos in ((state.main, 2, 3), (state.cfg_text, 0, 7), (state.cfg_img, 3, 11)): + cache = model._new_empty_cache() + if length > 0: + cache.key_cache[0] = torch.ones(length, 1, 1) + cache.value_cache[0] = torch.ones(length, 1, 1) + branch.install_cache( + cache=cache, + cache_len=length, + next_position_id=torch.tensor(next_pos), + device=model.device, + ) - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - model = BagelQwen2MoTAccelerated(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) - _install_three_branch_caches(model) - query = torch.zeros(5, int(model.config.hidden_size)) captured: dict[str, object] = {} def _capture_forward_inference(self, **kwargs): @@ -345,7 +225,7 @@ def _capture_forward_inference(self, **kwargs): monkeypatch.setattr(type(model), "forward_inference", _capture_forward_inference) tail = ConversationItem( type="output", - value=query, + value=torch.zeros(5, int(model.config.hidden_size)), role="assistant", meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": 0.5}, ) @@ -355,335 +235,3 @@ def _capture_forward_inference(self, **kwargs): assert int(captured["packed_query_sequence"].shape[0]) == 15 assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN assert int(tail.value.shape[0]) == 15 - - -def _fake_cfg_branch_count(generation_kwargs: dict | None) -> int: - """Return the number of active CFG branches implied by ``generation_kwargs``.""" - branch_count = 1 - if float((generation_kwargs or {}).get("cfg_text_scale", 1.0)) > 1.0: - branch_count += 1 - if float((generation_kwargs or {}).get("cfg_img_scale", 1.0)) > 1.0: - branch_count += 1 - return branch_count - - -class _StubConfig(OmniModuleConfig): - model_type = "bagel_generation_stub" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - -class _StubModule(PretrainedOmniModule, BaseMixin, InferenceModuleMixin): - """Weightless graph participant: ``OmniModel`` only takes ``PretrainedOmniModule``.""" - - config_class = _StubConfig - - def _check_and_adjust_attn_implementation(self, attn_implementation, **kwargs): - return attn_implementation - - def __init__(self): - super().__init__(_StubConfig()) - - -class _NoopBagelSiglip(_StubModule): - def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - return {"conversation_list": conversation_list} - - -class _CountingInferGenBagelSiglip(_StubModule): - def __init__(self): - super().__init__() - self.calls = 0 - - def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - self.calls += 1 - assert not any( - item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - for item in conversation_list - ) - for item in conversation_list: - if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT: - item.value = torch.ones(2, 8) - return {"conversation_list": conversation_list} - - -class _InferGenTextEncoder(_StubModule): - def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - return {"conversation_list": conversation_list} - - def encode_image_markers(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - for item in conversation_list: - if item.type not in {"image", "output"} or not torch.is_tensor(item.value) or item.value.dim() != 2: - continue - item.value = torch.cat([torch.zeros(1, 8), item.value, torch.ones(1, 8)], dim=0) - return {"conversation_list": conversation_list} - - -class _InferGenBagelQwen(_StubModule): - def generate( - self, - conversation_list: list[ConversationItem] | None = None, - generation_kwargs: dict | None = None, - **kwargs, - ): - del kwargs - assert conversation_list is not None - if not conversation_list or conversation_list[-1].type != "output": - return {"conversation_list": conversation_list} - tail = conversation_list[-1] - if tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_QUERY: - tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN - tail.value = tail.value.repeat(_fake_cfg_branch_count(generation_kwargs), 1) - return {"conversation_list": conversation_list} - if tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_VELOCITY: - tail.value = torch.zeros(16, 4) - return {"conversation_list": conversation_list} - return {"conversation_list": conversation_list} - - def denoise_branch( - self, - conversation_list: list[ConversationItem] | None = None, - generation_kwargs: dict | None = None, - **kwargs, - ): - del kwargs - assert conversation_list is not None - tail = conversation_list[-1] - assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_QUERY - tail.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_HIDDEN - tail.value = tail.value.repeat(_fake_cfg_branch_count(generation_kwargs), 1) - return {"conversation_list": conversation_list} - - def collect_velocity( - self, - conversation_list: list[ConversationItem] | None = None, - generation_kwargs: dict | None = None, - **kwargs, - ): - del kwargs - assert conversation_list is not None - tail = conversation_list[-1] - assert tail.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_VELOCITY - assert tail.value.shape[0] == 18 * _fake_cfg_branch_count(generation_kwargs) - tail.value = torch.zeros(16, 4) - return {"conversation_list": conversation_list} - - -class _InferGenBagelFlow(_StubModule): - def prepare_denoise_query(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - if ( - conversation_list - and conversation_list[-1].type == "output" - and not torch.is_tensor(conversation_list[-1].value) - ): - item = conversation_list[-1] - else: - item = conversation_list[-1] if conversation_list and conversation_list[-1].type == "output" else None - if item is None: - conversation_list.append( - ConversationItem( - type="output", - value=torch.zeros(16, 8), - role="assistant", - meta={BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": torch.tensor(0.5)}, - ) - ) - else: - item.value = torch.zeros(16, 8) - item.meta = {BAGEL_PHASE_KEY: BAGEL_FLOW_QUERY, "timestep": torch.tensor(0.5)} - return {"conversation_list": conversation_list} - - def decode_velocity_from_hidden(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - item = conversation_list[-1] - assert item.meta.get(BAGEL_PHASE_KEY) == BAGEL_FLOW_HIDDEN - assert item.value.shape[0] % 18 == 0 - item.value = torch.zeros(item.value.shape[0], 4) - item.meta[BAGEL_PHASE_KEY] = BAGEL_FLOW_VELOCITY - return {"conversation_list": conversation_list} - - def advance_denoise(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - item = conversation_list[-1] - item.value = torch.zeros(1, 4, 4) - item.meta[BAGEL_PHASE_KEY] = BAGEL_GENERATED_LATENT - item.meta.pop("timestep", None) - return {"conversation_list": conversation_list, FSM_SIGNAL_KEY: "image_complete"} - - -class _InferGenBagelVAE(_StubModule): - def decode_generated(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - assert "timestep" not in conversation_list[-1].meta - return { - "conversation_list": conversation_list, - "generated": {"type": "image", "value": Image.new("RGB", (1, 1)), "meta": {}}, - } - - -class _InferEditBagelVAE(_InferGenBagelVAE): - def encode_context(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - for item in conversation_list: - if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT: - item.value = torch.zeros(4, 4, 4) - return {"conversation_list": conversation_list} - - -class _InferEditBagelFlow(_InferGenBagelFlow): - def embed_context_latents(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - for item in conversation_list: - if ( - item.type == "image" - and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - and torch.is_tensor(item.value) - ): - item.value = torch.zeros(16, 8) - item.meta = {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} - return {"conversation_list": conversation_list} - - -def test_bagel_native_omni_model_and_processor_propagate_resolved_infer_type_without_duplication(tmp_path): - """Verify native ``OmniProcessor`` and ``OmniModel.generate`` propagate ``config.infer_type`` to preprocessing and module calls without caller duplication.""" - from tests.seed_omni.modules.bagel.test_processing import _bagel_template, _FakeBagelTokenizer - from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( - BagelSiglipNavitPreprocessor, - BagelSiglipNavitProcessor, - ) - from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor - from veomni.models.seed_omni.modules.bagel.vae.processing import BagelVAEPreprocessor, BagelVAEProcessor - from veomni.models.seed_omni.processing_omni import OmniProcessor - - image_path = tmp_path / "sample.png" - Image.new("RGB", (8, 8), color=(12, 34, 56)).save(str(image_path)) - - def _build_preprocessors(): - return { - "bagel_vae": BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.float32, - ), - "bagel_siglip_navit": BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), - dtype=torch.float32, - ), - "bagel_text_encoder": BagelTextEncoderPreprocessor(_bagel_template()), - } - - # 1. Default converted checkpoint scenario (infer_und): real BagelTextEncoder + BagelQwen2MoT - und_cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_und.yaml", - ) - und_cfg.training_graphs = {} - assert und_cfg.infer_type == "infer_und" - - BagelTextEncoder = native_model_cls("bagel_text_encoder") - BagelTextEncoderConfig = config_cls("bagel_text_encoder") - BagelQwen2MoT = native_model_cls("bagel_qwen2_mot") - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - - qwen_kwargs = { - **tiny_bagel_qwen2_cfg(), - "hidden_size": 8, - "intermediate_size": 16, - "num_attention_heads": 2, - "num_key_value_heads": 2, - } - hidden_size = int(qwen_kwargs["hidden_size"]) - - def _make_real_text_and_mot(): - text_enc = BagelTextEncoder( - BagelTextEncoderConfig(vocab_size=int(qwen_kwargs["vocab_size"]), hidden_size=hidden_size) - ).eval() - text_enc._tokenizer = _FakeBagelTokenizer() - text_enc._chat_template = _bagel_template() - text_enc._sample_token = lambda hidden_states, **kwargs: text_enc._chat_template.eos_token_id - mot = BagelQwen2MoT(BagelQwen2MoTConfig(**qwen_kwargs)).eval() - return text_enc, mot - - class _DimAlignedSiglip(_NoopBagelSiglip): - def generate(self, conversation_list: list[ConversationItem] | None = None, **kwargs): - del kwargs - assert conversation_list is not None - for item in conversation_list: - if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT: - item.value = torch.zeros(4, hidden_size) - return {"conversation_list": conversation_list} - - und_text_encoder, und_qwen_mot = _make_real_text_and_mot() - und_model = OmniModel( - und_cfg, - { - "bagel_text_encoder": und_text_encoder, - "bagel_siglip_navit": _DimAlignedSiglip(), - "bagel_qwen2_mot": und_qwen_mot, - "bagel_flow_connector": _InferGenBagelFlow(), - "bagel_vae": _InferGenBagelVAE(), - }, - ).eval() - und_processor = OmniProcessor(_build_preprocessors(), infer_type=und_model.config.infer_type) - - # Caller passes neither infer_type to processor(...) nor infer_type in generation_kwargs - und_request = und_processor(text="hi", images=str(image_path)) - assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_request["conversation_list"] if item.type == "image"] == [ - BAGEL_SIGLIP_CONTEXT - ] - und_model.reset() - und_generated = und_model.generate(und_request, generation_kwargs={"max_new_tokens": 4}) - assert any(item["type"] == "text" for item in und_generated) - assert und_qwen_mot._generation_state.main.cache is not None - - # 2. Image edit scenario (infer_edit): processor routes both VAE + SigLIP and real text_encoder + qwen_mot run edit FSM - edit_cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_edit.yaml", - ) - edit_cfg.training_graphs = {} - assert edit_cfg.infer_type == "infer_edit" - - edit_text_encoder, edit_qwen_mot = _make_real_text_and_mot() - edit_model = OmniModel( - edit_cfg, - { - "bagel_text_encoder": edit_text_encoder, - "bagel_siglip_navit": _DimAlignedSiglip(), - "bagel_qwen2_mot": edit_qwen_mot, - "bagel_flow_connector": _InferEditBagelFlow(), - "bagel_vae": _InferEditBagelVAE(), - }, - ).eval() - edit_processor = OmniProcessor(_build_preprocessors(), infer_type=edit_model.config.infer_type) - - # Caller passes neither infer_type to processor(...) nor infer_type in generation_kwargs - edit_request = edit_processor(text="edit", images=str(image_path)) - assert [ - item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_request["conversation_list"] if item.type == "image" - ] == [ - BAGEL_VAE_CONTEXT, - BAGEL_SIGLIP_CONTEXT, - ] - edit_model.reset() - edit_generated = edit_model.generate( - edit_request, - generation_kwargs={"max_new_tokens": 8, "do_sample": False, "image_height": 64, "image_width": 64}, - ) - assert any(item["type"] == "image" for item in edit_generated) - assert edit_qwen_mot._generation_state.main.cache is not None diff --git a/tests/seed_omni/modules/bagel/test_graph_config.py b/tests/seed_omni/modules/bagel/test_graph_config.py deleted file mode 100644 index 090eec0f90..0000000000 --- a/tests/seed_omni/modules/bagel/test_graph_config.py +++ /dev/null @@ -1,131 +0,0 @@ -from __future__ import annotations - -import pytest -import yaml - -from scripts.ci.accelerator.launch_plan import LaunchSpec -from tests.seed_omni.modules.bagel.helpers import bagel_cfg_dir, load_module_runtime_args, load_omni_config -from veomni.models.seed_omni.graphs.generation_graph import GenerationGraph - - -pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) - - -def test_bagel_train_yaml_loads_with_v2_module_names(): - """Verify that the BAGEL training YAML resolves all five module entries and graph endpoints.""" - cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / "infer/graph_infer_gen.yaml", - ) - - assert set(cfg._module_entries) == { - "bagel_text_encoder", - "bagel_siglip_navit", - "bagel_qwen2_mot", - "bagel_flow_connector", - "bagel_vae", - } - assert isinstance(cfg.training_graph, list) and cfg.training_graph - endpoints = {e["from"] for e in cfg.training_graph} | {e["to"] for e in cfg.training_graph} - assert "bagel_text_encoder.encode" in endpoints - assert "bagel_siglip_navit" in endpoints - assert "bagel_vae.encode" in endpoints - assert "bagel_flow_connector.embed_latent" in endpoints - assert "bagel_text_encoder.decode" in endpoints - assert "bagel_flow_connector.decode_velocity" in endpoints - assert "end" in endpoints - runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") - assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") - assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True - for infer_modules in ("infer/modules_infer_eager.yaml", "infer/modules_infer_fsdp.yaml"): - infer_runtime = load_module_runtime_args(modules_path=bagel_cfg_dir() / infer_modules, for_inference=True) - assert infer_runtime["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") - - -def test_bagel_train_graph_fan_in_execution_order(): - """Verify topological execution order of the BAGEL training DAG.""" - from veomni.models.seed_omni.graphs.training_graph import TrainingGraph - - graph = TrainingGraph(yaml.safe_load((bagel_cfg_dir() / "train/graph_train.yaml").read_text())) - order = graph.execution_order - assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_text_encoder.encode") - assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_siglip_navit.forward") - assert order.index("bagel_qwen2_mot.forward") > order.index("bagel_flow_connector.embed_latent") - assert order.index("bagel_flow_connector.embed_latent") > order.index("bagel_vae.encode") - assert set(graph.sources) == { - "bagel_text_encoder.encode", - "bagel_siglip_navit.forward", - "bagel_vae.encode", - } - - -def test_bagel_infer_gen_graph_uses_siglip_context_without_vae_context(): - """Verify ``graph_infer_gen.yaml`` routes prompt encoding through text and SigLIP only.""" - data = yaml.safe_load((bagel_cfg_dir() / "infer/graph_infer_gen.yaml").read_text()) - prompt_body = data["states"]["prompt_encode"]["body"] - - assert {"from": "bagel_text_encoder", "to": "bagel_qwen2_mot"} in prompt_body - assert {"from": "bagel_siglip_navit", "to": "bagel_qwen2_mot"} in prompt_body - assert all(edge["from"] != "bagel_vae.encode_context" for edge in prompt_body) - assert all(edge["from"] != "bagel_flow_connector.embed_context_latents" for edge in prompt_body) - - -def test_bagel_edit_prompt_graph_exposes_independent_prompt_producers() -> None: - """Verify ``graph_infer_edit.yaml`` wires VAE context and SigLIP context as independent prompt producers.""" - graph_config = yaml.safe_load((bagel_cfg_dir() / "infer/graph_infer_edit.yaml").read_text()) - prompt_body = graph_config["states"]["prompt_encode"]["body"] - - assert {"from": "bagel_text_encoder", "to": "bagel_qwen2_mot"} in prompt_body - assert {"from": "bagel_siglip_navit", "to": "bagel_qwen2_mot"} in prompt_body - assert {"from": "bagel_vae.encode_context", "to": "bagel_flow_connector.embed_context_latents"} in prompt_body - assert {"from": "bagel_flow_connector.embed_context_latents", "to": "bagel_qwen2_mot"} in prompt_body - assert all(edge["to"] != "bagel_text_encoder.encode_image_markers" for edge in prompt_body) - - forbidden_edges = { - ("bagel_vae.encode_context", "bagel_siglip_navit"), - ("bagel_siglip_navit", "bagel_vae.encode_context"), - ("bagel_siglip_navit", "bagel_flow_connector.embed_context_latents"), - } - assert not forbidden_edges.intersection({(edge["from"], edge["to"]) for edge in prompt_body}) - - graph = GenerationGraph(graph_config) - sequence = graph.state_node_sequence("prompt_encode") - assert "bagel_vae.encode_context" in sequence - assert "bagel_siglip_navit.generate" in sequence - - -@pytest.mark.parametrize( - "infer_graph", - ["infer/graph_infer_und.yaml", "infer/graph_infer_gen.yaml", "infer/graph_infer_edit.yaml"], -) -def test_bagel_train_plus_infer_merges_generation_graph(infer_graph: str): - """Verify that each BAGEL inference graph merges cleanly onto the training config and reaches ``done``.""" - cfg = load_omni_config( - modules_path=bagel_cfg_dir() / "train/modules_train.yaml", - train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", - infer_graph_path=bagel_cfg_dir() / infer_graph, - ) - assert set(cfg._module_entries) == { - "bagel_text_encoder", - "bagel_siglip_navit", - "bagel_qwen2_mot", - "bagel_flow_connector", - "bagel_vae", - } - assert cfg.generation_graph is not None - runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") - assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == ("veomni_flex_attention") - assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True - assert cfg.generation_graph["initial"] == "prompt_encode" - assert "done" not in cfg.generation_graph["states"] - assert any( - t.get("next_state") == "done" - for state in cfg.generation_graph["states"].values() - for t in state.get("transitions", []) - ), f"{infer_graph} has no transition to `done`." - for state_name, state in cfg.generation_graph["states"].items(): - for e in state.get("body", []): - assert isinstance(e, dict) and "from" in e and "to" in e, ( - f"state '{state_name}' body item must be a `{{from, to}}` dict: {e!r}" - ) diff --git a/tests/seed_omni/modules/bagel/test_masking.py b/tests/seed_omni/modules/bagel/test_masking.py deleted file mode 100644 index 838cdc5364..0000000000 --- a/tests/seed_omni/modules/bagel/test_masking.py +++ /dev/null @@ -1,100 +0,0 @@ -"""BAGEL Qwen2-MoT mask materializers: Flex, Magi, and SDPA share one metadata.""" - -from __future__ import annotations - -import pytest -import torch -from torch.nn.attention.flex_attention import BlockMask - -from scripts.ci.accelerator.launch_plan import LaunchSpec -from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAttentionAccelerated -from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( - build_mot_attention_metadata, - build_mot_magi_mask, - build_mot_sdpa_mask, - pad_mot_attention_metadata, -) -from veomni.ops.mask import MagiAttentionMask - - -pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) - - -def _expand_magi_mask(mask: MagiAttentionMask, sequence_length: int) -> torch.Tensor: - """Materialize a ``MagiAttentionMask`` into a dense ``[S, S]`` boolean visibility matrix.""" - visible = torch.zeros(sequence_length, sequence_length, dtype=torch.bool, device=mask.q_ranges.device) - attn_types = mask.attn_type_map - if attn_types is None: - attn_types = torch.zeros(mask.q_ranges.shape[0], dtype=torch.int32, device=mask.q_ranges.device) - for range_index in range(mask.q_ranges.shape[0]): - query_start, query_end = (int(mask.q_ranges[range_index, 0]), int(mask.q_ranges[range_index, 1])) - key_start, key_end = (int(mask.k_ranges[range_index, 0]), int(mask.k_ranges[range_index, 1])) - attn_type = int(attn_types[range_index]) - if attn_type == 0: - visible[query_start:query_end, key_start:key_end] = True - continue - if attn_type != 1: - raise AssertionError(f"Unexpected Magi attn type {attn_type} in BAGEL mask tests.") - query_idx = torch.arange(query_start, query_end, device=visible.device) - key_idx = torch.arange(key_start, key_end, device=visible.device) - visible[query_start:query_end, key_start:key_end] |= query_idx[:, None] >= key_idx[None, :] - return visible - - -@pytest.mark.parametrize( - ("sample_splits", "sample_attn_modes"), - [ - ([[8]], [["causal"]]), - ([[4, 8]], [["full", "causal"]]), - ([[8, 6]], [["causal", "noise"]]), - ([[4, 8, 6]], [["full", "causal", "noise"]]), - ([[3, 4, 5]], [["causal", "noise", "causal"]]), - ([[4], [5]], [["causal"], ["causal"]]), - ([[2, 3], [4, 2]], [["causal", "full"], ["causal", "noise"]]), - ], -) -def test_magi_mask_matches_sdpa_visibility( - sample_splits: list[list[int]], - sample_attn_modes: list[list[str]], -) -> None: - metadata = build_mot_attention_metadata(sample_splits, sample_attn_modes, device=torch.device("cpu")) - sdpa = build_mot_sdpa_mask(metadata) - magi = build_mot_magi_mask(metadata) - visible = _expand_magi_mask(magi, int(metadata.shape[1])) - torch.testing.assert_close(visible[None, None], sdpa) - - -def test_magi_mask_keeps_padding_as_isolated_full_document() -> None: - """Verify that SP-padded tokens form an isolated document invisible to real query tokens.""" - metadata = build_mot_attention_metadata([[4]], [["causal"]], device=torch.device("cpu")) - padded = pad_mot_attention_metadata(metadata, padded_length=8) - sdpa = build_mot_sdpa_mask(padded) - magi = build_mot_magi_mask(padded) - visible = _expand_magi_mask(magi, 8) - torch.testing.assert_close(visible[None, None], sdpa) - assert not bool(visible[:4, 4:].any()) - assert bool(visible[4:, 4:].all()) - - -def test_accelerated_build_attention_mask_dispatches_flex_and_magi() -> None: - """Verify that ``BagelQwen2MoTAttentionAccelerated.build_attention_mask`` dispatches Flex and Magi.""" - metadata = build_mot_attention_metadata([[4, 4]], [["full", "causal"]], device=torch.device("cpu")) - flex_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask( - metadata, - attn_implementation="veomni_flex_attention", - ) - magi_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask( - metadata, - attn_implementation="veomni_magi_attention", - ) - assert isinstance(flex_mask, BlockMask) - assert isinstance(magi_mask, MagiAttentionMask) - default_mask = BagelQwen2MoTAttentionAccelerated.build_attention_mask(metadata) - assert isinstance(default_mask, BlockMask) - - -def test_accelerated_build_attention_mask_rejects_unsupported_backend() -> None: - """Verify that ``build_attention_mask`` rejects non-packed attention backends.""" - metadata = build_mot_attention_metadata([[2]], [["causal"]], device=torch.device("cpu")) - with pytest.raises(ValueError, match="packed fused attention"): - BagelQwen2MoTAttentionAccelerated.build_attention_mask(metadata, attn_implementation="sdpa") diff --git a/tests/seed_omni/modules/bagel/test_processing.py b/tests/seed_omni/modules/bagel/test_processing.py index f6af9579bd..ac48a1f003 100644 --- a/tests/seed_omni/modules/bagel/test_processing.py +++ b/tests/seed_omni/modules/bagel/test_processing.py @@ -1,8 +1,9 @@ +"""BAGEL preprocessing, MoT packing, and offline VAE cache round-trip tests.""" + from __future__ import annotations import copy -import pickle -from inspect import Parameter, signature +from pathlib import Path from types import SimpleNamespace import pytest @@ -10,22 +11,15 @@ from scripts.ci.accelerator.launch_plan import LaunchSpec from tests.seed_omni.modules.bagel.helpers import ( + bagel_template, config_cls, model_cls, - native_model_cls, tiny_bagel_qwen2_cfg, ) -from veomni.models.seed_omni import ( - OMNI_ACCELERATED_MODEL_REGISTRY, - OMNI_CONFIG_REGISTRY, - OMNI_MODEL_REGISTRY, - OMNI_PROCESSOR_REGISTRY, -) +from veomni.data.dataset import get_length_by_labels_fn, get_length_fn +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated as mot_acc from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import TrainingMixin from veomni.models.seed_omni.modules.bagel.qwen2_mot.processing import preprocess_mot_inputs -from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( - _OMNI_POSITION_IDS as BAGEL_SIGLIP_POSITION_IDS, -) from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import ( _OMNI_TOKEN_LEN as BAGEL_SIGLIP_TOKEN_LEN, ) @@ -33,141 +27,147 @@ BagelSiglipNavitPreprocessor, BagelSiglipNavitProcessor, ) -from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT -from veomni.models.seed_omni.modules.bagel.text_encoder.chat_template import BagelChatTemplate -from veomni.models.seed_omni.modules.bagel.text_encoder.modeling import ( - _OMNI_TOKENIZED as BAGEL_TOK, -) -from veomni.models.seed_omni.modules.bagel.text_encoder.processing import ( - BagelTextEncoderPreprocessor, +from veomni.models.seed_omni.modules.bagel.sources import ( + BAGEL_CONTEXT_KEY, + BAGEL_PHASE_KEY, + BAGEL_SIGLIP_CONTEXT, + BAGEL_START_TOKEN, + BAGEL_VAE_CONTEXT, ) +from veomni.models.seed_omni.modules.bagel.text_encoder.processing import BagelTextEncoderPreprocessor from veomni.models.seed_omni.modules.bagel.vae.processing import ( BAGEL_VAE_PIXEL_SHAPE, BAGEL_VAE_POSTERIOR, BagelVAEPreprocessor, BagelVAEProcessor, ) -from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem, iter_desired_items +from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, _OMNI_TOKEN_LEN, ConversationItem pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) -@pytest.mark.parametrize("conversation_list", [None, []]) -def test_bagel_mot_packing_rejects_missing_conversation_list(conversation_list): - """Verify ``preprocess_mot_inputs`` rejects ``None`` or empty ``conversation_list``.""" - with pytest.raises(ValueError, match="requires a non-empty conversation_list"): - preprocess_mot_inputs( - conversation_list, - device=torch.device("cpu"), - dtype=torch.float32, - hidden_size=4, - ) +def _tiny_vae(meta: bool = False, **config_overrides): + """Construct a tiny ``BagelVAE`` on CPU or meta device with a bound processor.""" + BagelVAE = model_cls("bagel_vae") + BagelVAEConfig = config_cls("bagel_vae") + config_kwargs = dict( + resolution=8, + ch=32, + ch_mult=[1], + num_res_blocks=1, + z_channels=2, + max_image_size=8, + min_image_size=4, + image_stride=4, + max_pixels=64, + downsample=1, + ) + config_kwargs.update(config_overrides) + with torch.device("meta" if meta else "cpu"): + model = BagelVAE(BagelVAEConfig(**config_kwargs)) + model._image_processor = BagelVAEProcessor.from_config(model.config) + return model -def test_bagel_mot_packing_rejects_incompatible_vae_img_tag(): - """Verify ``preprocess_mot_inputs`` rejects VAE context items tagged with ``_img_tag='und'``.""" - item = ConversationItem( - type="image", - value=torch.ones(2, 4), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "und"}, +def _build_preprocessors(): + return ( + BagelVAEPreprocessor( + BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), + dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), + ), + BagelSiglipNavitPreprocessor( + BagelSiglipNavitProcessor( + patch_size=2, + image_size=4, + min_image_size=2, + max_pixels=16, + vit_max_num_patch_per_side=2, + ), + dtype=torch.bfloat16, + dummy_pixel_values=torch.zeros(1, 12, dtype=torch.bfloat16), + ), + BagelTextEncoderPreprocessor(bagel_template()), ) - with pytest.raises(ValueError, match="_img_tag"): - preprocess_mot_inputs([[item]], device=torch.device("cpu"), dtype=torch.float32, hidden_size=4) +def test_bagel_preprocessors_route_training_and_inference_contexts() -> None: + """Verify training (``edit``/``gen`` + text-only dummy injection + ``dyn_bsz`` token lens) and inference routing.""" + vae_pre, siglip_pre, text_pre = _build_preprocessors() + user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) + assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) -def test_bagel_mot_packing_routes_tagged_edit_vae_through_generation_expert(): - """Verify ``edit`` and ``gen`` VAE context items route through the generation expert while SigLIP routes through und.""" - edit_context = ConversationItem( - type="image", - value=torch.ones(2, 4), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, - ) - gen_target = ConversationItem( - type="image", - value=torch.full((2, 4), 2), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, - ) - siglip_context = ConversationItem( - type="image", - value=torch.full((1, 4), 3), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "edit"}, - ) + # 1. Training batch: sample 0 has edit + gen images; sample 1 is text-only (receives both VAE and SigLIP dummies) + train_batch = [ + [ + ConversationItem(type="text", value="edit", role="user"), + ConversationItem(type="image", value=user_image.clone(), role="user", meta={_IMG_TAG_KEY: "edit"}), + ConversationItem( + type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} + ), + ], + [ + ConversationItem(type="text", value="hi", role="user"), + ConversationItem(type="text", value="ok", role="assistant"), + ], + ] + for pre in (vae_pre, siglip_pre, text_pre): + pre({"conversation_list": train_batch}) - packed = preprocess_mot_inputs( - [[edit_context, gen_target, siglip_context]], - device=torch.device("cpu"), - dtype=torch.float32, - hidden_size=4, + edit_sample, text_sample = train_batch + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in edit_sample if item.type == "image"] == [ + BAGEL_VAE_CONTEXT, + BAGEL_SIGLIP_CONTEXT, + BAGEL_VAE_CONTEXT, + ] + assert [item.meta.get(_IMG_TAG_KEY) for item in edit_sample if item.type == "image"] == ["edit", "edit", "gen"] + dummies = [item for item in text_sample if item.is_dummy and item.type == "image"] + assert {item.meta.get(BAGEL_CONTEXT_KEY) for item in dummies} == {BAGEL_VAE_CONTEXT, BAGEL_SIGLIP_CONTEXT} + for sample in train_batch: + assert all( + isinstance(item.meta.get(_OMNI_TOKEN_LEN), int) and item.meta[_OMNI_TOKEN_LEN] > 0 for item in sample + ) + assert ( + get_length_fn({"conversation_list": edit_sample}) + > get_length_by_labels_fn({"conversation_list": edit_sample}) + > 0 ) - assert packed is not None - assert torch.equal(packed.packed_token_type_ids, torch.tensor([1, 1, 1, 1, 0])) - assert packed.sample_splits == [[2, 2, 1]] + # 2. Inference routing: infer_und routes to SigLIP + appends BAGEL_START_TOKEN; infer_edit expands to VAE + SigLIP + und_batch = [[ConversationItem(type="image", value=user_image.clone(), role="user")]] + for pre in (vae_pre, siglip_pre, text_pre): + pre({"conversation_list": und_batch}, inference=True, generation_kwargs={"infer_type": "infer_und"}) + assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in und_batch[0] if item.type == "image"] == [ + BAGEL_SIGLIP_CONTEXT + ] + assert und_batch[0][1].meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 + assert und_batch[0][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN -def test_bagel_mot_forward_pre_returns_sample_local_tensor_contract(): - """Verify ``BagelQwen2MoTAccelerated.forward_pre`` returns the packed tensor contract expected by ``forward``.""" +def test_bagel_mot_packing_sp_padding_and_scatter(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify ``preprocess_mot_inputs`` expert routing, ``forward_pre`` SP padding, and ``forward_post`` triplet scatter.""" BagelQwen2MoT = model_cls("bagel_qwen2_mot") BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train() hidden_size = int(model.config.hidden_size) - text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") - gen_target = ConversationItem( - type="image", - value=torch.ones(3, hidden_size), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, - ) - - inputs = model.forward_pre(conversation_list=[[text, gen_target]]) - - # The accelerated class inherits the graph's generic ``forward(**kwargs)`` - # trampoline, so the explicit packed-tensor contract lives on the native one. - forward_parameters = signature(native_model_cls("bagel_qwen2_mot").forward).parameters - assert "packed_token_type_ids" in forward_parameters - assert "packed_und_token_indexes" not in forward_parameters - assert "packed_gen_token_indexes" not in forward_parameters - assert all(parameter.kind is not Parameter.VAR_KEYWORD for parameter in forward_parameters.values()) - assert set(inputs) == { - "packed_sequence", - "packed_position_ids", - "packed_token_type_ids", - "packed_attention_metadata", - } - assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1])) - assert inputs["packed_attention_metadata"].shape == (3, 5) - assert inputs["packed_attention_metadata"].dtype == torch.int32 - assert inputs["packed_attention_metadata"].is_contiguous() - -def test_bagel_mot_forward_pre_keeps_metadata_full_and_marks_sequence_padding(monkeypatch): - """Verify ``forward_pre`` under Ulysses SP pads sequence tensors with ``-1`` token types and keeps full metadata.""" - from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated - - BagelQwen2MoT = model_cls("bagel_qwen2_mot") - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - model = BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())).train() monkeypatch.setattr( - accelerated, + mot_acc, "get_parallel_state", lambda: SimpleNamespace(sp_size=4, cp_size=1, ulysses_size=4, sp_group=object()), ) - def fake_sp_pad(tensor, dim, pad_value): + def _fake_sp_pad(tensor, dim, pad_value): pad_shape = list(tensor.shape) pad_shape[dim] = 3 padding = torch.full(pad_shape, pad_value, device=tensor.device, dtype=tensor.dtype) return torch.cat((tensor, padding), dim=dim) - monkeypatch.setattr(accelerated, "sp_pad", fake_sp_pad) - monkeypatch.setattr(accelerated, "slice_input_tensor", lambda tensor, **kwargs: tensor) - hidden_size = int(model.config.hidden_size) + monkeypatch.setattr(mot_acc, "sp_pad", _fake_sp_pad) + monkeypatch.setattr(mot_acc, "slice_input_tensor", lambda tensor, **kwargs: tensor) + text = ConversationItem(type="text", value=torch.ones(2, hidden_size), role="user") image = ConversationItem( type="image", @@ -175,241 +175,94 @@ def fake_sp_pad(tensor, dim, pad_value): role="assistant", meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, ) - inputs = model.forward_pre(conversation_list=[[text, image]]) - - assert inputs["packed_attention_metadata"].shape == (3, 8) - assert inputs["packed_sequence"].shape[0] == 8 - assert set(inputs) == { - "packed_sequence", - "packed_position_ids", - "packed_token_type_ids", - "packed_attention_metadata", - } assert torch.equal(inputs["packed_token_type_ids"], torch.tensor([0, 0, 1, 1, 1, -1, -1, -1])) - assert torch.equal(inputs["packed_attention_metadata"][0, 5:], torch.tensor([5, 5, 5], dtype=torch.int32)) - assert torch.equal(inputs["packed_attention_metadata"][1, 5:], torch.tensor([5, 5, 5], dtype=torch.int32)) assert torch.equal(inputs["packed_attention_metadata"][2, 5:], torch.tensor([-1, -1, -1], dtype=torch.int32)) - -def test_bagel_flow_training_embed_treats_edit_context_as_clean_and_gen_as_target(): - """Verify ``embed_latent_pre`` leaves ``edit`` context at timestep 0 without targets and noises ``gen`` targets.""" - BagelFlowConnector = model_cls("bagel_flow_connector") - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - model = BagelFlowConnector( - BagelFlowConnectorConfig( - hidden_size=4, - z_channels=1, - latent_patch_size=1, - patch_latent_dim=1, - max_latent_size=4, - timestep_frequency_embedding_size=4, - timestep_shift=1.0, - ) - ) - edit_context = ConversationItem( - type="image", - value=torch.ones(1, 1, 2), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, - ) - gen_target = ConversationItem( - type="image", - value=torch.full((1, 1, 2), 2.0), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, - ) - - inputs = model.embed_latent_pre(conversation_list=[[edit_context, gen_target]]) - - assert model._embed_items == [edit_context, gen_target] - assert model._embed_lengths == [2, 2] - assert torch.equal(inputs["latents"][:2], torch.ones(2, 1)) - assert torch.equal(inputs["timesteps"][:2], torch.zeros(2)) - assert "flow_velocity_target" not in edit_context.meta - assert "timestep" not in edit_context.meta - assert gen_target.meta["flow_velocity_target"].shape == (2, 1) - assert gen_target.meta["timestep"].shape == (2,) - assert torch.equal(gen_target.meta["timestep"][0], gen_target.meta["timestep"][1]) - assert torch.equal(inputs["timesteps"][2], inputs["timesteps"][3]) - - -def test_bagel_flow_preprocess_latent_embed_samples_one_shared_timestep_per_image() -> None: - """Verify ``preprocess_latent_embed`` samples one scalar timestep per image and expands it across patch tokens.""" - from veomni.models.seed_omni.modules.bagel.flow_connector.processing import preprocess_latent_embed - - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - cfg = BagelFlowConnectorConfig( - hidden_size=4, - z_channels=1, - latent_patch_size=1, - patch_latent_dim=1, - max_latent_size=4, - timestep_frequency_embedding_size=4, - timestep_shift=3.0, - ) - first = ConversationItem( - type="image", - value=torch.zeros(1, 2, 2), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, - ) - second = ConversationItem( - type="image", - value=torch.ones(1, 2, 3), - role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen"}, - ) - torch.manual_seed(1234) - inputs, lengths = preprocess_latent_embed( - [first, second], - config=cfg, - device=torch.device("cpu"), - dtype=torch.float32, - timestep_shift=3.0, - ) - assert lengths == [4, 6] - assert torch.all(first.meta["timestep"] == first.meta["timestep"][0]) - assert torch.all(second.meta["timestep"] == second.meta["timestep"][0]) - assert not torch.equal(first.meta["timestep"][0], second.meta["timestep"][0]) - assert torch.equal(inputs["timesteps"][:4], first.meta["timestep"]) - assert torch.equal(inputs["timesteps"][4:], second.meta["timestep"]) - - -def test_mot_forward_post_scatters_virtual_marker_triplet_hidden_states() -> None: - """Verify ``forward_post`` scatters packed hidden states back across ``(start, image, end)`` triplet items.""" - hidden_size = 4 - markers = torch.tensor([[101.0, 102.0, 103.0, 104.0], [201.0, 202.0, 203.0, 204.0]]) - - def marker(row: int, context: str) -> ConversationItem: + # Triplet scatter in forward_post + def _marker(context: str) -> ConversationItem: return ConversationItem( type="text", - value=markers[row : row + 1], + value=torch.zeros(1, 4), role="user", meta={"labels": torch.full((1,), -100), BAGEL_CONTEXT_KEY: context}, ) latent_item = ConversationItem( type="image", - value=torch.arange(8, dtype=torch.float32).reshape(2, hidden_size) + 20, + value=torch.zeros(2, 4), role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit"}, ) siglip_item = ConversationItem( type="image", - value=torch.arange(12, dtype=torch.float32).reshape(3, hidden_size), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ) - text_item = ConversationItem( - type="text", - value=torch.arange(8, dtype=torch.float32).reshape(2, hidden_size) + 40, + value=torch.zeros(3, 4), role="user", + meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT, _IMG_TAG_KEY: "edit"}, ) sample = [ - marker(0, BAGEL_VAE_CONTEXT), + _marker(BAGEL_VAE_CONTEXT), latent_item, - marker(1, BAGEL_VAE_CONTEXT), - marker(0, BAGEL_SIGLIP_CONTEXT), + _marker(BAGEL_VAE_CONTEXT), + _marker(BAGEL_SIGLIP_CONTEXT), siglip_item, - marker(1, BAGEL_SIGLIP_CONTEXT), - text_item, + _marker(BAGEL_SIGLIP_CONTEXT), ] - packed = preprocess_mot_inputs( - [sample], - device=torch.device("cpu"), - dtype=torch.float32, - hidden_size=hidden_size, - ) + packed = preprocess_mot_inputs([sample], device=torch.device("cpu"), dtype=torch.float32, hidden_size=4) assert packed is not None - - model = SimpleNamespace( - device=torch.device("cpu"), - _conversation_carrier=[sample], - _packed_training=packed, + assert torch.equal(packed.packed_token_type_ids, torch.tensor([0, 1, 1, 0, 0, 0, 0, 0, 0])) + hidden_states = torch.arange(9 * 4, dtype=torch.float32).reshape(9, 4) + monkeypatch.setattr( + mot_acc, + "get_parallel_state", + lambda: SimpleNamespace(sp_size=1, cp_size=1, ulysses_size=1, sp_group=None), ) - hidden_states = torch.arange(11 * hidden_size, dtype=torch.float32).reshape(11, hidden_size) - - out = TrainingMixin.forward_post(model, hidden_states) - - assert out["conversation_list"] == [sample] - assert torch.equal(sample[0].value, hidden_states[0:1]) + stub = SimpleNamespace(device=torch.device("cpu"), _conversation_carrier=[sample], _packed_training=packed) + TrainingMixin.forward_post(stub, hidden_states) assert torch.equal(latent_item.value, hidden_states[1:3]) - assert torch.equal(sample[2].value, hidden_states[3:4]) - assert torch.equal(sample[3].value, hidden_states[4:5]) assert torch.equal(siglip_item.value, hidden_states[5:8]) - assert torch.equal(sample[5].value, hidden_states[8:9]) - assert torch.equal(text_item.value, hidden_states[9:11]) -def test_bagel_vae_online_process_runs_on_a_meta_built_model() -> None: - """``offline_training`` builds the VAE on meta with no weights, because it mixes in ``OfflineEncodingMixin``.""" - encode_model = _tiny_vae() - process_model = _tiny_vae(meta=True) - encoded_cache = encode_model.offline_encode(pixel_values=torch.zeros(1, 3, 8, 8))["encoded_cache"] - item_cache = encoded_cache[0].reshape(2, process_model.config.z_channels, *encoded_cache.shape[-2:]) +def test_bagel_vae_encode_and_offline_cache_roundtrip(tmp_path: Path) -> None: + """Verify ragged-shape VAE ``encode``/``offline_encode`` and parquet cache replay into ``BagelFlowConnector``.""" + from datasets import load_dataset - assert process_model.encoder.conv_in.weight.is_meta - latents = process_model.online_process(encoded_cache=item_cache)["latents"] - assert isinstance(latents, list) - assert latents[0].shape == item_cache.shape[1:] - assert not latents[0].is_meta + from veomni.data.seed_omni.seedomni_transform import process_seedomni_cached_example + from veomni.models.seed_omni.utils.offline_cache import SeedOmniOfflineCacheWriter + vae = _tiny_vae(ch_mult=[1, 1], downsample=2) + meta_vae = _tiny_vae(meta=True, ch_mult=[1, 1], downsample=2) -def test_bagel_vae_online_process_consumes_variable_size_cache_items_without_padding() -> None: - """Verify ``online_process`` samples variable-size cached posteriors without spatial batch padding.""" - model = _tiny_vae(meta=True) - first = ConversationItem( + edit_img = ConversationItem( type="image", - value=torch.zeros(2, 2, 2, 1), + value=torch.zeros(3, 4, 8), role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "edit", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 8])}, ) - second = ConversationItem( + gen_img = ConversationItem( type="image", - value=torch.zeros(2, 2, 2, 2), + value=torch.ones(3, 8, 4), role="assistant", - meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, + meta={BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, _IMG_TAG_KEY: "gen", BAGEL_VAE_PIXEL_SHAPE: torch.tensor([8, 4])}, ) - conversation = [[first], [second]] - - pre = model.online_process_pre(conversation_list=conversation) - out = model.online_process(encoded_cache=pre["encoded_cache"]) - post = model.post_forward("online_process", latents=out["latents"]) - - assert isinstance(pre["encoded_cache"], list) - assert [tuple(cache.shape) for cache in pre["encoded_cache"]] == [(2, 2, 2, 1), (2, 2, 2, 2)] - assert post["conversation_list"] is conversation - assert first.value.shape == (2, 2, 1) - assert second.value.shape == (2, 2, 2) - - -def test_bagel_vae_offline_cache_replay_keeps_img_tag_for_flow_connector(tmp_path) -> None: - """Verify offline VAE cache parquet round-trip preserves ``_img_tag`` for downstream flow connector routing.""" - from datasets import load_dataset - from veomni.data.seed_omni.seedomni_transform import process_seedomni_cached_example - from veomni.models.seed_omni.utils.offline_cache import SeedOmniOfflineCacheWriter - - encode_model = _tiny_vae() - process_model = _tiny_vae(meta=True) - - def vae_image(tag: str) -> ConversationItem: - return ConversationItem( - type="image", - value=torch.zeros(3, 4, 4), - role="assistant", - meta={ - BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, - _IMG_TAG_KEY: tag, - BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 4]), - }, - ) + # 1. Ragged micro-batch offline_encode matches single-item posteriors without spatial padding + single_edit = copy.deepcopy([[edit_img]]) + vae.post_forward( + "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_edit)) + ) + single_gen = copy.deepcopy([[gen_img]]) + vae.post_forward( + "offline_encode", **vae.offline_encode(**vae.pre_forward("offline_encode", conversation_list=single_gen)) + ) - pre = encode_model.pre_forward("offline_encode", conversation_list=[[vae_image("edit"), vae_image("gen")]]) - out = encode_model.offline_encode(**pre) - encoded = encode_model.post_forward("offline_encode", **out)["conversation_list"] + batch = [[copy.deepcopy(edit_img), copy.deepcopy(gen_img)]] + pre = vae.pre_forward("offline_encode", conversation_list=batch) + encoded = vae.post_forward("offline_encode", **vae.offline_encode(**pre))["conversation_list"] + torch.testing.assert_close(encoded[0][0].value, single_edit[0][0].value) + torch.testing.assert_close(encoded[0][1].value, single_gen[0][0].value) + # 2. Parquet round-trip -> meta-built online_process -> BagelFlowConnector.embed_latent_pre writer = SeedOmniOfflineCacheWriter(str(tmp_path)) writer.open() writer.save_conversation_list(encoded) @@ -418,980 +271,28 @@ def vae_image(tag: str) -> ConversationItem: replayed = [process_seedomni_cached_example(dataset[0])[0]["conversation_list"]] assert all(item.meta[BAGEL_VAE_POSTERIOR] for item in replayed[0]) - pre = process_model.pre_forward("online_process", conversation_list=replayed) - out = process_model.online_process(**pre) - processed = process_model.post_forward("online_process", **out)["conversation_list"] - + assert meta_vae.encoder.conv_in.weight.is_meta + processed = meta_vae.post_forward( + "online_process", + **meta_vae.online_process(**meta_vae.pre_forward("online_process", conversation_list=replayed)), + )["conversation_list"] assert [item.meta[_IMG_TAG_KEY] for item in processed[0]] == ["edit", "gen"] - assert all(item.meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT for item in processed[0]) - BagelFlowConnector = model_cls("bagel_flow_connector") - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - flow_connector = BagelFlowConnector( - BagelFlowConnectorConfig( + flow = model_cls("bagel_flow_connector")( + config_cls("bagel_flow_connector")( hidden_size=4, z_channels=2, latent_patch_size=1, patch_latent_dim=2, max_latent_size=4, timestep_frequency_embedding_size=4, - timestep_shift=1.0, + timestep_shift=3.0, ) ) edit_item, gen_item = processed[0] - inputs = flow_connector.embed_latent_pre(conversation_list=processed) - - assert flow_connector._embed_lengths == [16, 16] - assert torch.equal(inputs["timesteps"][:16], torch.zeros(16)) + inputs = flow.embed_latent_pre(conversation_list=processed) + assert flow._embed_lengths == [8, 8] + assert torch.equal(inputs["timesteps"][:8], torch.zeros(8)) assert "flow_velocity_target" not in edit_item.meta - assert gen_item.meta["flow_velocity_target"].shape == (16, 2) - - -def _tiny_vae(meta=False, **config_overrides): - """Construct a tiny ``BagelVAE`` on CPU or meta device with a bound processor.""" - BagelVAE = model_cls("bagel_vae") - BagelVAEConfig = config_cls("bagel_vae") - config_kwargs = dict( - resolution=8, - ch=32, - ch_mult=[1], - num_res_blocks=1, - z_channels=2, - max_image_size=8, - min_image_size=4, - image_stride=4, - max_pixels=64, - downsample=1, - ) - config_kwargs.update(config_overrides) - with torch.device("meta" if meta else "cpu"): - model = BagelVAE(BagelVAEConfig(**config_kwargs)) - model._image_processor = BagelVAEProcessor.from_config(model.config) - return model - - -def _worker_dummies(conversation_list, role): - """Return worker-appended ``is_dummy`` image placeholders standing in for ``role`` images.""" - return [it for it in iter_desired_items(conversation_list, types=["image"], roles=[role]) if it.is_dummy] - - -class _FakeBagelTokenizer: - """Char-ordinal tokenizer with the marker tokens/ids that ``BagelChatTemplate`` resolves.""" - - bos_token = "" - eos_token = "" - boi_token = "" - eoi_token = "" - bos_token_id = 1 - eos_token_id = 2 - boi_token_id = 3 - eoi_token_id = 4 - pad_token_id = 0 - unk_token_id = -1 - special_tokens = { - "": 1, - "": 2, - "": 3, - "": 4, - "<|im_start|>": 5, - "<|im_end|>": 2, - "<|vision_start|>": 7, - "<|vision_end|>": 8, - } - - def convert_tokens_to_ids(self, token): - return self.special_tokens.get(token, self.unk_token_id) - - def decode(self, token_ids, skip_special_tokens=True): - del skip_special_tokens - special_by_id = {v: k for k, v in self.special_tokens.items()} - return "".join(special_by_id.get(int(t), chr(max(32, int(t)))) for t in token_ids) - - def __call__(self, text, add_special_tokens=False): - del add_special_tokens - input_ids = [] - index = 0 - while index < len(text): - matched = False - for token, token_id in sorted(self.special_tokens.items(), key=lambda item: len(item[0]), reverse=True): - if text.startswith(token, index): - input_ids.append(token_id) - index += len(token) - matched = True - break - if matched: - continue - input_ids.append(ord(text[index])) - index += 1 - return {"input_ids": input_ids} - - -def _bagel_template() -> BagelChatTemplate: - return BagelChatTemplate(_FakeBagelTokenizer()) - - -def _b(conversation_list): - return {"conversation_list": conversation_list} - - -def test_bagel_registry_entries_and_processors() -> None: - """Verify all five BAGEL modules and image processors are registered in the SeedOmni registries.""" - expected = { - "bagel_text_encoder": ("BagelTextEncoder", "BagelTextEncoderAccelerated"), - "bagel_siglip_navit": ("BagelSiglipNavit", "BagelSiglipNavitAccelerated"), - "bagel_qwen2_mot": ("BagelQwen2MoT", "BagelQwen2MoTAccelerated"), - "bagel_flow_connector": ("BagelFlowConnector", "BagelFlowConnectorAccelerated"), - "bagel_vae": ("BagelVAE", "BagelVAEAccelerated"), - } - for model_type, (native_name, accel_name) in expected.items(): - assert model_type in OMNI_CONFIG_REGISTRY.valid_keys() - assert OMNI_MODEL_REGISTRY[model_type]().__name__ == native_name - assert OMNI_ACCELERATED_MODEL_REGISTRY[model_type]().__name__ == accel_name - assert {"bagel_siglip_navit", "bagel_vae"}.issubset(set(OMNI_PROCESSOR_REGISTRY.valid_keys())) - - -def test_bagel_text_preprocessor_tokenizes_plain_items_and_is_idempotent() -> None: - """Verify ``BagelTextEncoderPreprocessor`` tokenizes text items, wraps images with markers, and is idempotent.""" - pre = BagelTextEncoderPreprocessor(_bagel_template()) - batch = [ - [ - ConversationItem(type="text", value="hi", role="user"), - ConversationItem( - type="image", value=torch.zeros(3, 4, 4), role="user", meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT} - ), - ConversationItem(type="text", value="ok", role="assistant"), - ] - ] - - pre(_b(batch)) - user_text, image_start, image, image_end, assistant_text = batch[0] - - assert image.type == "image" - assert image.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert image_start.type == "text" - assert image_start.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert torch.equal(image_start.value, torch.tensor([7])) - assert image_end.type == "text" - assert image_end.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert torch.equal(image_end.value, torch.tensor([8])) - assert torch.equal(user_text.value, torch.tensor([5, ord("h"), ord("i"), 2])) - assert user_text.value.device.type == "cpu" - assert user_text.meta[BAGEL_TOK] is True - assert torch.equal(user_text.meta["labels"], torch.full_like(user_text.value, -100)) - assert torch.equal(user_text.meta["attention_mask"], torch.ones_like(user_text.value)) - - assert torch.equal(assistant_text.value, torch.tensor([5, ord("o"), ord("k"), 2])) - assert assistant_text.meta[BAGEL_TOK] is True - assert torch.equal(assistant_text.meta["labels"], assistant_text.value) - - snapshot = copy.deepcopy(batch) - pre(_b(batch)) - assert len(batch[0]) == len(snapshot[0]) - for actual, expected in zip(batch[0], snapshot[0]): - if isinstance(actual.value, torch.Tensor): - assert torch.equal(actual.value, expected.value) - - -def test_bagel_siglip_preprocessor_patchifies_and_tags_context() -> None: - """Verify ``BagelSiglipNavitPreprocessor`` patchifies user images and populates position/length metadata.""" - pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor( - patch_size=2, - image_size=4, - min_image_size=2, - max_pixels=16, - vit_max_num_patch_per_side=2, - ), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ) - batch = [ - [ - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 7, dtype=torch.uint8), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ) - ] - ] - - pre(_b(batch)) - item = batch[0][0] - assert item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert item.meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 - assert item.meta[BAGEL_SIGLIP_POSITION_IDS].tolist() == [0, 1, 2, 3] - assert item.value.shape == (4, 2 * 2 * 3) - assert item.value.dtype == torch.bfloat16 - - -def test_bagel_siglip_preprocessor_appends_per_sample_dummy_for_missing_context() -> None: - """Verify ``BagelSiglipNavitPreprocessor`` appends a dummy SigLIP image to samples without one.""" - pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor( - patch_size=2, - image_size=4, - min_image_size=2, - max_pixels=16, - vit_max_num_patch_per_side=2, - ), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ) - batch = [ - [ - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 7, dtype=torch.uint8), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ) - ], - [ConversationItem(type="text", value="text-only", role="user")], - ] - - pre(_b(batch)) - - assert len(_worker_dummies(batch, "user")) == 1 - assert len(batch[0]) == 1 - dummy = batch[1][-1] - assert dummy.type == "image" - assert dummy.role == "user" and dummy.is_dummy - assert dummy.meta[_IMG_TAG_KEY] == "und" - assert dummy.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT - assert dummy.meta[BAGEL_SIGLIP_TOKEN_LEN] == 1 - assert dummy.meta[BAGEL_SIGLIP_POSITION_IDS].tolist() == [0] - assert dummy.value.shape == (1, 2 * 2 * 3) - assert dummy.value.dtype == torch.bfloat16 - - -def test_bagel_siglip_preprocessor_keeps_bs4_sample_aligned_for_0_2_4_images() -> None: - """Verify ``BagelSiglipNavitPreprocessor`` maintains one SigLIP item per sample across mixed 4-sample batches.""" - pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor( - patch_size=2, - image_size=4, - min_image_size=2, - max_pixels=16, - vit_max_num_patch_per_side=2, - ), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ) - - for real_count in (0, 2, 4): - batch = [] - for index in range(4): - if index < real_count: - batch.append( - [ - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 7, dtype=torch.uint8), - role="user", - meta={BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ) - ] - ) - else: - batch.append([ConversationItem(type="text", value=f"text-{index}", role="user")]) - - pre(_b(batch)) - - assert len(_worker_dummies(batch, "user")) == 4 - real_count - for sample in batch: - context_items = [ - item - for item in sample - if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - ] - assert len(context_items) == 1 - - -def test_bagel_vae_preprocessor_leaves_cached_posteriors_alone() -> None: - """An ``offline_training`` batch: ``offline_encode`` already turned every VAE - row (its dummies included) into a posterior, so nothing is re-processed or added.""" - pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - posterior = torch.randn(2, 4, 1, 1) - meta = {_IMG_TAG_KEY: "gen", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, BAGEL_VAE_POSTERIOR: True} - batch = [ - [ConversationItem(type="image", value=posterior, role="assistant", meta=dict(meta))], - [ConversationItem(type="image", value=posterior.clone(), role="assistant", meta=dict(meta), is_dummy=True)], - ] - - pre(_b(batch)) - - assert [len(sample) for sample in batch] == [1, 1] - assert batch[0][0].value is posterior - - -def test_bagel_preprocessors_route_inference_edit_prompt_context() -> None: - """Verify ``infer_edit`` preprocessing expands a user image into both VAE and SigLIP context triplets.""" - text_pre = BagelTextEncoderPreprocessor(_bagel_template()) - siglip_pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ) - vae_pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) - assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) - batch = [ - [ - ConversationItem(type="text", value="hi", role="user"), - ConversationItem(type="image", value=user_image.clone(), role="user"), - ConversationItem( - type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} - ), - ] - ] - - for preprocessor in (vae_pre, siglip_pre, text_pre): - preprocessor(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_edit"}) - - sample = batch[0] - assert [item.type for item in sample] == [ - "text", - "text", - "image", - "text", - "text", - "image", - "text", - "text", - "image", - "text", - ] - assert torch.equal(sample[0].value, torch.tensor([5, ord("h"), ord("i"), 2])) - assert sample[2].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - assert sample[5].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert sample[8].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - assert sample[2].value.shape == (3, 4, 4) - assert sample[2].value.dtype == torch.bfloat16 - assert sample[5].value.shape == (4, 2 * 2 * 3) - assert sample[5].value.dtype == torch.bfloat16 - assert sample[5].meta[BAGEL_SIGLIP_TOKEN_LEN] == 4 - assert sample[8].value.shape == (3, 4, 4) - assert sample[8].value.dtype == torch.bfloat16 - assert [sample[i].meta.get(BAGEL_CONTEXT_KEY) for i in [1, 3, 4, 6, 7, 9]] == [ - BAGEL_VAE_CONTEXT, - BAGEL_VAE_CONTEXT, - BAGEL_SIGLIP_CONTEXT, - BAGEL_SIGLIP_CONTEXT, - BAGEL_VAE_CONTEXT, - BAGEL_VAE_CONTEXT, - ] - assert [int(sample[i].value.numel()) for i in [1, 3, 4, 6, 7, 9]] == [1, 1, 1, 1, 1, 1] - - -def test_bagel_preprocessors_route_tagged_edit_without_infer_type() -> None: - """Verify training preprocessing routes ``_img_tag='edit'`` images into VAE and SigLIP copies.""" - text_pre = BagelTextEncoderPreprocessor(_bagel_template()) - siglip_pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 12, dtype=torch.bfloat16), - ) - vae_pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - user_image = torch.full((3, 4, 4), 7, dtype=torch.uint8) - assistant_image = torch.full((3, 4, 4), 9, dtype=torch.uint8) - batch = [ - [ - ConversationItem(type="text", value="edit", role="user"), - ConversationItem(type="image", value=user_image.clone(), role="user", meta={_IMG_TAG_KEY: "edit"}), - ConversationItem( - type="image", value=assistant_image.clone(), role="assistant", meta={_IMG_TAG_KEY: "gen"} - ), - ] - ] - - for preprocessor in (vae_pre, siglip_pre, text_pre): - preprocessor(_b(batch)) - - sample = batch[0] - assert [item.meta.get(BAGEL_CONTEXT_KEY) for item in sample if item.type == "image"] == [ - BAGEL_VAE_CONTEXT, - BAGEL_SIGLIP_CONTEXT, - BAGEL_VAE_CONTEXT, - ] - assert [item.meta.get(_IMG_TAG_KEY) for item in sample if item.type == "image"] == ["edit", "edit", "gen"] - vae_source, siglip_source, vae_target = (item for item in sample if item.type == "image") - assert vae_source.value.shape == (3, 4, 4) - assert vae_source.value.dtype == torch.bfloat16 - assert siglip_source.value.shape == (4, 2 * 2 * 3) - assert siglip_source.value.dtype == torch.bfloat16 - assert vae_target.value.shape == (3, 4, 4) - assert vae_target.value.dtype == torch.bfloat16 - - -def test_bagel_preprocessors_route_inference_und_user_image_to_siglip_only() -> None: - """Verify ``infer_und`` preprocessing routes user images to SigLIP context only and appends the assistant start token.""" - from veomni.models.seed_omni.modules.bagel.sources import BAGEL_PHASE_KEY, BAGEL_START_TOKEN - - vae_pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - text_pre = BagelTextEncoderPreprocessor(_bagel_template()) - image = torch.full((3, 4, 4), 7, dtype=torch.uint8) - batch = [[ConversationItem(type="image", value=image.clone(), role="user")]] - - vae_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) - text_pre(_b(batch), inference=True, generation_kwargs={"infer_type": "infer_und"}) - - assert [item.type for item in batch[0]] == ["text", "image", "text", "text"] - assert batch[0][1].meta.get(BAGEL_CONTEXT_KEY) == BAGEL_SIGLIP_CONTEXT - assert torch.equal(batch[0][1].value, image) - assert batch[0][-1].role == "assistant" - assert batch[0][-1].meta.get(BAGEL_PHASE_KEY) == BAGEL_START_TOKEN - - -def test_bagel_vae_preprocessor_appends_per_sample_dummy_for_missing_context() -> None: - """Verify ``BagelVAEPreprocessor`` appends a shape-matched dummy VAE image to samples without one.""" - pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - batch = [ - [ - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 9, dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - ], - [ConversationItem(type="text", value="text-only", role="user")], - ] - - pre(_b(batch)) - - real = batch[0][0] - assert real.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - assert real.value.shape == (3, 4, 4) - assert real.value.dtype == torch.bfloat16 - assert real.meta[BAGEL_VAE_PIXEL_SHAPE].tolist() == [4, 4] - - assert len(_worker_dummies(batch, "assistant")) == 1 - dummy = batch[1][-1] - assert dummy.type == "image" - assert dummy.role == "assistant" and dummy.is_dummy - assert dummy.meta[_IMG_TAG_KEY] == "gen" - assert dummy.meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT - assert dummy.meta[BAGEL_VAE_PIXEL_SHAPE].tolist() == [4, 4] - assert dummy.value.shape == real.value.shape - assert dummy.value.dtype == torch.bfloat16 - torch.stack([real.value, dummy.value], dim=0) - - -def test_bagel_vae_preprocessor_keeps_bs4_sample_aligned_for_0_2_4_images() -> None: - """Verify ``BagelVAEPreprocessor`` maintains uniform tensor shapes across mixed 4-sample batches.""" - pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - - for real_count in (0, 2, 4): - batch = [] - for index in range(4): - if index < real_count: - batch.append( - [ - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 9, dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - ] - ) - else: - batch.append([ConversationItem(type="text", value=f"text-{index}", role="user")]) - - pre(_b(batch)) - - assert len(_worker_dummies(batch, "assistant")) == 4 - real_count - for sample in batch: - context_items = [ - item - for item in sample - if item.type == "image" and item.meta.get(BAGEL_CONTEXT_KEY) == BAGEL_VAE_CONTEXT - ] - assert len(context_items) == 1 - shapes = [tuple(sample[-1].value.shape) for sample in batch] - assert len(set(shapes)) == 1 - expected_shape = (3, 2, 2) if real_count == 0 else (3, 4, 4) - assert shapes[0] == expected_shape - - -def test_bagel_preprocessors_are_picklable() -> None: - """Verify all three BAGEL worker preprocessors round-trip through ``pickle``.""" - for pre in ( - BagelTextEncoderPreprocessor(_bagel_template()), - BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ), - BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ), - ): - restored = pickle.loads(pickle.dumps(pre)) - assert type(restored) is type(pre) - - -def test_bagel_preprocessors_populate_omni_token_len_for_dyn_bsz() -> None: - """Verify BAGEL text, SigLIP, and VAE preprocessors populate ``_omni_token_len`` on real and dummy items.""" - from veomni.data.dataset import get_length_by_labels_fn, get_length_fn - from veomni.models.seed_omni.utils.conversation import _OMNI_TOKEN_LEN - - text_pre = BagelTextEncoderPreprocessor(_bagel_template()) - siglip_pre = BagelSiglipNavitPreprocessor( - BagelSiglipNavitProcessor(patch_size=2, image_size=4, min_image_size=2, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(1, 2 * 2 * 3, dtype=torch.bfloat16), - ) - vae_pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=4, max_pixels=16), - dtype=torch.bfloat16, - dummy_pixel_values=torch.zeros(3, 2, 2, dtype=torch.bfloat16), - dummy_pixel_shape=torch.tensor([2, 2], dtype=torch.long), - ) - gen_sample = [ - ConversationItem(type="text", value="draw", role="user"), - ConversationItem( - type="image", - value=torch.full((3, 4, 4), 9, dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ), - ] - batch = [gen_sample] - for pre in (vae_pre, siglip_pre, text_pre): - pre(_b(batch)) - - assert all(isinstance(item.meta.get(_OMNI_TOKEN_LEN), int) and item.meta[_OMNI_TOKEN_LEN] > 0 for item in batch[0]) - # User text (6 tokens) + VAE start/end markers (2 tokens) + 4 VAE tokens + 1 dummy SigLIP token = 13 total - assert get_length_fn({"conversation_list": batch[0]}) == 13 - # Assistant VAE image (4 tokens) is supervised; user text and dummy SigLIP are not - assert get_length_by_labels_fn({"conversation_list": batch[0]}) == 4 - - -def test_bagel_siglip_preprocessor_from_pretrained_applies_config_overrides(tmp_path) -> None: - """Verify ``BagelSiglipNavitPreprocessor.from_pretrained`` applies ``config_overrides`` to processor geometry.""" - from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig - - BagelSiglipNavitConfig(image_size=28, min_image_size=14, patch_size=14).save_pretrained(tmp_path) - pre = BagelSiglipNavitPreprocessor.from_pretrained( - str(tmp_path), - config_overrides={"image_size": 56, "min_image_size": 28}, - ) - assert pre._image_processor.image_size == 56 - assert pre._image_processor.min_image_size == 28 - - -def test_bagel_vae_encode_and_offline_encode_preserve_native_shapes_and_match_independent_posteriors() -> None: - """Verify ``encode`` and ``offline_encode`` group equal shapes without spatial padding and match independent encoding.""" - pre = BagelVAEPreprocessor( - BagelVAEProcessor(image_stride=2, min_image_size=4, max_image_size=8, max_pixels=64), - dtype=torch.float32, - dummy_pixel_values=torch.zeros(3, 4, 4, dtype=torch.float32), - dummy_pixel_shape=torch.tensor([4, 4], dtype=torch.long), - ) - torch.manual_seed(42) - sample_rect_a = [ - ConversationItem( - type="image", - value=torch.randint(0, 255, (3, 4, 8), dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - ] - sample_rect_b = [ - ConversationItem( - type="image", - value=torch.randint(0, 255, (3, 8, 4), dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - ] - sample_rect_c = [ - ConversationItem( - type="image", - value=torch.randint(0, 255, (3, 4, 8), dtype=torch.uint8), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - ] - sample_text_only = [ConversationItem(type="text", value="hello", role="user")] - for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): - pre(_b([sample])) - - assert sample_rect_a[0].value.shape == (3, 4, 8) - assert sample_rect_b[0].value.shape == (3, 8, 4) - assert sample_rect_c[0].value.shape == (3, 4, 8) - assert sample_text_only[-1].is_dummy - assert sample_text_only[-1].value.shape == (3, 4, 4) - - vae = _tiny_vae(ch_mult=[1, 1], downsample=2) - - # Independent single-image offline_encode oracle - independent_posteriors = [] - for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): - single = copy.deepcopy([sample]) - single_pre = vae.pre_forward("offline_encode", conversation_list=single) - single_out = vae.offline_encode(**single_pre) - vae.post_forward("offline_encode", **single_out) - independent_posteriors.append(single[0][-1].value) - - # 1. Offline encode with ragged real + dummy micro-batch - offline_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only]) - off_pre = vae.pre_forward("offline_encode", conversation_list=offline_batch) - assert isinstance(off_pre["pixel_values"], list) - assert [tuple(t.shape) for t in off_pre["pixel_values"]] == [(3, 4, 8), (3, 8, 4), (3, 4, 8), (3, 4, 4)] - off_out = vae.offline_encode(**off_pre) - vae.post_forward("offline_encode", **off_out) - assert offline_batch[0][0].value.shape == (2, 2, 2, 4) - assert offline_batch[1][0].value.shape == (2, 2, 4, 2) - assert offline_batch[2][0].value.shape == (2, 2, 2, 4) - assert offline_batch[3][-1].value.shape == (2, 2, 2, 2) - for idx, expected_posterior in enumerate(independent_posteriors): - torch.testing.assert_close(offline_batch[idx][-1].value, expected_posterior) - - # 2. Online encode with ragged real + dummy micro-batch (matching RNG sequence) - torch.manual_seed(777) - independent_latents = [] - for sample in (sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only): - single = copy.deepcopy([sample]) - single_pre = vae.pre_forward("encode", conversation_list=single) - single_out = vae.encode(**single_pre) - vae.post_forward("encode", **single_out) - independent_latents.append(single[0][-1].value) - - torch.manual_seed(777) - online_batch = copy.deepcopy([sample_rect_a, sample_rect_b, sample_rect_c, sample_text_only]) - pre_inputs = vae.pre_forward("encode", conversation_list=online_batch) - encoded_out = vae.encode(**pre_inputs) - vae.post_forward("encode", **encoded_out) - assert online_batch[0][0].value.shape == (2, 2, 4) - assert online_batch[1][0].value.shape == (2, 4, 2) - assert online_batch[2][0].value.shape == (2, 2, 4) - assert online_batch[3][-1].value.shape == (2, 2, 2) - for idx, expected_latent in enumerate(independent_latents): - torch.testing.assert_close(online_batch[idx][-1].value, expected_latent) - - -def test_bagel_flow_decode_velocity_reduces_loss_and_token_count_over_fsdp_group( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Verify ``decode_velocity_post`` reduces flow loss and valid-token counts over ``fsdp_group`` including zero-target ranks.""" - import torch.distributed as dist - - from veomni.models.seed_omni.modules.bagel.flow_connector.accelerated import accelerated as flow_acc - - BagelFlowConnector = model_cls("bagel_flow_connector") - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - model = BagelFlowConnector( - BagelFlowConnectorConfig( - hidden_size=4, - z_channels=1, - latent_patch_size=1, - patch_latent_dim=2, - max_latent_size=4, - timestep_frequency_embedding_size=4, - timestep_shift=1.0, - ) - ) - fake_fsdp_group = object() - monkeypatch.setattr( - flow_acc, - "get_parallel_state", - lambda: SimpleNamespace(sp_size=1, cp_size=1, sp_group=None, fsdp_group=fake_fsdp_group), - ) - monkeypatch.setattr(dist, "get_world_size", lambda group=None: 2) - - # Case 1: Generation rank (4 tokens) paired with a text-only rank (0 tokens on peer) - peer_loss_sum = 0.0 - peer_tokens = 0 - - def _fake_all_reduce_gen_vs_text(tensor: torch.Tensor, group=None): - assert group is fake_fsdp_group - if tensor.dtype == torch.long: - tensor.add_(peer_tokens) - else: - tensor.add_(peer_loss_sum) - - monkeypatch.setattr(dist, "all_reduce", _fake_all_reduce_gen_vs_text) - - gen_item = ConversationItem( - type="image", - value=torch.ones(4, 4), - role="assistant", - meta={ - BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, - _IMG_TAG_KEY: "gen", - "flow_velocity_target": torch.zeros(4, 2), - }, - ) - gen_conv = [[gen_item]] - model.decode_velocity_pre(conversation_list=gen_conv) - pred_vel = torch.full((4, 2), 2.0, requires_grad=True) - gen_out = model.decode_velocity_post(pred_vel) - assert gen_out["_loss"].item() == pytest.approx(4.0) - gen_out["_loss"].backward() - # Single-rank unreduced grad per element is 2 * (2 - 0) / (4 * 2) = 0.5; - # with world_size=2 and peer_tokens=0, ReduceLoss scales backward by 2.0 so FSDP's /2 preserves 0.5. - torch.testing.assert_close(pred_vel.grad, torch.full_like(pred_vel, 1.0)) - - # Zero-target rank paired with the 4-token generation rank on peer (peer_loss_sum = 16.0, peer_tokens = 4) - peer_loss_sum = 16.0 - peer_tokens = 4 - text_conv = [[ConversationItem(type="text", value=torch.ones(2, 4), role="user")]] - model.decode_velocity_pre(conversation_list=text_conv) - dummy_vel = torch.ones(1, 2, requires_grad=True) - zero_out = model.decode_velocity_post(dummy_vel) - assert zero_out["_loss"].item() == pytest.approx(4.0) - zero_out["_loss"].backward() - torch.testing.assert_close(dummy_vel.grad, torch.zeros_like(dummy_vel)) - - -def test_bagel_chat_template_rejects_multi_token_start_or_end_markers() -> None: - """Verify ``BagelChatTemplate.tokenize`` raises when ``<|im_start|>`` or ``<|im_end|>`` splits into multiple tokens.""" - - class _SplittingTokenizer(_FakeBagelTokenizer): - def __call__(self, text: str, add_special_tokens: bool = False) -> dict[str, list[int]]: - assert add_special_tokens is False - if text in {"<|im_start|>", "<|im_end|>"}: - return {"input_ids": [1, 2]} - return super().__call__(text, add_special_tokens=add_special_tokens) - - template = BagelChatTemplate(_SplittingTokenizer()) - for marker in ("<|im_start|>", "<|im_end|>"): - with pytest.raises(ValueError, match="BAGEL start/end tokens must tokenize to exactly one token"): - template.tokenize([ConversationItem(type="text", value=marker, role="user")]) - - -def test_bagel_vae_aligns_encoder_passes_across_shard_group_and_keeps_blocks_unsplit( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Verify ``BagelVAE`` keeps ``_no_split_modules`` empty and aligns ``_encode_posterior_list`` pass counts across ``fsdp_group``.""" - import torch.distributed as dist - - from veomni.models.seed_omni.modules.bagel.vae.accelerated import accelerated as vae_acc - - native_vae = native_model_cls("bagel_vae")( - config_cls("bagel_vae")(resolution=8, ch=32, ch_mult=[1, 1], num_res_blocks=1, z_channels=2, downsample=2) - ) - assert set(native_vae._no_split_modules) == {"ResnetBlock", "AttnBlock"} - - vae = _tiny_vae(ch_mult=[1, 1], downsample=2) - assert not vae._no_split_modules - - # Reference single-rank posterior for two equal-shape images (1 shape group natively) - img_a = torch.randn(3, 4, 8) - img_b = torch.randn(3, 4, 8) - ref_out = vae.offline_encode([img_a, img_b])["encoded_cache"] - - fake_fsdp_group = object() - monkeypatch.setattr(dist, "is_initialized", lambda: True) - monkeypatch.setattr( - vae_acc, - "get_parallel_state", - lambda: SimpleNamespace(sp_size=2, cp_size=1, sp_group=object(), fsdp_group=fake_fsdp_group), - ) - - def _fake_all_reduce_max(tensor: torch.Tensor, op=None, group=None): - assert group is fake_fsdp_group - assert op == dist.ReduceOp.MAX - # Peer rank has 3 distinct image shapes, while this rank has only 1 shape group. - tensor.fill_(3) - - monkeypatch.setattr(dist, "all_reduce", _fake_all_reduce_max) - - encoder_calls = 0 - orig_encoder_forward = vae.encoder.forward - - def _counting_encoder_forward(x: torch.Tensor) -> torch.Tensor: - nonlocal encoder_calls - encoder_calls += 1 - return orig_encoder_forward(x) - - monkeypatch.setattr(vae.encoder, "forward", _counting_encoder_forward) - - aligned_out = vae.offline_encode([img_a, img_b])["encoded_cache"] - assert encoder_calls == 3 - for actual, expected in zip(aligned_out, ref_out, strict=True): - torch.testing.assert_close(actual, expected) - - -def test_bagel_vae_sp_samples_posteriors_after_gather_and_keeps_flow_rng_aligned( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Verify SP-sharded VAE encoding defers posterior sampling until after gather so uneven image sizes keep flow RNG aligned across SP ranks.""" - from veomni.models.seed_omni.modules.bagel.flow_connector.accelerated import accelerated as flow_acc - from veomni.models.seed_omni.modules.bagel.vae.accelerated import accelerated as vae_acc - - vae = _tiny_vae(ch_mult=[1, 1], downsample=2) - BagelFlowConnector = model_cls("bagel_flow_connector") - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - flow = BagelFlowConnector( - BagelFlowConnectorConfig( - hidden_size=4, - z_channels=2, - latent_patch_size=1, - patch_latent_dim=2, - max_latent_size=8, - timestep_frequency_embedding_size=4, - timestep_shift=3.0, - ) - ) - - torch.manual_seed(99) - small_img = torch.randn(3, 4, 4) - large_img = torch.randn(3, 8, 8) - - def _make_batch() -> list[list[ConversationItem]]: - return [ - [ - ConversationItem( - type="image", - value=small_img.clone(), - role="assistant", - meta={ - BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, - _IMG_TAG_KEY: "gen", - BAGEL_VAE_PIXEL_SHAPE: torch.tensor([4, 4], dtype=torch.long), - }, - ), - ConversationItem( - type="image", - value=large_img.clone(), - role="assistant", - meta={ - BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT, - _IMG_TAG_KEY: "gen", - BAGEL_VAE_PIXEL_SHAPE: torch.tensor([8, 8], dtype=torch.long), - }, - ), - ] - ] - - # 1. Baseline sp_size=1 run - monkeypatch.setattr( - vae_acc, - "get_parallel_state", - lambda: SimpleNamespace(sp_size=1, sp_rank=0, cp_size=1, sp_group=None, fsdp_group=None), - ) - monkeypatch.setattr( - flow_acc, - "get_parallel_state", - lambda: SimpleNamespace(sp_size=1, sp_rank=0, cp_size=1, sp_group=None, fsdp_group=None), - ) - ref_batch = _make_batch() - torch.manual_seed(2026) - ref_pre = vae.pre_forward("encode", conversation_list=ref_batch) - ref_enc = vae.encode(**ref_pre) - vae.post_forward("encode", **ref_enc) - ref_rng_after_vae = torch.get_rng_state() - ref_flow_inputs = flow.pre_forward("embed_latent", conversation_list=ref_batch) - - # 2. Simulate 2-rank SP where rank 0 encodes small_img (4x4) and rank 1 encodes large_img (8x8) - rank_caches: dict[int, list[torch.Tensor]] = {} - for sp_rank in (0, 1): - monkeypatch.setattr( - vae_acc, - "get_parallel_state", - lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), - ) - batch_r = _make_batch() - torch.manual_seed(2026) - pre_r = vae.pre_forward("encode", conversation_list=batch_r) - enc_r = vae.encode(**pre_r) - # No RNG must be consumed during the SP-sharded encode step before gather - assert torch.equal(torch.get_rng_state(), torch.manual_seed(2026).get_state()) - rank_caches[sp_rank] = enc_r["latents"] - - def _fake_gather_outputs(x: torch.Tensor, gather_dim: int = 0, group=None) -> torch.Tensor: - del group - assert gather_dim == 0 - # Pad rank 0's flat posterior tensor to match max_rank_numel (rank 1's 2*2*4*4 = 64 elements) - flat_0 = rank_caches[0][0].reshape(-1) - flat_1 = rank_caches[1][0].reshape(-1) - max_numel = max(int(flat_0.numel()), int(flat_1.numel())) - pad_0 = torch.nn.functional.pad(flat_0, (0, max_numel - int(flat_0.numel()))).unsqueeze(0) - pad_1 = torch.nn.functional.pad(flat_1, (0, max_numel - int(flat_1.numel()))).unsqueeze(0) - return torch.cat([pad_0, pad_1], dim=0) - - monkeypatch.setattr(vae_acc, "gather_outputs", _fake_gather_outputs) - monkeypatch.setattr( - flow_acc, - "slice_sp_token_inputs", - lambda method, **inputs: (int(inputs["latents"].shape[0]), inputs), - ) - - rank_batches: dict[int, list[list[ConversationItem]]] = {} - rank_flow_inputs: dict[int, dict[str, torch.Tensor]] = {} - for sp_rank in (0, 1): - monkeypatch.setattr( - vae_acc, - "get_parallel_state", - lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), - ) - monkeypatch.setattr( - flow_acc, - "get_parallel_state", - lambda r=sp_rank: SimpleNamespace(sp_size=2, sp_rank=r, cp_size=1, sp_group=object(), fsdp_group=None), - ) - batch_r = _make_batch() - torch.manual_seed(2026) - pre_r = vae.pre_forward("encode", conversation_list=batch_r) - enc_r = vae.encode(**pre_r) - vae.post_forward("encode", **enc_r) - assert torch.equal(torch.get_rng_state(), ref_rng_after_vae) - flow_in_r = flow.pre_forward("embed_latent", conversation_list=batch_r) - rank_batches[sp_rank] = batch_r - rank_flow_inputs[sp_rank] = flow_in_r - - for idx in (0, 1): - torch.testing.assert_close(rank_batches[0][0][idx].value, ref_batch[0][idx].value) - torch.testing.assert_close(rank_batches[1][0][idx].value, ref_batch[0][idx].value) - torch.testing.assert_close( - rank_batches[0][0][idx].meta["flow_velocity_target"], - rank_batches[1][0][idx].meta["flow_velocity_target"], - ) - torch.testing.assert_close( - rank_batches[0][0][idx].meta["flow_velocity_target"], - ref_batch[0][idx].meta["flow_velocity_target"], - ) - torch.testing.assert_close( - rank_batches[0][0][idx].meta["timestep"], - rank_batches[1][0][idx].meta["timestep"], - ) - for key in ("latents", "position_ids", "timesteps"): - torch.testing.assert_close(rank_flow_inputs[0][key], rank_flow_inputs[1][key]) - torch.testing.assert_close(rank_flow_inputs[0][key], ref_flow_inputs[key]) + assert gen_item.meta["flow_velocity_target"].shape == (8, 2) + assert torch.all(gen_item.meta["timestep"] == gen_item.meta["timestep"][0]) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index 50dc533c0b..cad2f2adba 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -1,26 +1,49 @@ -"""BAGEL Qwen2-MoT attention dispatch and QKV checkpoint schema.""" +"""BAGEL module registry, YAML graph configs, MoT attention masks, QKV checkpoint conversion, and ops.""" from __future__ import annotations +from pathlib import Path + import pytest import torch from safetensors import safe_open +from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict +from torch.nn.attention.flex_attention import BlockMask from transformers.utils import SAFE_WEIGHTS_NAME from scripts.ci.accelerator.launch_plan import LaunchSpec from tests.models.conftest import available_nvidia_ops # noqa: F401 -from tests.seed_omni.modules.bagel.helpers import config_cls, tiny_bagel_qwen2_cfg +from tests.seed_omni.modules.bagel.helpers import ( + bagel_cfg_dir, + load_module_runtime_args, + load_omni_config, + tiny_bagel_qwen2_cfg, +) from veomni.models.checkpoint.weights import load_model_weights -from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated -from veomni.models.seed_omni.modules.bagel.qwen2_mot.checkpoint_conversion import ( - BagelQwen2MoTCheckpointTensorConverter, +from veomni.models.seed_omni import ( + OMNI_ACCELERATED_MODEL_REGISTRY, + OMNI_CONFIG_REGISTRY, + OMNI_MODEL_REGISTRY, + OMNI_PROCESSOR_REGISTRY, ) +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated +from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated.accelerated import BagelQwen2MoTAttentionAccelerated from veomni.models.seed_omni.modules.bagel.qwen2_mot.configuration import BagelQwen2MoTConfig from veomni.models.seed_omni.modules.bagel.qwen2_mot.masking import ( build_mot_attention_metadata, - build_mot_block_mask, + build_mot_magi_mask, + build_mot_sdpa_mask, + pad_mot_attention_metadata, ) -from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import _sdpa_packed_attention +from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT, BagelQwen2RotaryEmbedding +from veomni.models.seed_omni.modules.bagel.siglip_navit.accelerated.accelerated import BagelSiglipNavitAccelerated +from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig +from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit +from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN +from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT +from veomni.models.seed_omni.utils.conversation import ConversationItem +from veomni.ops.mask import MagiAttentionMask +from veomni.utils.save_safetensor_utils import get_model_save_state pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) @@ -46,116 +69,94 @@ def _flex_config() -> BagelQwen2MoTConfig: ) -def test_cached_sdpa_causal_attention_is_bottom_right_aligned() -> None: - query = torch.zeros(1, 1, 1) - key = torch.zeros(4, 1, 1) - value = torch.tensor([1.0, 2.0, 4.0, 8.0]).reshape(4, 1, 1) - - output = _sdpa_packed_attention( - query, - key, - value, - is_causal=True, - cu_seq_lens_q=torch.tensor([0, 1], dtype=torch.int32), - cu_seq_lens_k=torch.tensor([0, 4], dtype=torch.int32), - scale=1.0, - enable_gqa=False, +def _expand_magi_mask(mask: MagiAttentionMask, sequence_length: int) -> torch.Tensor: + visible = torch.zeros(sequence_length, sequence_length, dtype=torch.bool, device=mask.q_ranges.device) + attn_types = ( + mask.attn_type_map + if mask.attn_type_map is not None + else torch.zeros(mask.q_ranges.shape[0], dtype=torch.int32, device=mask.q_ranges.device) ) - - # A single cached decode query is the final logical row, so it can attend - # to all four historical keys. Upper-left causal alignment would return 1. - torch.testing.assert_close(output, torch.tensor([[[3.75]]])) - - -def test_accelerated_training_attention_rejects_non_flex_backend() -> None: - """Verify ``BagelQwen2MoTAttentionAccelerated.forward_packed_train`` rejects non-packed backends.""" - config = config_cls("bagel_qwen2_mot")(**tiny_bagel_qwen2_cfg(), attn_implementation="sdpa") - attention = accelerated.BagelQwen2MoTAttentionAccelerated(config, layer_idx=0) - sequence_length = 2 - metadata = build_mot_attention_metadata([[2]], [["causal"]], device=torch.device("cpu")) - - with pytest.raises(ValueError, match="requires packed fused attention"): - attention.forward_packed_train( - packed_sequence=torch.randn(sequence_length, config.hidden_size), - attention_mask=build_mot_block_mask(metadata), - packed_position_cos=torch.ones(sequence_length, attention.head_dim), - packed_position_sin=torch.zeros(sequence_length, attention.head_dim), - packed_und_token_indexes=torch.arange(sequence_length), - packed_gen_token_indexes=torch.empty(0, dtype=torch.long), - ) - - -def test_native_gqa_rejects_invalid_global_head_ratio_at_config_init() -> None: - """Verify ``BagelQwen2MoTConfig`` rejects query head counts not divisible by KV head counts.""" - with pytest.raises(ValueError, match="query heads must be divisible"): - config_cls("bagel_qwen2_mot")( - **{ - **tiny_bagel_qwen2_cfg(), - "num_attention_heads": 6, - "num_key_value_heads": 4, - } - ) - - -def test_checkpoint_tensor_converter_combines_legacy_qkv_groups() -> None: - """Verify ``BagelQwen2MoTCheckpointTensorConverter`` merges split Q/K/V weights and biases.""" - converter = BagelQwen2MoTCheckpointTensorConverter() - prefix = "model.layers.3.self_attn." - tensors = { - f"{prefix}q_proj.weight": torch.full((64, 64), 1.0), - f"{prefix}k_proj.weight": torch.full((16, 64), 2.0), - f"{prefix}v_proj.weight": torch.full((16, 64), 3.0), - f"{prefix}q_proj_moe_gen.bias": torch.full((64,), 4.0), - f"{prefix}k_proj_moe_gen.bias": torch.full((16,), 5.0), - f"{prefix}v_proj_moe_gen.bias": torch.full((16,), 6.0), + for range_index in range(mask.q_ranges.shape[0]): + q_start, q_end = int(mask.q_ranges[range_index, 0]), int(mask.q_ranges[range_index, 1]) + k_start, k_end = int(mask.k_ranges[range_index, 0]), int(mask.k_ranges[range_index, 1]) + if int(attn_types[range_index]) == 0: + visible[q_start:q_end, k_start:k_end] = True + else: + q_idx = torch.arange(q_start, q_end, device=visible.device) + k_idx = torch.arange(k_start, k_end, device=visible.device) + visible[q_start:q_end, k_start:k_end] |= q_idx[:, None] >= k_idx[None, :] + return visible + + +@pytest.mark.parametrize( + "infer_graph", + ["infer/graph_infer_und.yaml", "infer/graph_infer_gen.yaml", "infer/graph_infer_edit.yaml"], +) +def test_bagel_registry_and_yaml_graphs_load(infer_graph: str) -> None: + """Verify all five BAGEL modules are registered and train + infer YAML configs resolve cleanly.""" + expected = { + "bagel_text_encoder": ("BagelTextEncoder", "BagelTextEncoderAccelerated"), + "bagel_siglip_navit": ("BagelSiglipNavit", "BagelSiglipNavitAccelerated"), + "bagel_qwen2_mot": ("BagelQwen2MoT", "BagelQwen2MoTAccelerated"), + "bagel_flow_connector": ("BagelFlowConnector", "BagelFlowConnectorAccelerated"), + "bagel_vae": ("BagelVAE", "BagelVAEAccelerated"), } + for model_type, (native_name, accel_name) in expected.items(): + assert model_type in OMNI_CONFIG_REGISTRY.valid_keys() + assert OMNI_MODEL_REGISTRY[model_type]().__name__ == native_name + assert OMNI_ACCELERATED_MODEL_REGISTRY[model_type]().__name__ == accel_name + assert {"bagel_siglip_navit", "bagel_vae"}.issubset(set(OMNI_PROCESSOR_REGISTRY.valid_keys())) + + cfg = load_omni_config( + modules_path=bagel_cfg_dir() / "train/modules_train.yaml", + train_graph_path=bagel_cfg_dir() / "train/graph_train.yaml", + infer_graph_path=bagel_cfg_dir() / infer_graph, + ) + assert set(cfg._module_entries) == set(expected) + assert cfg.generation_graph["initial"] == "prompt_encode" + assert any( + t.get("next_state") == "done" + for state in cfg.generation_graph["states"].values() + for t in state.get("transitions", []) + ) + runtime_modules = load_module_runtime_args(modules_path=bagel_cfg_dir() / "train/modules_train.yaml") + assert runtime_modules["bagel_qwen2_mot"].ops_implementation.attn_implementation == "veomni_flex_attention" + assert runtime_modules["bagel_qwen2_mot"].accelerator.enable_async is True - converted = [] - for name in reversed(tensors): - result = converter.convert(name, tensors[name]) - if result is not None: - converted.append(result) - assert converter.finalize() == [] - assert {result.name for result in converted} == { - f"{prefix}qkv_proj_und.weight", - f"{prefix}qkv_proj_gen.bias", - } - by_name = {result.name: result.tensor for result in converted} - und_weight = by_name[f"{prefix}qkv_proj_und.weight"] - gen_bias = by_name[f"{prefix}qkv_proj_gen.bias"] - torch.testing.assert_close(und_weight[:64], tensors[f"{prefix}q_proj.weight"]) - torch.testing.assert_close(und_weight[64:80], tensors[f"{prefix}k_proj.weight"]) - torch.testing.assert_close(und_weight[80:], tensors[f"{prefix}v_proj.weight"]) - torch.testing.assert_close(gen_bias[:64], tensors[f"{prefix}q_proj_moe_gen.bias"]) - torch.testing.assert_close(gen_bias[64:80], tensors[f"{prefix}k_proj_moe_gen.bias"]) - torch.testing.assert_close(gen_bias[80:], tensors[f"{prefix}v_proj_moe_gen.bias"]) +def test_mot_attention_masks_and_sp_padding() -> None: + """Verify SDPA, Magi, and Flex masks agree on multi-sample causal/full/noise visibility and SP padding.""" + metadata = build_mot_attention_metadata( + [[2, 3, 2], [4, 2]], + [["full", "causal", "noise"], ["causal", "noise"]], + device=torch.device("cpu"), + ) + padded = pad_mot_attention_metadata(metadata, padded_length=16) + sdpa = build_mot_sdpa_mask(padded) + magi = build_mot_magi_mask(padded) + visible = _expand_magi_mask(magi, 16) + torch.testing.assert_close(visible[None, None], sdpa) + assert not bool(visible[:13, 13:].any()) + assert bool(visible[13:, 13:].all()) + assert isinstance( + BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_flex_attention"), + BlockMask, + ) + assert isinstance( + BagelQwen2MoTAttentionAccelerated.build_attention_mask(padded, attn_implementation="veomni_magi_attention"), + MagiAttentionMask, + ) -def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> None: +def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path: Path) -> None: """Verify fused runtime QKV parameters keep fused keys in ``state_dict`` / DCP and split on HF safetensors export.""" - from torch.distributed.checkpoint.state_dict import get_model_state_dict, set_model_state_dict - - from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT - from veomni.utils.save_safetensor_utils import get_model_save_state - torch.manual_seed(3014) eager_cfg = BagelQwen2MoTConfig(**{**tiny_bagel_qwen2_cfg(), "num_key_value_heads": 1}) eager_ref = BagelQwen2MoT(eager_cfg) eager_split_state = eager_ref.state_dict() - assert "model.layers.0.self_attn.q_proj.weight" in eager_split_state - assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in eager_split_state reference = accelerated.BagelQwen2MoTAccelerated(_flex_config()) reference.load_state_dict(eager_split_state) - internal_parameter_names = set(dict(reference.named_parameters())) - assert "model.layers.0.self_attn.qkv_proj_und.weight" in internal_parameter_names - assert "model.layers.0.self_attn.qkv_proj_gen.weight" in internal_parameter_names - assert "model.layers.0.self_attn.q_proj.weight" not in internal_parameter_names - assert "model.layers.0.self_attn.q_proj_moe_gen.weight" not in internal_parameter_names - - # PyTorch state_dict() and DCP get_model_state_dict() keep fused parameter keys so - # FQN resolution works under DDP as well as FSDP2. runtime_state = reference.state_dict() dcp_state = get_model_state_dict(reference) assert "model.layers.0.self_attn.qkv_proj_und.weight" in runtime_state @@ -171,15 +172,7 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> torch.testing.assert_close(actual.weight, expected.weight) torch.testing.assert_close(actual.bias, expected.bias) - # Distributed HF export routes through BagelQwen2MoTCheckpointTensorConverter.export_weights - # and splits fused QKV back into q_proj/k_proj/v_proj keys. reverted_hf_state = get_model_save_state(reference, None) - assert "model.layers.0.self_attn.q_proj.weight" in reverted_hf_state - assert "model.layers.0.self_attn.k_proj.weight" in reverted_hf_state - assert "model.layers.0.self_attn.v_proj.weight" in reverted_hf_state - assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in reverted_hf_state - assert "model.layers.0.self_attn.qkv_proj_und.weight" not in reverted_hf_state - assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in reverted_hf_state for key in ( "model.layers.0.self_attn.q_proj.weight", "model.layers.0.self_attn.k_proj.weight", @@ -192,9 +185,7 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> with safe_open(tmp_path / SAFE_WEIGHTS_NAME, framework="pt", device="cpu") as checkpoint: checkpoint_keys = set(checkpoint.keys()) assert "model.layers.0.self_attn.q_proj.weight" in checkpoint_keys - assert "model.layers.0.self_attn.q_proj_moe_gen.weight" in checkpoint_keys assert "model.layers.0.self_attn.qkv_proj_und.weight" not in checkpoint_keys - assert "model.layers.0.self_attn.qkv_proj_gen.weight" not in checkpoint_keys hf_loaded = accelerated.BagelQwen2MoTAccelerated.from_pretrained(tmp_path, config=_flex_config()) veomni_loaded = accelerated.BagelQwen2MoTAccelerated(_flex_config()) @@ -202,14 +193,12 @@ def test_accelerated_qkv_preserves_hf_checkpoint_schema_and_loaders(tmp_path) -> for name in ("qkv_proj_und", "qkv_proj_gen"): expected = getattr(reference.model.layers[0].self_attn, name) torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).weight, expected.weight) - torch.testing.assert_close(getattr(hf_loaded.model.layers[0].self_attn, name).bias, expected.bias) torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).weight, expected.weight) - torch.testing.assert_close(getattr(veomni_loaded.model.layers[0].self_attn, name).bias, expected.bias) -def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config(available_nvidia_ops) -> None: # noqa: F811 - """Verify ``BagelQwen2MoTAccelerated`` binds ``VeomniOp`` handles from ``config.ops_config``.""" - cfg = BagelQwen2MoTConfig( +def test_bagel_modules_ops_dispatch_meta_rotary_and_flops(available_nvidia_ops) -> None: # noqa: F811 + """Verify module-level ``VeomniOp`` binding, ``meta -> to_empty`` FP32 rotary table, and multi-image SigLIP FLOPs.""" + default_cfg = BagelQwen2MoTConfig( vocab_size=128, hidden_size=64, intermediate_size=128, @@ -219,145 +208,24 @@ def test_bagel_qwen2_mot_selects_module_level_ops_without_global_ops_config(avai max_position_embeddings=64, bos_token_id=1, eos_token_id=2, - ops_implementation={ - "attn_implementation": "flex_attention", - "rms_norm_implementation": "eager", - "swiglu_mlp_implementation": "eager", - "rotary_pos_emb_implementation": "eager", - }, ) - model = accelerated.BagelQwen2MoTAccelerated(cfg) + assert default_cfg._attn_implementation == "veomni_flex_attention" - layer = model.model.layers[0] - assert layer.input_layernorm.veomni_rms_norm.impl == "eager" - assert layer.mlp.veomni_swiglu_mlp.impl == "eager" - assert layer.self_attn.veomni_rms_norm.impl == "eager" - assert layer.self_attn.veomni_rope.impl == "eager" + with torch.device("meta"): + meta_model = accelerated.BagelQwen2MoTAccelerated(_flex_config()) + materialized = meta_model.to_empty(device="cpu").to(dtype=torch.bfloat16) + layer = materialized.model.layers[0] assert layer.self_attn.veomni_async_ulysses_qkv.variant == "bagel" assert layer.self_attn.veomni_async_ulysses_o.variant == "bagel" - assert layer.self_attn.veomni_attn is not None assert layer.self_attn.veomni_attn.impl == "veomni_flex_attention" - assert layer.self_attn.veomni_denoise_attn is not None assert layer.self_attn.veomni_denoise_attn.impl == "veomni_flash_attention_2" + ref_rotary = BagelQwen2RotaryEmbedding(_flex_config()) + rotary = materialized.model.rotary_emb + assert rotary.inv_freq.dtype == torch.float32 + assert rotary.original_inv_freq.dtype == torch.float32 + torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) -def test_bagel_siglip_navit_selects_module_level_ops_and_forwards() -> None: - """Verify ``BagelSiglipNavit`` binds vision RoPE and attention ops from ``config.ops_config``.""" - from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig - from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit - - cfg = BagelSiglipNavitConfig( - hidden_size=32, - intermediate_size=64, - num_hidden_layers=1, - num_attention_heads=4, - image_size=28, - patch_size=14, - ops_implementation={ - "attn_implementation": "sdpa", - "rotary_pos_emb_vision_implementation": "eager", - }, - ) - model = BagelSiglipNavit(cfg).eval() - attn = model.vision_model.encoder.layers[0].self_attn - assert attn.veomni_attn.impl == "veomni_sdpa" - assert attn.veomni_rope.impl == "eager" - - packed_patches = torch.randn(4, 3 * 14 * 14) - position_ids = torch.arange(4, dtype=torch.long) - for cu_seqlens, max_seqlen in ( - (torch.tensor([0, 4], dtype=torch.int32), 4), - (torch.tensor([0, 2, 4], dtype=torch.int32), 2), - ): - out = model.vision_model( - patchified_pixel_values=packed_patches, - patchified_position_ids=position_ids, - cu_seqlens=cu_seqlens, - max_seqlen=max_seqlen, - ) - assert out.shape == (4, 32) - assert torch.isfinite(out).all() - - -class _CallCounter: - def __init__(self, inner): - self.inner = inner - self.calls = 0 - - @property - def impl(self): - return self.inner.impl - - def __call__(self, *args, **kwargs): - self.calls += 1 - return self.inner(*args, **kwargs) - - -@pytest.mark.parametrize("hidden_act", ["silu", "swish", "gelu"]) -def test_bagel_qwen2_mot_mlp_and_rms_norm_dispatch_through_veomni_ops(hidden_act: str) -> None: - """Verify ``BagelQwen2MoTAccelerated`` reuses patched Qwen2 ``rms_norm_cls`` and ``mlp_cls`` ``VeomniOp`` handles.""" - from transformers.models.qwen2.modeling_qwen2 import Qwen2MLP as HFQwen2MLP - from transformers.models.qwen2.modeling_qwen2 import Qwen2RMSNorm as HFQwen2RMSNorm - - from tests.models.compare import assert_module_forward_and_grads_match - - cfg = BagelQwen2MoTConfig( - **{ - **tiny_bagel_qwen2_cfg(), - "hidden_act": hidden_act, - "ops_implementation": { - "attn_implementation": "sdpa", - "rms_norm_implementation": "eager", - "swiglu_mlp_implementation": "eager", - "rotary_pos_emb_implementation": "eager", - }, - } - ) - - torch.manual_seed(0) - hf_norm = HFQwen2RMSNorm(cfg.hidden_size, eps=cfg.rms_norm_eps) - ours_norm = accelerated.BagelQwen2MoTAccelerated.rms_norm_cls(cfg.hidden_size, eps=cfg.rms_norm_eps, config=cfg) - ours_norm.load_state_dict(hf_norm.state_dict()) - norm_counter = _CallCounter(ours_norm.veomni_rms_norm) - ours_norm.veomni_rms_norm = norm_counter - assert_module_forward_and_grads_match(hf_norm, ours_norm, torch.randn(3, cfg.hidden_size)) - assert norm_counter.calls == 1 - empty_norm_out = ours_norm(torch.empty(0, cfg.hidden_size, requires_grad=True)) - assert empty_norm_out.shape == (0, cfg.hidden_size) - assert norm_counter.calls == 2 - - hf_mlp = HFQwen2MLP(cfg) - ours_mlp = accelerated.BagelQwen2MoTAccelerated.mlp_cls(cfg) - ours_mlp.load_state_dict(hf_mlp.state_dict()) - mlp_counter = _CallCounter(ours_mlp.veomni_swiglu_mlp) - ours_mlp.veomni_swiglu_mlp = mlp_counter - assert_module_forward_and_grads_match(hf_mlp, ours_mlp, torch.randn(3, cfg.hidden_size)) - if hidden_act in {"silu", "swish"}: - assert mlp_counter.calls == 1 - empty_mlp_out = ours_mlp(torch.empty(0, cfg.hidden_size, requires_grad=True)) - assert empty_mlp_out.shape == (0, cfg.hidden_size) - assert mlp_counter.calls == 2 - else: - assert mlp_counter.calls == 0 - - -def test_bagel_modules_keep_construction_ops_when_config_mutated() -> None: - """Verify ``BagelQwen2MoTAccelerated``, ``BagelSiglipNavit``, and ``BagelTextEncoderAccelerated`` keep construction-time ops after ``config.ops_implementation`` is mutated.""" - from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig - from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit - from veomni.models.seed_omni.modules.bagel.text_encoder.accelerated.accelerated import BagelTextEncoderAccelerated - from veomni.models.seed_omni.modules.bagel.text_encoder.configuration import BagelTextEncoderConfig - - poison_ops = { - "attn_implementation": "sdpa", - "rms_norm_implementation": "liger_kernel", - "swiglu_mlp_implementation": "liger_kernel", - "rotary_pos_emb_implementation": "liger_kernel", - "rotary_pos_emb_vision_implementation": "liger_kernel", - "cross_entropy_loss_implementation": "liger_kernel", - } - - # 1. BagelSiglipNavit forward isolation siglip_cfg = BagelSiglipNavitConfig( hidden_size=32, intermediate_size=64, @@ -365,147 +233,15 @@ def test_bagel_modules_keep_construction_ops_when_config_mutated() -> None: num_attention_heads=4, image_size=28, patch_size=14, - rope=True, - ops_implementation={ - "attn_implementation": "eager", - "rotary_pos_emb_vision_implementation": "eager", - }, + ops_implementation={"attn_implementation": "sdpa", "rotary_pos_emb_vision_implementation": "eager"}, ) siglip = BagelSiglipNavit(siglip_cfg).eval() - siglip_attn = siglip.vision_model.encoder.layers[0].self_attn - assert siglip_attn.veomni_attn.impl == "eager" - assert siglip_attn.veomni_rope.impl == "eager" - - patches = torch.randn(4, 3 * 14 * 14) - pos_ids = torch.arange(4, dtype=torch.long) - cu_seqlens = torch.tensor([0, 2, 4], dtype=torch.int32) - with torch.no_grad(): - siglip_ref = siglip.vision_model(patches, pos_ids, cu_seqlens=cu_seqlens, max_seqlen=2) - siglip_cfg.ops_implementation = dict(poison_ops) - assert siglip_attn.veomni_attn.impl == "eager" - assert siglip_attn.veomni_rope.impl == "eager" - siglip_out = siglip.vision_model(patches, pos_ids, cu_seqlens=cu_seqlens, max_seqlen=2) - torch.testing.assert_close(siglip_out, siglip_ref) - - # 2. BagelTextEncoderAccelerated decode loss isolation - text_cfg = BagelTextEncoderConfig( - vocab_size=32, - hidden_size=16, - ops_implementation={"cross_entropy_loss_implementation": "eager"}, - ) - text_enc = BagelTextEncoderAccelerated(text_cfg).eval() - assert text_enc.veomni_ce.impl == "eager" - hidden = torch.randn(4, 16) - shift_labels = torch.tensor([1, 2, 3, -100], dtype=torch.long) - with torch.no_grad(): - loss_ref = text_enc.decode(hidden_states=hidden, shift_labels=shift_labels)["loss"] - text_cfg.ops_implementation = dict(poison_ops) - assert text_enc.veomni_ce.impl == "eager" - loss_out = text_enc.decode(hidden_states=hidden, shift_labels=shift_labels)["loss"] - torch.testing.assert_close(loss_out, loss_ref) - - # 3. BagelQwen2MoTAccelerated RoPE / RMSNorm / SwiGLU / Attention isolation - mot_cfg = _flex_config() - mot = accelerated.BagelQwen2MoTAccelerated(mot_cfg).eval() - layer = mot.model.layers[0] - rope_counter = _CallCounter(layer.self_attn.veomni_rope) - layer.self_attn.veomni_rope = rope_counter - q = torch.randn(3, layer.self_attn.num_heads, layer.self_attn.head_dim) - k = torch.randn(3, layer.self_attn.num_key_value_heads, layer.self_attn.head_dim) - cos = torch.ones(3, layer.self_attn.head_dim) - sin = torch.zeros(3, layer.self_attn.head_dim) - with torch.no_grad(): - q_ref, k_ref = layer.self_attn.apply_rotary_pos_emb(q, k, cos, sin) - mlp_ref = layer.mlp(torch.randn(3, mot_cfg.hidden_size)) - mot_cfg.ops_implementation = dict(poison_ops) - assert layer.input_layernorm.veomni_rms_norm.impl == "eager" - assert layer.mlp.veomni_swiglu_mlp.impl == "eager" - assert layer.self_attn.veomni_rms_norm.impl == "eager" - assert layer.self_attn.veomni_rope.impl == "eager" - assert layer.self_attn.veomni_attn_impl == "veomni_flex_attention" - assert mot.model._bound_attn_implementation() == "veomni_flex_attention" - q_out, k_out = layer.self_attn.apply_rotary_pos_emb(q, k, cos, sin) - torch.testing.assert_close(q_out, q_ref) - torch.testing.assert_close(k_out, k_ref) - assert rope_counter.calls == 2 - assert torch.isfinite(mlp_ref).all() - - -@pytest.mark.parametrize("target_proj", ("qkv_proj_und", "qkv_proj_gen", "o_proj", "o_proj_moe_gen")) -def test_bagel_qwen2_mot_async_ulysses_falls_back_to_sync_on_lora_projections( - monkeypatch: pytest.MonkeyPatch, - target_proj: str, -) -> None: - """Verify ``BagelQwen2MoTAttentionAccelerated.forward_packed_train`` falls back to synchronous projections when LoRA wraps any projection layer.""" - from types import SimpleNamespace - - from veomni.lora.layers import LoraLinear - - cfg = _flex_config() - attn = accelerated.BagelQwen2MoTAttentionAccelerated(cfg, layer_idx=0).eval() - base_linear = getattr(attn, target_proj) - lora_linear = LoraLinear(base_linear, adapter_name="default", r=2, lora_alpha=4) - with torch.no_grad(): - lora_linear.lora_B["default"].weight.fill_(0.25) - setattr(attn, target_proj, lora_linear) - - def _fake_attn_op(module, q, k, v, mask, **kwargs): - del module, k, mask, kwargs - v_expanded = v.repeat_interleave(q.shape[1] // v.shape[1], dim=1) - return (q + v_expanded).transpose(1, 2).contiguous(), None - - attn.veomni_attn = _fake_attn_op - async_qkv_counter = _CallCounter(lambda **kwargs: (_ for _ in ()).throw(AssertionError("async qkv called"))) - async_o_counter = _CallCounter(lambda **kwargs: (_ for _ in ()).throw(AssertionError("async o called"))) - attn.veomni_async_ulysses_qkv = async_qkv_counter - attn.veomni_async_ulysses_o = async_o_counter - - packed_sequence = torch.randn(4, cfg.hidden_size) - cos = torch.ones(4, attn.head_dim) - sin = torch.zeros(4, attn.head_dim) - und_idx = torch.tensor([0, 2], dtype=torch.long) - gen_idx = torch.tensor([1, 3], dtype=torch.long) - - monkeypatch.setattr( - accelerated, - "get_parallel_state", - lambda: SimpleNamespace(ulysses_enabled=False, async_enabled=False, ulysses_size=1, ulysses_group=None), - ) - with torch.no_grad(): - sync_ref = attn.forward_packed_train(packed_sequence, None, cos, sin, und_idx, gen_idx) - - monkeypatch.setattr( - accelerated, - "get_parallel_state", - lambda: SimpleNamespace(ulysses_enabled=True, async_enabled=True, ulysses_size=2, ulysses_group=None), - ) - with torch.no_grad(): - async_with_lora = attn.forward_packed_train(packed_sequence, None, cos, sin, und_idx, gen_idx) - - assert async_qkv_counter.calls == 0 - assert async_o_counter.calls == 0 - torch.testing.assert_close(async_with_lora, sync_ref) + assert siglip.vision_model.encoder.layers[0].self_attn.veomni_attn.impl == "veomni_sdpa" - -def test_bagel_siglip_navit_multi_image_sample_flops_and_seqlens() -> None: - """Verify multi-image samples preserve per-sample token counts while using per-image lengths for attention FLOPs.""" - from veomni.models.seed_omni.modules.bagel.siglip_navit.accelerated.accelerated import BagelSiglipNavitAccelerated - from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig - from veomni.models.seed_omni.modules.bagel.siglip_navit.processing import _OMNI_POSITION_IDS, _OMNI_TOKEN_LEN - from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT - from veomni.models.seed_omni.utils.conversation import ConversationItem - - cfg = BagelSiglipNavitConfig( - num_hidden_layers=1, - ops_implementation={ - "attn_implementation": "eager", - "rotary_pos_emb_vision_implementation": "eager", - }, - ) - model = BagelSiglipNavitAccelerated(cfg) - model.config.num_hidden_layers = 26 - - patch_dim = cfg.num_channels * cfg.patch_size * cfg.patch_size + # Multi-image SigLIP FLOPs meter uses per-image lengths while keeping per-sample token counts + acc_siglip = BagelSiglipNavitAccelerated(siglip_cfg) + acc_siglip.config.num_hidden_layers = 26 + patch_dim = siglip_cfg.num_channels * siglip_cfg.patch_size * siglip_cfg.patch_size items = [ ConversationItem( type="image", @@ -519,124 +255,9 @@ def test_bagel_siglip_navit_multi_image_sample_flops_and_seqlens() -> None: ) for _ in range(2) ] - model.forward_pre(conversation_list=[items]) - model.metric_meter_add("forward") - flops, seqlens = model.metric_meter_collect() - + acc_siglip.forward_pre(conversation_list=[items]) + acc_siglip.metric_meter_add("forward") + flops, seqlens = acc_siglip.metric_meter_collect() assert seqlens == [2048] - expected_per_image_flops = model.estimate_flops([1024, 1024]) - overcounted_sample_flops = model.estimate_flops([2048]) - assert flops == pytest.approx(expected_per_image_flops) - assert flops == pytest.approx(5.834821, rel=1e-4) - assert overcounted_sample_flops == pytest.approx(6.588592, rel=1e-4) - assert flops < overcounted_sample_flops - - -def test_bagel_flow_connector_and_siglip_navit_support_lora_wrapped_projections() -> None: - """Verify ``BagelFlowConnector`` and ``BagelSiglipNavit`` run with ``LoraLinear``-wrapped projections.""" - from veomni.lora.layers import LoraLinear - from veomni.models.seed_omni.modules.bagel.flow_connector.configuration import BagelFlowConnectorConfig - from veomni.models.seed_omni.modules.bagel.flow_connector.modeling import BagelFlowConnector - from veomni.models.seed_omni.modules.bagel.siglip_navit.configuration import BagelSiglipNavitConfig - from veomni.models.seed_omni.modules.bagel.siglip_navit.modeling import BagelSiglipNavit - - flow_cfg = BagelFlowConnectorConfig( - hidden_size=16, - z_channels=4, - latent_patch_size=2, - max_latent_size=8, - timestep_frequency_embedding_size=8, - ) - flow = BagelFlowConnector(flow_cfg).eval() - flow.vae2llm = LoraLinear(flow.vae2llm, adapter_name="default", r=2, lora_alpha=4) - flow.llm2vae = LoraLinear(flow.llm2vae, adapter_name="default", r=2, lora_alpha=4) - flow.time_embedder.mlp[0] = LoraLinear(flow.time_embedder.mlp[0], adapter_name="default", r=2, lora_alpha=4) - flow.time_embedder.mlp[2] = LoraLinear(flow.time_embedder.mlp[2], adapter_name="default", r=2, lora_alpha=4) - with torch.no_grad(): - flow.vae2llm.lora_B["default"].weight.fill_(0.1) - flow.llm2vae.lora_B["default"].weight.fill_(0.1) - latent_out = flow.embed_latent( - latents=torch.randn(3, flow_cfg.patch_latent_dim), - position_ids=torch.tensor([0, 1, 2], dtype=torch.long), - timesteps=torch.tensor([0.2, 0.5, 0.8], dtype=torch.float32), - )["latent_embeds"] - vel_out = flow.decode_velocity(torch.randn(3, flow_cfg.hidden_size))["velocity"] - assert latent_out.shape == (3, flow_cfg.hidden_size) - assert vel_out.shape == (3, flow_cfg.patch_latent_dim) - assert torch.isfinite(latent_out).all() - assert torch.isfinite(vel_out).all() - - siglip_cfg = BagelSiglipNavitConfig( - hidden_size=16, - output_size=24, - intermediate_size=32, - num_hidden_layers=1, - num_attention_heads=4, - image_size=28, - patch_size=14, - rope=False, - ops_implementation={ - "attn_implementation": "eager", - "rotary_pos_emb_vision_implementation": "eager", - }, - ) - siglip = BagelSiglipNavit(siglip_cfg).eval() - siglip.vision_model.embeddings.patch_embedding = LoraLinear( - siglip.vision_model.embeddings.patch_embedding, adapter_name="default", r=2, lora_alpha=4 - ) - siglip.connector.fc1 = LoraLinear(siglip.connector.fc1, adapter_name="default", r=2, lora_alpha=4) - siglip.connector.fc2 = LoraLinear(siglip.connector.fc2, adapter_name="default", r=2, lora_alpha=4) - patch_dim = siglip_cfg.num_channels * siglip_cfg.patch_size * siglip_cfg.patch_size - with torch.no_grad(): - siglip_out = siglip( - patchified_pixel_values=torch.randn(4, patch_dim), - patchified_position_ids=torch.arange(4, dtype=torch.long), - cu_seqlens=torch.tensor([0, 2, 4], dtype=torch.int32), - max_seqlen=2, - token_lens=torch.tensor([2, 2], dtype=torch.int32), - ) - assert siglip_out["image_embeds"].shape == (4, siglip_cfg.output_size) - assert torch.isfinite(siglip_out["image_embeds"]).all() - - -def test_bagel_qwen2_rotary_embedding_to_empty_from_meta_rebuilds_fp32_table() -> None: - """Verify ``BagelQwen2RotaryEmbedding`` built on ``meta`` rebuilds valid fp32 buffers on ``to_empty``.""" - from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2RotaryEmbedding - - cfg = _flex_config() - ref_rotary = BagelQwen2RotaryEmbedding(cfg) - with torch.device("meta"): - meta_model = accelerated.BagelQwen2MoTAccelerated(cfg) - assert meta_model.model.rotary_emb.inv_freq.is_meta - assert meta_model.model.rotary_emb.original_inv_freq.is_meta - - materialized = meta_model.to_empty(device="cpu").to(dtype=torch.bfloat16) - rotary = materialized.model.rotary_emb - assert not rotary.inv_freq.is_meta - assert not rotary.original_inv_freq.is_meta - assert rotary.inv_freq.dtype == torch.float32 - assert rotary.original_inv_freq.dtype == torch.float32 - assert rotary.inv_freq.data_ptr() != rotary.original_inv_freq.data_ptr() - torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) - torch.testing.assert_close(rotary.original_inv_freq, ref_rotary.original_inv_freq) - - -def test_bagel_qwen2_mot_config_defaults_to_flex_attention(available_nvidia_ops) -> None: # noqa: F811 - """Verify ``BagelQwen2MoTConfig`` defaults to ``veomni_flex_attention`` when no attention backend is specified.""" - from veomni.models.seed_omni.modules.bagel.qwen2_mot.modeling import BagelQwen2MoT - - cfg = BagelQwen2MoTConfig( - vocab_size=128, - hidden_size=64, - intermediate_size=128, - num_hidden_layers=1, - num_attention_heads=4, - num_key_value_heads=1, - max_position_embeddings=64, - bos_token_id=1, - eos_token_id=2, - ) - assert cfg._attn_implementation == "veomni_flex_attention" - model = BagelQwen2MoT(cfg).eval() - assert model.model.layers[0].self_attn.veomni_attn is not None - assert model.model.layers[0].self_attn.veomni_attn.impl == "veomni_flex_attention" + assert flops == pytest.approx(acc_siglip.estimate_flops([1024, 1024])) + assert flops < acc_siglip.estimate_flops([2048]) diff --git a/tests/seed_omni/modules/bagel/test_source_routing.py b/tests/seed_omni/modules/bagel/test_source_routing.py deleted file mode 100644 index ab12387ab7..0000000000 --- a/tests/seed_omni/modules/bagel/test_source_routing.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace - -import pytest -import torch - -from scripts.ci.accelerator.launch_plan import LaunchSpec -from tests.seed_omni.modules.bagel.helpers import config_cls, model_cls, tiny_bagel_qwen2_cfg -from veomni.models.seed_omni.modules.bagel.sources import BAGEL_CONTEXT_KEY, BAGEL_SIGLIP_CONTEXT, BAGEL_VAE_CONTEXT -from veomni.models.seed_omni.modules.bagel.vae.processing import route_image_contexts -from veomni.models.seed_omni.utils.conversation import _IMG_TAG_KEY, ConversationItem - - -pytestmark = pytest.mark.launch_plan(provider=LaunchSpec(device="cpu")) - - -def test_bagel_route_image_contexts_prefers_img_tag() -> None: - """Verify ``route_image_contexts`` routes training images according to ``_img_tag``.""" - und = ConversationItem( - type="image", - value=torch.zeros(3, 2, 2), - role="user", - meta={_IMG_TAG_KEY: "und"}, - ) - gen = ConversationItem( - type="image", - value=torch.ones(3, 2, 2), - role="assistant", - meta={_IMG_TAG_KEY: "gen"}, - ) - edit = ConversationItem( - type="image", - value=torch.full((3, 2, 2), 2), - role="user", - meta={_IMG_TAG_KEY: "edit"}, - ) - sample = [und, gen, edit] - - route_image_contexts([sample], inference=False, infer_type=None) - - assert sample[0] is und - assert sample[0].meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT - assert sample[1] is gen - assert sample[1].meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT - assert sample[3] is edit - assert sample[2].meta == {_IMG_TAG_KEY: "edit", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} - assert sample[3].meta == {_IMG_TAG_KEY: "edit", BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT} - assert sample[2] is not sample[3] - sample[2].value[0, 0, 0] = 99 - assert int(sample[3].value[0, 0, 0]) == 2 - - -def test_bagel_route_image_contexts_requires_img_tag_for_raw_training_images() -> None: - """Verify ``route_image_contexts`` raises when a raw training image lacks ``_img_tag``.""" - image = ConversationItem(type="image", value=torch.zeros(3, 2, 2), role="assistant") - - with pytest.raises(ValueError, match="_img_tag"): - route_image_contexts([[image]], inference=False, infer_type=None) - - -def test_bagel_route_image_contexts_uses_infer_type_for_raw_inference_images() -> None: - """Verify ``route_image_contexts`` routes raw inference user images based on ``infer_type``.""" - edit_user = ConversationItem(type="image", value=torch.full((3, 2, 2), 3), role="user") - und_user = ConversationItem(type="image", value=torch.ones(3, 2, 2), role="user") - gen_user = ConversationItem(type="image", value=torch.zeros(3, 2, 2), role="user") - edit_sample = [edit_user] - und_sample = [und_user] - gen_sample = [gen_user] - - route_image_contexts([edit_sample], inference=True, infer_type="infer_edit") - route_image_contexts([und_sample], inference=True, infer_type="infer_und") - route_image_contexts([gen_sample], inference=True, infer_type="infer_gen") - - assert edit_sample[0].meta[BAGEL_CONTEXT_KEY] == BAGEL_VAE_CONTEXT - assert edit_sample[1] is edit_user - assert edit_sample[1].meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT - assert und_sample[0] is und_user - assert und_user.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT - assert gen_sample[0] is gen_user - assert gen_user.meta[BAGEL_CONTEXT_KEY] == BAGEL_SIGLIP_CONTEXT - - -def test_bagel_qwen2_mot_forward_pre_rejects_only_upstream_dummy_anchor(monkeypatch) -> None: - """Verify ``forward_pre`` raises when the batch contains only dummy anchors and no real tokens.""" - model = _tiny_qwen2_mot() - hidden_size = int(model.config.hidden_size) - siglip_dummy = ConversationItem( - type="image", - value=torch.ones(1, hidden_size, requires_grad=True), - role="user", - is_dummy=True, - meta={_IMG_TAG_KEY: "und", BAGEL_CONTEXT_KEY: BAGEL_SIGLIP_CONTEXT}, - ) - monkeypatch.setattr(model, "_has_valid_upstream_embeddings", lambda conversation_list: (True, False)) - - with pytest.raises(ValueError, match="got no packable tokens"): - model.forward_pre(conversation_list=[[siglip_dummy]]) - - -def test_bagel_qwen2_mot_rejects_context_parallel_training(monkeypatch) -> None: - """Verify ``BagelQwen2MoTAccelerated.forward_pre`` rejects ``cp_size > 1``.""" - from veomni.models.seed_omni.modules.bagel.qwen2_mot.accelerated import accelerated - - model = _tiny_qwen2_mot() - monkeypatch.setattr( - accelerated, - "get_parallel_state", - lambda: SimpleNamespace(sp_size=2, cp_size=2), - ) - conversation = [ - [ - ConversationItem( - type="text", - value=torch.zeros(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype), - role="user", - ) - ] - ] - - with pytest.raises(ValueError, match="Ulysses sequence parallelism only"): - model.forward_pre(conversation_list=conversation) - - -def test_bagel_flow_dummy_embed_anchors_to_vae_dummy_output() -> None: - """Verify dummy flow embedding preserves autograd connectivity to the upstream VAE dummy output.""" - model = _tiny_flow_connector() - vae_dummy = ConversationItem( - type="image", - value=torch.ones(1, 1, 1, requires_grad=True), - role="assistant", - is_dummy=True, - meta={_IMG_TAG_KEY: "gen", BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT}, - ) - conversation = [[vae_dummy]] - - inputs = model.embed_latent_pre(conversation_list=conversation) - latent_embeds = torch.ones(1, int(model.config.hidden_size), device=model.device, dtype=model.dtype) - latent_embeds = latent_embeds + inputs["latents"].sum() * 0.0 - out = model.embed_latent_post(latent_embeds) - - assert inputs["latents"].requires_grad - assert out["conversation_list"] is conversation - flow_dummy = conversation[0][-1] - assert flow_dummy.type == "output" - assert flow_dummy.is_dummy - assert flow_dummy.meta == {BAGEL_CONTEXT_KEY: BAGEL_VAE_CONTEXT} - assert torch.is_tensor(flow_dummy.value) - assert flow_dummy.value.requires_grad - - -def _tiny_flow_connector(): - BagelFlowConnector = model_cls("bagel_flow_connector") - BagelFlowConnectorConfig = config_cls("bagel_flow_connector") - return BagelFlowConnector( - BagelFlowConnectorConfig( - hidden_size=4, - z_channels=1, - latent_patch_size=1, - patch_latent_dim=1, - max_latent_size=4, - timestep_frequency_embedding_size=4, - resolution=16, - ) - ) - - -def _tiny_qwen2_mot(): - BagelQwen2MoT = model_cls("bagel_qwen2_mot") - BagelQwen2MoTConfig = config_cls("bagel_qwen2_mot") - return BagelQwen2MoT(BagelQwen2MoTConfig(**tiny_bagel_qwen2_cfg())) From c9eea58222271478db89ec747f9973e3842419ee Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 11 Nov 2026 12:39:08 +0800 Subject: [PATCH 21/22] fix: preserve BagelFlowConnector llm2vae zero init in _init_weights Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 19 +++++++++++++++++++ .../modules/bagel/flow_connector/modeling.py | 7 +++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index cad2f2adba..e93fb53333 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -226,6 +226,25 @@ def test_bagel_modules_ops_dispatch_meta_rotary_and_flops(available_nvidia_ops) assert rotary.original_inv_freq.dtype == torch.float32 torch.testing.assert_close(rotary.inv_freq, ref_rotary.inv_freq) + # BagelFlowConnector meta -> to_empty -> init_weights preserves zero-initialized llm2vae velocity head + flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()( + hidden_size=16, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + ) + eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + with torch.device("meta"): + meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + materialized_flow = meta_flow.to_empty(device="cpu") + materialized_flow.init_weights() + assert torch.count_nonzero(materialized_flow.llm2vae.weight) == 0 + assert torch.count_nonzero(materialized_flow.llm2vae.bias) == 0 + assert torch.count_nonzero(materialized_flow.vae2llm.weight) > 0 + torch.testing.assert_close(materialized_flow.latent_pos_embed.pos_embed, eager_flow.latent_pos_embed.pos_embed) + siglip_cfg = BagelSiglipNavitConfig( hidden_size=32, intermediate_size=64, diff --git a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py index 3990bd78df..4d052c526f 100644 --- a/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py +++ b/veomni/models/seed_omni/modules/bagel/flow_connector/modeling.py @@ -309,13 +309,16 @@ def __init__(self, config: BagelFlowConnectorConfig) -> None: self.latent_pos_embed = PositionEmbedding(config.max_latent_size, config.hidden_size) self._generation_state = FlowGenerationState() self.post_init() - nn.init.constant_(self.llm2vae.weight, 0) - nn.init.constant_(self.llm2vae.bias, 0) def _init_weights(self, module: nn.Module) -> None: if isinstance(module, PositionEmbedding): module.reset_parameters() return + if module is self.llm2vae: + nn.init.constant_(module.weight, 0) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + return super()._init_weights(module) @property From 2e7c6619e53dafad3acc6ef21c385f8b7e3ffb05 Mon Sep 17 00:00:00 2001 From: "chenzeren.1" Date: Wed, 11 Nov 2026 15:07:27 +0800 Subject: [PATCH 22/22] fix: shard PositionEmbedding reset_parameters across DTensor mesh Co-authored-by: Cursor --- .../seed_omni/modules/bagel/test_qwen2_mot.py | 79 +++++++++++++++++++ .../modules/bagel/position_embedding.py | 14 +++- 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/tests/seed_omni/modules/bagel/test_qwen2_mot.py b/tests/seed_omni/modules/bagel/test_qwen2_mot.py index e93fb53333..d0a8bf2dc6 100644 --- a/tests/seed_omni/modules/bagel/test_qwen2_mot.py +++ b/tests/seed_omni/modules/bagel/test_qwen2_mot.py @@ -280,3 +280,82 @@ def test_bagel_modules_ops_dispatch_meta_rotary_and_flops(available_nvidia_ops) assert seqlens == [2048] assert flops == pytest.approx(acc_siglip.estimate_flops([1024, 1024])) assert flops < acc_siglip.estimate_flops([2048]) + + +def _two_rank_fsdp2_pos_embed_materialize_worker(rank: int, rendezvous: str) -> None: + """Verify two-rank FSDP2 ``meta -> fully_shard -> to_empty -> init_weights`` materializes sharded ``PositionEmbedding``.""" + import torch.distributed as dist + from torch.distributed.fsdp import fully_shard + from torch.distributed.tensor import DTensor + + from veomni.distributed.parallel_state import _init_parallel_state, clear_parallel_state + + dist.init_process_group("gloo", init_method=f"file://{rendezvous}", world_size=2, rank=rank) + try: + ps = _init_parallel_state(dp_size=2, dp_shard_size=2, ulysses_size=1, dp_mode="fsdp2", device_type="cpu") + + flow_cfg = OMNI_CONFIG_REGISTRY["bagel_flow_connector"]()( + hidden_size=16, + z_channels=2, + latent_patch_size=1, + patch_latent_dim=2, + max_latent_size=4, + timestep_frequency_embedding_size=4, + ) + eager_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + with torch.device("meta"): + meta_flow = OMNI_MODEL_REGISTRY["bagel_flow_connector"]()(flow_cfg) + fully_shard(meta_flow, mesh=ps.fsdp_mesh) + meta_flow.to_empty(device="cpu") + meta_flow.init_weights() + + assert isinstance(meta_flow.latent_pos_embed.pos_embed.data, DTensor) + assert meta_flow.latent_pos_embed.pos_embed.to_local().shape == (8, 16) + torch.testing.assert_close( + meta_flow.latent_pos_embed.pos_embed.full_tensor(), + eager_flow.latent_pos_embed.pos_embed, + ) + assert torch.count_nonzero(meta_flow.llm2vae.weight.full_tensor()) == 0 + assert torch.count_nonzero(meta_flow.llm2vae.bias.full_tensor()) == 0 + assert torch.count_nonzero(meta_flow.vae2llm.weight.full_tensor()) > 0 + + siglip_cfg = BagelSiglipNavitConfig( + hidden_size=16, + output_size=16, + intermediate_size=32, + num_hidden_layers=1, + num_attention_heads=2, + image_size=8, + patch_size=4, + vit_max_num_patch_per_side=4, + ops_implementation={"attn_implementation": "eager", "rotary_pos_emb_vision_implementation": "eager"}, + ) + eager_siglip = BagelSiglipNavit(siglip_cfg) + with torch.device("meta"): + meta_siglip = BagelSiglipNavit(siglip_cfg) + fully_shard(meta_siglip, mesh=ps.fsdp_mesh) + meta_siglip.to_empty(device="cpu") + meta_siglip.init_weights() + + assert isinstance(meta_siglip.vit_pos_embed.pos_embed.data, DTensor) + assert meta_siglip.vit_pos_embed.pos_embed.to_local().shape == (8, 16) + torch.testing.assert_close( + meta_siglip.vit_pos_embed.pos_embed.full_tensor(), + eager_siglip.vit_pos_embed.pos_embed, + ) + finally: + clear_parallel_state() + dist.destroy_process_group() + + +@pytest.mark.launch_plan(provider=LaunchSpec(nproc=2, device="cpu")) +def test_bagel_pos_embed_two_rank_fsdp2_meta_materialization(tmp_path: Path) -> None: + """Verify two-rank FSDP2 config-only initialization distributes ``PositionEmbedding.pos_embed`` across shards.""" + import torch.multiprocessing as mp + + mp.spawn( + _two_rank_fsdp2_pos_embed_materialize_worker, + args=(str(tmp_path / "rendezvous"),), + nprocs=2, + join=True, + ) diff --git a/veomni/models/seed_omni/modules/bagel/position_embedding.py b/veomni/models/seed_omni/modules/bagel/position_embedding.py index 7364939074..39e179a417 100644 --- a/veomni/models/seed_omni/modules/bagel/position_embedding.py +++ b/veomni/models/seed_omni/modules/bagel/position_embedding.py @@ -5,6 +5,7 @@ import numpy as np import torch import torch.nn as nn +from torch.distributed.tensor import DTensor, distribute_tensor class PositionEmbedding(nn.Module): @@ -22,8 +23,19 @@ def __init__(self, max_num_patch_per_side: int, hidden_size: int) -> None: def reset_parameters(self) -> None: """Populate ``pos_embed`` with deterministic 2D sin-cos values.""" + if self.pos_embed.is_meta: + return new_data = _get_2d_sincos_pos_embed(self.hidden_size, self.max_num_patch_per_side) - if hasattr(self.pos_embed.data, "to_local"): + if isinstance(self.pos_embed.data, DTensor): + local_target = self.pos_embed.data.to_local() + sharded = distribute_tensor( + new_data.to(device=local_target.device, dtype=local_target.dtype), + self.pos_embed.data.device_mesh, + self.pos_embed.data.placements, + src_data_rank=None, + ) + local_target.copy_(sharded.to_local().to(device=local_target.device, dtype=local_target.dtype)) + elif hasattr(self.pos_embed.data, "to_local"): self.pos_embed.data.to_local().copy_(new_data) else: self.pos_embed.data.copy_(new_data)