Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/gpu_unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ jobs:
# TODO: run all tests under tests/utils/ once all util tests are verified
run: |
uv run --frozen pytest -s -x tests/utils/test_count_flops.py
uv run --frozen pytest -s -x tests/utils/test_token_accounting.py
uv run --frozen pytest -s -x tests/utils/test_hdfs_io.py
- name: Run Muon optimizer tests
run: |
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/npu_unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ jobs:
- name: Run utils tests
run: |
uv run --frozen pytest -v -s -x tests/utils/test_count_flops.py
uv run --frozen pytest -v -s -x tests/utils/test_token_accounting.py
uv run --frozen pytest -v -s -x tests/utils/test_hdfs_io.py
- name: Run LoRA save/load test
# peft ships under the `npu` extra, so the earlier
Expand Down
18 changes: 18 additions & 0 deletions docs/usage/trainer.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,24 @@ VeOmni includes several built-in callbacks:
- **[GlobalStateCallback](https://github.com/ByteDance-Seed/VeOmni/blob/main/veomni/trainer/callbacks/global_state_callback.py)**: Saves the job cursor per rank — the dataloader position as `loader/rank_{N}.pt`, the step counter, rng and meters as `extra_state/rank_{N}.pt` — separate from the model's `model/lr_scheduler.pt`. Runs last, and writes the step's `checkpoint_manifest.json`.
- **[EvaluateCallback](https://github.com/ByteDance-Seed/VeOmni/blob/main/veomni/trainer/callbacks/evaluate_callback.py)**: Runs evaluation on the validation set.

`EnvironMeterCallback` also reports topology-invariant packed-token metrics
under `token_accounting/*`. The counters are captured before sequence-parallel
slicing and reduced only across data-parallel ranks:

- `physical_window_tokens`: padded window capacity.
- `aligned_compute_tokens`: compute tokens after alignment, excluding known tail padding.
- `source_input_tokens`: input document tokens before alignment and padding.
- `loss_tokens`: labels that are not `IGNORE_INDEX`.
- `source_fill`, `aligned_compute_fill`, and the corresponding
`*_tokens_per_second(M)` metrics.

This distinction is important when a sequence-parallel attention backend uses
rank-local `cu_seq_lens_q`; kernel-local boundaries are not a valid source-token
throughput counter. When only rank-local `cu_seq_lens_q` is available under
CP/Ulysses, the legacy FLOPs meter leaves those boundaries unchanged; it does
not multiply each document by the SP degree. The collator counters above remain
the authoritative global token totals.

On-disk layout for weights, optimizer, scheduler and job cursor, and how a checkpoint from an earlier layout is still resumed: [Checkpoint layout](checkpoint.md).

### Custom Callbacks
Expand Down
190 changes: 190 additions & 0 deletions tests/utils/test_token_accounting.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
import types

import pytest
import torch

from veomni.utils.constants import IGNORE_INDEX
from veomni.utils.token_accounting import (
TOKEN_ACCOUNTING_KEY,
TokenAccounting,
build_token_accounting,
metrics_from_totals,
pop_token_accounting,
)


def _parallel_state(*, sp_size: int, sp_rank: int = 0):
return types.SimpleNamespace(
sp_enabled=sp_size > 1,
sp_size=sp_size,
sp_rank=sp_rank,
cp_enabled=False,
)


def _documents():
return [
{
"input_ids": torch.arange(5),
"attention_mask": torch.ones(5, dtype=torch.long),
"labels": torch.arange(5),
},
{
"input_ids": torch.arange(7),
"attention_mask": torch.ones(7, dtype=torch.long),
"labels": torch.arange(7),
},
]


def _collate(monkeypatch, *, sp_size: int):
import veomni.data.data_collator as data_collator

monkeypatch.setattr(data_collator, "get_parallel_state", lambda: _parallel_state(sp_size=sp_size))
batch = data_collator.MainCollator(pad_to_length=16)(_documents())
assert TOKEN_ACCOUNTING_KEY in batch
return pop_token_accounting(batch)


def test_token_accounting_validates_ordering():
stats = build_token_accounting(
physical_window_tokens=16,
aligned_compute_tokens=12,
source_input_tokens=12,
loss_tokens=10,
num_documents=2,
max_document_length=7,
sum_document_len_squared=74,
)
assert stats.source_input_tokens == 12

with pytest.raises(ValueError, match="ordering violated"):
build_token_accounting(
physical_window_tokens=10,
aligned_compute_tokens=12,
source_input_tokens=12,
loss_tokens=10,
num_documents=2,
max_document_length=7,
)


def test_training_step_gate_rejects_schema_only_zero_counters():
with pytest.raises(ValueError, match="training-step gate violated"):
TokenAccounting(16, 12, 0, 0, 0, 0).validate_training_step()

TokenAccounting(16, 12, 12, 10, 2, 7, 74).validate_training_step()


def test_observability_validation_is_nonfatal():
from veomni.utils.helper import _safe_token_accounting_metrics

metrics, valid = _safe_token_accounting_metrics(
TokenAccounting(16, 12, 0, 0, 0, 0),
delta_time=1.0,
)

assert valid is False
assert metrics == {}


def test_collator_accounting_validation_is_nonfatal():
import veomni.data.data_collator as data_collator

batch = {"position_ids": torch.arange(8).unsqueeze(0)}
data_collator._finalize_token_accounting(
batch,
partial={
"source_input_tokens": 12,
"num_documents": 2,
"max_document_length": 7,
"sum_document_len_squared": 74,
},
loss_tokens=10,
tail_padding_length=0,
)

assert TOKEN_ACCOUNTING_KEY not in batch


def test_metrics_distinguish_capacity_source_and_loss():
totals = TokenAccounting(16, 12, 12, 10, 2, 7, 74)
metrics = metrics_from_totals(totals, delta_time=2.0)

assert metrics["token_accounting/source_fill"] == pytest.approx(0.75)
assert metrics["token_accounting/loss_density"] == pytest.approx(10 / 12)
assert metrics["token_accounting/capacity_tokens_per_second(M)"] == pytest.approx(8e-6)
assert metrics["token_accounting/source_tokens_per_second(M)"] == pytest.approx(6e-6)


def test_collator_accounting_is_sequence_parallel_invariant(monkeypatch):
without_sp = _collate(monkeypatch, sp_size=1)
with_sp = _collate(monkeypatch, sp_size=2)

assert without_sp == with_sp
assert with_sp.physical_window_tokens == 16
assert with_sp.aligned_compute_tokens == 12
assert with_sp.source_input_tokens == 12
assert with_sp.loss_tokens == 10
assert with_sp.num_documents == 2
assert with_sp.max_document_length == 7
assert with_sp.sum_document_len_squared == 74


def test_compute_seqlens_prefers_global_linear_attention_boundaries(monkeypatch):
from veomni.utils import helper

state = types.SimpleNamespace(cp_enabled=True, sp_size=4)
monkeypatch.setattr(helper, "get_parallel_state", lambda: state)
micro_batch = {
"cu_seq_lens_q": torch.tensor([0, 3, 4], dtype=torch.int32),
"linear_attn_cu_seq_lens_q": torch.tensor([0, 12, 16], dtype=torch.int32),
"tail_padding_length": torch.tensor(4, dtype=torch.int32),
}

assert helper._compute_seqlens(micro_batch) == [12]


def test_compute_seqlens_does_not_scale_rank_local_documents_under_cp(monkeypatch):
from veomni.utils import helper

monkeypatch.setattr(
helper,
"get_parallel_state",
lambda: types.SimpleNamespace(cp_enabled=True, sp_size=4),
)
micro_batch = {"cu_seq_lens_q": torch.tensor([0, 3, 7], dtype=torch.int32)}

# CP boundaries are rank-local fragments. Multiplying each fragment would
# inflate the quadratic attention term used by the FLOPs estimator.
assert helper._compute_seqlens(micro_batch) == [3, 4]


def test_environ_meter_consumes_accounting_before_model_forward(monkeypatch):
from veomni.utils import helper

meter = object.__new__(helper.EnvironMeter)
meter.config = types.SimpleNamespace(condition_model_type=None)
meter.enable_multisource = False
meter.batch_token_accountings = []
meter.batch_accounting_expected = 0
meter.batch_accounting_observed = 0
meter.batch_seqlens = []
meter.images_seqlens = []
monkeypatch.setattr(helper, "_compute_seqlens", lambda _: [12])
monkeypatch.setattr(helper, "_compute_image_seqlens", lambda _: [])

batch = {TOKEN_ACCOUNTING_KEY: TokenAccounting(16, 12, 12, 10, 2, 7, 74).to_dict()}
meter.add(batch)

assert TOKEN_ACCOUNTING_KEY not in batch
assert meter.batch_accounting_expected == 1
assert meter.batch_accounting_observed == 1
assert meter.batch_token_accountings == [TokenAccounting(16, 12, 12, 10, 2, 7, 74)]


def test_loss_count_excludes_padding_and_pack_boundaries(monkeypatch):
accounting = _collate(monkeypatch, sp_size=1)
assert accounting.loss_tokens < accounting.source_input_tokens
assert accounting.loss_tokens == 10
assert IGNORE_INDEX == -100
70 changes: 70 additions & 0 deletions veomni/data/data_collator.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@
prepare_fa_kwargs_from_position_ids,
valid_seqlens_from_cu_seqlens,
)
from ..utils.token_accounting import (
attach_token_accounting,
build_token_accounting,
count_loss_tokens,
document_lengths_from_position_ids,
summarize_document_lengths,
)


# A model-provided hook that derives ``multimodal_metadata`` from a packed +
Expand All @@ -45,6 +52,7 @@
logger = logging.get_logger(__name__)

_LINEAR_ATTN_TAIL_PADDING_LENGTH = "_linear_attn_tail_padding_length"
_TOKEN_ACCOUNTING_PARTIAL = "_token_accounting_partial"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please revert all changes to veomni/data/data_collator.py (and veomni/trainer/base.py, veomni/trainer/text_dpo_trainer.py, veomni/trainer/vlm_trainer.py). SequenceParallelCollator already populates global cu_seq_lens_q and tail_padding_length before slicing position_ids, so EnvironMeter does not need a collator sidecar dict.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I found two more cases while validating the helper-only simplification against the current main:

  • valid_seqlens_from_cu_seqlens treats absent tail_padding_length as an ambiguous legacy tail and strips trailing length-1 segments. The standard collator omits that field when the known tail length is zero, so packed lengths [3,1,1] can be reported as [3] if the producer's no-padding contract is not retained. This also affects the existing meter/source accounting, not just the new display metrics. For arbitrary collators an absent field is genuinely ambiguous, so blindly treating it as zero is unsafe.
  • Sequence classification can have input-shaped labels with only the final position supervised; a shape-based choice of the causal shift reports zero for a one-token example. The objective cannot always be inferred from the tensor shape.

I agree that the collator sidecar and extra reductions should be removed where the existing metadata proves the layout. For ambiguous inputs, would you prefer an explicit producer/objective signal, or suppressing the new metrics until the metadata is provable? I am keeping this change off the PR until these cases are safe and tested.



def add_flash_attention_kwargs_from_position_ids(
Expand Down Expand Up @@ -294,6 +302,13 @@ def __call__(self, features: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.T
if pack_dim == -1:
batch[key] = batch[key].unsqueeze(0)

if "position_ids" in batch:
document_stats = summarize_document_lengths(document_lengths_from_position_ids(batch["position_ids"]))
else:
sequence_length = int(batch["input_ids"].shape[-1]) if "input_ids" in batch else 0
document_stats = summarize_document_lengths([sequence_length] if sequence_length else [])
batch[_TOKEN_ACCOUNTING_PARTIAL] = document_stats

linear_attn_tail_padding_length = 0
if self.pad_to_length:
input_ids_len_before = batch["input_ids"].shape[-1]
Expand All @@ -309,11 +324,55 @@ def __call__(self, features: List[Dict[str, torch.Tensor]]) -> Dict[str, torch.T
# the hook sees the SP-padded batch + per-modality pad counts.
if self.metadata_collate_func is not None:
self.metadata_collate_func(batch, {"pixel_values": 0, "pixel_values_videos": 0})
if not self.seq_classification and "labels" in batch:
shifted_labels = F.pad(batch["labels"][..., 1:].contiguous(), (0, 1), "constant", IGNORE_INDEX)
loss_tokens = count_loss_tokens(shifted_labels)
else:
loss_tokens = count_loss_tokens(batch["labels"]) if "labels" in batch else 0
partial_accounting = batch.pop(_TOKEN_ACCOUNTING_PARTIAL, None)
if partial_accounting is not None:
_finalize_token_accounting(
batch,
partial=partial_accounting,
loss_tokens=loss_tokens,
tail_padding_length=linear_attn_tail_padding_length,
)
elif linear_attn_tail_padding_length:
batch[_LINEAR_ATTN_TAIL_PADDING_LENGTH] = linear_attn_tail_padding_length
return batch


def _finalize_token_accounting(
batch: Dict[str, Any],
*,
partial: Dict[str, int],
loss_tokens: int,
tail_padding_length: int,
) -> None:
"""Attach global token counters before sequence-parallel slicing."""
if "position_ids" not in batch:
return
physical = int(batch["position_ids"].shape[0] * batch["position_ids"].shape[-1])
if "linear_attn_cu_seq_lens_q" in batch:
aligned = int(batch["linear_attn_cu_seq_lens_q"][-1].item()) - int(tail_padding_length)
else:
aligned = physical - int(tail_padding_length)
try:
stats = build_token_accounting(
physical_window_tokens=physical,
aligned_compute_tokens=aligned,
source_input_tokens=int(partial["source_input_tokens"]),
loss_tokens=loss_tokens,
num_documents=int(partial["num_documents"]),
max_document_length=int(partial["max_document_length"]),
sum_document_len_squared=int(partial["sum_document_len_squared"]),
)
except (KeyError, TypeError, ValueError) as exc:
logger.warning_rank0("Skipping invalid token-accounting metadata: %s", exc)
return
attach_token_accounting(batch, stats)


@dataclass
class SequenceParallelCollator(DataCollator):
collate_infos: Dict[str, DataCollateInfo] = field(default_factory=lambda: DEFAULT_DATA_COLLATE_INFO.copy())
Expand Down Expand Up @@ -370,11 +429,15 @@ def sp_padding(
return torch.cat((feature, pad), dim=dim)

def __call__(self, batch: Dict[str, Union[torch.Tensor, List[torch.Tensor]]]) -> Dict[str, torch.Tensor]:
partial_accounting = batch.pop(_TOKEN_ACCOUNTING_PARTIAL, None)
if not self.seq_classification:
# shift labels
labels = batch["labels"][..., 1:].contiguous()
labels = F.pad(labels, (0, 1), "constant", IGNORE_INDEX)
batch["labels"] = labels
loss_tokens = count_loss_tokens(labels)
else:
loss_tokens = count_loss_tokens(batch["labels"]) if "labels" in batch else 0

linear_attn_tail_padding_length = int(batch.pop(_LINEAR_ATTN_TAIL_PADDING_LENGTH, 0))

Expand Down Expand Up @@ -413,6 +476,13 @@ def __call__(self, batch: Dict[str, Union[torch.Tensor, List[torch.Tensor]]]) ->
batch[key] = self.sp_slice(key, batch[key], dim=pack_dim)

add_flash_attention_kwargs_from_position_ids(batch, linear_attn_tail_padding_length)
if partial_accounting is not None:
_finalize_token_accounting(
batch,
partial=partial_accounting,
loss_tokens=loss_tokens,
tail_padding_length=linear_attn_tail_padding_length,
)

batch["position_ids"] = self.sp_slice(
"position_ids", batch["position_ids"], dim=self.collate_infos["position_ids"].pack_dim
Expand Down
Loading
Loading