Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 40 additions & 11 deletions tests/functional_tests/python_test_utils/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,32 @@ class SkipMetricError(Exception):
"""Raised if metric shall be skipped"""


def _load_event_accumulators_with_scalars(
files: List[str],
) -> List[event_accumulator.EventAccumulator]:
"""Loads event-file accumulators that contain scalar data, preserving order.

A resumed training phase can emit a header-only TensorBoard event file with
zero scalars before the file that holds the actual metrics (for example when
the fault-tolerance launcher initializes a SummaryWriter ahead of logging).
Dropping scalar-less files keeps positional ``index`` selection aligned with
real run data instead of latching onto an empty file and yielding no metrics.

Args:
files: Event-file paths, ordered oldest-first.

Returns:
Reloaded accumulators that expose at least one scalar tag, in input order.
"""
accumulators = []
for event_file in files:
ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE)
ea.Reload()
if ea.Tags()["scalars"]:
accumulators.append(ea)
return accumulators


def read_tb_logs_as_list(
path, index: int = 0, train_iters: int = 50, start_idx: int = 1, step_size: int = 5
) -> Optional[Dict[str, GoldenValueMetric]]:
Expand All @@ -113,18 +139,21 @@ def read_tb_logs_as_list(
return None

files.sort(key=lambda x: os.path.getmtime(os.path.join(path, pathlib.Path(x).name)))
accumulators = []

if index == -1:
for event_file in files:
ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE)
ea.Reload()
accumulators.append(ea)
else:
event_file = files[index]
ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE)
ea.Reload()
accumulators.append(ea)
accumulators = _load_event_accumulators_with_scalars(files)

if not accumulators:
logger.error(f"No event file with scalar data found at: {path}")
return None

if index != -1:
if index >= len(accumulators):
logger.error(
f"Requested event-file index {index} but only {len(accumulators)} "
f"event file(s) with scalar data found at: {path}"
)
return None
accumulators = [accumulators[index]]

summaries = {}
for ea in accumulators:
Expand Down
Loading