diff --git a/tests/functional_tests/python_test_utils/common.py b/tests/functional_tests/python_test_utils/common.py index 6f528b9a95c..41877df4664 100644 --- a/tests/functional_tests/python_test_utils/common.py +++ b/tests/functional_tests/python_test_utils/common.py @@ -92,6 +92,32 @@ class SkipMetricError(Exception): """Raised if metric shall be skipped""" +def _load_event_accumulators_with_scalars( + files: List[str], +) -> List[event_accumulator.EventAccumulator]: + """Loads event-file accumulators that contain scalar data, preserving order. + + A resumed training phase can emit a header-only TensorBoard event file with + zero scalars before the file that holds the actual metrics (for example when + the fault-tolerance launcher initializes a SummaryWriter ahead of logging). + Dropping scalar-less files keeps positional ``index`` selection aligned with + real run data instead of latching onto an empty file and yielding no metrics. + + Args: + files: Event-file paths, ordered oldest-first. + + Returns: + Reloaded accumulators that expose at least one scalar tag, in input order. + """ + accumulators = [] + for event_file in files: + ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE) + ea.Reload() + if ea.Tags()["scalars"]: + accumulators.append(ea) + return accumulators + + def read_tb_logs_as_list( path, index: int = 0, train_iters: int = 50, start_idx: int = 1, step_size: int = 5 ) -> Optional[Dict[str, GoldenValueMetric]]: @@ -113,18 +139,21 @@ def read_tb_logs_as_list( return None files.sort(key=lambda x: os.path.getmtime(os.path.join(path, pathlib.Path(x).name))) - accumulators = [] - if index == -1: - for event_file in files: - ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE) - ea.Reload() - accumulators.append(ea) - else: - event_file = files[index] - ea = event_accumulator.EventAccumulator(event_file, size_guidance=SIZE_GUIDANCE) - ea.Reload() - accumulators.append(ea) + accumulators = _load_event_accumulators_with_scalars(files) + + if not accumulators: + logger.error(f"No event file with scalar data found at: {path}") + return None + + if index != -1: + if index >= len(accumulators): + logger.error( + f"Requested event-file index {index} but only {len(accumulators)} " + f"event file(s) with scalar data found at: {path}" + ) + return None + accumulators = [accumulators[index]] summaries = {} for ea in accumulators: