From a242f559e5f3b72f876239eb947faf9cbd375e77 Mon Sep 17 00:00:00 2001 From: Pablo Deymonnaz Date: Thu, 5 Feb 2026 16:19:31 -0300 Subject: [PATCH 1/2] Add per-phase timing breakdown to Slack notifications and run logs in the multisync monitoring script (docker_monitor.py). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync completion logs already contain per-phase completion markers (e.g. "✓ BLOCK HEADERS complete: 25,693,009 headers in 0:29:00") but this data was not surfaced in the Slack messages or run summaries. This adds a parse_phase_timings() function that reads saved container logs and extracts timing, count, and duration for all 8 snap sync phases: Block Headers, Account Ranges, Account Insertion, Storage Ranges, Storage Insertion, State Healing, Storage Healing, and Bytecodes. The breakdown is appended to both the Slack notification (as a code block per network instance) and the text-based run log (run_history.log and per-run summary.txt). When a phase did not complete (e.g. on a failed run), it is simply omitted from the breakdown. --- tooling/sync/docker_monitor.py | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tooling/sync/docker_monitor.py b/tooling/sync/docker_monitor.py index 331b26d3ab0..20afead5558 100644 --- a/tooling/sync/docker_monitor.py +++ b/tooling/sync/docker_monitor.py @@ -48,6 +48,18 @@ "block_processing": "📦", "success": "🎉", "failed": "❌" } +# Phase completion patterns for parsing sync logs +PHASE_COMPLETION_PATTERNS = { + "Block Headers": r"✓ BLOCK HEADERS complete: ([\d,]+) headers in (\d+:\d+:\d+)", + "Account Ranges": r"✓ ACCOUNT RANGES complete: ([\d,]+) accounts in (\d+:\d+:\d+)", + "Account Insertion": r"✓ ACCOUNT INSERTION complete: ([\d,]+) accounts inserted in (\d+:\d+:\d+)", + "Storage Ranges": r"✓ STORAGE RANGES complete: ([\d,]+) storage slots in (\d+:\d+:\d+)", + "Storage Insertion": r"✓ STORAGE INSERTION complete: ([\d,]+) storage slots inserted in (\d+:\d+:\d+)", + "State Healing": r"✓ STATE HEALING complete: ([\d,]+) state paths healed in (\d+:\d+:\d+)", + "Storage Healing": r"✓ STORAGE HEALING complete: ([\d,]+) storage accounts healed in (\d+:\d+:\d+)", + "Bytecodes": r"✓ BYTECODES complete: ([\d,]+) bytecodes in (\d+:\d+:\d+)", +} + @dataclass class Instance: @@ -262,6 +274,30 @@ def rpc_call(url: str, method: str) -> Optional[Any]: return None +def parse_phase_timings(run_id: str, container: str) -> list[tuple[str, str, str]]: + """Parse phase completion times from saved container logs. + + Returns list of (phase_name, count, duration) tuples. + """ + log_file = LOGS_DIR / f"run_{run_id}" / f"{container}.log" + if not log_file.exists(): + return [] + + try: + logs = log_file.read_text() + except Exception: + return [] + + phases = [] + for phase_name, pattern in PHASE_COMPLETION_PATTERNS.items(): + match = re.search(pattern, logs) + if match: + count = match.group(1) + duration = match.group(2) + phases.append((phase_name, count, duration)) + return phases + + def slack_notify(run_id: str, run_count: int, instances: list, hostname: str, branch: str, commit: str, build_profile: str = ""): """Send a single summary Slack message for the run.""" all_success = all(i.status == "success" for i in instances) @@ -319,6 +355,21 @@ def slack_notify(run_id: str, run_count: int, instances: list, hostname: str, br if i.error: line += f"\n Error: {i.error}" blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": line}}) + + # Add phase breakdown for each instance + for i in instances: + phases = parse_phase_timings(run_id, i.container) + if phases: + phase_lines = [f"📊 *Phase Breakdown — {i.name}*", "```"] + max_name_len = max(len(name) for name, _, _ in phases) + for name, count, duration in phases: + phase_lines.append(f"{name:<{max_name_len}} {duration} ({count})") + phase_lines.append("```") + blocks.append({ + "type": "section", + "text": {"type": "mrkdwn", "text": "\n".join(phase_lines)} + }) + try: requests.post(url, json={"blocks": blocks}, timeout=10) except Exception: @@ -417,6 +468,15 @@ def log_run_result(run_id: str, run_count: int, instances: list[Instance], hostn if inst.error: line += f"\n Error: {inst.error}" lines.append(line) + + # Add phase breakdown + phases = parse_phase_timings(run_id, inst.container) + if phases: + lines.append(f" Phase Breakdown:") + max_name_len = max(len(name) for name, _, _ in phases) + for name, count, duration in phases: + lines.append(f" {name:<{max_name_len}} {duration} ({count})") + lines.append("") # Append to log file with open(RUN_LOG_FILE, "a") as f: From 7042b839012a5c42b52caa2badd90cab4d4ee60b Mon Sep 17 00:00:00 2001 From: Pablo Deymonnaz Date: Thu, 5 Feb 2026 16:55:51 -0300 Subject: [PATCH 2/2] Update tooling/sync/docker_monitor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tooling/sync/docker_monitor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tooling/sync/docker_monitor.py b/tooling/sync/docker_monitor.py index 20afead5558..0ab1c4254c0 100644 --- a/tooling/sync/docker_monitor.py +++ b/tooling/sync/docker_monitor.py @@ -50,14 +50,14 @@ # Phase completion patterns for parsing sync logs PHASE_COMPLETION_PATTERNS = { - "Block Headers": r"✓ BLOCK HEADERS complete: ([\d,]+) headers in (\d+:\d+:\d+)", - "Account Ranges": r"✓ ACCOUNT RANGES complete: ([\d,]+) accounts in (\d+:\d+:\d+)", - "Account Insertion": r"✓ ACCOUNT INSERTION complete: ([\d,]+) accounts inserted in (\d+:\d+:\d+)", - "Storage Ranges": r"✓ STORAGE RANGES complete: ([\d,]+) storage slots in (\d+:\d+:\d+)", - "Storage Insertion": r"✓ STORAGE INSERTION complete: ([\d,]+) storage slots inserted in (\d+:\d+:\d+)", - "State Healing": r"✓ STATE HEALING complete: ([\d,]+) state paths healed in (\d+:\d+:\d+)", - "Storage Healing": r"✓ STORAGE HEALING complete: ([\d,]+) storage accounts healed in (\d+:\d+:\d+)", - "Bytecodes": r"✓ BYTECODES complete: ([\d,]+) bytecodes in (\d+:\d+:\d+)", + "Block Headers": r"✓ BLOCK HEADERS complete: ([\d,]+) headers in (\d+:\d{2}:\d{2})", + "Account Ranges": r"✓ ACCOUNT RANGES complete: ([\d,]+) accounts in (\d+:\d{2}:\d{2})", + "Account Insertion": r"✓ ACCOUNT INSERTION complete: ([\d,]+) accounts inserted in (\d+:\d{2}:\d{2})", + "Storage Ranges": r"✓ STORAGE RANGES complete: ([\d,]+) storage slots in (\d+:\d{2}:\d{2})", + "Storage Insertion": r"✓ STORAGE INSERTION complete: ([\d,]+) storage slots inserted in (\d+:\d{2}:\d{2})", + "State Healing": r"✓ STATE HEALING complete: ([\d,]+) state paths healed in (\d+:\d{2}:\d{2})", + "Storage Healing": r"✓ STORAGE HEALING complete: ([\d,]+) storage accounts healed in (\d+:\d{2}:\d{2})", + "Bytecodes": r"✓ BYTECODES complete: ([\d,]+) bytecodes in (\d+:\d{2}:\d{2})", }