From cbd57c1792db69dc3d4790659c7c45128d33824b Mon Sep 17 00:00:00 2001 From: litianjian Date: Mon, 3 Nov 2025 17:45:10 +0800 Subject: [PATCH 01/24] feat: add router replay --- megatron/core/transformer/moe/moe_utils.py | 147 +++++++++++++++++- megatron/core/transformer/moe/router.py | 6 + .../core/transformer/transformer_config.py | 3 + 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index dc857129834..638a6c48cce 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1,6 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import math +from enum import Enum from typing import List, Optional, Union import torch @@ -519,6 +520,102 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map +class RoutingMode(Enum): + NONE = "none" + RECORD = "record" + REPLAY_FORWARD = "replay_forward" + REPLAY_BACKWARD = "replay_backward" + FALLTHROUGH = "fallthrough" + + +class RouterReplay: + """ + A class to manage the recording and replaying of MoE routing decisions. + It holds all router instances and provides static methods to globally + control recording and replaying. + """ + + # Static variable to hold all router instances, one per MoE layer. + router_instances = [] + + @staticmethod + def set_replay_data(all_layers_topk_indices: list): + """ + Distributes the topk indices for all layers to their respective RouterReplay instances. + :param all_layers_topk_indices: A list of tensors, where each tensor contains the + topk indices for a specific layer. The order + must match the instantiation order of the routers. + """ + if len(all_layers_topk_indices) != len(RouterReplay.router_instances): + raise ValueError( + f"The number of replay tensors ({len(all_layers_topk_indices)}) " + f"does not match the number of router instances ({len(RouterReplay.router_instances)})." + ) + for i, router_instance in enumerate(RouterReplay.router_instances): + router_instance.set_target_indices(all_layers_topk_indices[i]) + + @staticmethod + def get_recorded_data() -> list: + """ + Collects the recorded topk indices from all RouterReplay instances. + :return: A list of tensors, each containing the recorded topk indices for a layer. + """ + return [router.get_recorded_indices() for router in RouterReplay.router_instances] + + @staticmethod + def clear_global_indices(): + """Clears the recorded and target topk indices in all instances.""" + for router in RouterReplay.router_instances: + router.clear_indices() + + def __init__(self): + """Initializes a RouterReplay instance for a specific layer.""" + self.target_topk_idx = None # For replay + self.recorded_topk_idx = None # For recording + self.routing_mode = None # Routing mode for this layer + self.replay_backward_list = [] # List of tensors for backward pass replay + RouterReplay.router_instances.append(self) + + def set_target_indices(self, topk_indices: torch.Tensor): + """Sets the target topk indices for replay.""" + self.target_topk_idx = topk_indices + self.replay_backward_list.append(topk_indices) + + def get_recorded_indices(self) -> Optional[torch.Tensor]: + """Returns the recorded topk indices.""" + return self.recorded_topk_idx + + def record_indices(self, topk_indices: torch.Tensor): + """Records the topk indices.""" + self.recorded_topk_idx = topk_indices + + def clear_indices(self): + """Clears the recorded and target topk indices.""" + self.recorded_topk_idx = None + self.target_topk_idx = None + self.replay_backward_list = [] + + def set_routing_mode(self, routing_mode: RoutingMode): + """Sets the routing mode for this layer.""" + self.routing_mode = routing_mode + + def clear_routing_mode(self): + """Clears the routing mode for this layer.""" + self.routing_mode = None + + @staticmethod + def set_global_routing_mode(routing_mode: RoutingMode): + """Sets the routing mode for all router instances.""" + for router in RouterReplay.router_instances: + router.set_routing_mode(routing_mode) + + @staticmethod + def clear_global_routing_mode(): + """Clears the routing mode for all router instances.""" + for router in RouterReplay.router_instances: + router.clear_routing_mode() + + def topk_routing_with_score_function( logits: torch.Tensor, topk: int, @@ -529,6 +626,7 @@ def topk_routing_with_score_function( score_function: str = "softmax", expert_bias: Optional[torch.Tensor] = None, fused: bool = False, + router_replay: Optional['RouterReplay'] = None, ): """Compute the routing probabilities and map for top-k selection with score function. Args: @@ -540,6 +638,10 @@ def topk_routing_with_score_function( scaling_factor (float): Scaling factor of routing score in top-k selection. score_function (str): The score function to use. Can be either "softmax" or "sigmoid". expert_bias (torch.Tensor): The bias added to logits for expert routing. + router_replay (Optional['RouterReplay']): For debugging and development, allows for + deterministic routing by replaying a previously + recorded routing sequence. + Returns: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - routing_probs (torch.Tensor): A tensor of shape [num_tokens, num_experts] containing @@ -566,7 +668,7 @@ def topk_routing_with_score_function( expert_bias=expert_bias, ) - def compute_topk(scores, topk, num_groups=None, group_topk=None): + def _compute_topk(scores, topk, num_groups=None, group_topk=None): if group_topk: return group_limited_topk( scores=scores, @@ -579,6 +681,49 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): else: return torch.topk(scores, k=topk, dim=1) + def compute_topk(scores, topk, num_groups=None, group_topk=None): + # Default behavior if no replay is active + + routing_action = router_replay.routing_mode.value if router_replay is not None else None + + if routing_action is None or routing_action == "fallthrough": + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + if routing_action == "record": + probs, top_indices = _compute_topk( + scores, topk, num_groups=num_groups, group_topk=group_topk + ) + if router_replay is not None: + router_replay.record_indices(top_indices) + return probs, top_indices + + elif routing_action == "forward_replay": + if router_replay is None or router_replay.target_topk_idx is None: + # Fallback if replay data is not available + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + # Use the provided indices for replay + top_indices = router_replay.target_topk_idx + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + elif routing_action == "backward_replay": + if router_replay is None or not router_replay.replay_backward_list: + # Fallback if replay data is not available + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + # Use the last recorded indices for backward replay + top_indices = router_replay.replay_backward_list.pop() + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + else: # Unknown action, fallback + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + if score_function == "softmax": if use_pre_softmax: scores = torch.softmax(logits, dim=-1, dtype=torch.float32).type_as(logits) diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 068d680c798..dee7ffa4711 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -10,6 +10,7 @@ from megatron.core.transformer.moe.moe_utils import ( MoEAuxLossAutoScaler, ProcessGroupCollection, + RouterReplay, apply_random_logits, apply_router_token_dropping, compute_routing_scores_for_aux_loss, @@ -198,6 +199,10 @@ def __init__( self.global_tokens_per_expert = None self.ga_steps = None + self.router_replay = None + if self.config.enable_routing_replay: + self.router_replay = RouterReplay() + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. @@ -497,6 +502,7 @@ def routing(self, logits: torch.Tensor): score_function=self.score_function, expert_bias=self.expert_bias, fused=self.config.moe_router_fusion, + router_replay=self.router_replay, ) # Apply token dropping to probs and routing_map. diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 147c5b23b3d..a129c9faecb 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -464,6 +464,9 @@ class TransformerConfig(ModelParallelConfig): moe_router_topk: int = 2 """Number of experts to route to for each token.""" + enable_routing_replay: bool = False + """If True, enable the routing replay feature for MoE layers.""" + moe_router_topk_limited_devices: Optional[int] = None """Number of EP ranks to consider for each token in group-limited routing, DEPRECATED and replaced by moe_router_num_groups and moe_router_group_topk. From bd32db8dee9a13e32c84d62cb8c3bf29d0e8110f Mon Sep 17 00:00:00 2001 From: litianjian Date: Mon, 24 Nov 2025 14:18:14 +0800 Subject: [PATCH 02/24] refactor(router): rename RouterMode to RouterReplayAction --- megatron/core/transformer/moe/moe_utils.py | 40 ++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index edeec5bcdb6..941932530f5 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -524,12 +524,10 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map -class RoutingMode(Enum): - NONE = "none" +class RouterReplayAction(Enum): RECORD = "record" REPLAY_FORWARD = "replay_forward" REPLAY_BACKWARD = "replay_backward" - FALLTHROUGH = "fallthrough" class RouterReplay: @@ -576,7 +574,7 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx = None # For replay self.recorded_topk_idx = None # For recording - self.routing_mode = None # Routing mode for this layer + self.router_replay_action = None # Router replay action for this layer self.replay_backward_list = [] # List of tensors for backward pass replay RouterReplay.router_instances.append(self) @@ -599,25 +597,25 @@ def clear_indices(self): self.target_topk_idx = None self.replay_backward_list = [] - def set_routing_mode(self, routing_mode: RoutingMode): - """Sets the routing mode for this layer.""" - self.routing_mode = routing_mode + def set_router_replay_action(self, router_replay_action: RouterReplayAction): + """Sets the router replay action for this layer.""" + self.router_replay_action = router_replay_action - def clear_routing_mode(self): - """Clears the routing mode for this layer.""" - self.routing_mode = None + def clear_router_replay_action(self): + """Clears the router replay action for this layer.""" + self.router_replay_action = None @staticmethod - def set_global_routing_mode(routing_mode: RoutingMode): - """Sets the routing mode for all router instances.""" + def set_global_router_replay_action(router_replay_action: RouterReplayAction): + """Sets the router replay action for all router instances.""" for router in RouterReplay.router_instances: - router.set_routing_mode(routing_mode) + router.set_router_replay_action(router_replay_action) @staticmethod - def clear_global_routing_mode(): - """Clears the routing mode for all router instances.""" + def clear_global_router_replay_action(): + """Clears the router replay action for all router instances.""" for router in RouterReplay.router_instances: - router.clear_routing_mode() + router.clear_router_replay_action() def topk_routing_with_score_function( @@ -688,12 +686,12 @@ def _compute_topk(scores, topk, num_groups=None, group_topk=None): def compute_topk(scores, topk, num_groups=None, group_topk=None): # Default behavior if no replay is active - routing_action = router_replay.routing_mode.value if router_replay is not None else None + routing_action = router_replay.router_replay_action if router_replay is not None else None - if routing_action is None or routing_action == "fallthrough": + if routing_action is None: return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - if routing_action == "record": + if routing_action == RouterReplayAction.RECORD: probs, top_indices = _compute_topk( scores, topk, num_groups=num_groups, group_topk=group_topk ) @@ -701,7 +699,7 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): router_replay.record_indices(top_indices) return probs, top_indices - elif routing_action == "forward_replay": + elif routing_action == RouterReplayAction.REPLAY_FORWARD: if router_replay is None or router_replay.target_topk_idx is None: # Fallback if replay data is not available return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) @@ -713,7 +711,7 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): # Gather the scores for the replayed indices to get the probabilities probs = scores.gather(1, top_indices) return probs, top_indices - elif routing_action == "backward_replay": + elif routing_action == RouterReplayAction.REPLAY_BACKWARD: if router_replay is None or not router_replay.replay_backward_list: # Fallback if replay data is not available return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) From 1aec041b418e2feb6bca9e415a74dc30173ef901 Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 16 Dec 2025 13:10:54 +0800 Subject: [PATCH 03/24] simplify compute topk function --- megatron/core/transformer/moe/moe_utils.py | 130 ++++++++++----------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index d81e93b291c..a4581b4d051 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -525,9 +525,9 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso class RouterReplayAction(Enum): - RECORD = "record" - REPLAY_FORWARD = "replay_forward" - REPLAY_BACKWARD = "replay_backward" + RECORD = "record" # Record the topk indices for replay + REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass + REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass class RouterReplay: @@ -538,45 +538,57 @@ class RouterReplay: """ # Static variable to hold all router instances, one per MoE layer. - router_instances = [] + global_router_replay_instances: List['RouterReplay'] = [] @staticmethod - def set_replay_data(all_layers_topk_indices: list): + def set_replay_data(all_layers_topk_indices: List[torch.Tensor]): """ Distributes the topk indices for all layers to their respective RouterReplay instances. :param all_layers_topk_indices: A list of tensors, where each tensor contains the topk indices for a specific layer. The order must match the instantiation order of the routers. """ - if len(all_layers_topk_indices) != len(RouterReplay.router_instances): + if len(all_layers_topk_indices) != len(RouterReplay.global_router_replay_instances): raise ValueError( f"The number of replay tensors ({len(all_layers_topk_indices)}) " - f"does not match the number of router instances ({len(RouterReplay.router_instances)})." + f"does not match router instances ({len(RouterReplay.global_router_replay_instances)})." ) - for i, router_instance in enumerate(RouterReplay.router_instances): + for i, router_instance in enumerate(RouterReplay.global_router_replay_instances): router_instance.set_target_indices(all_layers_topk_indices[i]) @staticmethod - def get_recorded_data() -> list: + def get_recorded_data() -> List[torch.Tensor]: """ Collects the recorded topk indices from all RouterReplay instances. :return: A list of tensors, each containing the recorded topk indices for a layer. """ - return [router.get_recorded_indices() for router in RouterReplay.router_instances] + return [router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances] @staticmethod def clear_global_indices(): """Clears the recorded and target topk indices in all instances.""" - for router in RouterReplay.router_instances: + for router in RouterReplay.global_router_replay_instances: router.clear_indices() + @staticmethod + def set_global_router_replay_action(router_replay_action: RouterReplayAction): + """Sets the router replay action for all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.set_router_replay_action(router_replay_action) + + @staticmethod + def clear_global_router_replay_action(): + """Clears the router replay action for all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_router_replay_action() + def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" - self.target_topk_idx = None # For replay - self.recorded_topk_idx = None # For recording - self.router_replay_action = None # Router replay action for this layer - self.replay_backward_list = [] # List of tensors for backward pass replay - RouterReplay.router_instances.append(self) + self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay + self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay + self.router_replay_action: Optional[RouterReplayAction] = None # Router replay action for this layer + self.replay_backward_list: List[torch.Tensor] = [] # List of tensors for backward pass replay + RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): """Sets the target topk indices for replay.""" @@ -605,18 +617,39 @@ def clear_router_replay_action(self): """Clears the router replay action for this layer.""" self.router_replay_action = None - @staticmethod - def set_global_router_replay_action(router_replay_action: RouterReplayAction): - """Sets the router replay action for all router instances.""" - for router in RouterReplay.router_instances: - router.set_router_replay_action(router_replay_action) - - @staticmethod - def clear_global_router_replay_action(): - """Clears the router replay action for all router instances.""" - for router in RouterReplay.router_instances: - router.clear_router_replay_action() - + def get_replay_topk( + self, + scores: torch.Tensor, + topk: int, + num_groups: Optional[int] = None, + group_topk: Optional[int] = None, + default_compute_topk: Callable[[torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor] = None, + ) -> torch.Tensor: + """Returns the target topk indices for replay.""" + if self.router_replay_action == RouterReplayAction.RECORD: + probs, top_indices = default_compute_topk( + scores, topk, num_groups=num_groups, group_topk=group_topk + ) + self.record_indices(top_indices) + return probs, top_indices + elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: + # Use the provided indices for replay + top_indices = router_replay.target_topk_idx + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: + # Use the last recorded indices for backward replay + top_indices = router_replay.replay_backward_list.pop(0) + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + else: + return default_compute_topk(scores, topk, num_groups, group_topk) def topk_routing_with_score_function( logits: torch.Tensor, @@ -686,45 +719,12 @@ def _compute_topk(scores, topk, num_groups=None, group_topk=None): def compute_topk(scores, topk, num_groups=None, group_topk=None): # Default behavior if no replay is active - routing_action = router_replay.router_replay_action if router_replay is not None else None - - if routing_action is None: + if router_replay is None: return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - if routing_action == RouterReplayAction.RECORD: - probs, top_indices = _compute_topk( - scores, topk, num_groups=num_groups, group_topk=group_topk + else: + return router_replay.get_replay_topk( + scores, topk, num_groups, group_topk, _compute_topk ) - if router_replay is not None: - router_replay.record_indices(top_indices) - return probs, top_indices - - elif routing_action == RouterReplayAction.REPLAY_FORWARD: - if router_replay is None or router_replay.target_topk_idx is None: - # Fallback if replay data is not available - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - # Use the provided indices for replay - top_indices = router_replay.target_topk_idx - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - elif routing_action == RouterReplayAction.REPLAY_BACKWARD: - if router_replay is None or not router_replay.replay_backward_list: - # Fallback if replay data is not available - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - # Use the last recorded indices for backward replay - top_indices = router_replay.replay_backward_list.pop() - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - else: # Unknown action, fallback - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) if score_function == "softmax": if use_pre_softmax: From 39fd47ae621d769ee649cfaed8ae77fc45facf23 Mon Sep 17 00:00:00 2001 From: litianjian Date: Wed, 17 Dec 2025 15:45:19 +0800 Subject: [PATCH 04/24] update router replay --- megatron/core/transformer/moe/moe_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index a4581b4d051..0ece25f1b45 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -633,16 +633,14 @@ def get_replay_topk( self.record_indices(top_indices) return probs, top_indices elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: - # Use the provided indices for replay - top_indices = router_replay.target_topk_idx + top_indices = self.target_topk_idx # Ensure indices are on the correct device top_indices = top_indices.to(scores.device) # Gather the scores for the replayed indices to get the probabilities probs = scores.gather(1, top_indices) return probs, top_indices elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: - # Use the last recorded indices for backward replay - top_indices = router_replay.replay_backward_list.pop(0) + top_indices = self.replay_backward_list.pop(0) # Ensure indices are on the correct device top_indices = top_indices.to(scores.device) # Gather the scores for the replayed indices to get the probabilities From 49da256654a25554fa72154165181d1fb44772bb Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 23 Dec 2025 17:22:42 +0800 Subject: [PATCH 05/24] add unit test and doc --- docs/api-guide/router_replay.md | 174 ++++++++++++++++++ megatron/core/transformer/moe/moe_utils.py | 25 ++- .../transformer/moe/test_router_replay.py | 110 +++++++++++ 3 files changed, 301 insertions(+), 8 deletions(-) create mode 100644 docs/api-guide/router_replay.md create mode 100644 tests/unit_tests/transformer/moe/test_router_replay.py diff --git a/docs/api-guide/router_replay.md b/docs/api-guide/router_replay.md new file mode 100644 index 00000000000..3772997530f --- /dev/null +++ b/docs/api-guide/router_replay.md @@ -0,0 +1,174 @@ +# Design Document: MoE Router Replay Feature + +### 1. Overview + +This document provides a detailed description of the "Router Replay" feature implemented within the Megatron-LM Core for Mixture-of-Experts (MoE) models. + +This feature is designed to enhance determinism and analyzability in MoE model training and inference. It enables the model to load routing decisions from a predefined file and enforce their use during the forward pass, thereby bypassing the real-time routing computation. + +### 2. Motivation + +* **Determinism & Reproducibility**: In distributed training, MoE routing decisions can exhibit minor variations due to factors like floating-point precision. By replaying a fixed routing table, the MoE computation path is guaranteed to be identical across runs, which facilitates debugging and reproducing experimental results. +* **Performance Profiling**: The router's own computation (e.g., logits calculation, top-k selection) incurs overhead. In replay mode, this part of the computation can be completely skipped, allowing for more precise isolation and profiling of performance bottlenecks within the Expert Layers themselves. +* **Debugging Aid**: When issues arise in the model, fixing the routing decisions helps to isolate variables, making it easier to determine whether the problem lies with the routing mechanism or the expert computations. + +### 3. Design and Architecture + +The design follows the principles of being non-intrusive and on-demand, with the core idea of activating the replay logic only when explicitly requested by the user. + +* **Core Components**: + * `RouterReplay` (located in `megatron/core/transformer/moe/moe_utils.py`): A utility class for replaying MoE routing decisions. When enabled via the `enable_routing_replay` flag, a separate instance of `RouterReplay` is created for each MoE layer's router. Each instance is responsible for loading routing data and providing the deterministic routing decisions for its corresponding layer during the forward pass. + * `enable_routing_replay` (located in `megatron/core/transformer/transformer_config.py`): A boolean global configuration flag that serves as the sole entry point for enabling this feature. + +* **Workflow**: + The feature supports different modes, such as recording and replaying, controlled by a `RouterReplayAction`. + + 1. **Enabling the Feature**: The user sets `enable_routing_replay` to `True` in the model configuration. + 2. **Initialization**: When `enable_routing_replay` is true, each `TopKRouter` creates its own `RouterReplay` instance. + 3. **Mode Configuration**: The user must programmatically set the desired router replay action (e.g., `record`, `forward_replay`, `backward_replay`) on the `RouterReplay` instances. + 4. **Execution Flow (within a mini-batch)**: + * **Forward Pass**: + * For each micro-batch, the `topk_routing_with_score_function` checks the `router_replay_action`. + * **In `record` mode**: The dynamically computed `top-k` expert indices are captured and stored. + * **In `forward_replay` mode**: The function retrieves pre-loaded expert indices from `target_topk_idx`. These indices are used for the forward computation and are also appended to the `replay_backward_list` to prepare for the backward pass. + * **Backward Pass**: + * For each micro-batch (processed in reverse order in pipeline parallelism), the `router_replay_action` is checked again. + * **In `backward_replay` mode**: The function retrieves the expert indices for the corresponding micro-batch by popping them from the `replay_backward_list`. This mode is intended for training recomputation (e.g., activation checkpointing and pipeline recompute) so the same routing decisions are used during recompute/backward as in forward, ensuring determinism and correctness. + +### 4. Implementation Details + +The implementation cleanly separates the replay logic from the router's core computation. + +* **`megatron/core/transformer/transformer_config.py`**: + * Adds the configuration option `enable_routing_replay: bool = False`. + +* **`megatron/core/transformer/moe/moe_utils.py`**: + * Introduces the `RouterReplay` class to manage the state for recording and replaying routing decisions for a single MoE layer. + * `target_topk_idx`: An attribute holding the expert indices for the current micro-batch during forward replay mode. + * `recorded_topk_idx`: An attribute for storing the computed expert indices when in record mode. + * `replay_backward_list`: A list that accumulates the `top-k` indices used during the forward passes of a mini-batch. This list is consumed in FIFO order during the backward pass to ensure correctness under pipeline parallelism. + * `set_target_indices()`: A method to load the replay indices into `target_topk_idx` for the forward pass. + * `record_indices()`: A method to save the computed indices. + * The `topk_routing_with_score_function` is modified to contain the core logic. It checks the `router_replay_action` on the `router_replay` instance and accordingly performs one of the following actions: computes and records indices, replays indices from `target_topk_idx` (for forward), replays indices from `replay_backward_list` (for backward), or falls through to the default dynamic routing. + +#### Training recompute usage +- During forward replay, `set_target_indices()` prepares `replay_backward_list` so each micro-batch’s indices are available for recomputation. +- During recompute/backward, set action to `REPLAY_BACKWARD` so indices are consumed in FIFO order to mirror the forward sequence. + +### 5. Usage Guide + +1. **Enable & Instantiate** + - Create one `RouterReplay` instance per MoE router layer when building the model. + - Optionally use the global helpers to set/clear actions across all layers. +2. **Record Routing Decisions** + - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD)`. + - Run the model; retrieve per-layer indices via `RouterReplay.get_recorded_data()` and persist. +3. **Forward Replay** + - Load indices and distribute: `RouterReplay.set_replay_data(list_of_tensors)`. + - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD)`. + - Run the model; dynamic top‑k is bypassed and target indices are used. +4. **Backward Replay** + - For training recomputation (activation checkpointing or pipeline recompute), set action: `REPLAY_BACKWARD` during recomputation. + - Per micro‑batch indices are consumed from `replay_backward_list` in FIFO order. +5. **Cleanup** + - Use `RouterReplay.clear_global_indices()` and `RouterReplay.clear_global_router_replay_action()` to restore default behavior. + +#### Quick usage with `topk_routing_with_score_function` + +```python +import torch +from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, RouterReplayAction, topk_routing_with_score_function, +) + +rr = RouterReplay() + +# Record +RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) +logits = torch.randn(8, 16) +probs_rec, routing_map_rec = topk_routing_with_score_function( + logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr, +) +recorded = rr.get_recorded_indices() +torch.save(recorded, "/tmp/replay.pt") + +# Forward replay +rr.clear_router_replay_action() +rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) +target = torch.load("/tmp/replay.pt") +rr.set_target_indices(target) +probs_rep, routing_map_rep = topk_routing_with_score_function( + logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr, +) + +RouterReplay.clear_global_router_replay_action() +RouterReplay.clear_global_indices() +``` + +### 6. Minimal Demo + +Here is a minimal code example showing how to use RouterReplay for recording and replaying: + +```python +import torch +import torch.distributed as dist +from megatron.core.config import TransformerConfig +from megatron.core.transformer.moe.router import TopKRouter +from megatron.core.transformer.moe.moe_utils import RouterReplayAction, RouterReplay + + +# Initialize distributed training +if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + +# Create a transformer config with RouterReplay enabled +config = TransformerConfig( + num_experts=8, + expert_model_parallel_size=1, + num_top_k=2, + enable_routing_replay=True +) + +# Create a TopKRouter instance +router = TopKRouter(config) + +# Generate sample input (batch_size, sequence_length, hidden_size) +logits = torch.randn(16, 32, 8).to(torch.cuda.current_device()) + +# ----------------- +# 1. Recording Mode +# ----------------- +print("=== Recording Mode ===") +# Set global router replay action to RECORD +RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + +# Perform routing +routing_output = router.forward(logits) +print(f"Recorded top-k indices shape: {routing_output.top_k_idx.shape}") + +# ----------------- +# 2. Forward Replay Mode +# ----------------- +print("\n=== Forward Replay Mode ===") +# Save recorded indices to a file +torch.save(routing_output.top_k_idx, "/tmp/replay.pt") + +# Load indices from file and set as target for replay +replay_indices = torch.load("/tmp/replay.pt") +for router_instance in RouterReplay.router_instances: + router_instance.target_topk_idx = replay_indices + +# Set global router replay action to REPLAY_FORWARD +RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + +# Perform routing again - this will use the replayed indices +replay_routing_output = router.forward(logits) +print(f"Replayed top-k indices shape: {replay_routing_output.top_k_idx.shape}") +print(f"Are indices the same? {torch.equal(routing_output.top_k_idx, replay_routing_output.top_k_idx)}") + + +# Clean up +RouterReplay.clear_global_router_replay_action() +if dist.is_initialized(): + dist.destroy_process_group() +``` diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 0ece25f1b45..360cef9789d 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -2,7 +2,7 @@ import math from enum import Enum -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union import torch @@ -525,9 +525,9 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso class RouterReplayAction(Enum): - RECORD = "record" # Record the topk indices for replay - REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass - REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass + RECORD = "record" # Record the topk indices for replay + REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass + REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass class RouterReplay: @@ -562,7 +562,9 @@ def get_recorded_data() -> List[torch.Tensor]: Collects the recorded topk indices from all RouterReplay instances. :return: A list of tensors, each containing the recorded topk indices for a layer. """ - return [router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances] + return [ + router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances + ] @staticmethod def clear_global_indices(): @@ -586,8 +588,12 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay - self.router_replay_action: Optional[RouterReplayAction] = None # Router replay action for this layer - self.replay_backward_list: List[torch.Tensor] = [] # List of tensors for backward pass replay + self.router_replay_action: Optional[RouterReplayAction] = ( + None # Router replay action for this layer + ) + self.replay_backward_list: List[torch.Tensor] = ( + [] + ) # List of tensors for backward pass replay RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): @@ -623,7 +629,9 @@ def get_replay_topk( topk: int, num_groups: Optional[int] = None, group_topk: Optional[int] = None, - default_compute_topk: Callable[[torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor] = None, + default_compute_topk: Callable[ + [torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor + ] = None, ) -> torch.Tensor: """Returns the target topk indices for replay.""" if self.router_replay_action == RouterReplayAction.RECORD: @@ -649,6 +657,7 @@ def get_replay_topk( else: return default_compute_topk(scores, topk, num_groups, group_topk) + def topk_routing_with_score_function( logits: torch.Tensor, topk: int, diff --git a/tests/unit_tests/transformer/moe/test_router_replay.py b/tests/unit_tests/transformer/moe/test_router_replay.py new file mode 100644 index 00000000000..00732a63ee1 --- /dev/null +++ b/tests/unit_tests/transformer/moe/test_router_replay.py @@ -0,0 +1,110 @@ +import torch +import pytest + +from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, + RouterReplayAction, + topk_routing_with_score_function, +) + + +def setup_function(): + RouterReplay.global_router_replay_instances.clear() + + +def teardown_function(): + RouterReplay.global_router_replay_instances.clear() + + +def test_record_mode_with_topk_routing_softmax_post(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.RECORD) + logits = torch.randn(4, 6) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=False, + router_replay=rr, + score_function="softmax", + ) + recorded = rr.get_recorded_indices() + expected_idx = torch.topk(logits, k=2, dim=1).indices + assert recorded is not None + assert torch.equal(recorded, expected_idx) + assert probs.shape == (4, 6) + assert routing_map.shape == (4, 6) + assert routing_map.sum(dim=1).eq(2).all() + + +def test_replay_forward_with_topk_routing_softmax_pre(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + logits = torch.randn(3, 5) + target = torch.tensor([[1, 2], [0, 3], [2, 4]], dtype=torch.long) + rr.set_target_indices(target) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=True, + router_replay=rr, + score_function="softmax", + ) + assert routing_map.sum(dim=1).eq(2).all() + scores = torch.softmax(logits, dim=-1) + assert torch.equal(probs.gather(1, target), scores.gather(1, target)) + + +def test_replay_forward_with_topk_routing_softmax_post(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + logits = torch.randn(3, 6) + target = torch.tensor([[1, 2], [0, 5], [3, 4]], dtype=torch.long) + rr.set_target_indices(target) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=False, + router_replay=rr, + score_function="softmax", + ) + selected = torch.softmax(logits.gather(1, target), dim=-1) + assert torch.equal(probs.gather(1, target), selected) + assert routing_map.sum(dim=1).eq(2).all() + + +def test_global_set_get_clear_indices(): + r1 = RouterReplay() + r2 = RouterReplay() + t1 = torch.tensor([[0, 1]], dtype=torch.long) + t2 = torch.tensor([[1, 0]], dtype=torch.long) + RouterReplay.set_replay_data([t1, t2]) + assert torch.equal(r1.target_topk_idx, t1) + assert torch.equal(r2.target_topk_idx, t2) + r1.record_indices(t1) + r2.record_indices(t2) + rec = RouterReplay.get_recorded_data() + assert len(rec) == 2 + assert torch.equal(rec[0], t1) + assert torch.equal(rec[1], t2) + RouterReplay.clear_global_indices() + assert r1.target_topk_idx is None and r2.target_topk_idx is None + assert r1.get_recorded_indices() is None and r2.get_recorded_indices() is None + + +def test_global_action_set_and_clear(): + r1 = RouterReplay() + r2 = RouterReplay() + RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + assert r1.router_replay_action == RouterReplayAction.REPLAY_FORWARD + assert r2.router_replay_action == RouterReplayAction.REPLAY_FORWARD + RouterReplay.clear_global_router_replay_action() + assert r1.router_replay_action is None and r2.router_replay_action is None + + +def test_set_replay_data_length_mismatch(): + _ = RouterReplay() + with pytest.raises(ValueError): + RouterReplay.set_replay_data([ + torch.tensor([[0, 1]], dtype=torch.long), + torch.tensor([[1, 0]], dtype=torch.long), + ]) From 590ce52924228d27ef013f90a73433488ece3497 Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 23 Dec 2025 17:27:56 +0800 Subject: [PATCH 06/24] format code --- .../transformer/moe/test_router_replay.py | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tests/unit_tests/transformer/moe/test_router_replay.py b/tests/unit_tests/transformer/moe/test_router_replay.py index 00732a63ee1..fec1d290b2d 100644 --- a/tests/unit_tests/transformer/moe/test_router_replay.py +++ b/tests/unit_tests/transformer/moe/test_router_replay.py @@ -21,11 +21,7 @@ def test_record_mode_with_topk_routing_softmax_post(): rr.set_router_replay_action(RouterReplayAction.RECORD) logits = torch.randn(4, 6) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=False, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax" ) recorded = rr.get_recorded_indices() expected_idx = torch.topk(logits, k=2, dim=1).indices @@ -43,11 +39,7 @@ def test_replay_forward_with_topk_routing_softmax_pre(): target = torch.tensor([[1, 2], [0, 3], [2, 4]], dtype=torch.long) rr.set_target_indices(target) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=True, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=True, router_replay=rr, score_function="softmax" ) assert routing_map.sum(dim=1).eq(2).all() scores = torch.softmax(logits, dim=-1) @@ -61,11 +53,7 @@ def test_replay_forward_with_topk_routing_softmax_post(): target = torch.tensor([[1, 2], [0, 5], [3, 4]], dtype=torch.long) rr.set_target_indices(target) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=False, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax" ) selected = torch.softmax(logits.gather(1, target), dim=-1) assert torch.equal(probs.gather(1, target), selected) @@ -104,7 +92,6 @@ def test_global_action_set_and_clear(): def test_set_replay_data_length_mismatch(): _ = RouterReplay() with pytest.raises(ValueError): - RouterReplay.set_replay_data([ - torch.tensor([[0, 1]], dtype=torch.long), - torch.tensor([[1, 0]], dtype=torch.long), - ]) + RouterReplay.set_replay_data( + [torch.tensor([[0, 1]], dtype=torch.long), torch.tensor([[1, 0]], dtype=torch.long)] + ) From 15395b8e567ba9317313da9543e8e445fd32f700 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 20 Jan 2026 15:57:16 -0800 Subject: [PATCH 07/24] first attempt --- .../core/inference/engines/dynamic_engine.py | 33 +++++++++++++++++++ megatron/core/inference/inference_request.py | 4 +++ .../text_generation_controller.py | 13 ++++++++ megatron/training/arguments.py | 5 +++ 4 files changed, 55 insertions(+) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index ca386a13c7e..d1c1821273a 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -789,6 +789,7 @@ def post_process_requests( sample: torch.Tensor, log_probs: torch.Tensor, top_n_logprobs: Optional[Dict[int, List[Tuple[torch.Tensor, torch.Tensor]]]] = None, + routing_indices: Optional[List[torch.Tensor]] = None, ) -> Tuple[List[DynamicInferenceRequest], List[DynamicInferenceRequest]]: """ Handles post-processing for requests after a step. @@ -801,6 +802,8 @@ def post_process_requests( log_probs: (List): Log probs for each request top_n_logprobs: (Dict): Top-n log probs for each request. Maps request_idx to list of (top_n_logprobs, top_n_indices) tuples. + routing_indices: (List[Tensor]): MoE routing indices per layer. Each tensor has + shape [total_tokens, topk]. Will be split per request. Returns: A list of active requests and completed requests as `DynamicInferenceRequest` objects @@ -812,6 +815,26 @@ def post_process_requests( log_probs_iter = log_probs if log_probs else repeat(None) + # Split routing indices per request if available + routing_indices_per_request = None + if routing_indices is not None: + # Get the query lengths for splitting per request + active_request_slice = slice( + self.context.paused_request_count, self.context.total_request_count + ) + active_query_lengths = self.context.request_query_lengths[active_request_slice].tolist() + + # Split each layer's routing tensor by request query lengths + routing_indices_per_request = {} + for req_idx in range(len(active_query_lengths)): + routing_indices_per_request[req_idx] = [] + + for layer_routing in routing_indices: + # layer_routing has shape [total_tokens, topk] + routing_splits = layer_routing.split(active_query_lengths, dim=0) + for req_idx, routing_split in enumerate(routing_splits): + routing_indices_per_request[req_idx].append(routing_split) + for req_idx, (request_id, token, request_log_probs) in enumerate( zip(request_ids.tolist(), sample.tolist(), log_probs_iter) ): @@ -916,6 +939,14 @@ def post_process_requests( else: request.generated_top_n_logprobs.append(logit_dict) + # Process routing indices if available + if routing_indices_per_request is not None and req_idx in routing_indices_per_request: + # Initialize routing_indices list if it doesn't exist + if request.routing_indices is None: + request.routing_indices = [] + # Append this step's routing data (list of tensors, one per MoE layer) + request.routing_indices.append(routing_indices_per_request[req_idx]) + return active_request_ids, finished_request_records def schedule_waiting_requests(self): @@ -1101,6 +1132,7 @@ async def async_bookkeep( sample = step_result["sample"] log_probs = step_result["log_probs"] top_n_logprobs = step_result.get("top_n_logprobs", None) + routing_indices = step_result.get("routing_indices", None) cuda_graph_request_count = step_result["cuda_graph_request_count"] # Add paused events. @@ -1118,6 +1150,7 @@ async def async_bookkeep( sample, log_probs, top_n_logprobs, + routing_indices, ) else: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 458fbad387f..f411aa1100a 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -246,6 +246,10 @@ class DynamicInferenceRequest(InferenceRequest): remaining_prompt_tokens: Optional[torch.Tensor] = None latency: Optional[float] = None finished_chunk_token_count = 0 + # routing_indices stores MoE routing decisions per layer for each token. + # Structure: List[List[torch.Tensor]] - outer list is per-step, inner list is per-layer, + # each tensor has shape [num_tokens_this_step, topk] + routing_indices: Optional[List[List[torch.Tensor]]] = None def __post_init__(self): self.sampling_params = copy.deepcopy(self.sampling_params) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 532cdbccdf0..dabdcd153a4 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,6 +27,7 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.transformer.moe.moe_layer import BaseMoELayer +from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model @@ -855,10 +856,22 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() + # Collect routing indices if routing replay is enabled + routing_indices = None + config = self.inference_wrapped_model.model.config + if getattr(config, 'enable_routing_replay', False): + routing_indices = RouterReplay.get_recorded_data() + # Filter out None entries (non-MoE layers) + if routing_indices: + routing_indices = [r for r in routing_indices if r is not None] + if not routing_indices: + routing_indices = None + ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, "top_n_logprobs": top_n_logprobs, + "routing_indices": routing_indices, "cuda_graph_request_count": cuda_graph_request_count, } ret.update(request_bookkeeping) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index c311e6af9dc..9b504c43f1b 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -3256,6 +3256,11 @@ def _add_moe_args(parser): help="some MoE routers have a D2H sync that will break cuda graphs. If this flag is set the router will switch" \ " to dropping and padding during decode time which does not have a D2H sync. The capacity factor is set to the" \ " max that an expert could see during inference so no tokens are actually dropped.") + group.add_argument('--enable-routing-replay', action='store_true', + help='Enable the routing replay feature for MoE layers. When enabled, each TopKRouter ' + 'creates a RouterReplay instance that can record or replay routing decisions. ' + 'Use RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) to record ' + 'routing decisions, and RouterReplay.get_recorded_data() to retrieve them.') return parser def _add_mla_args(parser): From 14f134712eaeb32b83ebc11815a45caffd434173 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 11:09:07 -0800 Subject: [PATCH 08/24] non cudagraphable implementation tested --- .../core/inference/engines/dynamic_engine.py | 66 ++++++++-------- megatron/core/inference/inference_request.py | 24 +++++- .../text_generation_controller.py | 79 ++++++++++++++++--- 3 files changed, 123 insertions(+), 46 deletions(-) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index e43e11f5227..ecf808f679f 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -804,7 +804,7 @@ def post_process_requests( sample: torch.Tensor, log_probs: torch.Tensor, top_n_logprobs: Optional[Dict[int, List[Tuple[torch.Tensor, torch.Tensor]]]] = None, - routing_indices: Optional[List[torch.Tensor]] = None, + routing_indices_per_request: Optional[Dict[int, torch.Tensor]] = None, ) -> Tuple[List[DynamicInferenceRequest], List[DynamicInferenceRequest]]: """ Handles post-processing for requests after a step. @@ -818,8 +818,9 @@ def post_process_requests( log_probs: (List): Log probs for each request top_n_logprobs: (Dict): Top-n log probs for each request. Maps request_idx to list of (top_n_logprobs, top_n_indices) tuples. - routing_indices: (List[Tensor]): MoE routing indices per layer. Each tensor has - shape [total_tokens, topk]. Will be split per request. + routing_indices_per_request: (Dict[int, Tensor]): MoE routing indices + pre-mapped by request_id. Each value is a tensor of shape + [num_tokens_this_step, num_layers, topk]. Returns: A list of active requests and completed requests as `DynamicInferenceRequest` objects @@ -833,26 +834,6 @@ def post_process_requests( log_probs_iter = log_probs if log_probs else repeat(None) - # Split routing indices per request if available - routing_indices_per_request = None - if routing_indices is not None: - # Get the query lengths for splitting per request - active_request_slice = slice( - self.context.paused_request_count, self.context.total_request_count - ) - active_query_lengths = self.context.request_query_lengths[active_request_slice].tolist() - - # Split each layer's routing tensor by request query lengths - routing_indices_per_request = {} - for req_idx in range(len(active_query_lengths)): - routing_indices_per_request[req_idx] = [] - - for layer_routing in routing_indices: - # layer_routing has shape [total_tokens, topk] - routing_splits = layer_routing.split(active_query_lengths, dim=0) - for req_idx, routing_split in enumerate(routing_splits): - routing_indices_per_request[req_idx].append(routing_split) - for req_idx, (request_id, token, request_log_probs) in enumerate( zip(request_ids.tolist(), sample.tolist(), log_probs_iter) ): @@ -969,13 +950,36 @@ def post_process_requests( else: request.generated_top_n_logprobs.append(logit_dict) - # Process routing indices if available - if routing_indices_per_request is not None and req_idx in routing_indices_per_request: - # Initialize routing_indices list if it doesn't exist + # Process routing indices if available (keyed by request_id) + # Each step's routing is a tensor of shape [num_tokens_this_step, num_layers, topk] + # We concatenate along dim=0 to accumulate: [total_tokens, num_layers, topk] + if routing_indices_per_request is not None and request_id in routing_indices_per_request: + step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] if request.routing_indices is None: - request.routing_indices = [] - # Append this step's routing data (list of tensors, one per MoE layer) - request.routing_indices.append(routing_indices_per_request[req_idx]) + request.routing_indices = step_routing + else: + request.routing_indices = torch.cat( + [request.routing_indices, step_routing], dim=0 + ) + + # Sanity check logging for routing indices (only log for first request, first few steps) + if req_idx == 0 and request.routing_indices.shape[0] <= 15: # first 15 tokens + rank = torch.distributed.get_rank() + logging.info( + f"[ROUTING DEBUG] rank={rank}, request_id={request_id}, " + f"total_tokens={request.routing_indices.shape[0]}, " + f"num_layers={request.routing_indices.shape[1]}, " + f"topk={request.routing_indices.shape[2]}, " + f"tokens_this_step={step_routing.shape[0]}" + ) + # Log first layer's routing for first few tokens + if step_routing.numel() > 0: + num_tokens_to_log = min(3, step_routing.shape[0]) + # step_routing is [num_tokens, num_layers, topk], get layer 0 + logging.info( + f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " + f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" + ) # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: @@ -1234,7 +1238,7 @@ async def async_bookkeep( sample = step_result["sample"] log_probs = step_result["log_probs"] top_n_logprobs = step_result.get("top_n_logprobs", None) - routing_indices = step_result.get("routing_indices", None) + routing_indices_per_request = step_result.get("routing_indices_per_request", None) cuda_graph_request_count = step_result["cuda_graph_request_count"] # Add paused events. @@ -1253,7 +1257,7 @@ async def async_bookkeep( sample, log_probs, top_n_logprobs, - routing_indices, + routing_indices_per_request, ) else: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 0a23accbaf1..b71a489c48b 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -248,10 +248,9 @@ class DynamicInferenceRequest(InferenceRequest): remaining_prompt_tokens: Optional[torch.Tensor] = None latency: Optional[float] = None finished_chunk_token_count = 0 - # routing_indices stores MoE routing decisions per layer for each token. - # Structure: List[List[torch.Tensor]] - outer list is per-step, inner list is per-layer, - # each tensor has shape [num_tokens_this_step, topk] - routing_indices: Optional[List[List[torch.Tensor]]] = None + # routing_indices stores MoE routing decisions for all tokens generated so far. + # Shape: [total_tokens, num_layers, topk] - accumulated across all generation steps + routing_indices: Optional[torch.Tensor] = None stop_word_ids: Optional[List[List[int]]] = None # Tokenized stop words (populated internally) def __post_init__(self): @@ -288,6 +287,18 @@ def serialize(self): """ obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] + + # Serialize routing_indices: Tensor [total_tokens - 1, num_layers, topk] -> 3D list + if self.routing_indices is not None: + total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) + # the last generated token does not undergo a forward pass + # hence we expect routing indices for total_tokens - 1 + assert self.routing_indices.shape[0] == total_tokens-1, ( + f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " + f"total tokens {total_tokens-1}." + ) + obj["routing_indices"] = self.routing_indices.tolist() + return obj @classmethod @@ -302,6 +313,11 @@ def deserialize(cls, obj: dict) -> "DynamicInferenceRequest": """ request = super().deserialize(obj) request.events = [DynamicInferenceEvent.deserialize(e) for e in obj["events"]] + + # Deserialize routing_indices: 3D list -> Tensor [total_tokens - 1, num_layers, topk] + if obj.get("routing_indices") is not None: + request.routing_indices = torch.tensor(obj["routing_indices"], dtype=torch.int64) + return request @property diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 93702a3422b..58424e78fe9 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -677,6 +677,63 @@ def _dynamic_step_log_probs_bookkeeping(self) -> Tuple[bool, bool]: return return_log_probs.any(), top_n_log_probs.any() + def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: + """Collect and map routing indices per request for MoE router recording. + + This method retrieves recorded routing decisions from RouterReplay and maps + them to individual requests using the context's request_ids and query_lengths. + Must be called while context attributes are still valid (before request transitions). + + Returns: + Optional[Dict[int, Tensor]]: A dictionary mapping request_id to a tensor of + shape [num_tokens, num_layers, topk]. Returns None if routing replay is + disabled or no routing data was recorded. + """ + config = self.inference_wrapped_model.model.config + if not getattr(config, 'enable_routing_replay', False): + return None + + raw_routing_indices = RouterReplay.get_recorded_data() + if not raw_routing_indices: + return None + + # Filter out None entries (non-MoE layers) + raw_routing_indices = [r for r in raw_routing_indices if r is not None] + if not raw_routing_indices: + return None + + # Get active request info from context + context = self.inference_wrapped_model.inference_context + active_request_slice = slice(context.paused_request_count, context.total_request_count) + active_request_ids = context.request_ids[active_request_slice].tolist() + active_query_lengths = context.request_query_lengths[active_request_slice].tolist() + + # When using CUDA graphs, tokens are padded to padded_active_token_count. + # The router records routing for all padded tokens, but we only want the + # real tokens (active_token_count). Slice down before splitting by request. + active_token_count = context.active_token_count + + # Slice each layer's routing to real tokens and split by request + # Each layer_routing has shape [padded_token_count, topk] + # After slicing: [active_token_count, topk] + # After splitting: list of [num_tokens_for_request, topk] per request + per_request_per_layer = {req_id: [] for req_id in active_request_ids} + for layer_routing in raw_routing_indices: + layer_routing = layer_routing[:active_token_count] + routing_splits = layer_routing.split(active_query_lengths, dim=0) + for req_id, routing_split in zip(active_request_ids, routing_splits): + per_request_per_layer[req_id].append(routing_split) + + # Stack layers to get [num_tokens, num_layers, topk] per request + routing_indices_per_request = {} + for req_id in active_request_ids: + layer_tensors = per_request_per_layer[req_id] # List of [num_tokens, topk] + # Stack along dim=1 to get [num_tokens, num_layers, topk] + stacked = torch.stack(layer_tensors, dim=1) + routing_indices_per_request[req_id] = stacked + + return routing_indices_per_request + def _dynamic_step_calculate_log_probs(self, logits: Tensor) -> Optional[Tensor]: """Calculate log probs from logits.""" context = self.inference_wrapped_model.inference_context @@ -891,8 +948,17 @@ async def async_generate_output_tokens_dynamic_batch( context.padded_active_request_count if context.is_decode_only() else None ) + # Enable routing recording before forward pass if routing replay is enabled + config = self.inference_wrapped_model.model.config + if getattr(config, 'enable_routing_replay', False): + RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + logits = self._dynamic_step_forward_logits(input_ids, position_ids) + # Collect routing indices per request (must be done before context transitions) + routing_indices_per_request = self._router_record_bookkeeping() + + # This is the best place to yield control back to event loop. # At this point we have enqueued FW pass GPU kernels asynchronously. # While they are running, we can do other useful CPU work. @@ -920,22 +986,13 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() - # Collect routing indices if routing replay is enabled - routing_indices = None - config = self.inference_wrapped_model.model.config - if getattr(config, 'enable_routing_replay', False): - routing_indices = RouterReplay.get_recorded_data() - # Filter out None entries (non-MoE layers) - if routing_indices: - routing_indices = [r for r in routing_indices if r is not None] - if not routing_indices: - routing_indices = None + ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, "top_n_logprobs": top_n_logprobs, - "routing_indices": routing_indices, + "routing_indices_per_request": routing_indices_per_request, "cuda_graph_request_count": cuda_graph_request_count, } ret.update(request_bookkeeping) From 4b804576a6ea97033caaddeabdd638f03d505885 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 12:02:58 -0800 Subject: [PATCH 09/24] make this work with sequence parallel + multiple prompts --- .../text_generation_controller.py | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 58424e78fe9..7b5dbdd088e 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,10 +27,13 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.transformer.enums import CudaGraphScope +from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region from megatron.core.transformer.moe.moe_layer import BaseMoELayer from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel +from megatron.core import parallel_state from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model +from megatron.core.utils import get_pg_size try: import transformer_engine as te # pylint: disable=unused-import @@ -702,35 +705,46 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: if not raw_routing_indices: return None + # raw_routing_indices is a list of length num_layers + # each entry is a Tensor of shape [local_token_count, topk] + # Now we will try to map these to requests + # Get active request info from context context = self.inference_wrapped_model.inference_context active_request_slice = slice(context.paused_request_count, context.total_request_count) active_request_ids = context.request_ids[active_request_slice].tolist() active_query_lengths = context.request_query_lengths[active_request_slice].tolist() - - # When using CUDA graphs, tokens are padded to padded_active_token_count. - # The router records routing for all padded tokens, but we only want the - # real tokens (active_token_count). Slice down before splitting by request. active_token_count = context.active_token_count - # Slice each layer's routing to real tokens and split by request - # Each layer_routing has shape [padded_token_count, topk] - # After slicing: [active_token_count, topk] - # After splitting: list of [num_tokens_for_request, topk] per request - per_request_per_layer = {req_id: [] for req_id in active_request_ids} - for layer_routing in raw_routing_indices: - layer_routing = layer_routing[:active_token_count] - routing_splits = layer_routing.split(active_query_lengths, dim=0) - for req_id, routing_split in zip(active_request_ids, routing_splits): - per_request_per_layer[req_id].append(routing_split) - - # Stack layers to get [num_tokens, num_layers, topk] per request + # Get TP group for all-gather if using sequence parallelism + # With sequence parallelism, each TP rank only sees a portion of the tokens, + # so we need to gather routing indices across all TP ranks. + tp_group = self.inference_wrapped_model.tp_group + tp_size = get_pg_size(tp_group) + + # Stack all layers first to do a single all-gather instead of per-layer all-gathers + # Each layer_routing has shape [local_token_count, topk] + # After stacking: [local_token_count, num_layers, topk] + stacked_routing = torch.stack(raw_routing_indices, dim=1) + + # All-gather across TP group if using sequence parallelism (tp_size > 1) + if tp_size > 1 and get_model_config(self.inference_wrapped_model.model).sequence_parallel: + # gather_from_sequence_parallel_region gathers along dim 0 + # [local_token_count, num_layers, topk] -> [global_token_count, num_layers, topk] + stacked_routing = gather_from_sequence_parallel_region(stacked_routing, group=tp_group) + + # Slice to real tokens (remove CUDA padding) + stacked_routing = stacked_routing[:active_token_count] + + # Split by request along token dimension + # stacked_routing has shape [active_token_count, num_layers, topk] + routing_splits = stacked_routing.split(active_query_lengths, dim=0) + + # Map to request IDs routing_indices_per_request = {} - for req_id in active_request_ids: - layer_tensors = per_request_per_layer[req_id] # List of [num_tokens, topk] - # Stack along dim=1 to get [num_tokens, num_layers, topk] - stacked = torch.stack(layer_tensors, dim=1) - routing_indices_per_request[req_id] = stacked + for req_id, routing_split in zip(active_request_ids, routing_splits): + # routing_split has shape [num_tokens_for_request, num_layers, topk] + routing_indices_per_request[req_id] = routing_split return routing_indices_per_request From 36e850a1f1c0e1555dea7b62dcb20d7243bbe65a Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 12:55:46 -0800 Subject: [PATCH 10/24] extract number of moe layers --- .../inference/gpt/gpt_dynamic_inference.py | 12 +++++ .../inference/contexts/dynamic_context.py | 6 +++ megatron/core/transformer/moe/moe_utils.py | 50 +++++++++++++------ 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference.py b/examples/inference/gpt/gpt_dynamic_inference.py index 679dd78b42b..9e966e57d99 100644 --- a/examples/inference/gpt/gpt_dynamic_inference.py +++ b/examples/inference/gpt/gpt_dynamic_inference.py @@ -44,6 +44,7 @@ TextGenerationController, ) from megatron.core.tokenizers.text.utils.build_tokenizer import build_tokenizer +from megatron.core.transformer.moe.moe_utils import get_num_moe_layers from megatron.core.transformer.module import MegatronModule from megatron.core.utils import get_mamba_inference_state_config_from_model @@ -159,6 +160,15 @@ def get_inference_context( if args.inference_logging_step_interval > 0 and args.inference_wandb_logging: metrics_writer = get_wandb_writer() + # Calculate number of MoE layers from moe_layer_freq. + num_moe_layers = None + moe_router_topk = None + if args.num_experts is not None: + num_moe_layers = get_num_moe_layers( + args.num_layers, args.moe_layer_freq + ) // args.pipeline_model_parallel_size + moe_router_topk = args.moe_router_topk + # Inference context. context = DynamicInferenceContext( params_dtype=args.params_dtype, @@ -191,6 +201,8 @@ def get_inference_context( cuda_graph_max_tokens=args.inference_dynamic_batching_cuda_graph_max_tokens, cuda_graph_mixed_prefill_count=args.inference_dynamic_batching_cuda_graph_mixed_prefill_count, metrics_writer=metrics_writer, + num_moe_layers=num_moe_layers, + moe_router_topk=moe_router_topk, ) return context diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 4267f9d0952..0b0d87e8001 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -286,9 +286,15 @@ def __init__( metrics_writer: Optional['WandbModule'] = None, request_metadata_types: Optional[List[Tuple[str, torch.dtype, bool]]] = None, persist_cuda_graphs: Optional[bool] = False, + num_moe_layers: Optional[int] = None, + moe_router_topk: Optional[int] = None, ): super().__init__(materialize_only_last_token_logits=materialize_only_last_token_logits) + # MoE routing replay parameters. + self.num_moe_layers = num_moe_layers + self.moe_router_topk = moe_router_topk + self.cache_mla_latent = cache_mla_latent if self.cache_mla_latent: assert ( diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 0d77d558b08..27b8db07cea 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -958,6 +958,41 @@ def reduce_aux_losses_tracker_across_ranks( ) +def get_num_moe_layers( + num_layers: int, + moe_layer_freq: Optional[Union[int, List[int]]] = None, + mtp_num_layers: Optional[int] = None, +) -> int: + """Calculate the number of MoE layers based on moe_layer_freq. + + Args: + num_layers (int): Total number of transformer layers. + moe_layer_freq (Optional[Union[int, List[int]]]): Frequency of MoE layers. + If int, every moe_layer_freq-th layer is an MoE layer. + If list, it's a binary pattern indicating which layers are MoE. + If None, all layers are assumed to be MoE layers. + mtp_num_layers (Optional[int]): Number of MTP (multi-token prediction) layers to add. + + Returns: + int: The number of MoE layers. + """ + if moe_layer_freq is None: + num_moe_layers = num_layers + elif isinstance(moe_layer_freq, int): + assert isinstance(num_layers, int) + moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] + num_moe_layers = sum(moe_layer_pattern) + elif isinstance(moe_layer_freq, list): + num_moe_layers = sum(moe_layer_freq) + else: + raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") + + if mtp_num_layers is not None: + num_moe_layers += mtp_num_layers + + return num_moe_layers + + def track_moe_metrics( loss_scale: float, iteration: int, @@ -987,20 +1022,7 @@ def track_moe_metrics( tracker[key]["reduce_group_has_dp"] = False reduce_aux_losses_tracker_across_ranks(track_names, pg_collection=pg_collection) - # Get number of MoE layers - if moe_layer_freq is None: - num_moe_layers = num_layers - elif isinstance(moe_layer_freq, int): - assert isinstance(num_layers, int) - moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] - num_moe_layers = sum(moe_layer_pattern) - elif isinstance(moe_layer_freq, list): - num_moe_layers = sum(moe_layer_freq) - else: - raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") - - if mtp_num_layers is not None: - num_moe_layers += mtp_num_layers + num_moe_layers = get_num_moe_layers(num_layers, moe_layer_freq, mtp_num_layers) aux_losses = {k: v['values'].float() * loss_scale for k, v in tracker.items()} for name, loss_list in aux_losses.items(): From c3e785451a5ca7252cd07f0eb84959ea5aede26b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 14:39:51 -0800 Subject: [PATCH 11/24] cuda graphability --- .../inference/contexts/dynamic_context.py | 17 ++++ .../inference/contexts/routing_metadata.py | 88 +++++++++++++++++++ .../text_generation_controller.py | 27 +++--- megatron/core/transformer/moe/moe_utils.py | 50 ++++++++++- 4 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 megatron/core/inference/contexts/routing_metadata.py diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 0b0d87e8001..ffa8029ec31 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -702,6 +702,17 @@ def allocate_mamba_states(): allocate_memory_buffer() allocate_mamba_states() + # Allocate routing metadata for MoE models. + if self.moe_router_topk is not None: + from megatron.core.inference.contexts.routing_metadata import RoutingMetadata + + self.routing_metadata = RoutingMetadata( + context=self, + moe_router_topk=self.moe_router_topk, + ) + else: + self.routing_metadata = None + # Reset attention and Mamba state. self.reset_attention_state() self.reset_mamba_state() @@ -1413,6 +1424,12 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) + if self.routing_metadata is not None: + if self.using_cuda_graph_this_step(): + self.routing_metadata.enable_static_buffer_recording() + else: + self.routing_metadata.disable_static_buffer_recording() + def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py new file mode 100644 index 00000000000..dbbdc760eb9 --- /dev/null +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +from typing import TYPE_CHECKING, Optional + +import torch + +if TYPE_CHECKING: + from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext + + +class RoutingMetadata: + """Manages routing indices metadata for MoE layers during inference. + + This class provides static buffers for CUDA graph compatibility when + recording routing decisions. It holds a reference to the inference context + to automatically determine whether to use static buffers based on CUDA graph state. + + Args: + context (DynamicInferenceContext): The inference context. + moe_router_topk (int): Number of experts selected per token. + """ + + def __init__( + self, + context: 'DynamicInferenceContext', + moe_router_topk: int, + ): + from megatron.core.transformer.moe.moe_utils import RouterReplay + + self.context = context + self.max_tokens = context.max_tokens + # Get actual number of MoE layers from RouterReplay instances on this rank. + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) + self.moe_router_topk = moe_router_topk + self.device = torch.cuda.current_device() + + # Static buffer for CUDA graph compatibility. + # Shape: [max_tokens, num_moe_layers, moe_router_topk] + self.routing_indices_buffer = torch.empty( + (self.max_tokens, self.num_moe_layers, self.moe_router_topk), + dtype=torch.int32, + device=self.device, + ) + + def get_routing_indices(self) -> Optional[torch.Tensor]: + """Get the recorded routing indices. + + Automatically uses the static buffer when CUDA graphs are active, + otherwise retrieves from RouterReplay utility. + + Returns: + Tensor of shape [num_tokens, num_moe_layers, topk] or None if not available. + """ + if self.context.using_cuda_graph_this_step(): + # Return view of static buffer up to current token count. + return self.routing_indices_buffer[:self.context.active_token_count] + else: + # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. + from megatron.core.transformer.moe.moe_utils import RouterReplay + + recorded_data = RouterReplay.get_recorded_data() + if recorded_data is None or len(recorded_data) == 0: + return None + if recorded_data[0] is None: + return None + # Stack: list of [num_tokens, topk] -> [num_tokens, num_layers, topk] + return torch.stack(recorded_data, dim=1) + + def enable_static_buffer_recording(self) -> None: + """Enable recording into the static buffer for CUDA graph compatibility. + + This sets up RouterReplay instances to copy routing indices into our + pre-allocated static buffer instead of creating new tensors. + """ + from megatron.core.transformer.moe.moe_utils import RouterReplay + + RouterReplay.set_global_static_buffers(self.routing_indices_buffer) + + def disable_static_buffer_recording(self) -> None: + """Disable static buffer recording, reverting to normal tensor assignment.""" + from megatron.core.transformer.moe.moe_utils import RouterReplay + + RouterReplay.clear_global_static_buffers() + + def reset(self) -> None: + """Reset the routing metadata state.""" + pass + self.current_token_count = 0 diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 7b5dbdd088e..6e496a883a8 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -683,8 +683,9 @@ def _dynamic_step_log_probs_bookkeeping(self) -> Tuple[bool, bool]: def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: """Collect and map routing indices per request for MoE router recording. - This method retrieves recorded routing decisions from RouterReplay and maps - them to individual requests using the context's request_ids and query_lengths. + This method retrieves recorded routing decisions and maps them to individual + requests using the context's request_ids and query_lengths. Uses the context's + routing_metadata when available (which handles CUDA graph static buffers automatically). Must be called while context attributes are still valid (before request transitions). Returns: @@ -696,37 +697,29 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: if not getattr(config, 'enable_routing_replay', False): return None - raw_routing_indices = RouterReplay.get_recorded_data() - if not raw_routing_indices: + # Get routing indices - use routing_metadata if available (handles CUDA graph static buffers) + context = self.inference_wrapped_model.inference_context + if context.routing_metadata is None: return None + + stacked_routing = context.routing_metadata.get_routing_indices() - # Filter out None entries (non-MoE layers) - raw_routing_indices = [r for r in raw_routing_indices if r is not None] - if not raw_routing_indices: + if stacked_routing is None: return None - # raw_routing_indices is a list of length num_layers - # each entry is a Tensor of shape [local_token_count, topk] - # Now we will try to map these to requests - # Get active request info from context - context = self.inference_wrapped_model.inference_context active_request_slice = slice(context.paused_request_count, context.total_request_count) active_request_ids = context.request_ids[active_request_slice].tolist() active_query_lengths = context.request_query_lengths[active_request_slice].tolist() active_token_count = context.active_token_count + # Get TP group for all-gather if using sequence parallelism # With sequence parallelism, each TP rank only sees a portion of the tokens, # so we need to gather routing indices across all TP ranks. tp_group = self.inference_wrapped_model.tp_group tp_size = get_pg_size(tp_group) - # Stack all layers first to do a single all-gather instead of per-layer all-gathers - # Each layer_routing has shape [local_token_count, topk] - # After stacking: [local_token_count, num_layers, topk] - stacked_routing = torch.stack(raw_routing_indices, dim=1) - # All-gather across TP group if using sequence parallelism (tp_size > 1) if tp_size > 1 and get_model_config(self.inference_wrapped_model.model).sequence_parallel: # gather_from_sequence_parallel_region gathers along dim 0 diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 27b8db07cea..a7bf666698a 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -578,6 +578,29 @@ def clear_global_indices(): for router in RouterReplay.global_router_replay_instances: router.clear_indices() + @staticmethod + def set_global_static_buffers(static_buffer: torch.Tensor): + """Sets static buffers for all router instances from a combined buffer. + + Args: + static_buffer: Tensor of shape [max_tokens, num_layers, topk]. + Each layer's RouterReplay gets a slice [:, layer_idx, :]. + """ + num_layers = len(RouterReplay.global_router_replay_instances) + assert static_buffer.shape[1] == num_layers, ( + f"Buffer has {static_buffer.shape[1]} layers but there are " + f"{num_layers} RouterReplay instances." + ) + for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): + # Each layer gets a view of shape [max_tokens, topk] + router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) + + @staticmethod + def clear_global_static_buffers(): + """Clears static buffers from all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_static_buffer() + @staticmethod def set_global_router_replay_action(router_replay_action: RouterReplayAction): """Sets the router replay action for all router instances.""" @@ -594,6 +617,7 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay + self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph recording self.router_replay_action: Optional[RouterReplayAction] = ( None # Router replay action for this layer ) @@ -602,6 +626,18 @@ def __init__(self): ) # List of tensors for backward pass replay RouterReplay.global_router_replay_instances.append(self) + def set_static_buffer(self, buffer: torch.Tensor): + """Sets a static buffer for CUDA graph compatible recording. + + Args: + buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. + """ + self.static_buffer = buffer + + def clear_static_buffer(self): + """Clears the static buffer.""" + self.static_buffer = None + def set_target_indices(self, topk_indices: torch.Tensor): """Sets the target topk indices for replay.""" self.target_topk_idx = topk_indices @@ -612,8 +648,18 @@ def get_recorded_indices(self) -> Optional[torch.Tensor]: return self.recorded_topk_idx def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices.""" - self.recorded_topk_idx = topk_indices + """Records the topk indices. + + If a static buffer is set (for CUDA graph compatibility), copies into it. + Otherwise, just stores the tensor reference. + """ + if self.static_buffer is not None: + # Copy into static buffer for CUDA graph compatibility. + num_tokens = topk_indices.shape[0] + self.static_buffer[:num_tokens].copy_(topk_indices) + self.recorded_topk_idx = self.static_buffer[:num_tokens] + else: + self.recorded_topk_idx = topk_indices def clear_indices(self): """Clears the recorded and target topk indices.""" From 0a72c80ee36bbfe11207000ef259305930ea1e21 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 15:55:28 -0800 Subject: [PATCH 12/24] make this work with cuda graphs --- .../gpt_dynamic_inference_with_coordinator.py | 3 ++ .../inference/contexts/routing_metadata.py | 34 +++++++++++++++---- .../core/inference/engines/dynamic_engine.py | 14 +++++++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index f354b122a7e..c3a4f521b83 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -137,6 +137,9 @@ async def main( # While we wait for the requests to complete, the engine runs in the background. results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) + for i, res in enumerate(results): + routing_indices = res.requests[0].routing_indices + logging.info(f"Result {i} has routing indices with shape {routing_indices}.") if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index dbbdc760eb9..aa87e448992 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -25,15 +25,33 @@ def __init__( context: 'DynamicInferenceContext', moe_router_topk: int, ): - from megatron.core.transformer.moe.moe_utils import RouterReplay - self.context = context self.max_tokens = context.max_tokens - # Get actual number of MoE layers from RouterReplay instances on this rank. - self.num_moe_layers = len(RouterReplay.global_router_replay_instances) self.moe_router_topk = moe_router_topk self.device = torch.cuda.current_device() + # Static buffer allocated lazily in _ensure_buffer_allocated(). + # We defer allocation because RouterReplay instances don't exist yet at init time. + self.routing_indices_buffer: Optional[torch.Tensor] = None + self.num_moe_layers: Optional[int] = None + + def _ensure_buffer_allocated(self) -> None: + """Allocate the static buffer if not already allocated. + + Gets the actual number of MoE layers from RouterReplay instances. + """ + if self.routing_indices_buffer is not None: + return + + from megatron.core.transformer.moe.moe_utils import RouterReplay + + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) + print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " + f"(from {len(RouterReplay.global_router_replay_instances)} RouterReplay instances)") + + if self.num_moe_layers == 0: + return + # Static buffer for CUDA graph compatibility. # Shape: [max_tokens, num_moe_layers, moe_router_topk] self.routing_indices_buffer = torch.empty( @@ -53,6 +71,8 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: """ if self.context.using_cuda_graph_this_step(): # Return view of static buffer up to current token count. + if self.routing_indices_buffer is None: + return None return self.routing_indices_buffer[:self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. @@ -71,10 +91,13 @@ def enable_static_buffer_recording(self) -> None: This sets up RouterReplay instances to copy routing indices into our pre-allocated static buffer instead of creating new tensors. + Allocates the buffer lazily on first call. """ from megatron.core.transformer.moe.moe_utils import RouterReplay - RouterReplay.set_global_static_buffers(self.routing_indices_buffer) + self._ensure_buffer_allocated() + if self.routing_indices_buffer is not None: + RouterReplay.set_global_static_buffers(self.routing_indices_buffer) def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" @@ -85,4 +108,3 @@ def disable_static_buffer_recording(self) -> None: def reset(self) -> None: """Reset the routing metadata state.""" pass - self.current_token_count = 0 diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index ecf808f679f..5ba66148300 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -308,6 +308,16 @@ def create_cuda_graphs(self, reset_context: bool = True): f"{tbar_idx}/{len(context.cuda_graph_batch_dimensions_list)}. {tbar_str}" ) + # Enable routing recording during warmup if routing replay is enabled. + # This ensures the record_indices copy operation is captured in the CUDA graph. + model_config = controller.inference_wrapped_model.model.config + if getattr(model_config, 'enable_routing_replay', False): + from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, + RouterReplayAction, + ) + RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + # Forward pass -> logits. controller._dynamic_step_forward_logits(input_ids, position_ids) @@ -956,7 +966,7 @@ def post_process_requests( if routing_indices_per_request is not None and request_id in routing_indices_per_request: step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] if request.routing_indices is None: - request.routing_indices = step_routing + request.routing_indices = step_routing.clone() else: request.routing_indices = torch.cat( [request.routing_indices, step_routing], dim=0 @@ -980,6 +990,8 @@ def post_process_requests( f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" ) + + # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: From d62038d1446b722796ce607290b72465f592bc5d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 23 Jan 2026 08:18:19 -0800 Subject: [PATCH 13/24] save router routing in functional tests and correctly handle inference request record merge --- .../inference/gpt/gpt_dynamic_inference_with_coordinator.py | 4 +++- megatron/core/inference/inference_request.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index c3a4f521b83..7ec826c74a8 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -139,7 +139,7 @@ async def main( results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) for i, res in enumerate(results): routing_indices = res.requests[0].routing_indices - logging.info(f"Result {i} has routing indices with shape {routing_indices}.") + logging.info(f"Result {i} has routing indices with shape {routing_indices.shape}.") if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. @@ -159,6 +159,8 @@ async def main( result_dict["logprobs"] = req.prompt_log_probs + req.generated_log_probs throughput = len(req.generated_tokens) / req.latency throughputs.append(throughput) + if req.routing_indices is not None: + result_dict["routing_indices"] = req.routing_indices.tolist() json_results[req.request_id] = result_dict throughput_dict = {"throughput": throughputs} if args.throughput_check_only: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index b71a489c48b..de75123866c 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -481,7 +481,7 @@ def checkpoint(self, tokenizer: MegatronTokenizer | None = None): new_request = DynamicInferenceRequest( request_id=old_request.request_id, prompt_tokens=new_prompt_tokens, - sampling_params=new_sampling_params, + sampling_params=new_sampling_params ) self.requests.append(new_request) @@ -503,6 +503,8 @@ def merge_lists(key): prompt_tokens = self.requests[0].prompt_tokens prompt_text = self.requests[0].prompt + if self.requests[0].routing_indices is not None: + routing_indices = torch.cat([r.routing_indices for r in self.requests]) generated_tokens = merge_lists("generated_tokens") try: generated_text = "".join(r.generated_text for r in self.requests) @@ -526,6 +528,7 @@ def merge_lists(key): status=self.requests[-1].status, latency=self.latency, events=merge_lists("events"), + routing_indices=routing_indices, ) return request From 0666c2ef9eb26460df4fa6c4f850e3d1640d4a33 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 14:47:58 -0800 Subject: [PATCH 14/24] make it work with merge --- .../gpt_dynamic_inference_with_coordinator.py | 5 +- .../inference/contexts/dynamic_context.py | 26 +- .../inference/contexts/routing_metadata.py | 10 +- .../core/inference/engines/dynamic_engine.py | 12 +- megatron/core/inference/inference_request.py | 5 +- .../text_generation_controller.py | 10 +- megatron/core/transformer/moe/moe_utils.py | 234 ++---------------- megatron/core/transformer/moe/router.py | 4 +- .../core/transformer/moe/router_replay.py | 51 ++++ .../model_config.yaml | 1 + 10 files changed, 99 insertions(+), 259 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index a0d2e1cfdf3..771b3eca519 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -125,10 +125,7 @@ async def main( await asyncio.sleep(0) # While we wait for the requests to complete, the engine runs in the background. - results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) - for i, res in enumerate(results): - routing_indices = res.requests[0].routing_indices - logging.info(f"Result {i} has routing indices with shape {routing_indices.shape}.") + results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 07e2e53b74f..4e3dbb78043 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -35,6 +35,7 @@ from .attention_context.mha_metadata import GraphedMHAMetadata, NonGraphedMHAMetadata from .base_context import BaseInferenceContext from .dynamic_block_allocator import BlockAllocator +from .routing_metadata import RoutingMetadata try: from .fused_kv_append_kernel import triton_append_key_value_cache @@ -469,6 +470,14 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC max_seqlen=self.max_sequence_length, ) + self.moe_enable_routing_replay = model_config.moe_enable_routing_replay + if self.moe_enable_routing_replay: + assert model_config.num_moe_experts is not None, "Router recording/replay requested but no MoE experts specified!" + self.moe_routing_metadata = RoutingMetadata( + self, model_config.moe_router_topk + ) + + # CUDA graph config list self.use_cuda_graphs_for_non_decode_steps = ( inference_config.use_cuda_graphs_for_non_decode_steps @@ -659,17 +668,6 @@ def allocate_mamba_states(): allocate_memory_buffer() allocate_mamba_states() - # Allocate routing metadata for MoE models. - if self.moe_router_topk is not None: - from megatron.core.inference.contexts.routing_metadata import RoutingMetadata - - self.routing_metadata = RoutingMetadata( - context=self, - moe_router_topk=self.moe_router_topk, - ) - else: - self.routing_metadata = None - # Reset attention and Mamba state. self.reset_attention_state() self.reset_mamba_state() @@ -1305,11 +1303,11 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) - if self.routing_metadata is not None: + if self.moe_enable_routing_replay is not None: if self.using_cuda_graph_this_step(): - self.routing_metadata.enable_static_buffer_recording() + self.moe_routing_metadata.enable_static_buffer_recording() else: - self.routing_metadata.disable_static_buffer_recording() + self.moe_routing_metadata.disable_static_buffer_recording() def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index aa87e448992..faa184c24b7 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext +from megatron.core.transformer.moe.router_replay import RouterReplay class RoutingMetadata: """Manages routing indices metadata for MoE layers during inference. @@ -43,7 +44,6 @@ def _ensure_buffer_allocated(self) -> None: if self.routing_indices_buffer is not None: return - from megatron.core.transformer.moe.moe_utils import RouterReplay self.num_moe_layers = len(RouterReplay.global_router_replay_instances) print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " @@ -73,11 +73,11 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: # Return view of static buffer up to current token count. if self.routing_indices_buffer is None: return None + # Only return up to active token count, to skip entries + # for padding tokens. return self.routing_indices_buffer[:self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. - from megatron.core.transformer.moe.moe_utils import RouterReplay - recorded_data = RouterReplay.get_recorded_data() if recorded_data is None or len(recorded_data) == 0: return None @@ -93,16 +93,12 @@ def enable_static_buffer_recording(self) -> None: pre-allocated static buffer instead of creating new tensors. Allocates the buffer lazily on first call. """ - from megatron.core.transformer.moe.moe_utils import RouterReplay - self._ensure_buffer_allocated() if self.routing_indices_buffer is not None: RouterReplay.set_global_static_buffers(self.routing_indices_buffer) def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" - from megatron.core.transformer.moe.moe_utils import RouterReplay - RouterReplay.clear_global_static_buffers() def reset(self) -> None: diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 2cc98481e69..9faff001071 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -38,6 +38,10 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.transformer.moe.router_replay import ( + RouterReplay, + RouterReplayAction, + ) from megatron.core.inference.utils import Counter, await_process_call from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.cuda_graphs import delete_cuda_graphs @@ -52,6 +56,8 @@ trace_async_exceptions, ) + + from .async_zmq_communicator import AsyncZMQCommunicator try: @@ -300,11 +306,7 @@ def create_cuda_graphs(self, reset_context: bool = True): # Enable routing recording during warmup if routing replay is enabled. # This ensures the record_indices copy operation is captured in the CUDA graph. model_config = controller.inference_wrapped_model.model.config - if getattr(model_config, 'enable_routing_replay', False): - from megatron.core.transformer.moe.moe_utils import ( - RouterReplay, - RouterReplayAction, - ) + if model_config.moe_enable_routing_replay: RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) # Forward pass -> logits. diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 4356eb33bf9..3fd66633742 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -313,7 +313,7 @@ def serialize(self) -> dict: obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] - # Serialize routing_indices: Tensor [total_tokens - 1, num_layers, topk] -> 3D list + # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] if self.routing_indices is not None: total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) # the last generated token does not undergo a forward pass @@ -322,8 +322,7 @@ def serialize(self) -> dict: f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " f"total tokens {total_tokens-1}." ) - obj["routing_indices"] = self.routing_indices.tolist() - + torch.cuda.nvtx.range_pop() return obj diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index b58fd749b41..8e70055a88f 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -30,7 +30,7 @@ from megatron.core.transformer.enums import CudaGraphScope from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region from megatron.core.transformer.moe.moe_layer import BaseMoELayer -from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction +from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel from megatron.core import parallel_state from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model @@ -693,15 +693,15 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: disabled or no routing data was recorded. """ config = self.inference_wrapped_model.model.config - if not getattr(config, 'enable_routing_replay', False): + if not config.moe_enable_routing_replay: return None # Get routing indices - use routing_metadata if available (handles CUDA graph static buffers) context = self.inference_wrapped_model.inference_context - if context.routing_metadata is None: + if context.moe_routing_metadata is None: return None - stacked_routing = context.routing_metadata.get_routing_indices() + stacked_routing = context.moe_routing_metadata.get_routing_indices() if stacked_routing is None: return None @@ -956,7 +956,7 @@ async def async_generate_output_tokens_dynamic_batch( # Enable routing recording before forward pass if routing replay is enabled config = self.inference_wrapped_model.model.config - if getattr(config, 'enable_routing_replay', False): + if config.moe_enable_routing_replay: RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) logits = self._dynamic_step_forward_logits(input_ids, position_ids) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index c0d41993daf..666d6e16c38 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -2,8 +2,6 @@ import functools import math -from enum import Enum -from typing import Callable, List, Optional, Union from dataclasses import dataclass from typing import List, Optional, Tuple, Union @@ -655,186 +653,6 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map -class RouterReplayAction(Enum): - RECORD = "record" # Record the topk indices for replay - REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass - REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass - - -class RouterReplay: - """ - A class to manage the recording and replaying of MoE routing decisions. - It holds all router instances and provides static methods to globally - control recording and replaying. - """ - - # Static variable to hold all router instances, one per MoE layer. - global_router_replay_instances: List['RouterReplay'] = [] - - @staticmethod - def set_replay_data(all_layers_topk_indices: List[torch.Tensor]): - """ - Distributes the topk indices for all layers to their respective RouterReplay instances. - :param all_layers_topk_indices: A list of tensors, where each tensor contains the - topk indices for a specific layer. The order - must match the instantiation order of the routers. - """ - if len(all_layers_topk_indices) != len(RouterReplay.global_router_replay_instances): - raise ValueError( - f"The number of replay tensors ({len(all_layers_topk_indices)}) " - f"does not match router instances ({len(RouterReplay.global_router_replay_instances)})." - ) - for i, router_instance in enumerate(RouterReplay.global_router_replay_instances): - router_instance.set_target_indices(all_layers_topk_indices[i]) - - @staticmethod - def get_recorded_data() -> List[torch.Tensor]: - """ - Collects the recorded topk indices from all RouterReplay instances. - :return: A list of tensors, each containing the recorded topk indices for a layer. - """ - return [ - router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances - ] - - @staticmethod - def clear_global_indices(): - """Clears the recorded and target topk indices in all instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_indices() - - @staticmethod - def set_global_static_buffers(static_buffer: torch.Tensor): - """Sets static buffers for all router instances from a combined buffer. - - Args: - static_buffer: Tensor of shape [max_tokens, num_layers, topk]. - Each layer's RouterReplay gets a slice [:, layer_idx, :]. - """ - num_layers = len(RouterReplay.global_router_replay_instances) - assert static_buffer.shape[1] == num_layers, ( - f"Buffer has {static_buffer.shape[1]} layers but there are " - f"{num_layers} RouterReplay instances." - ) - for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): - # Each layer gets a view of shape [max_tokens, topk] - router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) - - @staticmethod - def clear_global_static_buffers(): - """Clears static buffers from all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_static_buffer() - - @staticmethod - def set_global_router_replay_action(router_replay_action: RouterReplayAction): - """Sets the router replay action for all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.set_router_replay_action(router_replay_action) - - @staticmethod - def clear_global_router_replay_action(): - """Clears the router replay action for all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_router_replay_action() - - def __init__(self): - """Initializes a RouterReplay instance for a specific layer.""" - self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay - self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay - self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph recording - self.router_replay_action: Optional[RouterReplayAction] = ( - None # Router replay action for this layer - ) - self.replay_backward_list: List[torch.Tensor] = ( - [] - ) # List of tensors for backward pass replay - RouterReplay.global_router_replay_instances.append(self) - - def set_static_buffer(self, buffer: torch.Tensor): - """Sets a static buffer for CUDA graph compatible recording. - - Args: - buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. - """ - self.static_buffer = buffer - - def clear_static_buffer(self): - """Clears the static buffer.""" - self.static_buffer = None - - def set_target_indices(self, topk_indices: torch.Tensor): - """Sets the target topk indices for replay.""" - self.target_topk_idx = topk_indices - self.replay_backward_list.append(topk_indices) - - def get_recorded_indices(self) -> Optional[torch.Tensor]: - """Returns the recorded topk indices.""" - return self.recorded_topk_idx - - def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices. - - If a static buffer is set (for CUDA graph compatibility), copies into it. - Otherwise, just stores the tensor reference. - """ - if self.static_buffer is not None: - # Copy into static buffer for CUDA graph compatibility. - num_tokens = topk_indices.shape[0] - self.static_buffer[:num_tokens].copy_(topk_indices) - self.recorded_topk_idx = self.static_buffer[:num_tokens] - else: - self.recorded_topk_idx = topk_indices - - def clear_indices(self): - """Clears the recorded and target topk indices.""" - self.recorded_topk_idx = None - self.target_topk_idx = None - self.replay_backward_list = [] - - def set_router_replay_action(self, router_replay_action: RouterReplayAction): - """Sets the router replay action for this layer.""" - self.router_replay_action = router_replay_action - - def clear_router_replay_action(self): - """Clears the router replay action for this layer.""" - self.router_replay_action = None - - def get_replay_topk( - self, - scores: torch.Tensor, - topk: int, - num_groups: Optional[int] = None, - group_topk: Optional[int] = None, - default_compute_topk: Callable[ - [torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor - ] = None, - ) -> torch.Tensor: - """Returns the target topk indices for replay.""" - if self.router_replay_action == RouterReplayAction.RECORD: - probs, top_indices = default_compute_topk( - scores, topk, num_groups=num_groups, group_topk=group_topk - ) - self.record_indices(top_indices) - return probs, top_indices - elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: - top_indices = self.target_topk_idx - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: - top_indices = self.replay_backward_list.pop(0) - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - else: - return default_compute_topk(scores, topk, num_groups, group_topk) - - def topk_routing_with_score_function( logits: torch.Tensor, topk: int, @@ -1182,41 +1000,6 @@ def reduce_aux_losses_tracker_across_ranks( ) -def get_num_moe_layers( - num_layers: int, - moe_layer_freq: Optional[Union[int, List[int]]] = None, - mtp_num_layers: Optional[int] = None, -) -> int: - """Calculate the number of MoE layers based on moe_layer_freq. - - Args: - num_layers (int): Total number of transformer layers. - moe_layer_freq (Optional[Union[int, List[int]]]): Frequency of MoE layers. - If int, every moe_layer_freq-th layer is an MoE layer. - If list, it's a binary pattern indicating which layers are MoE. - If None, all layers are assumed to be MoE layers. - mtp_num_layers (Optional[int]): Number of MTP (multi-token prediction) layers to add. - - Returns: - int: The number of MoE layers. - """ - if moe_layer_freq is None: - num_moe_layers = num_layers - elif isinstance(moe_layer_freq, int): - assert isinstance(num_layers, int) - moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] - num_moe_layers = sum(moe_layer_pattern) - elif isinstance(moe_layer_freq, list): - num_moe_layers = sum(moe_layer_freq) - else: - raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") - - if mtp_num_layers is not None: - num_moe_layers += mtp_num_layers - - return num_moe_layers - - def track_moe_metrics( loss_scale: float, iteration: int, @@ -1266,7 +1049,20 @@ def track_moe_metrics( tracker[key]["reduce_group_has_dp"] = False reduce_aux_losses_tracker_across_ranks(track_names, pg_collection=pg_collection) - num_moe_layers = get_num_moe_layers(num_layers, moe_layer_freq, mtp_num_layers) + # Get number of MoE layers + if moe_layer_freq is None: + num_moe_layers = num_layers + elif isinstance(moe_layer_freq, int): + assert isinstance(num_layers, int) + moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] + num_moe_layers = sum(moe_layer_pattern) + elif isinstance(moe_layer_freq, list): + num_moe_layers = sum(moe_layer_freq) + else: + raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") + + if mtp_num_layers is not None: + num_moe_layers += mtp_num_layers aux_losses = {k: v['values'].float() * loss_scale for k, v in tracker.items()} for name, loss_list in aux_losses.items(): @@ -1777,4 +1573,4 @@ def wrapped_func(moe_layer, *args, **kwargs): return wrapped_func - return decorator + return decorator \ No newline at end of file diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 6ee98bbfd3a..9d1b557d2e4 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -10,7 +10,6 @@ from megatron.core.transformer.moe.moe_utils import ( MoEAuxLossAutoScaler, ProcessGroupCollection, - RouterReplay, apply_random_logits, apply_router_token_dropping, compute_routing_scores_for_aux_loss, @@ -213,9 +212,10 @@ def __init__( self.ga_steps = None self.router_replay = None + if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() - + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index b6b8e26a0a6..75fbb9009a3 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -74,6 +74,29 @@ def clear_global_router_replay_instances(): """Clear the global list of router replay instances to prevent memory leaks.""" RouterReplay.global_router_replay_instances.clear() + @staticmethod + def set_global_static_buffers(static_buffer: torch.Tensor): + """Sets static buffers for all router instances from a combined buffer. + + Args: + static_buffer: Tensor of shape [max_tokens, num_layers, topk]. + Each layer's RouterReplay gets a slice [:, layer_idx, :]. + """ + num_layers = len(RouterReplay.global_router_replay_instances) + assert static_buffer.shape[1] == num_layers, ( + f"Buffer has {static_buffer.shape[1]} layers but there are " + f"{num_layers} RouterReplay instances." + ) + for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): + # Each layer gets a view of shape [max_tokens, topk] + router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) + + @staticmethod + def clear_global_static_buffers(): + """Clears static buffers from all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_static_buffer() + def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay @@ -84,6 +107,7 @@ def __init__(self): self.replay_backward_list: List[torch.Tensor] = ( [] ) # List of tensors for backward pass replay + self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): @@ -159,3 +183,30 @@ def get_replay_topk( return probs, top_indices else: return default_compute_topk(scores, topk, num_groups, group_topk) + + + def set_static_buffer(self, buffer: torch.Tensor): + """Sets a static buffer for CUDA graph compatible recording. + + Args: + buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. + """ + self.static_buffer = buffer + + def clear_static_buffer(self): + """Clears the static buffer.""" + self.static_buffer = None + + def record_indices(self, topk_indices: torch.Tensor): + """Records the topk indices. + + If a static buffer is set (for CUDA graph compatibility), copies into it. + Otherwise, just stores the tensor reference. + """ + if self.static_buffer is not None: + # Copy into static buffer for CUDA graph compatibility. + num_tokens = topk_indices.shape[0] + self.static_buffer[:num_tokens].copy_(topk_indices) + self.recorded_topk_idx = self.static_buffer[:num_tokens] + else: + self.recorded_topk_idx = topk_indices diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml index edc5fc2eb32..d33a854e49e 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml @@ -84,6 +84,7 @@ MODEL_ARGS: --inference-dynamic-batching-max-requests: 512 --inference-logging-step-interval: 1 --sequence-parallel: true + --moe-enable-routing-replay: true METRICS: - "generated_tokens" From 9109ca196a310243d15d50ba48b56112f918ee58 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:42:04 -0800 Subject: [PATCH 15/24] hook upto openAI API --- megatron/core/inference/inference_request.py | 1 + .../endpoints/chat_completions.py | 100 ++++++----- .../endpoints/completions.py | 170 +++++++++--------- 3 files changed, 147 insertions(+), 124 deletions(-) diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 3fd66633742..d67e8570033 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -513,6 +513,7 @@ def merge_lists(key): prompt_tokens = self.requests[0].prompt_tokens prompt_text = self.requests[0].prompt + routing_indices = None if self.requests[0].routing_indices is not None: routing_indices = torch.cat([r.routing_indices for r in self.requests]) generated_tokens = merge_lists("generated_tokens") diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 0c3379bc53f..e9d16e08ca0 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -99,50 +99,62 @@ async def chat_completions(): request_idx = 0 for record in batch_results: - for result in record.requests: - text_output = result.generated_text - - logprobs_content = None - if sampling_params.return_log_probs: - token_logprobs = getattr(result, 'log_probs', []) - tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] - - # Get top_n_logprobs if available - generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) - - logprobs_content = [] - for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): - # Build top_logprobs list for this token position - top_logprobs_list = [] - if generated_top_n_logprobs and i < len(generated_top_n_logprobs): - top_n_dict = generated_top_n_logprobs[i] - for token_str, logprob in top_n_dict.items(): - top_logprobs_list.append( - { - "token": token_str, - "logprob": logprob, - "bytes": list(token_str.encode("utf-8")), - } - ) - - entry = { - "token": tok, - "logprob": lp, - "bytes": list(tok.encode("utf-8")), - "top_logprobs": top_logprobs_list, - } - logprobs_content.append(entry) - - choice_data = { - "index": 0, - "message": {"role": "assistant", "content": text_output}, - # 'logprobs' in chat API is an object containing 'content' - "logprobs": {"content": logprobs_content} if logprobs_content else None, - "finish_reason": "length", # Original code hardcoded this. - } - choices.append(choice_data) - total_completion_tokens += len(result.generated_tokens) - request_idx += 0 + assert len(record.requests) == 1, "Each record should contain one request result." + result = record.merge() + text_output = result.generated_text + + logprobs_content = None + if sampling_params.return_log_probs: + token_logprobs = getattr(result, 'log_probs', []) + tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] + + # Get top_n_logprobs if available + generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) + + logprobs_content = [] + for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): + # Build top_logprobs list for this token position + top_logprobs_list = [] + if generated_top_n_logprobs and i < len(generated_top_n_logprobs): + top_n_dict = generated_top_n_logprobs[i] + for token_str, logprob in top_n_dict.items(): + top_logprobs_list.append( + { + "token": token_str, + "logprob": logprob, + "bytes": list(token_str.encode("utf-8")), + } + ) + + entry = { + "token": tok, + "logprob": lp, + "bytes": list(tok.encode("utf-8")), + "top_logprobs": top_logprobs_list, + } + logprobs_content.append(entry) + + choice_data = { + "index": 0, + "message": {"role": "assistant", "content": text_output}, + # 'logprobs' in chat API is an object containing 'content' + "logprobs": {"content": logprobs_content} if logprobs_content else None, + "finish_reason": "length", # Original code hardcoded this. + } + logging.info(result) + if result.routing_indices is not None: + choice_data["moe_topk_indices"] = result.routing_indices.tolist() + prompt_length = ( + len(result.prompt_tokens) + if result.prompt_tokens is not None + else 0 + ) + if prompt_length: + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices.append(choice_data) + total_completion_tokens += len(result.generated_tokens) + request_idx += 0 + response = { "choices": choices, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py index b749205cdfd..cb0adc718d3 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py @@ -125,88 +125,98 @@ async def completions(): request_idx = 0 for record in batch_results: - for result in record.requests: - full_text = result.generated_text or "" - text_output = (prompts_as_strings[request_idx] + full_text) if echo else full_text - - logprobs_data = None - if sampling_params.return_log_probs: - # Get prompt tokens and logprobs - prompt_tokens_list = [] - if result.prompt_tokens is not None: - if hasattr(result.prompt_tokens, 'tolist'): - prompt_tokens_list = result.prompt_tokens.tolist() - else: - prompt_tokens_list = list(result.prompt_tokens) - - prompt_log_probs = getattr(result, 'prompt_log_probs', None) or [] - prompt_top_n_logprobs = getattr(result, 'prompt_top_n_logprobs', None) or [] - - # Get generated tokens and logprobs - generated_tokens_list = ( - list(result.generated_tokens) if result.generated_tokens else [] - ) - generated_log_probs = getattr(result, 'generated_log_probs', None) or [] - generated_top_n_logprobs = ( - getattr(result, 'generated_top_n_logprobs', None) or [] - ) - - if echo: - # When echo=True, include prompt tokens and their logprobs - # Prompt logprobs are for tokens [1:] (first token has no logprob) - all_token_ids = prompt_tokens_list + generated_tokens_list - tokens = [tokenizer.detokenize([tok]) for tok in all_token_ids] - - # Build token_logprobs: [None] for first token, then prompt logprobs, - # then generated logprobs - token_logprobs = [None] + list(prompt_log_probs) + list(generated_log_probs) - - # Build top_logprobs: [None] for first token, then prompt top_n, - # then generated top_n - top_logprobs = None - if prompt_top_n_logprobs or generated_top_n_logprobs: - top_logprobs = ( - [None] - + list(prompt_top_n_logprobs) - + list(generated_top_n_logprobs) - ) - - # Calculate text_offset: cumulative character positions starting from 0 - text_offset = [] - current_offset = 0 - for tok_str in tokens: - text_offset.append(current_offset) - current_offset += len(tok_str) + result = record.merge() + full_text = result.generated_text or "" + text_output = (prompts_as_strings[request_idx] + full_text) if echo else full_text + + logprobs_data = None + if sampling_params.return_log_probs: + # Get prompt tokens and logprobs + prompt_tokens_list = [] + if result.prompt_tokens is not None: + if hasattr(result.prompt_tokens, 'tolist'): + prompt_tokens_list = result.prompt_tokens.tolist() else: - # When echo=False, only return generated tokens and their logprobs - tokens = [tokenizer.detokenize([tok]) for tok in generated_tokens_list] - - # Prepend [None] to match OpenAI format - token_logprobs = [None] + list(generated_log_probs) - - # Build top_logprobs - top_logprobs = None - if generated_top_n_logprobs: - top_logprobs = [None] + list(generated_top_n_logprobs) - - # Calculate text_offset for generated tokens only - text_offset = [] - current_offset = 0 - for tok_str in tokens: - text_offset.append(current_offset) - current_offset += len(tok_str) - - logprobs_data = { - "token_logprobs": token_logprobs, - "tokens": tokens, - "text_offset": text_offset, - "top_logprobs": top_logprobs, - } - - choices.append( - {"index": request_idx, "text": text_output, "logprobs": logprobs_data} + prompt_tokens_list = list(result.prompt_tokens) + + prompt_log_probs = getattr(result, 'prompt_log_probs', None) or [] + prompt_top_n_logprobs = getattr(result, 'prompt_top_n_logprobs', None) or [] + + # Get generated tokens and logprobs + generated_tokens_list = ( + list(result.generated_tokens) if result.generated_tokens else [] + ) + generated_log_probs = getattr(result, 'generated_log_probs', None) or [] + generated_top_n_logprobs = ( + getattr(result, 'generated_top_n_logprobs', None) or [] ) - request_idx += 1 + + if echo: + # When echo=True, include prompt tokens and their logprobs + # Prompt logprobs are for tokens [1:] (first token has no logprob) + all_token_ids = prompt_tokens_list + generated_tokens_list + tokens = [tokenizer.detokenize([tok]) for tok in all_token_ids] + + # Build token_logprobs: [None] for first token, then prompt logprobs, + # then generated logprobs + token_logprobs = [None] + list(prompt_log_probs) + list(generated_log_probs) + + # Build top_logprobs: [None] for first token, then prompt top_n, + # then generated top_n + top_logprobs = None + if prompt_top_n_logprobs or generated_top_n_logprobs: + top_logprobs = ( + [None] + + list(prompt_top_n_logprobs) + + list(generated_top_n_logprobs) + ) + + # Calculate text_offset: cumulative character positions starting from 0 + text_offset = [] + current_offset = 0 + for tok_str in tokens: + text_offset.append(current_offset) + current_offset += len(tok_str) + else: + # When echo=False, only return generated tokens and their logprobs + tokens = [tokenizer.detokenize([tok]) for tok in generated_tokens_list] + + # Prepend [None] to match OpenAI format + token_logprobs = [None] + list(generated_log_probs) + + # Build top_logprobs + top_logprobs = None + if generated_top_n_logprobs: + top_logprobs = [None] + list(generated_top_n_logprobs) + + # Calculate text_offset for generated tokens only + text_offset = [] + current_offset = 0 + for tok_str in tokens: + text_offset.append(current_offset) + current_offset += len(tok_str) + + logprobs_data = { + "token_logprobs": token_logprobs, + "tokens": tokens, + "text_offset": text_offset, + "top_logprobs": top_logprobs, + } + + choices.append( + {"index": request_idx, "text": text_output, "logprobs": logprobs_data} + ) + if result.routing_indices is not None: + choices[-1]["moe_topk_indices"] = result.routing_indices.tolist() + prompt_length = ( + len(result.prompt_tokens) + if result.prompt_tokens is not None + else 0 + ) + if prompt_length: + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + + request_idx += 1 return jsonify({"choices": choices}) From 7feee53d11ae6c9ba3c1fff52c3a5764ee4ef4bd Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:53:55 -0800 Subject: [PATCH 16/24] format --- .../inference/contexts/dynamic_context.py | 11 +++--- .../inference/contexts/routing_metadata.py | 34 ++++++++---------- .../core/inference/engines/dynamic_engine.py | 36 +++++-------------- megatron/core/inference/inference_request.py | 12 +++---- .../text_generation_controller.py | 14 +++----- .../endpoints/chat_completions.py | 11 +++--- .../endpoints/completions.py | 22 ++++-------- megatron/core/transformer/moe/moe_utils.py | 2 +- megatron/core/transformer/moe/router.py | 2 +- .../core/transformer/moe/router_replay.py | 7 ++-- 10 files changed, 53 insertions(+), 98 deletions(-) diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 4e3dbb78043..47b0bb50748 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -472,11 +472,10 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC self.moe_enable_routing_replay = model_config.moe_enable_routing_replay if self.moe_enable_routing_replay: - assert model_config.num_moe_experts is not None, "Router recording/replay requested but no MoE experts specified!" - self.moe_routing_metadata = RoutingMetadata( - self, model_config.moe_router_topk - ) - + assert ( + model_config.num_moe_experts is not None + ), "Router recording/replay requested but no MoE experts specified!" + self.moe_routing_metadata = RoutingMetadata(self, model_config.moe_router_topk) # CUDA graph config list self.use_cuda_graphs_for_non_decode_steps = ( @@ -1308,7 +1307,7 @@ def initialize_attention_state( self.moe_routing_metadata.enable_static_buffer_recording() else: self.moe_routing_metadata.disable_static_buffer_recording() - + def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index faa184c24b7..07abe0c3469 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -9,23 +9,20 @@ from megatron.core.transformer.moe.router_replay import RouterReplay + class RoutingMetadata: """Manages routing indices metadata for MoE layers during inference. - + This class provides static buffers for CUDA graph compatibility when recording routing decisions. It holds a reference to the inference context to automatically determine whether to use static buffers based on CUDA graph state. - + Args: context (DynamicInferenceContext): The inference context. moe_router_topk (int): Number of experts selected per token. """ - def __init__( - self, - context: 'DynamicInferenceContext', - moe_router_topk: int, - ): + def __init__(self, context: 'DynamicInferenceContext', moe_router_topk: int): self.context = context self.max_tokens = context.max_tokens self.moe_router_topk = moe_router_topk @@ -38,20 +35,17 @@ def __init__( def _ensure_buffer_allocated(self) -> None: """Allocate the static buffer if not already allocated. - + Gets the actual number of MoE layers from RouterReplay instances. """ if self.routing_indices_buffer is not None: return - - + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) - print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " - f"(from {len(RouterReplay.global_router_replay_instances)} RouterReplay instances)") - + if self.num_moe_layers == 0: return - + # Static buffer for CUDA graph compatibility. # Shape: [max_tokens, num_moe_layers, moe_router_topk] self.routing_indices_buffer = torch.empty( @@ -62,10 +56,10 @@ def _ensure_buffer_allocated(self) -> None: def get_routing_indices(self) -> Optional[torch.Tensor]: """Get the recorded routing indices. - + Automatically uses the static buffer when CUDA graphs are active, otherwise retrieves from RouterReplay utility. - + Returns: Tensor of shape [num_tokens, num_moe_layers, topk] or None if not available. """ @@ -73,9 +67,9 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: # Return view of static buffer up to current token count. if self.routing_indices_buffer is None: return None - # Only return up to active token count, to skip entries - # for padding tokens. - return self.routing_indices_buffer[:self.context.active_token_count] + # Only return up to active token count, to skip entries + # for padding tokens. + return self.routing_indices_buffer[: self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. recorded_data = RouterReplay.get_recorded_data() @@ -88,7 +82,7 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: def enable_static_buffer_recording(self) -> None: """Enable recording into the static buffer for CUDA graph compatibility. - + This sets up RouterReplay instances to copy routing indices into our pre-allocated static buffer instead of creating new tensors. Allocates the buffer lazily on first call. diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 9faff001071..729588506d4 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -38,13 +38,10 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) -from megatron.core.transformer.moe.router_replay import ( - RouterReplay, - RouterReplayAction, - ) from megatron.core.inference.utils import Counter, await_process_call from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.cuda_graphs import delete_cuda_graphs +from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.utils import ( deprecate_args, experimental_api, @@ -56,8 +53,6 @@ trace_async_exceptions, ) - - from .async_zmq_communicator import AsyncZMQCommunicator try: @@ -998,8 +993,13 @@ def post_process_requests( # Process routing indices if available (keyed by request_id) # Each step's routing is a tensor of shape [num_tokens_this_step, num_layers, topk] # We concatenate along dim=0 to accumulate: [total_tokens, num_layers, topk] - if routing_indices_per_request is not None and request_id in routing_indices_per_request: - step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] + if ( + routing_indices_per_request is not None + and request_id in routing_indices_per_request + ): + step_routing = routing_indices_per_request[ + request_id + ] # [num_tokens, num_layers, topk] if request.routing_indices is None: request.routing_indices = step_routing.clone() else: @@ -1007,26 +1007,6 @@ def post_process_requests( [request.routing_indices, step_routing], dim=0 ) - # Sanity check logging for routing indices (only log for first request, first few steps) - if req_idx == 0 and request.routing_indices.shape[0] <= 15: # first 15 tokens - rank = torch.distributed.get_rank() - logging.info( - f"[ROUTING DEBUG] rank={rank}, request_id={request_id}, " - f"total_tokens={request.routing_indices.shape[0]}, " - f"num_layers={request.routing_indices.shape[1]}, " - f"topk={request.routing_indices.shape[2]}, " - f"tokens_this_step={step_routing.shape[0]}" - ) - # Log first layer's routing for first few tokens - if step_routing.numel() > 0: - num_tokens_to_log = min(3, step_routing.shape[0]) - # step_routing is [num_tokens, num_layers, topk], get layer 0 - logging.info( - f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " - f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" - ) - - # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index d67e8570033..d8179614a3c 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -312,17 +312,17 @@ def serialize(self) -> dict: torch.cuda.nvtx.range_push("DynamicInferenceRequest.serialize") obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] - - # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] + + # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] if self.routing_indices is not None: total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) - # the last generated token does not undergo a forward pass + # the last generated token does not undergo a forward pass # hence we expect routing indices for total_tokens - 1 - assert self.routing_indices.shape[0] == total_tokens-1, ( + assert self.routing_indices.shape[0] == total_tokens - 1, ( f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " f"total tokens {total_tokens-1}." ) - + torch.cuda.nvtx.range_pop() return obj @@ -491,7 +491,7 @@ def checkpoint(self, tokenizer: MegatronTokenizer | None = None): new_request = DynamicInferenceRequest( request_id=old_request.request_id, prompt_tokens=new_prompt_tokens, - sampling_params=new_sampling_params + sampling_params=new_sampling_params, ) self.requests.append(new_request) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 8e70055a88f..f56e5b1c761 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,14 +27,12 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.models.multimodal.llava_model import LLaVAModel -from megatron.core.transformer.enums import CudaGraphScope from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region +from megatron.core.transformer.enums import CudaGraphScope from megatron.core.transformer.moe.moe_layer import BaseMoELayer from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel -from megatron.core import parallel_state -from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model -from megatron.core.utils import get_pg_size +from megatron.core.utils import get_asyncio_loop, get_model_config, get_pg_size, unwrap_model try: import transformer_engine as te # pylint: disable=unused-import @@ -700,7 +698,7 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: context = self.inference_wrapped_model.inference_context if context.moe_routing_metadata is None: return None - + stacked_routing = context.moe_routing_metadata.get_routing_indices() if stacked_routing is None: @@ -712,7 +710,6 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: active_query_lengths = context.request_query_lengths[active_request_slice].tolist() active_token_count = context.active_token_count - # Get TP group for all-gather if using sequence parallelism # With sequence parallelism, each TP rank only sees a portion of the tokens, # so we need to gather routing indices across all TP ranks. @@ -961,10 +958,9 @@ async def async_generate_output_tokens_dynamic_batch( logits = self._dynamic_step_forward_logits(input_ids, position_ids) - # Collect routing indices per request (must be done before context transitions) + # Collect routing indices per request (must be done before context transitions) routing_indices_per_request = self._router_record_bookkeeping() - # This is the best place to yield control back to event loop. # At this point we have enqueued FW pass GPU kernels asynchronously. # While they are running, we can do other useful CPU work. @@ -992,8 +988,6 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() - - ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index e9d16e08ca0..34c3b954074 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -144,17 +144,14 @@ async def chat_completions(): logging.info(result) if result.routing_indices is not None: choice_data["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = ( - len(result.prompt_tokens) - if result.prompt_tokens is not None - else 0 - ) + prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 if prompt_length: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ + :prompt_length + ].tolist() choices.append(choice_data) total_completion_tokens += len(result.generated_tokens) request_idx += 0 - response = { "choices": choices, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py index cb0adc718d3..072848e3507 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py @@ -147,9 +147,7 @@ async def completions(): list(result.generated_tokens) if result.generated_tokens else [] ) generated_log_probs = getattr(result, 'generated_log_probs', None) or [] - generated_top_n_logprobs = ( - getattr(result, 'generated_top_n_logprobs', None) or [] - ) + generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) or [] if echo: # When echo=True, include prompt tokens and their logprobs @@ -166,9 +164,7 @@ async def completions(): top_logprobs = None if prompt_top_n_logprobs or generated_top_n_logprobs: top_logprobs = ( - [None] - + list(prompt_top_n_logprobs) - + list(generated_top_n_logprobs) + [None] + list(prompt_top_n_logprobs) + list(generated_top_n_logprobs) ) # Calculate text_offset: cumulative character positions starting from 0 @@ -203,18 +199,14 @@ async def completions(): "top_logprobs": top_logprobs, } - choices.append( - {"index": request_idx, "text": text_output, "logprobs": logprobs_data} - ) + choices.append({"index": request_idx, "text": text_output, "logprobs": logprobs_data}) if result.routing_indices is not None: choices[-1]["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = ( - len(result.prompt_tokens) - if result.prompt_tokens is not None - else 0 - ) + prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 if prompt_length: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ + :prompt_length + ].tolist() request_idx += 1 diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 666d6e16c38..47debdd27df 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1573,4 +1573,4 @@ def wrapped_func(moe_layer, *args, **kwargs): return wrapped_func - return decorator \ No newline at end of file + return decorator diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 9d1b557d2e4..ca9372670f5 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -215,7 +215,7 @@ def __init__( if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() - + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index 75fbb9009a3..e3c5d40fa3c 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -77,7 +77,7 @@ def clear_global_router_replay_instances(): @staticmethod def set_global_static_buffers(static_buffer: torch.Tensor): """Sets static buffers for all router instances from a combined buffer. - + Args: static_buffer: Tensor of shape [max_tokens, num_layers, topk]. Each layer's RouterReplay gets a slice [:, layer_idx, :]. @@ -184,10 +184,9 @@ def get_replay_topk( else: return default_compute_topk(scores, topk, num_groups, group_topk) - def set_static_buffer(self, buffer: torch.Tensor): """Sets a static buffer for CUDA graph compatible recording. - + Args: buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. """ @@ -199,7 +198,7 @@ def clear_static_buffer(self): def record_indices(self, topk_indices: torch.Tensor): """Records the topk indices. - + If a static buffer is set (for CUDA graph compatibility), copies into it. Otherwise, just stores the tensor reference. """ From 473a65238a9e05d838004a56f849362daf4baad4 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:55:21 -0800 Subject: [PATCH 17/24] minor --- .../inference/gpt/gpt_dynamic_inference_with_coordinator.py | 2 +- megatron/core/transformer/moe/router.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index 771b3eca519..5c58b4d52e3 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -125,7 +125,7 @@ async def main( await asyncio.sleep(0) # While we wait for the requests to complete, the engine runs in the background. - results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) + results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index ca9372670f5..e42fd1ca8aa 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -212,7 +212,6 @@ def __init__( self.ga_steps = None self.router_replay = None - if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() From 452acb20c2316546de2ad2cfe2363100ef48b75c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 07:06:27 -0800 Subject: [PATCH 18/24] remove unnecessary methods --- megatron/core/inference/contexts/routing_metadata.py | 4 ---- megatron/core/transformer/moe/router_replay.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index 07abe0c3469..6664681a7ce 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -94,7 +94,3 @@ def enable_static_buffer_recording(self) -> None: def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" RouterReplay.clear_global_static_buffers() - - def reset(self) -> None: - """Reset the routing metadata state.""" - pass diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index e3c5d40fa3c..5430f75568f 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -119,10 +119,6 @@ def get_recorded_indices(self) -> Optional[torch.Tensor]: """Returns the recorded topk indices.""" return self.recorded_topk_idx - def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices.""" - self.recorded_topk_idx = topk_indices - def clear_indices(self): """Clears the recorded and target topk indices.""" self.recorded_topk_idx = None From bf27426eea2ae0781a90c78a4d25847a16516499 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:14:18 -0800 Subject: [PATCH 19/24] miinor bugfix --- megatron/core/inference/contexts/dynamic_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 47b0bb50748..e9b324cc0b3 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -1302,7 +1302,7 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) - if self.moe_enable_routing_replay is not None: + if self.moe_enable_routing_replay: if self.using_cuda_graph_this_step(): self.moe_routing_metadata.enable_static_buffer_recording() else: From 26002fc161c4e78925c290e40881740298c81a43 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:14:48 -0800 Subject: [PATCH 20/24] attempt to reactivate functional test --- .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index 513aa92834b..b9d78097bbd 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -63,6 +63,6 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [flaky] + scope: [mr] platforms: [dgx_h100] From db37b3b27f4f73b0c3cb8af44a504e0788079a26 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:35:09 -0800 Subject: [PATCH 21/24] update functional test to run router recording --- .../gpt_dynamic_inference_with_coordinator.py | 1 + .../test_inference_regular_pipeline.py | 8 + .../golden_values_dev_dgx_h100.json | 25510 +++++++++++++++- 3 files changed, 25518 insertions(+), 1 deletion(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index 5c58b4d52e3..536f533eccd 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -147,6 +147,7 @@ async def main( throughputs.append(throughput) if req.routing_indices is not None: result_dict["routing_indices"] = req.routing_indices.tolist() + json_results[req.request_id] = result_dict throughput_dict = {"throughput": throughputs} if args.throughput_check_only: diff --git a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py index 346b464b79d..a0d7216edf8 100644 --- a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py @@ -156,5 +156,13 @@ def test_inference_pipeline(golden_values_path: str, test_values_path: str) -> N f"\nCurrent (truncated to {min_len} chars): {generated_text_current[:min_len]}" ) + if "routing_indices" in groundtruth_results: + at_least_one_test_loop = True + routing_indices_groundtruth = groundtruth_results["routing_indices"] + routing_indices_current = current_results["routing_indices"] + assert ( + routing_indices_groundtruth == routing_indices_current + ), f"Routing indices mismatch:\nGround truth: {routing_indices_groundtruth}\nCurrent: {routing_indices_current}" + if not at_least_one_test_loop: raise AssertionError(f"No test performed for output {groundtruth_results}") diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json index b239ac96c3d..ab970730d9b 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json @@ -34,7 +34,7 @@ 1394, 1636 ], - "latency": 1.974948332994245, + "latency": 1.972374301403761, "logprobs": [ -10.737512588500977, -3.724862575531006, @@ -153,6 +153,25514 @@ -1.2551532983779907, -1.256169080734253, -0.49199968576431274 + ], + "routing_indices": [ + [ + [ + 33, + 50, + 36, + 4, + 25, + 63 + ], + [ + 0, + 16, + 3, + 26, + 9, + 54 + ], + [ + 62, + 60, + 8, + 16, + 58, + 52 + ], + [ + 58, + 10, + 6, + 45, + 16, + 32 + ], + [ + 43, + 49, + 18, + 54, + 55, + 13 + ], + [ + 27, + 19, + 26, + 44, + 12, + 28 + ], + [ + 53, + 42, + 3, + 27, + 26, + 19 + ], + [ + 6, + 47, + 1, + 8, + 19, + 22 + ], + [ + 51, + 27, + 1, + 38, + 16, + 62 + ], + [ + 41, + 49, + 21, + 57, + 16, + 24 + ], + [ + 13, + 28, + 38, + 22, + 49, + 48 + ], + [ + 52, + 58, + 6, + 25, + 29, + 17 + ], + [ + 49, + 50, + 25, + 41, + 54, + 58 + ], + [ + 27, + 41, + 3, + 1, + 26, + 29 + ], + [ + 41, + 18, + 34, + 45, + 1, + 33 + ], + [ + 17, + 26, + 59, + 22, + 19, + 4 + ], + [ + 1, + 43, + 62, + 57, + 61, + 21 + ], + [ + 11, + 46, + 15, + 28, + 61, + 50 + ], + [ + 14, + 30, + 58, + 26, + 38, + 53 + ], + [ + 59, + 20, + 63, + 54, + 47, + 61 + ], + [ + 9, + 5, + 43, + 33, + 15, + 46 + ], + [ + 16, + 52, + 33, + 61, + 49, + 11 + ], + [ + 52, + 35, + 40, + 43, + 29, + 36 + ], + [ + 57, + 34, + 38, + 44, + 20, + 18 + ], + [ + 44, + 51, + 2, + 63, + 7, + 22 + ], + [ + 32, + 47, + 58, + 9, + 54, + 5 + ], + [ + 12, + 59, + 54, + 33, + 50, + 6 + ] + ], + [ + [ + 49, + 43, + 18, + 28, + 23, + 25 + ], + [ + 34, + 24, + 23, + 60, + 2, + 18 + ], + [ + 33, + 40, + 30, + 3, + 59, + 48 + ], + [ + 16, + 58, + 17, + 48, + 6, + 45 + ], + [ + 23, + 58, + 46, + 37, + 34, + 48 + ], + [ + 28, + 26, + 35, + 33, + 22, + 43 + ], + [ + 6, + 31, + 13, + 46, + 41, + 37 + ], + [ + 60, + 29, + 44, + 36, + 39, + 15 + ], + [ + 51, + 18, + 62, + 5, + 27, + 35 + ], + [ + 62, + 45, + 32, + 56, + 25, + 3 + ], + [ + 6, + 20, + 3, + 16, + 49, + 41 + ], + [ + 36, + 41, + 50, + 45, + 35, + 48 + ], + [ + 39, + 46, + 48, + 25, + 21, + 33 + ], + [ + 30, + 38, + 11, + 22, + 15, + 9 + ], + [ + 24, + 2, + 32, + 56, + 63, + 3 + ], + [ + 36, + 55, + 35, + 17, + 32, + 44 + ], + [ + 59, + 46, + 32, + 44, + 24, + 14 + ], + [ + 51, + 15, + 61, + 43, + 30, + 22 + ], + [ + 32, + 2, + 5, + 39, + 11, + 50 + ], + [ + 28, + 42, + 6, + 30, + 57, + 37 + ], + [ + 28, + 55, + 45, + 0, + 7, + 5 + ], + [ + 48, + 40, + 34, + 3, + 49, + 22 + ], + [ + 25, + 6, + 62, + 50, + 18, + 53 + ], + [ + 13, + 22, + 20, + 28, + 25, + 59 + ], + [ + 58, + 39, + 36, + 47, + 29, + 37 + ], + [ + 4, + 33, + 41, + 12, + 3, + 17 + ], + [ + 13, + 22, + 20, + 52, + 24, + 62 + ] + ], + [ + [ + 17, + 10, + 57, + 54, + 6, + 15 + ], + [ + 33, + 43, + 13, + 1, + 16, + 62 + ], + [ + 63, + 1, + 35, + 43, + 27, + 10 + ], + [ + 47, + 4, + 38, + 50, + 51, + 0 + ], + [ + 11, + 51, + 57, + 23, + 14, + 34 + ], + [ + 10, + 43, + 35, + 33, + 20, + 22 + ], + [ + 36, + 48, + 35, + 19, + 21, + 28 + ], + [ + 14, + 8, + 7, + 46, + 35, + 13 + ], + [ + 18, + 44, + 63, + 6, + 4, + 37 + ], + [ + 62, + 29, + 15, + 38, + 39, + 34 + ], + [ + 1, + 6, + 16, + 46, + 22, + 13 + ], + [ + 36, + 8, + 16, + 37, + 10, + 14 + ], + [ + 8, + 0, + 32, + 3, + 43, + 10 + ], + [ + 25, + 0, + 22, + 30, + 60, + 57 + ], + [ + 60, + 58, + 55, + 32, + 2, + 7 + ], + [ + 55, + 11, + 19, + 5, + 24, + 43 + ], + [ + 9, + 47, + 36, + 39, + 5, + 42 + ], + [ + 31, + 20, + 9, + 43, + 18, + 41 + ], + [ + 12, + 5, + 32, + 50, + 3, + 31 + ], + [ + 40, + 18, + 30, + 63, + 7, + 25 + ], + [ + 14, + 7, + 3, + 38, + 59, + 54 + ], + [ + 31, + 20, + 1, + 22, + 47, + 6 + ], + [ + 51, + 15, + 18, + 53, + 40, + 52 + ], + [ + 27, + 54, + 8, + 38, + 3, + 59 + ], + [ + 11, + 16, + 39, + 59, + 9, + 23 + ], + [ + 7, + 37, + 51, + 30, + 18, + 48 + ], + [ + 0, + 36, + 33, + 40, + 46, + 48 + ] + ], + [ + [ + 48, + 33, + 52, + 53, + 24, + 38 + ], + [ + 3, + 40, + 45, + 19, + 22, + 35 + ], + [ + 46, + 15, + 39, + 60, + 25, + 31 + ], + [ + 49, + 62, + 61, + 48, + 55, + 46 + ], + [ + 41, + 35, + 53, + 52, + 13, + 10 + ], + [ + 52, + 15, + 45, + 63, + 46, + 10 + ], + [ + 14, + 15, + 35, + 60, + 49, + 31 + ], + [ + 44, + 3, + 6, + 8, + 23, + 48 + ], + [ + 32, + 15, + 44, + 27, + 4, + 41 + ], + [ + 51, + 25, + 62, + 53, + 48, + 10 + ], + [ + 8, + 50, + 5, + 19, + 16, + 22 + ], + [ + 36, + 49, + 60, + 44, + 15, + 33 + ], + [ + 13, + 27, + 53, + 30, + 56, + 43 + ], + [ + 26, + 22, + 14, + 7, + 32, + 17 + ], + [ + 26, + 60, + 2, + 58, + 54, + 10 + ], + [ + 50, + 55, + 17, + 51, + 47, + 14 + ], + [ + 0, + 12, + 16, + 4, + 23, + 9 + ], + [ + 10, + 23, + 27, + 46, + 56, + 55 + ], + [ + 3, + 37, + 4, + 60, + 16, + 59 + ], + [ + 38, + 3, + 29, + 40, + 25, + 50 + ], + [ + 31, + 16, + 62, + 54, + 42, + 5 + ], + [ + 47, + 35, + 11, + 37, + 46, + 32 + ], + [ + 51, + 27, + 20, + 50, + 16, + 55 + ], + [ + 63, + 55, + 46, + 27, + 48, + 12 + ], + [ + 53, + 50, + 30, + 2, + 39, + 20 + ], + [ + 53, + 44, + 24, + 8, + 51, + 14 + ], + [ + 19, + 49, + 37, + 14, + 44, + 36 + ] + ], + [ + [ + 0, + 52, + 16, + 12, + 54, + 7 + ], + [ + 42, + 25, + 51, + 61, + 35, + 58 + ], + [ + 51, + 42, + 19, + 57, + 28, + 8 + ], + [ + 49, + 62, + 5, + 2, + 46, + 21 + ], + [ + 3, + 41, + 53, + 25, + 39, + 37 + ], + [ + 45, + 15, + 37, + 48, + 19, + 60 + ], + [ + 14, + 15, + 47, + 17, + 35, + 24 + ], + [ + 3, + 52, + 63, + 16, + 28, + 47 + ], + [ + 32, + 27, + 15, + 24, + 62, + 23 + ], + [ + 62, + 25, + 51, + 53, + 0, + 20 + ], + [ + 8, + 19, + 50, + 16, + 32, + 22 + ], + [ + 36, + 13, + 42, + 49, + 60, + 44 + ], + [ + 30, + 27, + 53, + 56, + 0, + 10 + ], + [ + 22, + 7, + 14, + 26, + 32, + 17 + ], + [ + 60, + 7, + 26, + 2, + 58, + 13 + ], + [ + 55, + 51, + 17, + 60, + 62, + 47 + ], + [ + 53, + 16, + 18, + 4, + 50, + 5 + ], + [ + 28, + 39, + 41, + 1, + 55, + 18 + ], + [ + 59, + 49, + 16, + 23, + 42, + 11 + ], + [ + 17, + 30, + 46, + 55, + 25, + 0 + ], + [ + 54, + 9, + 45, + 0, + 6, + 19 + ], + [ + 51, + 9, + 22, + 23, + 16, + 25 + ], + [ + 62, + 50, + 43, + 51, + 55, + 27 + ], + [ + 13, + 43, + 27, + 1, + 14, + 52 + ], + [ + 61, + 26, + 1, + 17, + 32, + 63 + ], + [ + 37, + 46, + 63, + 20, + 24, + 4 + ], + [ + 63, + 11, + 12, + 61, + 31, + 22 + ] + ], + [ + [ + 49, + 54, + 56, + 11, + 38, + 3 + ], + [ + 53, + 46, + 49, + 38, + 57, + 17 + ], + [ + 9, + 48, + 31, + 12, + 56, + 6 + ], + [ + 49, + 62, + 23, + 5, + 12, + 63 + ], + [ + 36, + 26, + 38, + 7, + 20, + 23 + ], + [ + 37, + 33, + 41, + 58, + 57, + 32 + ], + [ + 10, + 14, + 15, + 31, + 8, + 43 + ], + [ + 16, + 52, + 3, + 2, + 34, + 14 + ], + [ + 15, + 32, + 35, + 27, + 62, + 54 + ], + [ + 51, + 62, + 25, + 53, + 10, + 20 + ], + [ + 19, + 8, + 22, + 50, + 1, + 5 + ], + [ + 13, + 49, + 36, + 60, + 42, + 20 + ], + [ + 27, + 30, + 53, + 54, + 26, + 43 + ], + [ + 14, + 32, + 26, + 22, + 7, + 17 + ], + [ + 26, + 60, + 7, + 2, + 52, + 54 + ], + [ + 55, + 17, + 51, + 47, + 62, + 60 + ], + [ + 16, + 18, + 44, + 4, + 53, + 50 + ], + [ + 1, + 47, + 39, + 45, + 28, + 56 + ], + [ + 23, + 16, + 55, + 49, + 8, + 32 + ], + [ + 17, + 55, + 30, + 62, + 31, + 23 + ], + [ + 9, + 54, + 0, + 44, + 14, + 56 + ], + [ + 23, + 9, + 51, + 22, + 31, + 50 + ], + [ + 50, + 27, + 14, + 51, + 8, + 34 + ], + [ + 12, + 13, + 33, + 1, + 5, + 43 + ], + [ + 26, + 32, + 1, + 50, + 37, + 57 + ], + [ + 37, + 47, + 63, + 46, + 4, + 5 + ], + [ + 63, + 11, + 12, + 19, + 33, + 61 + ] + ], + [ + [ + 49, + 54, + 40, + 56, + 3, + 11 + ], + [ + 37, + 15, + 12, + 33, + 59, + 17 + ], + [ + 38, + 49, + 14, + 46, + 35, + 59 + ], + [ + 25, + 20, + 39, + 62, + 49, + 12 + ], + [ + 26, + 51, + 16, + 36, + 18, + 8 + ], + [ + 37, + 51, + 41, + 33, + 32, + 60 + ], + [ + 10, + 14, + 59, + 8, + 15, + 40 + ], + [ + 16, + 52, + 19, + 61, + 32, + 2 + ], + [ + 32, + 15, + 27, + 24, + 62, + 35 + ], + [ + 51, + 25, + 62, + 0, + 49, + 60 + ], + [ + 8, + 50, + 19, + 48, + 16, + 54 + ], + [ + 13, + 49, + 36, + 42, + 11, + 60 + ], + [ + 53, + 27, + 7, + 26, + 30, + 21 + ], + [ + 14, + 22, + 26, + 37, + 32, + 4 + ], + [ + 26, + 40, + 60, + 2, + 52, + 7 + ], + [ + 55, + 51, + 17, + 46, + 13, + 62 + ], + [ + 38, + 16, + 53, + 44, + 4, + 18 + ], + [ + 39, + 1, + 4, + 14, + 56, + 57 + ], + [ + 55, + 23, + 32, + 14, + 13, + 16 + ], + [ + 55, + 17, + 2, + 30, + 62, + 12 + ], + [ + 13, + 54, + 0, + 62, + 61, + 25 + ], + [ + 9, + 51, + 5, + 22, + 19, + 16 + ], + [ + 50, + 51, + 19, + 7, + 48, + 53 + ], + [ + 12, + 1, + 5, + 43, + 61, + 13 + ], + [ + 26, + 32, + 30, + 37, + 34, + 20 + ], + [ + 37, + 63, + 46, + 4, + 47, + 8 + ], + [ + 63, + 12, + 11, + 31, + 33, + 61 + ] + ], + [ + [ + 47, + 34, + 30, + 25, + 31, + 3 + ], + [ + 15, + 24, + 46, + 21, + 8, + 6 + ], + [ + 34, + 21, + 18, + 62, + 28, + 55 + ], + [ + 35, + 32, + 20, + 39, + 59, + 54 + ], + [ + 26, + 27, + 15, + 48, + 60, + 47 + ], + [ + 37, + 8, + 50, + 18, + 54, + 61 + ], + [ + 35, + 31, + 8, + 24, + 14, + 15 + ], + [ + 16, + 52, + 34, + 29, + 48, + 36 + ], + [ + 11, + 32, + 62, + 27, + 46, + 26 + ], + [ + 61, + 62, + 25, + 56, + 46, + 53 + ], + [ + 56, + 50, + 63, + 3, + 45, + 28 + ], + [ + 11, + 36, + 5, + 60, + 35, + 50 + ], + [ + 21, + 26, + 41, + 51, + 46, + 53 + ], + [ + 14, + 22, + 33, + 19, + 41, + 16 + ], + [ + 2, + 52, + 34, + 60, + 21, + 49 + ], + [ + 59, + 55, + 29, + 8, + 61, + 22 + ], + [ + 51, + 44, + 2, + 59, + 47, + 53 + ], + [ + 39, + 25, + 18, + 12, + 51, + 56 + ], + [ + 34, + 53, + 32, + 12, + 9, + 38 + ], + [ + 30, + 53, + 56, + 7, + 40, + 62 + ], + [ + 40, + 49, + 14, + 28, + 23, + 55 + ], + [ + 15, + 48, + 40, + 47, + 9, + 1 + ], + [ + 50, + 41, + 53, + 25, + 18, + 0 + ], + [ + 22, + 1, + 59, + 3, + 55, + 10 + ], + [ + 1, + 32, + 53, + 26, + 47, + 3 + ], + [ + 4, + 33, + 28, + 37, + 55, + 54 + ], + [ + 30, + 22, + 57, + 12, + 33, + 63 + ] + ], + [ + [ + 16, + 11, + 0, + 46, + 31, + 21 + ], + [ + 49, + 13, + 31, + 5, + 11, + 14 + ], + [ + 36, + 13, + 56, + 27, + 46, + 3 + ], + [ + 24, + 44, + 62, + 29, + 15, + 13 + ], + [ + 17, + 2, + 50, + 8, + 45, + 1 + ], + [ + 8, + 7, + 49, + 0, + 62, + 13 + ], + [ + 35, + 61, + 58, + 23, + 36, + 0 + ], + [ + 16, + 48, + 42, + 4, + 32, + 1 + ], + [ + 63, + 18, + 32, + 4, + 34, + 62 + ], + [ + 57, + 62, + 54, + 27, + 25, + 53 + ], + [ + 1, + 59, + 60, + 29, + 22, + 14 + ], + [ + 31, + 36, + 11, + 14, + 20, + 47 + ], + [ + 34, + 2, + 19, + 14, + 8, + 37 + ], + [ + 57, + 22, + 40, + 14, + 62, + 48 + ], + [ + 44, + 60, + 7, + 14, + 45, + 2 + ], + [ + 8, + 6, + 55, + 25, + 50, + 59 + ], + [ + 13, + 47, + 23, + 61, + 42, + 39 + ], + [ + 23, + 25, + 4, + 14, + 46, + 53 + ], + [ + 8, + 45, + 32, + 53, + 10, + 54 + ], + [ + 15, + 38, + 53, + 55, + 7, + 30 + ], + [ + 41, + 14, + 28, + 27, + 5, + 58 + ], + [ + 11, + 41, + 57, + 10, + 1, + 47 + ], + [ + 50, + 0, + 53, + 51, + 34, + 55 + ], + [ + 1, + 14, + 55, + 25, + 8, + 3 + ], + [ + 11, + 49, + 1, + 9, + 0, + 3 + ], + [ + 50, + 51, + 6, + 42, + 4, + 54 + ], + [ + 17, + 37, + 31, + 40, + 5, + 36 + ] + ], + [ + [ + 22, + 53, + 47, + 6, + 57, + 21 + ], + [ + 27, + 11, + 14, + 6, + 57, + 16 + ], + [ + 1, + 11, + 29, + 26, + 41, + 17 + ], + [ + 14, + 22, + 38, + 31, + 29, + 36 + ], + [ + 14, + 59, + 29, + 61, + 45, + 52 + ], + [ + 30, + 8, + 0, + 21, + 47, + 58 + ], + [ + 58, + 35, + 4, + 61, + 23, + 36 + ], + [ + 42, + 20, + 48, + 16, + 9, + 4 + ], + [ + 47, + 29, + 4, + 18, + 63, + 32 + ], + [ + 19, + 54, + 62, + 53, + 57, + 29 + ], + [ + 1, + 60, + 14, + 59, + 29, + 4 + ], + [ + 36, + 0, + 47, + 3, + 31, + 8 + ], + [ + 2, + 19, + 36, + 8, + 20, + 37 + ], + [ + 57, + 22, + 40, + 49, + 31, + 14 + ], + [ + 44, + 37, + 2, + 5, + 60, + 21 + ], + [ + 6, + 43, + 24, + 5, + 2, + 59 + ], + [ + 13, + 61, + 19, + 47, + 50, + 39 + ], + [ + 58, + 14, + 28, + 4, + 22, + 11 + ], + [ + 35, + 32, + 46, + 10, + 31, + 45 + ], + [ + 15, + 13, + 55, + 45, + 18, + 63 + ], + [ + 15, + 27, + 28, + 14, + 5, + 60 + ], + [ + 57, + 41, + 47, + 19, + 36, + 10 + ], + [ + 34, + 10, + 53, + 55, + 22, + 19 + ], + [ + 38, + 55, + 39, + 27, + 3, + 25 + ], + [ + 11, + 39, + 0, + 9, + 3, + 49 + ], + [ + 51, + 6, + 43, + 18, + 50, + 53 + ], + [ + 55, + 43, + 9, + 36, + 40, + 5 + ] + ], + [ + [ + 18, + 9, + 1, + 36, + 61, + 44 + ], + [ + 56, + 34, + 19, + 42, + 3, + 5 + ], + [ + 39, + 20, + 15, + 60, + 46, + 32 + ], + [ + 60, + 22, + 31, + 27, + 14, + 55 + ], + [ + 59, + 58, + 10, + 7, + 46, + 18 + ], + [ + 43, + 2, + 57, + 62, + 11, + 30 + ], + [ + 54, + 19, + 9, + 21, + 48, + 56 + ], + [ + 46, + 24, + 7, + 14, + 3, + 8 + ], + [ + 47, + 0, + 4, + 18, + 31, + 29 + ], + [ + 54, + 62, + 47, + 38, + 4, + 32 + ], + [ + 1, + 14, + 15, + 22, + 59, + 38 + ], + [ + 16, + 36, + 42, + 55, + 15, + 18 + ], + [ + 49, + 8, + 20, + 14, + 0, + 33 + ], + [ + 18, + 39, + 25, + 2, + 62, + 22 + ], + [ + 62, + 5, + 58, + 37, + 7, + 32 + ], + [ + 43, + 5, + 42, + 63, + 55, + 37 + ], + [ + 47, + 33, + 15, + 63, + 50, + 12 + ], + [ + 60, + 0, + 7, + 16, + 32, + 13 + ], + [ + 12, + 39, + 32, + 61, + 16, + 45 + ], + [ + 52, + 34, + 15, + 18, + 62, + 30 + ], + [ + 28, + 26, + 46, + 40, + 6, + 38 + ], + [ + 1, + 19, + 17, + 20, + 4, + 21 + ], + [ + 41, + 40, + 4, + 53, + 55, + 19 + ], + [ + 25, + 38, + 27, + 34, + 52, + 46 + ], + [ + 11, + 29, + 52, + 44, + 13, + 35 + ], + [ + 50, + 51, + 41, + 16, + 4, + 14 + ], + [ + 19, + 6, + 23, + 36, + 60, + 0 + ] + ], + [ + [ + 17, + 10, + 57, + 27, + 5, + 54 + ], + [ + 33, + 9, + 43, + 40, + 56, + 11 + ], + [ + 63, + 1, + 35, + 43, + 10, + 27 + ], + [ + 51, + 47, + 20, + 21, + 28, + 61 + ], + [ + 25, + 11, + 58, + 23, + 55, + 46 + ], + [ + 43, + 10, + 12, + 2, + 30, + 62 + ], + [ + 48, + 21, + 19, + 8, + 7, + 54 + ], + [ + 14, + 7, + 24, + 8, + 46, + 2 + ], + [ + 4, + 47, + 37, + 0, + 44, + 27 + ], + [ + 54, + 38, + 62, + 47, + 15, + 14 + ], + [ + 1, + 46, + 15, + 22, + 51, + 38 + ], + [ + 36, + 16, + 42, + 24, + 55, + 37 + ], + [ + 49, + 10, + 0, + 3, + 43, + 8 + ], + [ + 39, + 58, + 0, + 62, + 22, + 25 + ], + [ + 58, + 38, + 7, + 62, + 55, + 56 + ], + [ + 19, + 42, + 55, + 43, + 11, + 37 + ], + [ + 9, + 47, + 43, + 52, + 18, + 50 + ], + [ + 31, + 41, + 32, + 25, + 20, + 13 + ], + [ + 12, + 32, + 61, + 3, + 21, + 43 + ], + [ + 36, + 13, + 40, + 7, + 62, + 16 + ], + [ + 14, + 53, + 50, + 47, + 51, + 1 + ], + [ + 1, + 38, + 19, + 30, + 18, + 16 + ], + [ + 0, + 19, + 51, + 18, + 52, + 15 + ], + [ + 8, + 52, + 27, + 34, + 38, + 3 + ], + [ + 27, + 53, + 59, + 9, + 40, + 4 + ], + [ + 37, + 3, + 26, + 48, + 8, + 16 + ], + [ + 46, + 18, + 11, + 40, + 33, + 44 + ] + ], + [ + [ + 48, + 62, + 61, + 26, + 0, + 50 + ], + [ + 3, + 45, + 40, + 35, + 29, + 54 + ], + [ + 56, + 31, + 23, + 28, + 2, + 53 + ], + [ + 62, + 49, + 20, + 61, + 6, + 41 + ], + [ + 18, + 25, + 50, + 0, + 14, + 57 + ], + [ + 58, + 4, + 10, + 43, + 56, + 46 + ], + [ + 35, + 15, + 25, + 24, + 3, + 7 + ], + [ + 14, + 23, + 8, + 12, + 7, + 24 + ], + [ + 29, + 17, + 35, + 44, + 24, + 4 + ], + [ + 62, + 15, + 38, + 20, + 58, + 17 + ], + [ + 19, + 1, + 46, + 26, + 22, + 63 + ], + [ + 36, + 60, + 42, + 16, + 11, + 55 + ], + [ + 17, + 7, + 49, + 14, + 16, + 26 + ], + [ + 45, + 47, + 22, + 62, + 0, + 58 + ], + [ + 58, + 38, + 48, + 63, + 2, + 49 + ], + [ + 55, + 0, + 1, + 37, + 30, + 10 + ], + [ + 12, + 43, + 21, + 9, + 23, + 47 + ], + [ + 32, + 57, + 42, + 25, + 63, + 43 + ], + [ + 3, + 32, + 49, + 61, + 21, + 12 + ], + [ + 5, + 36, + 16, + 22, + 62, + 42 + ], + [ + 53, + 7, + 46, + 61, + 14, + 52 + ], + [ + 55, + 30, + 3, + 5, + 31, + 53 + ], + [ + 0, + 44, + 15, + 18, + 19, + 28 + ], + [ + 8, + 52, + 51, + 11, + 28, + 29 + ], + [ + 27, + 40, + 4, + 9, + 35, + 39 + ], + [ + 14, + 26, + 3, + 48, + 16, + 21 + ], + [ + 60, + 54, + 35, + 20, + 53, + 12 + ] + ], + [ + [ + 21, + 50, + 29, + 41, + 34, + 60 + ], + [ + 28, + 51, + 60, + 33, + 14, + 45 + ], + [ + 31, + 2, + 46, + 33, + 24, + 49 + ], + [ + 42, + 3, + 18, + 62, + 39, + 49 + ], + [ + 25, + 15, + 62, + 27, + 12, + 11 + ], + [ + 44, + 50, + 36, + 57, + 55, + 41 + ], + [ + 41, + 37, + 22, + 15, + 2, + 40 + ], + [ + 36, + 62, + 53, + 30, + 14, + 57 + ], + [ + 16, + 58, + 2, + 29, + 4, + 3 + ], + [ + 41, + 38, + 26, + 16, + 45, + 46 + ], + [ + 45, + 46, + 32, + 41, + 56, + 26 + ], + [ + 17, + 53, + 21, + 11, + 36, + 35 + ], + [ + 11, + 16, + 28, + 14, + 51, + 39 + ], + [ + 9, + 35, + 33, + 22, + 52, + 62 + ], + [ + 58, + 50, + 63, + 30, + 7, + 27 + ], + [ + 55, + 3, + 8, + 41, + 63, + 37 + ], + [ + 3, + 51, + 32, + 46, + 15, + 6 + ], + [ + 32, + 12, + 10, + 25, + 5, + 16 + ], + [ + 34, + 2, + 37, + 61, + 63, + 39 + ], + [ + 42, + 22, + 27, + 53, + 11, + 56 + ], + [ + 53, + 12, + 0, + 47, + 61, + 49 + ], + [ + 39, + 45, + 53, + 17, + 48, + 14 + ], + [ + 6, + 0, + 4, + 53, + 25, + 11 + ], + [ + 51, + 11, + 1, + 63, + 54, + 59 + ], + [ + 40, + 56, + 37, + 53, + 35, + 5 + ], + [ + 59, + 28, + 41, + 10, + 1, + 45 + ], + [ + 27, + 30, + 28, + 24, + 32, + 57 + ] + ], + [ + [ + 24, + 56, + 6, + 0, + 19, + 45 + ], + [ + 11, + 57, + 59, + 25, + 46, + 30 + ], + [ + 11, + 26, + 37, + 29, + 14, + 52 + ], + [ + 3, + 32, + 7, + 38, + 36, + 24 + ], + [ + 61, + 2, + 24, + 14, + 51, + 44 + ], + [ + 20, + 47, + 0, + 63, + 30, + 58 + ], + [ + 4, + 36, + 29, + 58, + 16, + 3 + ], + [ + 20, + 0, + 45, + 14, + 28, + 44 + ], + [ + 29, + 56, + 47, + 35, + 16, + 4 + ], + [ + 33, + 61, + 55, + 41, + 51, + 38 + ], + [ + 58, + 1, + 38, + 14, + 4, + 19 + ], + [ + 0, + 36, + 14, + 18, + 52, + 42 + ], + [ + 29, + 36, + 45, + 25, + 8, + 6 + ], + [ + 6, + 57, + 50, + 40, + 58, + 61 + ], + [ + 44, + 58, + 29, + 19, + 61, + 56 + ], + [ + 23, + 18, + 28, + 55, + 5, + 37 + ], + [ + 13, + 9, + 19, + 43, + 37, + 3 + ], + [ + 32, + 22, + 63, + 14, + 57, + 41 + ], + [ + 10, + 61, + 3, + 1, + 19, + 32 + ], + [ + 16, + 55, + 10, + 41, + 59, + 22 + ], + [ + 53, + 7, + 29, + 38, + 27, + 46 + ], + [ + 24, + 47, + 18, + 53, + 39, + 30 + ], + [ + 0, + 33, + 19, + 51, + 5, + 17 + ], + [ + 51, + 8, + 11, + 45, + 44, + 41 + ], + [ + 40, + 4, + 23, + 11, + 27, + 19 + ], + [ + 16, + 18, + 3, + 48, + 51, + 21 + ], + [ + 43, + 46, + 60, + 19, + 53, + 12 + ] + ], + [ + [ + 48, + 62, + 61, + 30, + 0, + 2 + ], + [ + 45, + 3, + 35, + 54, + 29, + 2 + ], + [ + 56, + 31, + 53, + 23, + 49, + 28 + ], + [ + 60, + 14, + 57, + 46, + 41, + 48 + ], + [ + 18, + 61, + 59, + 14, + 44, + 32 + ], + [ + 45, + 58, + 47, + 20, + 4, + 30 + ], + [ + 54, + 13, + 25, + 36, + 26, + 47 + ], + [ + 20, + 12, + 0, + 47, + 30, + 45 + ], + [ + 56, + 29, + 47, + 17, + 35, + 16 + ], + [ + 33, + 61, + 55, + 11, + 38, + 48 + ], + [ + 58, + 19, + 14, + 1, + 38, + 36 + ], + [ + 14, + 36, + 0, + 60, + 11, + 52 + ], + [ + 29, + 44, + 7, + 36, + 16, + 45 + ], + [ + 6, + 47, + 50, + 33, + 42, + 62 + ], + [ + 44, + 58, + 61, + 38, + 29, + 56 + ], + [ + 23, + 55, + 18, + 0, + 57, + 37 + ], + [ + 9, + 12, + 43, + 19, + 13, + 6 + ], + [ + 32, + 22, + 63, + 57, + 42, + 29 + ], + [ + 3, + 61, + 1, + 10, + 49, + 32 + ], + [ + 5, + 16, + 55, + 36, + 22, + 59 + ], + [ + 53, + 7, + 29, + 46, + 9, + 14 + ], + [ + 24, + 30, + 18, + 39, + 55, + 53 + ], + [ + 33, + 0, + 19, + 44, + 51, + 5 + ], + [ + 51, + 8, + 53, + 41, + 11, + 4 + ], + [ + 40, + 4, + 27, + 19, + 23, + 16 + ], + [ + 16, + 14, + 48, + 3, + 21, + 26 + ], + [ + 60, + 54, + 35, + 53, + 12, + 43 + ] + ], + [ + [ + 19, + 41, + 8, + 7, + 13, + 2 + ], + [ + 48, + 46, + 62, + 29, + 5, + 41 + ], + [ + 12, + 5, + 59, + 3, + 58, + 49 + ], + [ + 60, + 3, + 42, + 39, + 14, + 18 + ], + [ + 42, + 12, + 27, + 11, + 25, + 19 + ], + [ + 50, + 36, + 44, + 26, + 33, + 37 + ], + [ + 41, + 54, + 22, + 52, + 37, + 35 + ], + [ + 62, + 30, + 36, + 53, + 10, + 14 + ], + [ + 2, + 16, + 58, + 29, + 7, + 41 + ], + [ + 18, + 32, + 45, + 16, + 22, + 38 + ], + [ + 45, + 56, + 41, + 10, + 3, + 46 + ], + [ + 50, + 21, + 36, + 35, + 53, + 12 + ], + [ + 11, + 28, + 16, + 41, + 39, + 46 + ], + [ + 16, + 9, + 33, + 38, + 28, + 52 + ], + [ + 58, + 50, + 63, + 62, + 27, + 52 + ], + [ + 55, + 59, + 8, + 13, + 41, + 43 + ], + [ + 51, + 3, + 15, + 46, + 57, + 47 + ], + [ + 32, + 10, + 30, + 12, + 25, + 18 + ], + [ + 34, + 2, + 27, + 61, + 53, + 39 + ], + [ + 42, + 22, + 56, + 53, + 44, + 34 + ], + [ + 53, + 12, + 49, + 41, + 44, + 8 + ], + [ + 45, + 1, + 48, + 47, + 16, + 61 + ], + [ + 4, + 0, + 53, + 25, + 24, + 11 + ], + [ + 1, + 44, + 11, + 34, + 45, + 51 + ], + [ + 40, + 5, + 53, + 6, + 22, + 18 + ], + [ + 28, + 10, + 1, + 15, + 3, + 4 + ], + [ + 30, + 27, + 24, + 57, + 32, + 16 + ] + ], + [ + [ + 24, + 56, + 6, + 0, + 19, + 45 + ], + [ + 11, + 57, + 59, + 46, + 25, + 30 + ], + [ + 26, + 11, + 37, + 14, + 29, + 49 + ], + [ + 38, + 36, + 3, + 24, + 18, + 20 + ], + [ + 61, + 51, + 14, + 2, + 24, + 1 + ], + [ + 20, + 0, + 47, + 30, + 8, + 35 + ], + [ + 4, + 58, + 36, + 54, + 29, + 12 + ], + [ + 20, + 58, + 44, + 28, + 45, + 9 + ], + [ + 56, + 47, + 10, + 29, + 35, + 27 + ], + [ + 61, + 33, + 55, + 54, + 4, + 36 + ], + [ + 58, + 1, + 14, + 4, + 38, + 52 + ], + [ + 14, + 0, + 36, + 63, + 15, + 52 + ], + [ + 29, + 36, + 44, + 8, + 16, + 2 + ], + [ + 6, + 40, + 27, + 57, + 50, + 42 + ], + [ + 19, + 44, + 58, + 61, + 37, + 38 + ], + [ + 23, + 18, + 17, + 57, + 13, + 5 + ], + [ + 13, + 9, + 19, + 37, + 50, + 15 + ], + [ + 32, + 14, + 57, + 58, + 29, + 22 + ], + [ + 10, + 61, + 1, + 3, + 14, + 59 + ], + [ + 55, + 16, + 34, + 18, + 22, + 7 + ], + [ + 53, + 27, + 38, + 28, + 23, + 44 + ], + [ + 24, + 47, + 18, + 41, + 30, + 62 + ], + [ + 33, + 51, + 19, + 5, + 0, + 34 + ], + [ + 51, + 8, + 25, + 53, + 27, + 55 + ], + [ + 40, + 11, + 27, + 4, + 23, + 19 + ], + [ + 16, + 18, + 51, + 48, + 3, + 47 + ], + [ + 46, + 43, + 36, + 9, + 5, + 33 + ] + ], + [ + [ + 37, + 10, + 46, + 60, + 61, + 55 + ], + [ + 35, + 53, + 34, + 43, + 19, + 57 + ], + [ + 49, + 56, + 45, + 30, + 6, + 12 + ], + [ + 60, + 27, + 14, + 48, + 46, + 57 + ], + [ + 61, + 59, + 14, + 41, + 16, + 1 + ], + [ + 45, + 4, + 3, + 24, + 58, + 47 + ], + [ + 54, + 13, + 9, + 43, + 16, + 26 + ], + [ + 47, + 23, + 12, + 20, + 63, + 30 + ], + [ + 23, + 44, + 56, + 29, + 47, + 17 + ], + [ + 33, + 60, + 61, + 48, + 41, + 14 + ], + [ + 58, + 63, + 19, + 11, + 38, + 9 + ], + [ + 60, + 63, + 0, + 36, + 15, + 9 + ], + [ + 29, + 36, + 30, + 59, + 11, + 27 + ], + [ + 6, + 7, + 47, + 62, + 50, + 57 + ], + [ + 27, + 58, + 19, + 46, + 29, + 56 + ], + [ + 29, + 60, + 56, + 55, + 23, + 26 + ], + [ + 53, + 59, + 6, + 9, + 16, + 43 + ], + [ + 41, + 32, + 57, + 63, + 18, + 37 + ], + [ + 42, + 61, + 3, + 10, + 34, + 59 + ], + [ + 4, + 43, + 17, + 16, + 52, + 60 + ], + [ + 45, + 53, + 61, + 56, + 16, + 7 + ], + [ + 55, + 9, + 61, + 18, + 60, + 3 + ], + [ + 60, + 47, + 53, + 33, + 12, + 27 + ], + [ + 43, + 51, + 11, + 8, + 45, + 63 + ], + [ + 6, + 40, + 15, + 27, + 26, + 23 + ], + [ + 49, + 14, + 9, + 58, + 21, + 12 + ], + [ + 63, + 24, + 60, + 31, + 12, + 34 + ] + ], + [ + [ + 16, + 13, + 44, + 4, + 23, + 46 + ], + [ + 16, + 50, + 9, + 13, + 23, + 36 + ], + [ + 11, + 35, + 21, + 7, + 59, + 9 + ], + [ + 1, + 3, + 25, + 15, + 60, + 39 + ], + [ + 54, + 61, + 31, + 35, + 55, + 1 + ], + [ + 51, + 52, + 46, + 15, + 4, + 45 + ], + [ + 60, + 54, + 59, + 44, + 10, + 7 + ], + [ + 12, + 22, + 14, + 47, + 0, + 30 + ], + [ + 42, + 29, + 23, + 56, + 33, + 47 + ], + [ + 33, + 61, + 20, + 60, + 0, + 49 + ], + [ + 35, + 58, + 63, + 14, + 51, + 24 + ], + [ + 29, + 33, + 36, + 60, + 49, + 0 + ], + [ + 29, + 17, + 30, + 31, + 12, + 36 + ], + [ + 6, + 0, + 61, + 50, + 48, + 3 + ], + [ + 8, + 6, + 58, + 37, + 29, + 19 + ], + [ + 60, + 39, + 27, + 19, + 1, + 57 + ], + [ + 56, + 9, + 30, + 6, + 43, + 10 + ], + [ + 32, + 20, + 13, + 57, + 63, + 49 + ], + [ + 8, + 42, + 21, + 61, + 37, + 4 + ], + [ + 45, + 49, + 16, + 13, + 2, + 58 + ], + [ + 53, + 29, + 14, + 50, + 61, + 3 + ], + [ + 37, + 57, + 27, + 54, + 46, + 9 + ], + [ + 52, + 19, + 22, + 0, + 18, + 5 + ], + [ + 14, + 49, + 30, + 33, + 53, + 34 + ], + [ + 13, + 9, + 4, + 40, + 23, + 39 + ], + [ + 27, + 43, + 47, + 36, + 49, + 3 + ], + [ + 59, + 43, + 40, + 28, + 0, + 33 + ] + ], + [ + [ + 48, + 50, + 42, + 63, + 34, + 38 + ], + [ + 3, + 40, + 61, + 62, + 2, + 6 + ], + [ + 39, + 7, + 36, + 6, + 45, + 40 + ], + [ + 41, + 35, + 46, + 13, + 63, + 56 + ], + [ + 6, + 1, + 54, + 37, + 38, + 34 + ], + [ + 59, + 46, + 51, + 31, + 4, + 52 + ], + [ + 60, + 44, + 11, + 54, + 4, + 24 + ], + [ + 12, + 0, + 2, + 63, + 50, + 47 + ], + [ + 33, + 42, + 29, + 23, + 16, + 56 + ], + [ + 20, + 61, + 33, + 60, + 53, + 0 + ], + [ + 35, + 58, + 63, + 9, + 8, + 19 + ], + [ + 59, + 48, + 36, + 60, + 10, + 14 + ], + [ + 29, + 44, + 7, + 17, + 36, + 12 + ], + [ + 47, + 27, + 6, + 62, + 42, + 48 + ], + [ + 18, + 58, + 49, + 46, + 42, + 38 + ], + [ + 60, + 34, + 27, + 18, + 23, + 55 + ], + [ + 43, + 40, + 9, + 50, + 18, + 45 + ], + [ + 32, + 57, + 48, + 42, + 29, + 39 + ], + [ + 42, + 61, + 49, + 3, + 32, + 1 + ], + [ + 23, + 37, + 1, + 16, + 36, + 39 + ], + [ + 53, + 21, + 7, + 61, + 50, + 31 + ], + [ + 8, + 60, + 18, + 24, + 9, + 30 + ], + [ + 51, + 33, + 28, + 5, + 44, + 8 + ], + [ + 51, + 52, + 8, + 4, + 41, + 45 + ], + [ + 40, + 4, + 27, + 9, + 60, + 19 + ], + [ + 3, + 61, + 16, + 26, + 48, + 12 + ], + [ + 54, + 61, + 35, + 1, + 53, + 20 + ] + ], + [ + [ + 62, + 28, + 1, + 42, + 8, + 55 + ], + [ + 18, + 12, + 8, + 41, + 40, + 31 + ], + [ + 12, + 6, + 50, + 4, + 23, + 45 + ], + [ + 43, + 35, + 8, + 20, + 42, + 46 + ], + [ + 39, + 41, + 29, + 22, + 3, + 56 + ], + [ + 61, + 45, + 46, + 48, + 28, + 51 + ], + [ + 44, + 4, + 11, + 25, + 54, + 59 + ], + [ + 12, + 33, + 56, + 52, + 30, + 17 + ], + [ + 55, + 29, + 17, + 42, + 23, + 14 + ], + [ + 60, + 12, + 18, + 61, + 33, + 28 + ], + [ + 35, + 58, + 37, + 63, + 6, + 27 + ], + [ + 48, + 59, + 10, + 36, + 58, + 60 + ], + [ + 17, + 7, + 28, + 31, + 29, + 27 + ], + [ + 47, + 42, + 50, + 6, + 8, + 14 + ], + [ + 39, + 58, + 56, + 37, + 18, + 59 + ], + [ + 60, + 18, + 57, + 9, + 55, + 23 + ], + [ + 43, + 63, + 18, + 60, + 22, + 19 + ], + [ + 1, + 32, + 42, + 57, + 35, + 63 + ], + [ + 42, + 61, + 3, + 32, + 1, + 50 + ], + [ + 37, + 36, + 10, + 23, + 16, + 57 + ], + [ + 53, + 61, + 7, + 57, + 21, + 23 + ], + [ + 9, + 39, + 30, + 18, + 14, + 17 + ], + [ + 33, + 44, + 8, + 5, + 0, + 19 + ], + [ + 51, + 53, + 49, + 52, + 4, + 41 + ], + [ + 40, + 4, + 27, + 6, + 9, + 16 + ], + [ + 3, + 16, + 48, + 26, + 12, + 4 + ], + [ + 61, + 14, + 12, + 54, + 20, + 35 + ] + ], + [ + [ + 47, + 23, + 63, + 11, + 61, + 55 + ], + [ + 17, + 44, + 28, + 39, + 47, + 27 + ], + [ + 34, + 53, + 50, + 38, + 29, + 5 + ], + [ + 11, + 10, + 17, + 52, + 47, + 42 + ], + [ + 15, + 41, + 27, + 20, + 12, + 6 + ], + [ + 34, + 44, + 50, + 39, + 36, + 61 + ], + [ + 37, + 41, + 52, + 29, + 46, + 47 + ], + [ + 62, + 36, + 34, + 30, + 39, + 22 + ], + [ + 62, + 16, + 58, + 5, + 2, + 8 + ], + [ + 32, + 41, + 56, + 12, + 46, + 8 + ], + [ + 10, + 35, + 45, + 41, + 3, + 56 + ], + [ + 50, + 48, + 35, + 53, + 36, + 12 + ], + [ + 39, + 11, + 46, + 7, + 23, + 51 + ], + [ + 9, + 47, + 19, + 22, + 52, + 34 + ], + [ + 35, + 18, + 56, + 50, + 3, + 58 + ], + [ + 3, + 60, + 38, + 36, + 9, + 58 + ], + [ + 46, + 28, + 32, + 5, + 43, + 56 + ], + [ + 30, + 32, + 57, + 42, + 52, + 19 + ], + [ + 2, + 34, + 32, + 61, + 14, + 42 + ], + [ + 42, + 37, + 20, + 48, + 50, + 9 + ], + [ + 53, + 7, + 56, + 25, + 60, + 13 + ], + [ + 17, + 39, + 53, + 14, + 30, + 25 + ], + [ + 5, + 40, + 6, + 33, + 29, + 35 + ], + [ + 51, + 4, + 11, + 58, + 57, + 28 + ], + [ + 40, + 37, + 4, + 44, + 8, + 48 + ], + [ + 3, + 48, + 26, + 9, + 12, + 41 + ], + [ + 47, + 61, + 26, + 24, + 20, + 53 + ] + ], + [ + [ + 31, + 43, + 41, + 47, + 11, + 25 + ], + [ + 50, + 25, + 31, + 40, + 24, + 46 + ], + [ + 23, + 9, + 62, + 15, + 20, + 53 + ], + [ + 4, + 47, + 44, + 58, + 48, + 25 + ], + [ + 2, + 19, + 12, + 52, + 0, + 40 + ], + [ + 49, + 15, + 24, + 34, + 60, + 42 + ], + [ + 12, + 46, + 17, + 29, + 41, + 3 + ], + [ + 39, + 60, + 44, + 41, + 33, + 36 + ], + [ + 21, + 60, + 16, + 44, + 51, + 57 + ], + [ + 24, + 41, + 12, + 33, + 13, + 21 + ], + [ + 43, + 62, + 3, + 12, + 28, + 45 + ], + [ + 58, + 19, + 39, + 17, + 49, + 42 + ], + [ + 25, + 54, + 4, + 11, + 7, + 6 + ], + [ + 35, + 34, + 4, + 42, + 62, + 19 + ], + [ + 43, + 41, + 42, + 35, + 40, + 32 + ], + [ + 21, + 63, + 3, + 17, + 20, + 50 + ], + [ + 58, + 46, + 44, + 1, + 25, + 20 + ], + [ + 26, + 32, + 16, + 25, + 46, + 41 + ], + [ + 37, + 63, + 61, + 28, + 56, + 12 + ], + [ + 33, + 42, + 40, + 37, + 48, + 50 + ], + [ + 11, + 53, + 25, + 39, + 4, + 61 + ], + [ + 12, + 54, + 4, + 27, + 50, + 14 + ], + [ + 47, + 19, + 42, + 17, + 35, + 40 + ], + [ + 54, + 40, + 60, + 63, + 45, + 57 + ], + [ + 44, + 56, + 40, + 62, + 37, + 3 + ], + [ + 59, + 41, + 57, + 34, + 48, + 22 + ], + [ + 56, + 13, + 59, + 51, + 58, + 23 + ] + ], + [ + [ + 35, + 32, + 8, + 40, + 51, + 52 + ], + [ + 52, + 5, + 22, + 21, + 6, + 33 + ], + [ + 22, + 58, + 11, + 25, + 3, + 51 + ], + [ + 63, + 2, + 56, + 4, + 23, + 54 + ], + [ + 39, + 12, + 23, + 32, + 30, + 46 + ], + [ + 50, + 34, + 36, + 58, + 26, + 28 + ], + [ + 46, + 41, + 3, + 2, + 22, + 16 + ], + [ + 60, + 36, + 53, + 30, + 54, + 39 + ], + [ + 16, + 51, + 3, + 39, + 2, + 26 + ], + [ + 45, + 26, + 18, + 41, + 32, + 46 + ], + [ + 45, + 3, + 10, + 56, + 36, + 35 + ], + [ + 21, + 36, + 35, + 50, + 11, + 19 + ], + [ + 28, + 11, + 46, + 59, + 41, + 15 + ], + [ + 23, + 16, + 38, + 19, + 15, + 22 + ], + [ + 27, + 7, + 34, + 58, + 3, + 42 + ], + [ + 9, + 22, + 36, + 46, + 26, + 41 + ], + [ + 3, + 51, + 40, + 56, + 46, + 8 + ], + [ + 12, + 25, + 21, + 50, + 17, + 62 + ], + [ + 27, + 34, + 61, + 13, + 60, + 11 + ], + [ + 53, + 12, + 56, + 0, + 42, + 33 + ], + [ + 53, + 37, + 12, + 24, + 25, + 63 + ], + [ + 45, + 55, + 18, + 26, + 17, + 1 + ], + [ + 4, + 25, + 32, + 48, + 1, + 53 + ], + [ + 17, + 27, + 63, + 4, + 62, + 44 + ], + [ + 6, + 52, + 62, + 40, + 46, + 23 + ], + [ + 10, + 42, + 28, + 49, + 3, + 1 + ], + [ + 45, + 27, + 41, + 21, + 16, + 47 + ] + ], + [ + [ + 44, + 24, + 56, + 33, + 15, + 41 + ], + [ + 38, + 26, + 24, + 29, + 19, + 53 + ], + [ + 12, + 15, + 29, + 9, + 1, + 63 + ], + [ + 58, + 38, + 50, + 0, + 43, + 61 + ], + [ + 24, + 51, + 31, + 34, + 60, + 7 + ], + [ + 0, + 7, + 22, + 43, + 35, + 1 + ], + [ + 63, + 36, + 11, + 1, + 16, + 4 + ], + [ + 8, + 50, + 56, + 4, + 30, + 55 + ], + [ + 43, + 16, + 42, + 29, + 60, + 35 + ], + [ + 34, + 0, + 9, + 22, + 18, + 26 + ], + [ + 54, + 51, + 45, + 35, + 2, + 36 + ], + [ + 37, + 36, + 43, + 60, + 11, + 59 + ], + [ + 56, + 38, + 10, + 28, + 14, + 43 + ], + [ + 30, + 0, + 58, + 62, + 22, + 19 + ], + [ + 7, + 55, + 42, + 58, + 30, + 38 + ], + [ + 11, + 33, + 1, + 39, + 19, + 16 + ], + [ + 55, + 20, + 40, + 9, + 18, + 30 + ], + [ + 18, + 20, + 57, + 45, + 32, + 1 + ], + [ + 43, + 61, + 12, + 32, + 31, + 30 + ], + [ + 23, + 25, + 7, + 28, + 40, + 19 + ], + [ + 14, + 51, + 48, + 58, + 53, + 25 + ], + [ + 18, + 30, + 1, + 49, + 41, + 9 + ], + [ + 2, + 51, + 22, + 0, + 52, + 5 + ], + [ + 53, + 4, + 47, + 52, + 51, + 40 + ], + [ + 40, + 16, + 9, + 47, + 23, + 11 + ], + [ + 47, + 3, + 43, + 46, + 26, + 53 + ], + [ + 8, + 40, + 18, + 46, + 33, + 63 + ] + ], + [ + [ + 48, + 38, + 50, + 42, + 63, + 36 + ], + [ + 3, + 26, + 10, + 2, + 6, + 61 + ], + [ + 39, + 44, + 45, + 40, + 6, + 7 + ], + [ + 41, + 5, + 20, + 49, + 56, + 13 + ], + [ + 6, + 1, + 30, + 37, + 28, + 38 + ], + [ + 59, + 46, + 22, + 35, + 61, + 0 + ], + [ + 1, + 63, + 35, + 3, + 60, + 49 + ], + [ + 8, + 12, + 2, + 50, + 5, + 55 + ], + [ + 42, + 33, + 43, + 16, + 32, + 29 + ], + [ + 9, + 34, + 0, + 20, + 41, + 31 + ], + [ + 51, + 54, + 8, + 19, + 63, + 9 + ], + [ + 37, + 56, + 36, + 11, + 59, + 43 + ], + [ + 38, + 10, + 28, + 17, + 56, + 63 + ], + [ + 27, + 30, + 42, + 19, + 0, + 22 + ], + [ + 7, + 55, + 49, + 42, + 58, + 38 + ], + [ + 29, + 34, + 39, + 33, + 47, + 45 + ], + [ + 55, + 40, + 20, + 18, + 7, + 5 + ], + [ + 43, + 57, + 39, + 54, + 48, + 28 + ], + [ + 12, + 43, + 61, + 42, + 32, + 49 + ], + [ + 23, + 36, + 1, + 7, + 59, + 28 + ], + [ + 14, + 53, + 21, + 7, + 57, + 37 + ], + [ + 18, + 1, + 24, + 60, + 30, + 9 + ], + [ + 51, + 0, + 33, + 2, + 44, + 5 + ], + [ + 52, + 29, + 4, + 41, + 54, + 58 + ], + [ + 40, + 19, + 16, + 9, + 46, + 47 + ], + [ + 61, + 3, + 47, + 22, + 21, + 53 + ], + [ + 35, + 60, + 54, + 1, + 5, + 40 + ] + ], + [ + [ + 17, + 18, + 8, + 53, + 25, + 43 + ], + [ + 9, + 38, + 24, + 47, + 25, + 63 + ], + [ + 20, + 24, + 5, + 12, + 54, + 28 + ], + [ + 43, + 10, + 20, + 42, + 11, + 8 + ], + [ + 53, + 61, + 30, + 39, + 29, + 18 + ], + [ + 61, + 56, + 25, + 40, + 5, + 22 + ], + [ + 62, + 17, + 24, + 1, + 47, + 33 + ], + [ + 41, + 16, + 34, + 39, + 29, + 8 + ], + [ + 39, + 16, + 36, + 42, + 29, + 23 + ], + [ + 9, + 11, + 41, + 63, + 56, + 31 + ], + [ + 48, + 51, + 10, + 62, + 63, + 45 + ], + [ + 36, + 11, + 37, + 42, + 58, + 46 + ], + [ + 51, + 38, + 25, + 63, + 29, + 44 + ], + [ + 4, + 56, + 44, + 62, + 58, + 30 + ], + [ + 3, + 7, + 46, + 42, + 33, + 35 + ], + [ + 39, + 9, + 33, + 58, + 60, + 29 + ], + [ + 40, + 37, + 20, + 16, + 55, + 25 + ], + [ + 54, + 19, + 11, + 57, + 0, + 39 + ], + [ + 12, + 43, + 61, + 25, + 49, + 32 + ], + [ + 4, + 23, + 54, + 36, + 28, + 7 + ], + [ + 40, + 25, + 26, + 14, + 2, + 58 + ], + [ + 18, + 24, + 58, + 1, + 22, + 46 + ], + [ + 2, + 63, + 22, + 6, + 44, + 56 + ], + [ + 52, + 29, + 51, + 4, + 32, + 40 + ], + [ + 40, + 17, + 15, + 16, + 46, + 57 + ], + [ + 9, + 61, + 3, + 47, + 24, + 11 + ], + [ + 2, + 39, + 24, + 42, + 0, + 44 + ] + ], + [ + [ + 0, + 10, + 49, + 23, + 62, + 44 + ], + [ + 28, + 0, + 36, + 26, + 47, + 52 + ], + [ + 30, + 4, + 16, + 48, + 40, + 10 + ], + [ + 61, + 32, + 26, + 16, + 33, + 62 + ], + [ + 30, + 39, + 53, + 5, + 57, + 20 + ], + [ + 5, + 37, + 61, + 15, + 25, + 6 + ], + [ + 15, + 17, + 24, + 60, + 49, + 62 + ], + [ + 34, + 39, + 61, + 0, + 58, + 40 + ], + [ + 16, + 39, + 36, + 51, + 2, + 29 + ], + [ + 9, + 11, + 41, + 31, + 56, + 52 + ], + [ + 10, + 48, + 24, + 45, + 62, + 51 + ], + [ + 11, + 38, + 36, + 37, + 6, + 42 + ], + [ + 51, + 50, + 15, + 30, + 25, + 38 + ], + [ + 4, + 19, + 24, + 35, + 31, + 48 + ], + [ + 7, + 46, + 3, + 58, + 30, + 41 + ], + [ + 58, + 9, + 39, + 32, + 29, + 40 + ], + [ + 40, + 37, + 20, + 8, + 25, + 55 + ], + [ + 19, + 0, + 54, + 52, + 17, + 39 + ], + [ + 25, + 43, + 12, + 61, + 11, + 14 + ], + [ + 23, + 4, + 54, + 36, + 28, + 33 + ], + [ + 40, + 2, + 25, + 58, + 36, + 53 + ], + [ + 18, + 46, + 35, + 22, + 53, + 16 + ], + [ + 2, + 6, + 63, + 14, + 42, + 11 + ], + [ + 35, + 7, + 52, + 40, + 29, + 57 + ], + [ + 40, + 15, + 19, + 57, + 17, + 23 + ], + [ + 9, + 11, + 47, + 22, + 49, + 1 + ], + [ + 24, + 39, + 42, + 2, + 16, + 22 + ] + ], + [ + [ + 55, + 39, + 9, + 43, + 21, + 46 + ], + [ + 56, + 0, + 63, + 39, + 30, + 41 + ], + [ + 20, + 1, + 26, + 58, + 34, + 19 + ], + [ + 54, + 24, + 32, + 51, + 26, + 44 + ], + [ + 30, + 53, + 56, + 39, + 34, + 40 + ], + [ + 5, + 37, + 25, + 50, + 6, + 61 + ], + [ + 24, + 49, + 37, + 15, + 6, + 29 + ], + [ + 34, + 16, + 30, + 61, + 10, + 36 + ], + [ + 16, + 29, + 2, + 5, + 51, + 26 + ], + [ + 9, + 56, + 11, + 31, + 46, + 45 + ], + [ + 10, + 45, + 56, + 62, + 25, + 36 + ], + [ + 11, + 6, + 35, + 36, + 1, + 52 + ], + [ + 51, + 50, + 41, + 46, + 38, + 4 + ], + [ + 19, + 33, + 41, + 16, + 31, + 52 + ], + [ + 34, + 7, + 17, + 47, + 63, + 3 + ], + [ + 58, + 9, + 22, + 61, + 59, + 8 + ], + [ + 40, + 37, + 3, + 51, + 22, + 57 + ], + [ + 12, + 52, + 21, + 54, + 25, + 19 + ], + [ + 34, + 53, + 27, + 43, + 14, + 29 + ], + [ + 56, + 53, + 44, + 43, + 60, + 24 + ], + [ + 12, + 53, + 40, + 2, + 49, + 62 + ], + [ + 18, + 39, + 61, + 26, + 23, + 44 + ], + [ + 0, + 4, + 53, + 41, + 25, + 21 + ], + [ + 1, + 7, + 25, + 10, + 40, + 56 + ], + [ + 40, + 22, + 6, + 29, + 19, + 48 + ], + [ + 28, + 10, + 47, + 55, + 42, + 44 + ], + [ + 30, + 27, + 57, + 16, + 50, + 59 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 3 + ], + [ + 8, + 60, + 10, + 59, + 43, + 6 + ], + [ + 51, + 45, + 28, + 59, + 63, + 34 + ], + [ + 4, + 16, + 20, + 58, + 44, + 28 + ], + [ + 50, + 31, + 57, + 24, + 51, + 53 + ], + [ + 58, + 9, + 0, + 61, + 35, + 41 + ], + [ + 16, + 63, + 11, + 61, + 23, + 36 + ], + [ + 4, + 47, + 42, + 53, + 8, + 30 + ], + [ + 44, + 14, + 16, + 33, + 3, + 20 + ], + [ + 34, + 28, + 26, + 57, + 22, + 18 + ], + [ + 20, + 35, + 19, + 59, + 2, + 38 + ], + [ + 12, + 60, + 43, + 63, + 32, + 62 + ], + [ + 28, + 12, + 29, + 11, + 14, + 50 + ], + [ + 23, + 29, + 33, + 22, + 11, + 19 + ], + [ + 23, + 60, + 51, + 50, + 7, + 22 + ], + [ + 44, + 46, + 49, + 7, + 1, + 12 + ], + [ + 2, + 54, + 27, + 61, + 18, + 5 + ], + [ + 17, + 50, + 51, + 32, + 33, + 34 + ], + [ + 5, + 19, + 61, + 27, + 32, + 11 + ], + [ + 6, + 0, + 5, + 13, + 41, + 57 + ], + [ + 27, + 33, + 53, + 45, + 38, + 32 + ], + [ + 26, + 36, + 55, + 59, + 61, + 18 + ], + [ + 47, + 46, + 3, + 37, + 57, + 49 + ], + [ + 20, + 22, + 4, + 16, + 51, + 11 + ], + [ + 62, + 11, + 21, + 34, + 4, + 1 + ], + [ + 34, + 18, + 7, + 60, + 33, + 32 + ], + [ + 45, + 52, + 4, + 36, + 21, + 9 + ] + ], + [ + [ + 18, + 8, + 20, + 49, + 30, + 23 + ], + [ + 1, + 27, + 26, + 22, + 59, + 36 + ], + [ + 43, + 26, + 15, + 58, + 0, + 46 + ], + [ + 55, + 1, + 35, + 28, + 16, + 32 + ], + [ + 59, + 9, + 10, + 53, + 12, + 21 + ], + [ + 9, + 2, + 27, + 11, + 61, + 43 + ], + [ + 16, + 57, + 63, + 23, + 19, + 12 + ], + [ + 46, + 45, + 26, + 4, + 30, + 37 + ], + [ + 43, + 44, + 20, + 16, + 14, + 9 + ], + [ + 34, + 47, + 42, + 43, + 26, + 51 + ], + [ + 2, + 42, + 38, + 45, + 20, + 36 + ], + [ + 18, + 7, + 12, + 2, + 43, + 60 + ], + [ + 1, + 28, + 12, + 3, + 29, + 33 + ], + [ + 25, + 13, + 0, + 63, + 2, + 62 + ], + [ + 18, + 36, + 6, + 29, + 19, + 15 + ], + [ + 1, + 42, + 63, + 41, + 57, + 19 + ], + [ + 57, + 54, + 5, + 27, + 18, + 31 + ], + [ + 50, + 6, + 13, + 32, + 17, + 20 + ], + [ + 17, + 5, + 27, + 32, + 1, + 55 + ], + [ + 49, + 0, + 61, + 10, + 30, + 5 + ], + [ + 29, + 53, + 51, + 13, + 33, + 46 + ], + [ + 29, + 17, + 21, + 30, + 14, + 40 + ], + [ + 5, + 17, + 33, + 32, + 18, + 28 + ], + [ + 4, + 51, + 20, + 54, + 58, + 41 + ], + [ + 47, + 4, + 27, + 48, + 37, + 60 + ], + [ + 3, + 26, + 12, + 59, + 2, + 53 + ], + [ + 46, + 43, + 20, + 18, + 9, + 53 + ] + ], + [ + [ + 45, + 6, + 57, + 43, + 40, + 55 + ], + [ + 38, + 63, + 36, + 27, + 54, + 33 + ], + [ + 37, + 14, + 19, + 41, + 58, + 63 + ], + [ + 9, + 12, + 2, + 55, + 28, + 23 + ], + [ + 39, + 59, + 7, + 13, + 33, + 43 + ], + [ + 45, + 9, + 63, + 27, + 32, + 58 + ], + [ + 16, + 57, + 10, + 63, + 11, + 23 + ], + [ + 51, + 45, + 25, + 4, + 21, + 30 + ], + [ + 21, + 44, + 14, + 16, + 39, + 33 + ], + [ + 42, + 44, + 43, + 5, + 37, + 34 + ], + [ + 42, + 19, + 20, + 2, + 38, + 61 + ], + [ + 4, + 12, + 2, + 62, + 63, + 36 + ], + [ + 32, + 55, + 0, + 11, + 47, + 28 + ], + [ + 43, + 13, + 2, + 44, + 26, + 50 + ], + [ + 49, + 33, + 15, + 28, + 29, + 35 + ], + [ + 44, + 41, + 7, + 2, + 22, + 63 + ], + [ + 48, + 6, + 54, + 20, + 2, + 27 + ], + [ + 50, + 51, + 32, + 3, + 17, + 36 + ], + [ + 5, + 61, + 57, + 48, + 19, + 32 + ], + [ + 21, + 0, + 6, + 31, + 29, + 47 + ], + [ + 33, + 9, + 53, + 27, + 17, + 36 + ], + [ + 29, + 26, + 55, + 19, + 17, + 62 + ], + [ + 12, + 46, + 5, + 37, + 57, + 3 + ], + [ + 20, + 51, + 4, + 22, + 16, + 41 + ], + [ + 21, + 11, + 62, + 46, + 23, + 48 + ], + [ + 32, + 60, + 37, + 18, + 3, + 7 + ], + [ + 9, + 11, + 36, + 48, + 0, + 45 + ] + ], + [ + [ + 49, + 42, + 28, + 23, + 33, + 61 + ], + [ + 4, + 2, + 12, + 8, + 6, + 55 + ], + [ + 12, + 0, + 26, + 41, + 6, + 27 + ], + [ + 9, + 57, + 6, + 23, + 51, + 28 + ], + [ + 40, + 7, + 20, + 33, + 16, + 15 + ], + [ + 45, + 59, + 63, + 62, + 32, + 3 + ], + [ + 10, + 39, + 57, + 13, + 16, + 19 + ], + [ + 45, + 23, + 51, + 33, + 25, + 46 + ], + [ + 28, + 21, + 44, + 11, + 16, + 59 + ], + [ + 5, + 42, + 44, + 24, + 43, + 47 + ], + [ + 42, + 53, + 30, + 18, + 2, + 27 + ], + [ + 2, + 62, + 4, + 43, + 10, + 36 + ], + [ + 0, + 56, + 55, + 47, + 32, + 49 + ], + [ + 43, + 25, + 2, + 5, + 3, + 49 + ], + [ + 4, + 28, + 15, + 8, + 49, + 58 + ], + [ + 2, + 42, + 44, + 41, + 7, + 63 + ], + [ + 48, + 27, + 54, + 20, + 2, + 18 + ], + [ + 50, + 51, + 0, + 36, + 3, + 32 + ], + [ + 57, + 5, + 61, + 19, + 32, + 38 + ], + [ + 21, + 0, + 6, + 63, + 23, + 51 + ], + [ + 33, + 53, + 27, + 36, + 9, + 38 + ], + [ + 29, + 26, + 55, + 62, + 18, + 31 + ], + [ + 46, + 56, + 12, + 53, + 29, + 0 + ], + [ + 20, + 16, + 22, + 4, + 51, + 17 + ], + [ + 21, + 62, + 11, + 31, + 46, + 33 + ], + [ + 37, + 60, + 18, + 7, + 32, + 44 + ], + [ + 11, + 9, + 36, + 0, + 48, + 63 + ] + ], + [ + [ + 41, + 32, + 49, + 39, + 44, + 61 + ], + [ + 47, + 26, + 16, + 21, + 36, + 22 + ], + [ + 4, + 30, + 37, + 42, + 60, + 54 + ], + [ + 9, + 57, + 26, + 32, + 50, + 20 + ], + [ + 56, + 3, + 40, + 33, + 36, + 54 + ], + [ + 11, + 38, + 2, + 32, + 61, + 30 + ], + [ + 39, + 57, + 19, + 10, + 16, + 42 + ], + [ + 46, + 21, + 35, + 39, + 45, + 25 + ], + [ + 21, + 37, + 12, + 20, + 11, + 28 + ], + [ + 5, + 47, + 44, + 10, + 42, + 23 + ], + [ + 18, + 42, + 61, + 2, + 38, + 31 + ], + [ + 54, + 4, + 2, + 7, + 22, + 16 + ], + [ + 5, + 3, + 17, + 56, + 32, + 55 + ], + [ + 55, + 0, + 2, + 25, + 43, + 5 + ], + [ + 22, + 28, + 15, + 6, + 5, + 49 + ], + [ + 2, + 57, + 19, + 54, + 41, + 30 + ], + [ + 7, + 48, + 20, + 54, + 27, + 0 + ], + [ + 3, + 56, + 13, + 37, + 43, + 59 + ], + [ + 45, + 55, + 57, + 61, + 48, + 52 + ], + [ + 21, + 5, + 0, + 16, + 27, + 23 + ], + [ + 25, + 42, + 17, + 54, + 23, + 14 + ], + [ + 21, + 44, + 15, + 20, + 42, + 18 + ], + [ + 35, + 12, + 25, + 53, + 61, + 2 + ], + [ + 38, + 54, + 48, + 53, + 21, + 36 + ], + [ + 13, + 31, + 48, + 33, + 18, + 55 + ], + [ + 38, + 27, + 19, + 6, + 44, + 3 + ], + [ + 29, + 62, + 43, + 59, + 46, + 5 + ] + ], + [ + [ + 57, + 9, + 19, + 51, + 18, + 41 + ], + [ + 28, + 57, + 36, + 8, + 48, + 60 + ], + [ + 2, + 51, + 59, + 5, + 34, + 9 + ], + [ + 9, + 55, + 59, + 26, + 4, + 2 + ], + [ + 49, + 56, + 35, + 42, + 30, + 23 + ], + [ + 18, + 30, + 22, + 29, + 19, + 52 + ], + [ + 39, + 34, + 33, + 51, + 56, + 3 + ], + [ + 32, + 21, + 1, + 7, + 46, + 49 + ], + [ + 33, + 54, + 23, + 21, + 12, + 17 + ], + [ + 5, + 30, + 60, + 47, + 15, + 18 + ], + [ + 4, + 18, + 46, + 27, + 20, + 22 + ], + [ + 22, + 59, + 54, + 48, + 19, + 4 + ], + [ + 17, + 5, + 56, + 31, + 49, + 4 + ], + [ + 29, + 47, + 55, + 2, + 53, + 60 + ], + [ + 8, + 22, + 11, + 44, + 36, + 15 + ], + [ + 60, + 44, + 30, + 57, + 54, + 39 + ], + [ + 7, + 44, + 27, + 20, + 2, + 61 + ], + [ + 48, + 17, + 37, + 21, + 32, + 57 + ], + [ + 48, + 32, + 46, + 6, + 61, + 42 + ], + [ + 4, + 57, + 1, + 36, + 0, + 30 + ], + [ + 7, + 17, + 61, + 53, + 21, + 63 + ], + [ + 60, + 14, + 53, + 35, + 18, + 42 + ], + [ + 10, + 15, + 33, + 51, + 36, + 5 + ], + [ + 11, + 4, + 19, + 51, + 21, + 52 + ], + [ + 47, + 19, + 48, + 43, + 4, + 58 + ], + [ + 3, + 33, + 26, + 21, + 52, + 19 + ], + [ + 24, + 45, + 60, + 35, + 49, + 1 + ] + ], + [ + [ + 16, + 4, + 44, + 23, + 22, + 35 + ], + [ + 16, + 50, + 23, + 9, + 32, + 13 + ], + [ + 11, + 35, + 21, + 7, + 48, + 59 + ], + [ + 55, + 15, + 1, + 11, + 8, + 40 + ], + [ + 35, + 61, + 30, + 59, + 31, + 62 + ], + [ + 51, + 29, + 15, + 52, + 38, + 61 + ], + [ + 60, + 0, + 55, + 34, + 59, + 33 + ], + [ + 12, + 22, + 56, + 63, + 54, + 55 + ], + [ + 42, + 54, + 23, + 33, + 27, + 47 + ], + [ + 30, + 60, + 20, + 5, + 4, + 22 + ], + [ + 4, + 35, + 22, + 46, + 23, + 19 + ], + [ + 29, + 22, + 59, + 49, + 24, + 28 + ], + [ + 17, + 31, + 4, + 56, + 5, + 9 + ], + [ + 61, + 29, + 0, + 48, + 59, + 50 + ], + [ + 8, + 6, + 22, + 60, + 55, + 31 + ], + [ + 60, + 39, + 19, + 57, + 53, + 27 + ], + [ + 56, + 30, + 22, + 10, + 5, + 55 + ], + [ + 20, + 31, + 26, + 1, + 61, + 45 + ], + [ + 8, + 23, + 7, + 46, + 48, + 4 + ], + [ + 45, + 23, + 51, + 17, + 4, + 13 + ], + [ + 17, + 13, + 61, + 14, + 29, + 55 + ], + [ + 14, + 27, + 43, + 21, + 56, + 42 + ], + [ + 52, + 51, + 15, + 58, + 8, + 5 + ], + [ + 4, + 51, + 49, + 14, + 21, + 34 + ], + [ + 4, + 9, + 13, + 47, + 8, + 61 + ], + [ + 27, + 3, + 16, + 43, + 31, + 47 + ], + [ + 59, + 43, + 29, + 61, + 0, + 18 + ] + ], + [ + [ + 48, + 21, + 18, + 49, + 41, + 23 + ], + [ + 36, + 4, + 60, + 8, + 49, + 44 + ], + [ + 20, + 39, + 30, + 59, + 45, + 55 + ], + [ + 35, + 46, + 15, + 48, + 33, + 2 + ], + [ + 61, + 37, + 8, + 15, + 54, + 10 + ], + [ + 46, + 51, + 6, + 29, + 58, + 4 + ], + [ + 28, + 11, + 44, + 60, + 0, + 1 + ], + [ + 63, + 12, + 13, + 27, + 10, + 0 + ], + [ + 33, + 42, + 54, + 44, + 23, + 14 + ], + [ + 18, + 60, + 30, + 22, + 40, + 14 + ], + [ + 35, + 4, + 61, + 9, + 18, + 33 + ], + [ + 59, + 45, + 48, + 28, + 62, + 22 + ], + [ + 17, + 56, + 7, + 53, + 5, + 36 + ], + [ + 8, + 47, + 29, + 59, + 1, + 6 + ], + [ + 9, + 8, + 18, + 22, + 60, + 15 + ], + [ + 46, + 60, + 22, + 44, + 30, + 57 + ], + [ + 61, + 2, + 27, + 34, + 7, + 60 + ], + [ + 48, + 21, + 37, + 17, + 50, + 57 + ], + [ + 44, + 42, + 5, + 2, + 48, + 61 + ], + [ + 37, + 57, + 4, + 36, + 17, + 59 + ], + [ + 7, + 17, + 44, + 61, + 53, + 33 + ], + [ + 60, + 45, + 14, + 42, + 18, + 9 + ], + [ + 10, + 8, + 36, + 33, + 15, + 58 + ], + [ + 19, + 20, + 11, + 4, + 49, + 51 + ], + [ + 61, + 37, + 47, + 23, + 12, + 3 + ], + [ + 17, + 33, + 3, + 40, + 19, + 16 + ], + [ + 61, + 45, + 49, + 1, + 14, + 63 + ] + ], + [ + [ + 13, + 40, + 55, + 63, + 26, + 4 + ], + [ + 5, + 35, + 49, + 40, + 17, + 46 + ], + [ + 38, + 17, + 59, + 49, + 2, + 58 + ], + [ + 40, + 8, + 1, + 16, + 0, + 11 + ], + [ + 37, + 62, + 51, + 10, + 8, + 38 + ], + [ + 9, + 42, + 61, + 29, + 35, + 33 + ], + [ + 63, + 53, + 11, + 16, + 33, + 60 + ], + [ + 63, + 37, + 5, + 13, + 17, + 39 + ], + [ + 44, + 20, + 31, + 54, + 38, + 21 + ], + [ + 43, + 21, + 30, + 34, + 18, + 49 + ], + [ + 2, + 11, + 19, + 35, + 4, + 9 + ], + [ + 22, + 60, + 43, + 2, + 4, + 49 + ], + [ + 29, + 5, + 17, + 22, + 24, + 55 + ], + [ + 2, + 59, + 29, + 5, + 55, + 41 + ], + [ + 23, + 8, + 36, + 15, + 22, + 54 + ], + [ + 12, + 44, + 41, + 45, + 5, + 22 + ], + [ + 54, + 7, + 41, + 11, + 53, + 1 + ], + [ + 6, + 50, + 2, + 9, + 21, + 37 + ], + [ + 13, + 19, + 5, + 10, + 48, + 8 + ], + [ + 41, + 32, + 6, + 21, + 47, + 0 + ], + [ + 38, + 33, + 36, + 53, + 31, + 17 + ], + [ + 3, + 26, + 7, + 62, + 18, + 59 + ], + [ + 56, + 57, + 46, + 12, + 35, + 48 + ], + [ + 20, + 16, + 22, + 24, + 27, + 42 + ], + [ + 36, + 21, + 46, + 34, + 3, + 11 + ], + [ + 33, + 34, + 45, + 60, + 7, + 59 + ], + [ + 56, + 34, + 52, + 58, + 26, + 48 + ] + ], + [ + [ + 54, + 23, + 53, + 11, + 58, + 3 + ], + [ + 11, + 30, + 59, + 58, + 63, + 4 + ], + [ + 20, + 29, + 58, + 17, + 42, + 4 + ], + [ + 1, + 35, + 40, + 45, + 53, + 59 + ], + [ + 40, + 55, + 21, + 33, + 38, + 49 + ], + [ + 45, + 29, + 61, + 27, + 63, + 62 + ], + [ + 33, + 57, + 11, + 28, + 53, + 34 + ], + [ + 11, + 63, + 39, + 10, + 45, + 14 + ], + [ + 30, + 54, + 57, + 59, + 33, + 26 + ], + [ + 43, + 23, + 5, + 18, + 21, + 42 + ], + [ + 11, + 18, + 2, + 9, + 34, + 6 + ], + [ + 22, + 60, + 28, + 63, + 2, + 17 + ], + [ + 5, + 41, + 6, + 17, + 56, + 29 + ], + [ + 55, + 6, + 5, + 2, + 48, + 59 + ], + [ + 23, + 19, + 62, + 22, + 11, + 9 + ], + [ + 12, + 45, + 41, + 27, + 8, + 42 + ], + [ + 11, + 53, + 41, + 44, + 51, + 7 + ], + [ + 21, + 2, + 6, + 36, + 50, + 56 + ], + [ + 13, + 10, + 48, + 53, + 61, + 19 + ], + [ + 47, + 21, + 56, + 44, + 6, + 31 + ], + [ + 44, + 12, + 3, + 55, + 41, + 53 + ], + [ + 44, + 47, + 28, + 43, + 45, + 63 + ], + [ + 1, + 25, + 53, + 11, + 39, + 19 + ], + [ + 1, + 59, + 38, + 3, + 37, + 63 + ], + [ + 45, + 3, + 0, + 21, + 14, + 22 + ], + [ + 10, + 28, + 42, + 49, + 11, + 3 + ], + [ + 30, + 57, + 15, + 16, + 56, + 41 + ] + ], + [ + [ + 53, + 15, + 34, + 0, + 46, + 33 + ], + [ + 8, + 12, + 41, + 19, + 39, + 32 + ], + [ + 56, + 31, + 36, + 13, + 23, + 9 + ], + [ + 36, + 51, + 30, + 21, + 1, + 11 + ], + [ + 13, + 58, + 50, + 2, + 53, + 54 + ], + [ + 49, + 52, + 32, + 7, + 23, + 47 + ], + [ + 61, + 38, + 23, + 39, + 0, + 35 + ], + [ + 42, + 27, + 9, + 20, + 17, + 57 + ], + [ + 34, + 1, + 29, + 4, + 35, + 45 + ], + [ + 54, + 57, + 27, + 19, + 38, + 62 + ], + [ + 59, + 1, + 60, + 26, + 38, + 22 + ], + [ + 25, + 31, + 51, + 36, + 32, + 8 + ], + [ + 14, + 62, + 2, + 19, + 37, + 11 + ], + [ + 57, + 40, + 13, + 22, + 37, + 46 + ], + [ + 45, + 34, + 58, + 44, + 42, + 16 + ], + [ + 50, + 16, + 6, + 5, + 33, + 43 + ], + [ + 42, + 39, + 61, + 13, + 5, + 15 + ], + [ + 46, + 23, + 27, + 28, + 4, + 63 + ], + [ + 62, + 31, + 10, + 45, + 35, + 30 + ], + [ + 15, + 13, + 38, + 63, + 4, + 19 + ], + [ + 34, + 15, + 38, + 57, + 27, + 19 + ], + [ + 41, + 62, + 36, + 57, + 19, + 47 + ], + [ + 34, + 22, + 53, + 10, + 46, + 45 + ], + [ + 24, + 51, + 4, + 47, + 39, + 10 + ], + [ + 11, + 57, + 51, + 50, + 54, + 6 + ], + [ + 51, + 7, + 11, + 50, + 43, + 18 + ], + [ + 39, + 37, + 9, + 42, + 40, + 44 + ] + ], + [ + [ + 57, + 17, + 62, + 42, + 23, + 60 + ], + [ + 18, + 7, + 53, + 43, + 26, + 60 + ], + [ + 60, + 5, + 3, + 53, + 23, + 57 + ], + [ + 17, + 10, + 22, + 19, + 11, + 31 + ], + [ + 10, + 15, + 12, + 27, + 17, + 4 + ], + [ + 2, + 44, + 39, + 36, + 25, + 54 + ], + [ + 52, + 62, + 37, + 21, + 41, + 42 + ], + [ + 62, + 7, + 46, + 30, + 36, + 14 + ], + [ + 50, + 4, + 10, + 58, + 0, + 16 + ], + [ + 32, + 47, + 38, + 54, + 8, + 41 + ], + [ + 41, + 32, + 3, + 18, + 1, + 22 + ], + [ + 16, + 50, + 53, + 7, + 44, + 12 + ], + [ + 11, + 39, + 3, + 35, + 25, + 48 + ], + [ + 18, + 9, + 15, + 13, + 28, + 63 + ], + [ + 62, + 58, + 13, + 5, + 17, + 3 + ], + [ + 3, + 31, + 43, + 53, + 35, + 57 + ], + [ + 24, + 51, + 15, + 46, + 5, + 59 + ], + [ + 13, + 30, + 0, + 32, + 5, + 59 + ], + [ + 2, + 39, + 32, + 38, + 34, + 22 + ], + [ + 42, + 26, + 34, + 28, + 37, + 54 + ], + [ + 28, + 43, + 53, + 41, + 13, + 23 + ], + [ + 14, + 15, + 34, + 1, + 48, + 40 + ], + [ + 5, + 25, + 4, + 33, + 39, + 53 + ], + [ + 58, + 4, + 17, + 57, + 51, + 12 + ], + [ + 47, + 37, + 35, + 31, + 63, + 29 + ], + [ + 15, + 3, + 28, + 33, + 23, + 9 + ], + [ + 23, + 6, + 58, + 47, + 30, + 56 + ] + ], + [ + [ + 47, + 29, + 14, + 6, + 51, + 43 + ], + [ + 30, + 29, + 39, + 7, + 52, + 3 + ], + [ + 63, + 34, + 41, + 2, + 7, + 47 + ], + [ + 4, + 28, + 54, + 45, + 52, + 58 + ], + [ + 29, + 7, + 12, + 15, + 41, + 6 + ], + [ + 34, + 29, + 48, + 3, + 43, + 40 + ], + [ + 30, + 29, + 16, + 47, + 42, + 45 + ], + [ + 33, + 39, + 25, + 60, + 41, + 3 + ], + [ + 50, + 26, + 4, + 25, + 13, + 17 + ], + [ + 5, + 43, + 54, + 16, + 12, + 53 + ], + [ + 18, + 6, + 3, + 35, + 21, + 1 + ], + [ + 56, + 46, + 48, + 10, + 16, + 44 + ], + [ + 9, + 35, + 7, + 24, + 47, + 57 + ], + [ + 53, + 42, + 15, + 56, + 47, + 59 + ], + [ + 39, + 11, + 36, + 32, + 35, + 18 + ], + [ + 46, + 20, + 53, + 38, + 56, + 26 + ], + [ + 58, + 29, + 14, + 26, + 17, + 49 + ], + [ + 24, + 25, + 39, + 16, + 1, + 57 + ], + [ + 24, + 41, + 4, + 1, + 63, + 28 + ], + [ + 42, + 37, + 48, + 34, + 26, + 41 + ], + [ + 11, + 28, + 16, + 7, + 32, + 56 + ], + [ + 14, + 42, + 6, + 16, + 22, + 15 + ], + [ + 33, + 56, + 42, + 8, + 25, + 38 + ], + [ + 4, + 58, + 48, + 33, + 11, + 28 + ], + [ + 37, + 47, + 29, + 48, + 30, + 53 + ], + [ + 12, + 41, + 3, + 4, + 48, + 46 + ], + [ + 14, + 13, + 61, + 6, + 62, + 1 + ] + ], + [ + [ + 45, + 10, + 44, + 43, + 53, + 33 + ], + [ + 32, + 63, + 22, + 27, + 30, + 29 + ], + [ + 54, + 35, + 37, + 32, + 26, + 30 + ], + [ + 24, + 63, + 0, + 17, + 25, + 45 + ], + [ + 40, + 7, + 0, + 57, + 29, + 22 + ], + [ + 10, + 34, + 20, + 22, + 43, + 33 + ], + [ + 42, + 30, + 5, + 25, + 19, + 34 + ], + [ + 33, + 18, + 35, + 51, + 7, + 57 + ], + [ + 50, + 28, + 25, + 4, + 10, + 9 + ], + [ + 5, + 38, + 16, + 43, + 54, + 12 + ], + [ + 18, + 3, + 21, + 6, + 39, + 53 + ], + [ + 56, + 16, + 53, + 39, + 46, + 42 + ], + [ + 9, + 35, + 57, + 11, + 47, + 13 + ], + [ + 56, + 15, + 3, + 59, + 9, + 28 + ], + [ + 4, + 62, + 39, + 58, + 63, + 36 + ], + [ + 20, + 53, + 57, + 8, + 51, + 35 + ], + [ + 51, + 49, + 11, + 26, + 15, + 14 + ], + [ + 0, + 25, + 62, + 59, + 21, + 13 + ], + [ + 39, + 34, + 48, + 53, + 61, + 33 + ], + [ + 47, + 26, + 28, + 34, + 21, + 39 + ], + [ + 28, + 43, + 12, + 53, + 41, + 32 + ], + [ + 14, + 52, + 17, + 1, + 15, + 38 + ], + [ + 25, + 4, + 5, + 11, + 58, + 50 + ], + [ + 58, + 4, + 17, + 10, + 25, + 57 + ], + [ + 29, + 47, + 35, + 31, + 52, + 48 + ], + [ + 55, + 28, + 23, + 15, + 3, + 24 + ], + [ + 6, + 30, + 57, + 32, + 34, + 62 + ] + ], + [ + [ + 39, + 5, + 30, + 17, + 61, + 15 + ], + [ + 11, + 63, + 0, + 23, + 61, + 10 + ], + [ + 61, + 15, + 53, + 22, + 7, + 57 + ], + [ + 50, + 57, + 58, + 63, + 45, + 47 + ], + [ + 55, + 31, + 57, + 24, + 60, + 5 + ], + [ + 22, + 7, + 43, + 1, + 10, + 0 + ], + [ + 14, + 58, + 1, + 34, + 19, + 45 + ], + [ + 50, + 8, + 14, + 7, + 57, + 9 + ], + [ + 43, + 0, + 4, + 10, + 45, + 46 + ], + [ + 38, + 0, + 4, + 55, + 54, + 10 + ], + [ + 51, + 54, + 46, + 39, + 1, + 38 + ], + [ + 16, + 37, + 33, + 36, + 21, + 63 + ], + [ + 10, + 38, + 57, + 58, + 3, + 63 + ], + [ + 30, + 0, + 63, + 13, + 22, + 18 + ], + [ + 55, + 58, + 62, + 38, + 6, + 36 + ], + [ + 53, + 19, + 11, + 1, + 57, + 25 + ], + [ + 11, + 6, + 51, + 16, + 18, + 30 + ], + [ + 20, + 57, + 32, + 36, + 13, + 56 + ], + [ + 3, + 12, + 61, + 26, + 32, + 1 + ], + [ + 25, + 36, + 34, + 51, + 59, + 37 + ], + [ + 13, + 28, + 53, + 16, + 14, + 9 + ], + [ + 14, + 42, + 15, + 55, + 22, + 38 + ], + [ + 62, + 58, + 29, + 33, + 5, + 34 + ], + [ + 21, + 4, + 51, + 49, + 58, + 12 + ], + [ + 61, + 60, + 40, + 35, + 59, + 47 + ], + [ + 37, + 46, + 3, + 48, + 12, + 53 + ], + [ + 63, + 8, + 46, + 33, + 1, + 53 + ] + ], + [ + [ + 62, + 9, + 61, + 0, + 41, + 6 + ], + [ + 45, + 3, + 29, + 35, + 2, + 7 + ], + [ + 56, + 31, + 53, + 28, + 23, + 2 + ], + [ + 13, + 41, + 46, + 49, + 5, + 45 + ], + [ + 18, + 25, + 57, + 55, + 50, + 15 + ], + [ + 58, + 22, + 4, + 46, + 19, + 12 + ], + [ + 1, + 25, + 58, + 22, + 43, + 35 + ], + [ + 23, + 50, + 12, + 8, + 9, + 16 + ], + [ + 29, + 17, + 16, + 43, + 10, + 4 + ], + [ + 38, + 55, + 0, + 40, + 20, + 10 + ], + [ + 51, + 19, + 54, + 39, + 46, + 1 + ], + [ + 60, + 16, + 37, + 11, + 56, + 36 + ], + [ + 10, + 7, + 38, + 57, + 54, + 44 + ], + [ + 45, + 0, + 13, + 30, + 47, + 63 + ], + [ + 58, + 48, + 55, + 38, + 36, + 29 + ], + [ + 29, + 53, + 0, + 1, + 55, + 57 + ], + [ + 11, + 12, + 6, + 29, + 5, + 40 + ], + [ + 57, + 32, + 42, + 8, + 20, + 36 + ], + [ + 3, + 61, + 49, + 32, + 26, + 1 + ], + [ + 42, + 5, + 36, + 16, + 39, + 51 + ], + [ + 9, + 13, + 7, + 53, + 46, + 28 + ], + [ + 14, + 55, + 42, + 24, + 33, + 47 + ], + [ + 33, + 58, + 62, + 15, + 36, + 5 + ], + [ + 51, + 4, + 49, + 58, + 21, + 41 + ], + [ + 40, + 60, + 19, + 35, + 61, + 15 + ], + [ + 14, + 21, + 3, + 12, + 5, + 37 + ], + [ + 35, + 60, + 1, + 10, + 53, + 54 + ] + ], + [ + [ + 40, + 28, + 60, + 31, + 59, + 23 + ], + [ + 42, + 12, + 26, + 34, + 0, + 61 + ], + [ + 16, + 5, + 62, + 3, + 32, + 0 + ], + [ + 13, + 3, + 18, + 39, + 42, + 52 + ], + [ + 25, + 12, + 39, + 0, + 57, + 15 + ], + [ + 33, + 50, + 58, + 36, + 6, + 26 + ], + [ + 41, + 43, + 1, + 15, + 2, + 25 + ], + [ + 36, + 16, + 53, + 14, + 30, + 58 + ], + [ + 16, + 29, + 3, + 2, + 10, + 5 + ], + [ + 38, + 16, + 32, + 26, + 45, + 11 + ], + [ + 45, + 19, + 3, + 46, + 56, + 53 + ], + [ + 21, + 36, + 11, + 9, + 16, + 41 + ], + [ + 11, + 16, + 10, + 50, + 8, + 63 + ], + [ + 35, + 38, + 33, + 51, + 25, + 22 + ], + [ + 58, + 48, + 27, + 50, + 29, + 26 + ], + [ + 29, + 57, + 53, + 3, + 54, + 4 + ], + [ + 3, + 23, + 6, + 46, + 51, + 11 + ], + [ + 32, + 5, + 57, + 54, + 52, + 30 + ], + [ + 34, + 61, + 40, + 0, + 3, + 48 + ], + [ + 22, + 53, + 12, + 33, + 39, + 60 + ], + [ + 20, + 53, + 47, + 40, + 12, + 8 + ], + [ + 55, + 17, + 52, + 14, + 45, + 56 + ], + [ + 1, + 4, + 13, + 11, + 39, + 33 + ], + [ + 17, + 10, + 51, + 44, + 55, + 4 + ], + [ + 35, + 6, + 45, + 21, + 52, + 37 + ], + [ + 10, + 42, + 9, + 1, + 53, + 24 + ], + [ + 27, + 41, + 32, + 45, + 10, + 47 + ] + ], + [ + [ + 11, + 34, + 44, + 51, + 41, + 12 + ], + [ + 20, + 34, + 3, + 25, + 63, + 16 + ], + [ + 40, + 26, + 37, + 22, + 15, + 54 + ], + [ + 4, + 16, + 25, + 28, + 45, + 58 + ], + [ + 33, + 10, + 32, + 24, + 3, + 4 + ], + [ + 0, + 9, + 58, + 3, + 34, + 15 + ], + [ + 16, + 63, + 43, + 59, + 42, + 25 + ], + [ + 16, + 48, + 36, + 33, + 25, + 58 + ], + [ + 20, + 50, + 16, + 23, + 42, + 29 + ], + [ + 60, + 51, + 16, + 22, + 38, + 48 + ], + [ + 31, + 19, + 38, + 3, + 2, + 43 + ], + [ + 56, + 9, + 21, + 40, + 18, + 44 + ], + [ + 11, + 13, + 10, + 45, + 27, + 57 + ], + [ + 56, + 14, + 35, + 44, + 45, + 13 + ], + [ + 58, + 43, + 26, + 33, + 31, + 50 + ], + [ + 13, + 38, + 51, + 46, + 28, + 37 + ], + [ + 58, + 63, + 42, + 10, + 16, + 0 + ], + [ + 0, + 1, + 53, + 52, + 24, + 59 + ], + [ + 17, + 29, + 0, + 40, + 60, + 24 + ], + [ + 17, + 24, + 2, + 26, + 35, + 23 + ], + [ + 44, + 20, + 22, + 4, + 50, + 40 + ], + [ + 9, + 61, + 17, + 16, + 27, + 37 + ], + [ + 63, + 13, + 11, + 16, + 2, + 24 + ], + [ + 44, + 48, + 23, + 49, + 56, + 45 + ], + [ + 6, + 5, + 49, + 33, + 31, + 63 + ], + [ + 49, + 47, + 1, + 57, + 4, + 53 + ], + [ + 2, + 56, + 19, + 63, + 39, + 50 + ] + ], + [ + [ + 16, + 22, + 23, + 35, + 3, + 13 + ], + [ + 16, + 23, + 50, + 9, + 13, + 59 + ], + [ + 11, + 35, + 21, + 7, + 59, + 24 + ], + [ + 53, + 32, + 7, + 60, + 1, + 13 + ], + [ + 24, + 33, + 57, + 35, + 0, + 39 + ], + [ + 15, + 38, + 0, + 45, + 32, + 58 + ], + [ + 59, + 16, + 25, + 49, + 35, + 5 + ], + [ + 45, + 40, + 16, + 14, + 12, + 30 + ], + [ + 20, + 16, + 42, + 6, + 17, + 14 + ], + [ + 60, + 10, + 22, + 50, + 48, + 38 + ], + [ + 31, + 19, + 35, + 27, + 38, + 53 + ], + [ + 33, + 59, + 48, + 16, + 60, + 39 + ], + [ + 9, + 57, + 10, + 11, + 7, + 44 + ], + [ + 13, + 0, + 59, + 61, + 19, + 33 + ], + [ + 18, + 39, + 58, + 21, + 50, + 36 + ], + [ + 57, + 60, + 49, + 19, + 53, + 23 + ], + [ + 6, + 36, + 17, + 38, + 41, + 10 + ], + [ + 63, + 1, + 57, + 20, + 36, + 31 + ], + [ + 40, + 41, + 7, + 0, + 61, + 32 + ], + [ + 39, + 16, + 23, + 14, + 17, + 59 + ], + [ + 51, + 30, + 50, + 7, + 61, + 3 + ], + [ + 9, + 14, + 42, + 15, + 1, + 55 + ], + [ + 33, + 19, + 56, + 58, + 11, + 15 + ], + [ + 9, + 26, + 4, + 41, + 51, + 10 + ], + [ + 4, + 60, + 35, + 16, + 48, + 47 + ], + [ + 3, + 52, + 31, + 16, + 26, + 39 + ], + [ + 4, + 1, + 3, + 61, + 0, + 54 + ] + ], + [ + [ + 38, + 45, + 52, + 43, + 32, + 39 + ], + [ + 29, + 48, + 22, + 60, + 55, + 57 + ], + [ + 5, + 49, + 8, + 20, + 14, + 55 + ], + [ + 35, + 46, + 49, + 32, + 7, + 48 + ], + [ + 24, + 29, + 49, + 16, + 41, + 0 + ], + [ + 56, + 3, + 35, + 15, + 4, + 40 + ], + [ + 49, + 59, + 4, + 16, + 33, + 11 + ], + [ + 12, + 51, + 30, + 56, + 40, + 27 + ], + [ + 23, + 14, + 46, + 55, + 42, + 17 + ], + [ + 60, + 22, + 38, + 6, + 21, + 50 + ], + [ + 11, + 19, + 31, + 35, + 38, + 22 + ], + [ + 59, + 48, + 19, + 33, + 16, + 9 + ], + [ + 9, + 28, + 45, + 57, + 55, + 16 + ], + [ + 59, + 1, + 25, + 47, + 6, + 3 + ], + [ + 9, + 39, + 50, + 18, + 58, + 21 + ], + [ + 60, + 57, + 38, + 53, + 50, + 35 + ], + [ + 41, + 53, + 6, + 38, + 10, + 25 + ], + [ + 1, + 63, + 15, + 24, + 36, + 29 + ], + [ + 41, + 40, + 7, + 42, + 32, + 62 + ], + [ + 4, + 10, + 14, + 37, + 39, + 17 + ], + [ + 61, + 30, + 7, + 50, + 57, + 3 + ], + [ + 9, + 14, + 58, + 30, + 42, + 8 + ], + [ + 33, + 56, + 19, + 60, + 24, + 59 + ], + [ + 9, + 26, + 4, + 51, + 25, + 41 + ], + [ + 4, + 35, + 47, + 41, + 48, + 60 + ], + [ + 39, + 52, + 3, + 26, + 8, + 16 + ], + [ + 4, + 1, + 10, + 61, + 0, + 12 + ] + ], + [ + [ + 51, + 53, + 33, + 13, + 28, + 48 + ], + [ + 63, + 31, + 41, + 39, + 40, + 49 + ], + [ + 42, + 14, + 3, + 24, + 50, + 44 + ], + [ + 11, + 39, + 52, + 10, + 17, + 42 + ], + [ + 7, + 60, + 58, + 15, + 12, + 27 + ], + [ + 34, + 36, + 44, + 28, + 29, + 40 + ], + [ + 52, + 17, + 47, + 42, + 37, + 41 + ], + [ + 62, + 33, + 38, + 39, + 41, + 36 + ], + [ + 50, + 58, + 37, + 10, + 16, + 62 + ], + [ + 41, + 59, + 12, + 62, + 49, + 32 + ], + [ + 3, + 35, + 10, + 41, + 6, + 25 + ], + [ + 39, + 53, + 58, + 21, + 19, + 54 + ], + [ + 42, + 35, + 7, + 39, + 63, + 21 + ], + [ + 15, + 9, + 3, + 54, + 51, + 32 + ], + [ + 3, + 35, + 32, + 36, + 18, + 13 + ], + [ + 20, + 3, + 35, + 45, + 32, + 4 + ], + [ + 46, + 24, + 32, + 33, + 14, + 17 + ], + [ + 16, + 57, + 30, + 19, + 61, + 36 + ], + [ + 41, + 25, + 2, + 26, + 24, + 4 + ], + [ + 42, + 37, + 47, + 48, + 33, + 50 + ], + [ + 54, + 13, + 56, + 53, + 63, + 0 + ], + [ + 34, + 14, + 7, + 51, + 42, + 6 + ], + [ + 6, + 43, + 25, + 58, + 11, + 39 + ], + [ + 0, + 26, + 44, + 4, + 51, + 9 + ], + [ + 37, + 63, + 4, + 13, + 15, + 61 + ], + [ + 9, + 59, + 41, + 1, + 52, + 25 + ], + [ + 56, + 58, + 47, + 1, + 62, + 51 + ] + ], + [ + [ + 52, + 47, + 27, + 36, + 38, + 29 + ], + [ + 43, + 56, + 4, + 25, + 52, + 21 + ], + [ + 25, + 54, + 35, + 18, + 11, + 63 + ], + [ + 17, + 4, + 1, + 18, + 50, + 39 + ], + [ + 36, + 7, + 32, + 4, + 30, + 60 + ], + [ + 34, + 3, + 61, + 48, + 24, + 40 + ], + [ + 17, + 42, + 3, + 12, + 29, + 25 + ], + [ + 33, + 38, + 39, + 55, + 17, + 19 + ], + [ + 17, + 50, + 41, + 13, + 16, + 51 + ], + [ + 41, + 12, + 60, + 16, + 62, + 58 + ], + [ + 3, + 36, + 28, + 39, + 35, + 2 + ], + [ + 48, + 27, + 53, + 54, + 19, + 43 + ], + [ + 63, + 42, + 7, + 35, + 62, + 3 + ], + [ + 56, + 15, + 9, + 51, + 42, + 47 + ], + [ + 35, + 39, + 18, + 36, + 43, + 38 + ], + [ + 60, + 20, + 0, + 3, + 35, + 31 + ], + [ + 24, + 14, + 42, + 17, + 32, + 62 + ], + [ + 24, + 57, + 63, + 0, + 42, + 1 + ], + [ + 41, + 50, + 28, + 32, + 26, + 40 + ], + [ + 2, + 37, + 46, + 17, + 42, + 57 + ], + [ + 49, + 7, + 11, + 54, + 13, + 53 + ], + [ + 14, + 9, + 42, + 6, + 19, + 58 + ], + [ + 63, + 33, + 43, + 19, + 58, + 25 + ], + [ + 9, + 26, + 41, + 4, + 48, + 44 + ], + [ + 4, + 41, + 5, + 15, + 9, + 53 + ], + [ + 14, + 38, + 9, + 3, + 52, + 8 + ], + [ + 23, + 1, + 61, + 4, + 47, + 14 + ] + ], + [ + [ + 37, + 14, + 3, + 5, + 33, + 38 + ], + [ + 34, + 0, + 56, + 58, + 37, + 13 + ], + [ + 61, + 14, + 22, + 29, + 15, + 46 + ], + [ + 50, + 58, + 18, + 44, + 47, + 17 + ], + [ + 4, + 36, + 53, + 7, + 32, + 2 + ], + [ + 22, + 1, + 34, + 61, + 10, + 33 + ], + [ + 28, + 25, + 42, + 30, + 29, + 3 + ], + [ + 33, + 51, + 38, + 39, + 62, + 60 + ], + [ + 17, + 51, + 26, + 16, + 46, + 50 + ], + [ + 12, + 16, + 59, + 6, + 38, + 3 + ], + [ + 39, + 31, + 35, + 3, + 36, + 0 + ], + [ + 48, + 19, + 53, + 56, + 27, + 39 + ], + [ + 7, + 62, + 9, + 63, + 15, + 42 + ], + [ + 56, + 19, + 9, + 47, + 15, + 59 + ], + [ + 39, + 36, + 18, + 35, + 42, + 38 + ], + [ + 60, + 20, + 0, + 57, + 47, + 53 + ], + [ + 6, + 32, + 29, + 62, + 43, + 5 + ], + [ + 24, + 63, + 57, + 1, + 42, + 2 + ], + [ + 41, + 40, + 7, + 62, + 32, + 50 + ], + [ + 37, + 14, + 10, + 2, + 57, + 17 + ], + [ + 7, + 54, + 30, + 53, + 50, + 3 + ], + [ + 9, + 14, + 8, + 6, + 42, + 58 + ], + [ + 33, + 56, + 60, + 19, + 18, + 15 + ], + [ + 9, + 26, + 4, + 41, + 59, + 23 + ], + [ + 4, + 41, + 47, + 9, + 2, + 16 + ], + [ + 3, + 14, + 16, + 30, + 8, + 52 + ], + [ + 1, + 4, + 14, + 61, + 0, + 12 + ] + ], + [ + [ + 27, + 21, + 61, + 30, + 22, + 40 + ], + [ + 33, + 12, + 44, + 0, + 47, + 20 + ], + [ + 14, + 39, + 58, + 37, + 16, + 61 + ], + [ + 47, + 11, + 4, + 18, + 32, + 35 + ], + [ + 2, + 19, + 39, + 20, + 10, + 61 + ], + [ + 39, + 61, + 49, + 15, + 30, + 47 + ], + [ + 12, + 47, + 17, + 49, + 29, + 46 + ], + [ + 39, + 41, + 62, + 30, + 52, + 37 + ], + [ + 21, + 15, + 28, + 48, + 26, + 27 + ], + [ + 12, + 32, + 16, + 24, + 55, + 41 + ], + [ + 43, + 35, + 28, + 31, + 60, + 47 + ], + [ + 3, + 17, + 19, + 49, + 42, + 10 + ], + [ + 7, + 27, + 54, + 59, + 4, + 23 + ], + [ + 35, + 33, + 7, + 6, + 14, + 51 + ], + [ + 51, + 28, + 36, + 42, + 41, + 32 + ], + [ + 45, + 52, + 21, + 60, + 53, + 12 + ], + [ + 53, + 1, + 23, + 5, + 6, + 46 + ], + [ + 35, + 9, + 63, + 16, + 26, + 22 + ], + [ + 41, + 60, + 27, + 40, + 42, + 15 + ], + [ + 42, + 48, + 46, + 9, + 37, + 17 + ], + [ + 31, + 11, + 23, + 58, + 39, + 44 + ], + [ + 34, + 2, + 4, + 14, + 13, + 9 + ], + [ + 60, + 59, + 47, + 40, + 17, + 27 + ], + [ + 60, + 63, + 7, + 42, + 49, + 40 + ], + [ + 59, + 15, + 38, + 62, + 44, + 25 + ], + [ + 23, + 57, + 60, + 31, + 41, + 3 + ], + [ + 38, + 59, + 31, + 51, + 36, + 7 + ] + ], + [ + [ + 12, + 10, + 50, + 23, + 53, + 33 + ], + [ + 48, + 56, + 44, + 11, + 31, + 17 + ], + [ + 33, + 11, + 17, + 54, + 15, + 62 + ], + [ + 31, + 13, + 17, + 40, + 8, + 3 + ], + [ + 42, + 4, + 27, + 15, + 12, + 5 + ], + [ + 34, + 50, + 33, + 29, + 55, + 6 + ], + [ + 16, + 49, + 52, + 29, + 41, + 30 + ], + [ + 0, + 30, + 62, + 57, + 26, + 22 + ], + [ + 26, + 62, + 5, + 58, + 51, + 49 + ], + [ + 5, + 62, + 16, + 46, + 39, + 37 + ], + [ + 31, + 10, + 24, + 45, + 18, + 35 + ], + [ + 50, + 19, + 48, + 54, + 16, + 35 + ], + [ + 59, + 39, + 62, + 7, + 35, + 28 + ], + [ + 20, + 9, + 19, + 6, + 22, + 15 + ], + [ + 35, + 18, + 50, + 32, + 36, + 39 + ], + [ + 3, + 4, + 13, + 38, + 60, + 26 + ], + [ + 46, + 32, + 28, + 3, + 37, + 33 + ], + [ + 30, + 63, + 16, + 19, + 24, + 42 + ], + [ + 41, + 17, + 4, + 2, + 32, + 34 + ], + [ + 42, + 37, + 48, + 3, + 33, + 24 + ], + [ + 56, + 7, + 25, + 11, + 39, + 44 + ], + [ + 42, + 14, + 26, + 13, + 12, + 22 + ], + [ + 25, + 11, + 6, + 42, + 13, + 38 + ], + [ + 0, + 9, + 26, + 41, + 4, + 57 + ], + [ + 37, + 4, + 63, + 41, + 2, + 44 + ], + [ + 9, + 52, + 3, + 41, + 16, + 59 + ], + [ + 58, + 47, + 56, + 1, + 26, + 62 + ] + ], + [ + [ + 54, + 30, + 22, + 26, + 3, + 55 + ], + [ + 44, + 7, + 49, + 50, + 25, + 5 + ], + [ + 54, + 4, + 48, + 58, + 26, + 32 + ], + [ + 20, + 25, + 3, + 9, + 55, + 28 + ], + [ + 7, + 27, + 42, + 12, + 58, + 32 + ], + [ + 50, + 34, + 6, + 42, + 29, + 55 + ], + [ + 16, + 49, + 40, + 3, + 27, + 11 + ], + [ + 51, + 30, + 26, + 62, + 57, + 53 + ], + [ + 3, + 5, + 49, + 28, + 26, + 50 + ], + [ + 16, + 22, + 46, + 6, + 49, + 45 + ], + [ + 31, + 45, + 11, + 10, + 56, + 3 + ], + [ + 21, + 51, + 50, + 19, + 9, + 61 + ], + [ + 28, + 41, + 59, + 13, + 34, + 53 + ], + [ + 23, + 20, + 16, + 9, + 38, + 19 + ], + [ + 50, + 34, + 58, + 20, + 27, + 35 + ], + [ + 8, + 59, + 61, + 29, + 35, + 53 + ], + [ + 3, + 46, + 51, + 10, + 25, + 37 + ], + [ + 62, + 21, + 25, + 9, + 18, + 56 + ], + [ + 34, + 27, + 10, + 29, + 53, + 59 + ], + [ + 56, + 33, + 44, + 24, + 9, + 18 + ], + [ + 44, + 12, + 19, + 8, + 30, + 49 + ], + [ + 47, + 62, + 12, + 51, + 16, + 17 + ], + [ + 11, + 25, + 37, + 58, + 35, + 13 + ], + [ + 4, + 9, + 10, + 39, + 37, + 40 + ], + [ + 22, + 21, + 11, + 48, + 45, + 47 + ], + [ + 10, + 28, + 55, + 12, + 24, + 23 + ], + [ + 30, + 16, + 27, + 32, + 57, + 15 + ] + ], + [ + [ + 16, + 11, + 31, + 46, + 35, + 49 + ], + [ + 13, + 49, + 54, + 5, + 6, + 10 + ], + [ + 36, + 13, + 27, + 46, + 3, + 18 + ], + [ + 24, + 61, + 15, + 0, + 13, + 63 + ], + [ + 17, + 2, + 58, + 50, + 35, + 19 + ], + [ + 8, + 7, + 49, + 52, + 47, + 23 + ], + [ + 61, + 16, + 58, + 20, + 38, + 23 + ], + [ + 20, + 42, + 9, + 51, + 35, + 16 + ], + [ + 25, + 47, + 4, + 1, + 50, + 63 + ], + [ + 54, + 38, + 57, + 27, + 33, + 28 + ], + [ + 1, + 59, + 30, + 60, + 14, + 31 + ], + [ + 31, + 51, + 25, + 14, + 41, + 55 + ], + [ + 34, + 2, + 14, + 11, + 19, + 28 + ], + [ + 40, + 57, + 13, + 61, + 59, + 18 + ], + [ + 44, + 58, + 50, + 45, + 37, + 26 + ], + [ + 5, + 8, + 62, + 24, + 57, + 38 + ], + [ + 13, + 47, + 15, + 3, + 5, + 39 + ], + [ + 25, + 23, + 9, + 18, + 63, + 33 + ], + [ + 1, + 10, + 20, + 8, + 53, + 4 + ], + [ + 38, + 15, + 37, + 34, + 60, + 0 + ], + [ + 41, + 27, + 30, + 57, + 19, + 8 + ], + [ + 11, + 62, + 41, + 14, + 46, + 44 + ], + [ + 11, + 33, + 46, + 31, + 0, + 45 + ], + [ + 4, + 51, + 47, + 16, + 9, + 12 + ], + [ + 11, + 21, + 45, + 47, + 18, + 61 + ], + [ + 10, + 50, + 51, + 12, + 18, + 7 + ], + [ + 17, + 31, + 36, + 5, + 19, + 1 + ] + ], + [ + [ + 22, + 6, + 57, + 39, + 29, + 47 + ], + [ + 27, + 6, + 14, + 17, + 51, + 32 + ], + [ + 1, + 29, + 11, + 26, + 47, + 51 + ], + [ + 14, + 38, + 22, + 31, + 29, + 53 + ], + [ + 14, + 61, + 59, + 29, + 1, + 49 + ], + [ + 30, + 8, + 21, + 47, + 52, + 0 + ], + [ + 4, + 58, + 61, + 23, + 20, + 29 + ], + [ + 20, + 9, + 42, + 35, + 7, + 24 + ], + [ + 47, + 25, + 4, + 1, + 29, + 0 + ], + [ + 54, + 19, + 38, + 29, + 33, + 44 + ], + [ + 14, + 1, + 59, + 40, + 60, + 20 + ], + [ + 51, + 0, + 14, + 62, + 16, + 52 + ], + [ + 2, + 36, + 20, + 29, + 19, + 52 + ], + [ + 57, + 13, + 40, + 22, + 60, + 6 + ], + [ + 37, + 44, + 58, + 5, + 8, + 50 + ], + [ + 24, + 5, + 43, + 62, + 23, + 59 + ], + [ + 13, + 19, + 47, + 39, + 61, + 15 + ], + [ + 58, + 33, + 9, + 7, + 4, + 28 + ], + [ + 1, + 35, + 10, + 19, + 31, + 20 + ], + [ + 15, + 55, + 63, + 18, + 34, + 38 + ], + [ + 27, + 15, + 38, + 30, + 57, + 42 + ], + [ + 62, + 41, + 16, + 29, + 6, + 46 + ], + [ + 46, + 33, + 45, + 10, + 34, + 23 + ], + [ + 4, + 47, + 51, + 1, + 16, + 41 + ], + [ + 11, + 21, + 18, + 7, + 48, + 28 + ], + [ + 18, + 51, + 7, + 50, + 6, + 32 + ], + [ + 9, + 11, + 36, + 55, + 43, + 48 + ] + ], + [ + [ + 47, + 8, + 36, + 61, + 21, + 45 + ], + [ + 46, + 2, + 15, + 32, + 0, + 51 + ], + [ + 24, + 15, + 33, + 61, + 2, + 43 + ], + [ + 60, + 22, + 31, + 27, + 14, + 11 + ], + [ + 59, + 58, + 39, + 57, + 46, + 3 + ], + [ + 57, + 43, + 2, + 31, + 7, + 62 + ], + [ + 9, + 42, + 54, + 19, + 4, + 55 + ], + [ + 46, + 14, + 7, + 24, + 43, + 35 + ], + [ + 47, + 4, + 0, + 37, + 12, + 13 + ], + [ + 54, + 38, + 4, + 47, + 25, + 6 + ], + [ + 47, + 14, + 15, + 24, + 1, + 61 + ], + [ + 16, + 54, + 5, + 0, + 7, + 63 + ], + [ + 49, + 3, + 33, + 13, + 11, + 10 + ], + [ + 18, + 13, + 10, + 39, + 58, + 60 + ], + [ + 58, + 62, + 10, + 33, + 5, + 26 + ], + [ + 48, + 5, + 63, + 53, + 2, + 43 + ], + [ + 24, + 47, + 51, + 15, + 0, + 53 + ], + [ + 44, + 0, + 6, + 3, + 7, + 34 + ], + [ + 58, + 38, + 53, + 61, + 0, + 54 + ], + [ + 55, + 9, + 15, + 27, + 42, + 3 + ], + [ + 43, + 1, + 51, + 41, + 4, + 5 + ], + [ + 49, + 20, + 15, + 6, + 37, + 46 + ], + [ + 11, + 25, + 52, + 5, + 4, + 39 + ], + [ + 38, + 40, + 44, + 51, + 10, + 14 + ], + [ + 13, + 8, + 52, + 63, + 2, + 23 + ], + [ + 23, + 38, + 59, + 57, + 55, + 41 + ], + [ + 23, + 6, + 62, + 0, + 7, + 28 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 61 + ], + [ + 51, + 41, + 5, + 15, + 10, + 61 + ], + [ + 43, + 1, + 29, + 55, + 21, + 60 + ], + [ + 24, + 53, + 25, + 13, + 51, + 32 + ], + [ + 41, + 31, + 57, + 49, + 34, + 11 + ], + [ + 17, + 4, + 35, + 30, + 10, + 38 + ], + [ + 34, + 7, + 56, + 42, + 21, + 19 + ], + [ + 14, + 46, + 7, + 27, + 25, + 52 + ], + [ + 0, + 4, + 6, + 12, + 47, + 60 + ], + [ + 54, + 25, + 4, + 38, + 47, + 6 + ], + [ + 24, + 61, + 15, + 46, + 7, + 22 + ], + [ + 5, + 16, + 57, + 0, + 22, + 55 + ], + [ + 49, + 3, + 26, + 17, + 57, + 52 + ], + [ + 13, + 10, + 61, + 60, + 0, + 58 + ], + [ + 58, + 6, + 49, + 10, + 5, + 2 + ], + [ + 25, + 19, + 2, + 11, + 54, + 53 + ], + [ + 47, + 30, + 27, + 5, + 18, + 10 + ], + [ + 44, + 55, + 63, + 13, + 22, + 31 + ], + [ + 58, + 7, + 38, + 35, + 32, + 40 + ], + [ + 13, + 34, + 16, + 45, + 49, + 55 + ], + [ + 51, + 35, + 30, + 58, + 55, + 4 + ], + [ + 46, + 57, + 15, + 36, + 30, + 27 + ], + [ + 52, + 33, + 23, + 10, + 51, + 15 + ], + [ + 40, + 41, + 4, + 51, + 10, + 31 + ], + [ + 8, + 4, + 59, + 48, + 34, + 9 + ], + [ + 43, + 3, + 27, + 26, + 31, + 19 + ], + [ + 46, + 18, + 8, + 4, + 50, + 29 + ] + ], + [ + [ + 22, + 36, + 35, + 63, + 43, + 23 + ], + [ + 54, + 30, + 4, + 36, + 35, + 55 + ], + [ + 28, + 19, + 23, + 49, + 50, + 59 + ], + [ + 62, + 5, + 50, + 53, + 48, + 42 + ], + [ + 0, + 3, + 61, + 57, + 41, + 49 + ], + [ + 60, + 4, + 29, + 16, + 53, + 30 + ], + [ + 34, + 32, + 33, + 9, + 56, + 35 + ], + [ + 12, + 53, + 14, + 36, + 25, + 61 + ], + [ + 23, + 48, + 35, + 29, + 4, + 16 + ], + [ + 22, + 25, + 4, + 54, + 62, + 5 + ], + [ + 24, + 19, + 27, + 55, + 47, + 25 + ], + [ + 18, + 5, + 22, + 34, + 63, + 30 + ], + [ + 17, + 49, + 30, + 28, + 11, + 42 + ], + [ + 29, + 13, + 54, + 25, + 45, + 47 + ], + [ + 27, + 2, + 10, + 5, + 54, + 33 + ], + [ + 55, + 2, + 57, + 54, + 56, + 22 + ], + [ + 53, + 3, + 60, + 27, + 5, + 50 + ], + [ + 17, + 8, + 47, + 61, + 50, + 44 + ], + [ + 27, + 38, + 32, + 14, + 61, + 39 + ], + [ + 22, + 43, + 32, + 57, + 39, + 34 + ], + [ + 16, + 44, + 37, + 23, + 61, + 27 + ], + [ + 45, + 40, + 55, + 32, + 31, + 3 + ], + [ + 32, + 28, + 41, + 15, + 1, + 52 + ], + [ + 21, + 22, + 31, + 10, + 4, + 40 + ], + [ + 29, + 35, + 62, + 60, + 41, + 1 + ], + [ + 39, + 58, + 1, + 63, + 3, + 35 + ], + [ + 10, + 62, + 31, + 45, + 27, + 7 + ] + ], + [ + [ + 17, + 5, + 10, + 57, + 14, + 27 + ], + [ + 43, + 9, + 33, + 56, + 1, + 20 + ], + [ + 63, + 1, + 35, + 43, + 27, + 10 + ], + [ + 47, + 18, + 6, + 3, + 38, + 15 + ], + [ + 11, + 51, + 61, + 34, + 44, + 55 + ], + [ + 10, + 63, + 53, + 60, + 37, + 58 + ], + [ + 51, + 45, + 63, + 34, + 18, + 60 + ], + [ + 35, + 51, + 52, + 53, + 38, + 45 + ], + [ + 44, + 0, + 35, + 55, + 38, + 9 + ], + [ + 42, + 5, + 43, + 25, + 21, + 6 + ], + [ + 42, + 13, + 27, + 25, + 38, + 32 + ], + [ + 43, + 5, + 1, + 38, + 2, + 22 + ], + [ + 17, + 49, + 0, + 37, + 28, + 30 + ], + [ + 2, + 29, + 54, + 5, + 13, + 60 + ], + [ + 2, + 27, + 10, + 58, + 40, + 28 + ], + [ + 2, + 49, + 54, + 62, + 53, + 57 + ], + [ + 60, + 53, + 12, + 27, + 28, + 0 + ], + [ + 17, + 50, + 51, + 33, + 3, + 11 + ], + [ + 38, + 5, + 19, + 13, + 27, + 32 + ], + [ + 43, + 22, + 0, + 57, + 40, + 63 + ], + [ + 27, + 16, + 35, + 52, + 38, + 39 + ], + [ + 29, + 31, + 55, + 40, + 62, + 27 + ], + [ + 41, + 48, + 32, + 46, + 40, + 57 + ], + [ + 20, + 54, + 4, + 22, + 0, + 55 + ], + [ + 62, + 23, + 25, + 33, + 28, + 20 + ], + [ + 7, + 18, + 60, + 22, + 58, + 3 + ], + [ + 36, + 9, + 11, + 0, + 48, + 31 + ] + ], + [ + [ + 11, + 59, + 52, + 28, + 6, + 45 + ], + [ + 7, + 23, + 15, + 17, + 55, + 61 + ], + [ + 28, + 0, + 29, + 46, + 58, + 14 + ], + [ + 62, + 49, + 39, + 18, + 6, + 13 + ], + [ + 36, + 61, + 7, + 40, + 35, + 33 + ], + [ + 8, + 16, + 29, + 43, + 57, + 37 + ], + [ + 30, + 19, + 21, + 57, + 42, + 14 + ], + [ + 35, + 11, + 53, + 51, + 36, + 33 + ], + [ + 48, + 35, + 4, + 50, + 11, + 40 + ], + [ + 5, + 3, + 21, + 47, + 43, + 38 + ], + [ + 21, + 18, + 2, + 6, + 3, + 10 + ], + [ + 18, + 43, + 5, + 45, + 22, + 4 + ], + [ + 33, + 28, + 37, + 41, + 49, + 10 + ], + [ + 56, + 55, + 22, + 54, + 62, + 15 + ], + [ + 2, + 27, + 57, + 5, + 63, + 19 + ], + [ + 55, + 8, + 54, + 2, + 48, + 59 + ], + [ + 0, + 53, + 51, + 3, + 50, + 52 + ], + [ + 21, + 12, + 62, + 60, + 18, + 43 + ], + [ + 38, + 13, + 53, + 27, + 14, + 28 + ], + [ + 22, + 56, + 44, + 43, + 51, + 53 + ], + [ + 29, + 43, + 12, + 16, + 41, + 52 + ], + [ + 51, + 1, + 35, + 44, + 34, + 48 + ], + [ + 32, + 25, + 4, + 41, + 53, + 54 + ], + [ + 48, + 17, + 25, + 1, + 60, + 62 + ], + [ + 22, + 29, + 5, + 18, + 53, + 20 + ], + [ + 28, + 55, + 15, + 1, + 8, + 49 + ], + [ + 30, + 57, + 6, + 7, + 31, + 50 + ] + ], + [ + [ + 11, + 16, + 31, + 46, + 35, + 49 + ], + [ + 13, + 49, + 31, + 16, + 34, + 10 + ], + [ + 36, + 13, + 27, + 34, + 42, + 18 + ], + [ + 24, + 7, + 61, + 63, + 15, + 34 + ], + [ + 17, + 35, + 2, + 48, + 44, + 62 + ], + [ + 8, + 7, + 23, + 47, + 51, + 26 + ], + [ + 61, + 58, + 29, + 38, + 33, + 45 + ], + [ + 20, + 9, + 42, + 27, + 54, + 29 + ], + [ + 25, + 47, + 34, + 1, + 4, + 38 + ], + [ + 54, + 27, + 57, + 4, + 28, + 25 + ], + [ + 1, + 30, + 59, + 60, + 17, + 22 + ], + [ + 31, + 25, + 51, + 5, + 58, + 49 + ], + [ + 34, + 2, + 19, + 14, + 29, + 36 + ], + [ + 57, + 40, + 6, + 9, + 32, + 14 + ], + [ + 8, + 44, + 34, + 16, + 45, + 47 + ], + [ + 8, + 25, + 40, + 37, + 14, + 59 + ], + [ + 13, + 15, + 19, + 51, + 25, + 37 + ], + [ + 23, + 4, + 44, + 52, + 19, + 15 + ], + [ + 8, + 10, + 1, + 45, + 31, + 53 + ], + [ + 38, + 59, + 15, + 2, + 34, + 0 + ], + [ + 41, + 25, + 57, + 27, + 55, + 24 + ], + [ + 11, + 44, + 41, + 39, + 53, + 62 + ], + [ + 31, + 30, + 42, + 37, + 34, + 59 + ], + [ + 33, + 14, + 12, + 10, + 54, + 51 + ], + [ + 11, + 18, + 45, + 20, + 33, + 48 + ], + [ + 50, + 51, + 54, + 23, + 10, + 2 + ], + [ + 17, + 31, + 37, + 5, + 19, + 36 + ] + ], + [ + [ + 31, + 35, + 51, + 18, + 53, + 61 + ], + [ + 7, + 40, + 39, + 41, + 31, + 37 + ], + [ + 16, + 29, + 26, + 50, + 33, + 10 + ], + [ + 14, + 22, + 37, + 17, + 6, + 25 + ], + [ + 60, + 20, + 46, + 4, + 3, + 57 + ], + [ + 11, + 2, + 49, + 21, + 9, + 27 + ], + [ + 42, + 30, + 54, + 58, + 19, + 56 + ], + [ + 46, + 33, + 38, + 41, + 35, + 49 + ], + [ + 50, + 48, + 38, + 37, + 57, + 10 + ], + [ + 59, + 63, + 47, + 28, + 10, + 35 + ], + [ + 28, + 3, + 2, + 52, + 33, + 43 + ], + [ + 58, + 19, + 17, + 16, + 57, + 63 + ], + [ + 42, + 4, + 3, + 23, + 45, + 57 + ], + [ + 18, + 42, + 51, + 3, + 20, + 15 + ], + [ + 32, + 43, + 12, + 3, + 0, + 35 + ], + [ + 15, + 45, + 4, + 7, + 53, + 21 + ], + [ + 28, + 24, + 14, + 15, + 60, + 31 + ], + [ + 26, + 30, + 0, + 54, + 5, + 36 + ], + [ + 63, + 37, + 11, + 41, + 51, + 4 + ], + [ + 48, + 9, + 33, + 0, + 42, + 54 + ], + [ + 22, + 60, + 11, + 39, + 1, + 49 + ], + [ + 17, + 4, + 34, + 2, + 27, + 53 + ], + [ + 17, + 6, + 40, + 58, + 42, + 39 + ], + [ + 60, + 54, + 35, + 39, + 0, + 32 + ], + [ + 44, + 15, + 37, + 13, + 8, + 25 + ], + [ + 41, + 59, + 25, + 45, + 13, + 6 + ], + [ + 23, + 58, + 13, + 19, + 29, + 62 + ] + ], + [ + [ + 52, + 47, + 27, + 36, + 33, + 38 + ], + [ + 43, + 56, + 4, + 21, + 25, + 6 + ], + [ + 25, + 54, + 35, + 18, + 11, + 57 + ], + [ + 17, + 16, + 1, + 6, + 33, + 45 + ], + [ + 32, + 36, + 0, + 60, + 46, + 57 + ], + [ + 11, + 1, + 3, + 21, + 2, + 34 + ], + [ + 30, + 42, + 58, + 54, + 19, + 13 + ], + [ + 33, + 56, + 35, + 42, + 38, + 45 + ], + [ + 50, + 37, + 36, + 14, + 20, + 48 + ], + [ + 10, + 28, + 4, + 37, + 32, + 6 + ], + [ + 3, + 18, + 28, + 61, + 2, + 44 + ], + [ + 16, + 19, + 56, + 27, + 43, + 46 + ], + [ + 42, + 33, + 57, + 3, + 58, + 26 + ], + [ + 56, + 15, + 51, + 55, + 50, + 13 + ], + [ + 12, + 43, + 40, + 13, + 16, + 29 + ], + [ + 15, + 7, + 28, + 53, + 5, + 20 + ], + [ + 14, + 58, + 24, + 60, + 31, + 51 + ], + [ + 0, + 60, + 3, + 24, + 19, + 44 + ], + [ + 36, + 28, + 11, + 63, + 53, + 15 + ], + [ + 2, + 26, + 9, + 34, + 0, + 3 + ], + [ + 49, + 28, + 43, + 41, + 30, + 11 + ], + [ + 25, + 51, + 12, + 6, + 61, + 16 + ], + [ + 39, + 17, + 41, + 50, + 40, + 21 + ], + [ + 25, + 58, + 48, + 12, + 60, + 33 + ], + [ + 31, + 49, + 5, + 52, + 63, + 3 + ], + [ + 15, + 55, + 38, + 47, + 1, + 49 + ], + [ + 23, + 6, + 32, + 19, + 62, + 7 + ] + ], + [ + [ + 44, + 24, + 56, + 33, + 15, + 7 + ], + [ + 38, + 26, + 24, + 29, + 53, + 19 + ], + [ + 12, + 15, + 29, + 9, + 1, + 63 + ], + [ + 38, + 61, + 58, + 50, + 45, + 6 + ], + [ + 24, + 34, + 4, + 36, + 57, + 31 + ], + [ + 1, + 22, + 43, + 21, + 10, + 7 + ], + [ + 20, + 19, + 54, + 58, + 18, + 42 + ], + [ + 56, + 33, + 14, + 21, + 51, + 18 + ], + [ + 60, + 50, + 14, + 36, + 43, + 4 + ], + [ + 6, + 10, + 9, + 63, + 4, + 38 + ], + [ + 54, + 39, + 46, + 18, + 3, + 2 + ], + [ + 30, + 16, + 37, + 53, + 56, + 43 + ], + [ + 56, + 10, + 42, + 58, + 57, + 26 + ], + [ + 30, + 56, + 13, + 50, + 51, + 0 + ], + [ + 55, + 40, + 62, + 12, + 13, + 30 + ], + [ + 53, + 28, + 11, + 61, + 7, + 19 + ], + [ + 55, + 14, + 17, + 47, + 30, + 5 + ], + [ + 18, + 31, + 20, + 60, + 57, + 32 + ], + [ + 43, + 12, + 53, + 26, + 32, + 61 + ], + [ + 25, + 16, + 26, + 61, + 53, + 3 + ], + [ + 49, + 28, + 51, + 59, + 55, + 11 + ], + [ + 6, + 56, + 32, + 14, + 21, + 10 + ], + [ + 2, + 15, + 58, + 17, + 13, + 62 + ], + [ + 53, + 51, + 4, + 29, + 50, + 25 + ], + [ + 61, + 31, + 19, + 15, + 60, + 49 + ], + [ + 46, + 19, + 44, + 36, + 8, + 40 + ], + [ + 8, + 29, + 46, + 7, + 53, + 20 + ] + ], + [ + [ + 48, + 42, + 38, + 63, + 50, + 7 + ], + [ + 3, + 40, + 2, + 33, + 14, + 60 + ], + [ + 39, + 7, + 45, + 40, + 6, + 44 + ], + [ + 41, + 5, + 20, + 56, + 13, + 0 + ], + [ + 6, + 30, + 37, + 1, + 38, + 52 + ], + [ + 59, + 46, + 4, + 22, + 5, + 6 + ], + [ + 20, + 1, + 44, + 35, + 13, + 3 + ], + [ + 12, + 56, + 8, + 50, + 31, + 2 + ], + [ + 33, + 60, + 41, + 43, + 37, + 36 + ], + [ + 9, + 10, + 0, + 55, + 37, + 40 + ], + [ + 54, + 39, + 9, + 8, + 61, + 46 + ], + [ + 30, + 56, + 53, + 24, + 16, + 11 + ], + [ + 7, + 58, + 57, + 42, + 10, + 52 + ], + [ + 27, + 30, + 42, + 25, + 59, + 13 + ], + [ + 9, + 11, + 55, + 49, + 46, + 0 + ], + [ + 34, + 53, + 54, + 29, + 57, + 37 + ], + [ + 55, + 17, + 5, + 60, + 18, + 31 + ], + [ + 57, + 48, + 43, + 39, + 32, + 36 + ], + [ + 12, + 43, + 42, + 49, + 7, + 61 + ], + [ + 23, + 36, + 1, + 0, + 16, + 61 + ], + [ + 21, + 35, + 11, + 31, + 55, + 7 + ], + [ + 60, + 8, + 42, + 24, + 14, + 6 + ], + [ + 15, + 51, + 58, + 2, + 33, + 30 + ], + [ + 52, + 51, + 4, + 28, + 21, + 19 + ], + [ + 19, + 60, + 15, + 34, + 54, + 46 + ], + [ + 19, + 61, + 58, + 12, + 40, + 3 + ], + [ + 35, + 49, + 54, + 53, + 1, + 25 + ] + ], + [ + [ + 47, + 37, + 59, + 38, + 33, + 10 + ], + [ + 36, + 7, + 21, + 51, + 8, + 47 + ], + [ + 20, + 32, + 44, + 47, + 4, + 54 + ], + [ + 43, + 20, + 42, + 52, + 8, + 19 + ], + [ + 27, + 9, + 39, + 57, + 12, + 54 + ], + [ + 28, + 16, + 29, + 11, + 61, + 58 + ], + [ + 42, + 2, + 30, + 43, + 28, + 25 + ], + [ + 18, + 36, + 32, + 21, + 53, + 15 + ], + [ + 50, + 12, + 37, + 48, + 14, + 52 + ], + [ + 9, + 10, + 13, + 3, + 58, + 26 + ], + [ + 18, + 3, + 9, + 55, + 6, + 61 + ], + [ + 30, + 18, + 45, + 60, + 16, + 24 + ], + [ + 35, + 5, + 42, + 26, + 37, + 58 + ], + [ + 55, + 38, + 15, + 13, + 14, + 54 + ], + [ + 9, + 62, + 2, + 17, + 13, + 26 + ], + [ + 9, + 40, + 37, + 54, + 17, + 47 + ], + [ + 22, + 60, + 51, + 14, + 40, + 39 + ], + [ + 17, + 3, + 21, + 30, + 36, + 25 + ], + [ + 56, + 41, + 24, + 9, + 43, + 14 + ], + [ + 53, + 34, + 26, + 22, + 12, + 35 + ], + [ + 36, + 42, + 24, + 37, + 8, + 4 + ], + [ + 34, + 16, + 8, + 46, + 56, + 6 + ], + [ + 31, + 10, + 16, + 41, + 56, + 32 + ], + [ + 48, + 0, + 42, + 56, + 31, + 30 + ], + [ + 6, + 54, + 26, + 19, + 8, + 7 + ], + [ + 49, + 6, + 22, + 13, + 24, + 59 + ], + [ + 45, + 62, + 27, + 47, + 50, + 7 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 3 + ], + [ + 8, + 60, + 59, + 43, + 10, + 48 + ], + [ + 51, + 45, + 28, + 34, + 59, + 63 + ], + [ + 43, + 2, + 38, + 12, + 20, + 4 + ], + [ + 50, + 57, + 39, + 31, + 63, + 0 + ], + [ + 58, + 53, + 18, + 9, + 30, + 21 + ], + [ + 63, + 51, + 34, + 7, + 20, + 27 + ], + [ + 32, + 21, + 46, + 47, + 25, + 18 + ], + [ + 6, + 12, + 50, + 14, + 33, + 37 + ], + [ + 30, + 10, + 9, + 6, + 13, + 29 + ], + [ + 61, + 18, + 23, + 20, + 44, + 6 + ], + [ + 16, + 12, + 30, + 60, + 0, + 62 + ], + [ + 5, + 26, + 57, + 35, + 37, + 61 + ], + [ + 46, + 55, + 13, + 1, + 17, + 31 + ], + [ + 6, + 10, + 22, + 2, + 16, + 15 + ], + [ + 6, + 54, + 19, + 25, + 57, + 46 + ], + [ + 2, + 30, + 60, + 61, + 18, + 22 + ], + [ + 17, + 4, + 2, + 27, + 3, + 44 + ], + [ + 56, + 46, + 8, + 32, + 6, + 14 + ], + [ + 5, + 13, + 0, + 34, + 14, + 30 + ], + [ + 15, + 23, + 59, + 57, + 24, + 27 + ], + [ + 36, + 61, + 8, + 43, + 57, + 37 + ], + [ + 10, + 15, + 52, + 31, + 29, + 23 + ], + [ + 10, + 38, + 1, + 57, + 31, + 4 + ], + [ + 8, + 32, + 54, + 33, + 3, + 18 + ], + [ + 6, + 33, + 19, + 50, + 2, + 3 + ], + [ + 55, + 43, + 4, + 5, + 25, + 8 + ] + ], + [ + [ + 25, + 14, + 18, + 49, + 51, + 63 + ], + [ + 42, + 21, + 30, + 43, + 24, + 7 + ], + [ + 54, + 39, + 9, + 59, + 28, + 49 + ], + [ + 23, + 1, + 55, + 45, + 43, + 33 + ], + [ + 37, + 41, + 30, + 59, + 21, + 44 + ], + [ + 4, + 41, + 31, + 35, + 19, + 14 + ], + [ + 28, + 55, + 44, + 63, + 9, + 51 + ], + [ + 63, + 12, + 32, + 13, + 47, + 28 + ], + [ + 33, + 12, + 59, + 35, + 6, + 39 + ], + [ + 30, + 40, + 10, + 29, + 52, + 13 + ], + [ + 63, + 19, + 23, + 61, + 8, + 4 + ], + [ + 45, + 62, + 22, + 12, + 38, + 42 + ], + [ + 58, + 26, + 57, + 20, + 45, + 30 + ], + [ + 8, + 59, + 47, + 17, + 25, + 22 + ], + [ + 24, + 54, + 51, + 55, + 10, + 15 + ], + [ + 46, + 22, + 54, + 44, + 57, + 40 + ], + [ + 34, + 60, + 2, + 17, + 18, + 27 + ], + [ + 51, + 6, + 61, + 2, + 39, + 32 + ], + [ + 5, + 49, + 38, + 8, + 32, + 46 + ], + [ + 57, + 4, + 23, + 14, + 5, + 59 + ], + [ + 7, + 36, + 45, + 21, + 53, + 4 + ], + [ + 60, + 3, + 39, + 8, + 14, + 35 + ], + [ + 33, + 57, + 60, + 47, + 15, + 52 + ], + [ + 20, + 11, + 22, + 58, + 19, + 4 + ], + [ + 36, + 34, + 47, + 41, + 60, + 40 + ], + [ + 33, + 3, + 49, + 29, + 14, + 59 + ], + [ + 52, + 24, + 60, + 25, + 35, + 34 + ] + ], + [ + [ + 23, + 54, + 53, + 11, + 58, + 8 + ], + [ + 11, + 30, + 15, + 63, + 59, + 16 + ], + [ + 20, + 29, + 58, + 17, + 52, + 30 + ], + [ + 18, + 1, + 43, + 15, + 3, + 8 + ], + [ + 59, + 55, + 13, + 20, + 44, + 30 + ], + [ + 33, + 45, + 27, + 53, + 63, + 52 + ], + [ + 28, + 57, + 51, + 34, + 53, + 31 + ], + [ + 13, + 63, + 3, + 32, + 44, + 45 + ], + [ + 59, + 33, + 12, + 57, + 6, + 35 + ], + [ + 23, + 30, + 42, + 10, + 29, + 13 + ], + [ + 63, + 23, + 18, + 42, + 38, + 19 + ], + [ + 62, + 2, + 22, + 19, + 45, + 6 + ], + [ + 6, + 26, + 20, + 30, + 5, + 25 + ], + [ + 3, + 6, + 8, + 31, + 17, + 37 + ], + [ + 59, + 2, + 10, + 54, + 55, + 19 + ], + [ + 45, + 7, + 57, + 42, + 54, + 46 + ], + [ + 60, + 55, + 53, + 34, + 41, + 18 + ], + [ + 36, + 2, + 8, + 31, + 32, + 46 + ], + [ + 36, + 45, + 38, + 32, + 61, + 25 + ], + [ + 57, + 39, + 6, + 18, + 19, + 30 + ], + [ + 37, + 39, + 59, + 48, + 53, + 14 + ], + [ + 52, + 28, + 46, + 55, + 47, + 61 + ], + [ + 1, + 28, + 52, + 15, + 31, + 18 + ], + [ + 22, + 21, + 8, + 10, + 46, + 39 + ], + [ + 45, + 41, + 35, + 9, + 54, + 33 + ], + [ + 42, + 39, + 58, + 61, + 24, + 3 + ], + [ + 41, + 46, + 10, + 3, + 15, + 33 + ] + ], + [ + [ + 48, + 63, + 38, + 42, + 7, + 37 + ], + [ + 3, + 10, + 26, + 6, + 2, + 62 + ], + [ + 39, + 7, + 44, + 6, + 45, + 40 + ], + [ + 21, + 39, + 1, + 18, + 15, + 41 + ], + [ + 6, + 33, + 9, + 13, + 3, + 27 + ], + [ + 59, + 33, + 53, + 62, + 21, + 45 + ], + [ + 28, + 10, + 33, + 7, + 50, + 57 + ], + [ + 50, + 13, + 12, + 49, + 3, + 55 + ], + [ + 33, + 59, + 26, + 35, + 48, + 38 + ], + [ + 23, + 30, + 20, + 43, + 10, + 58 + ], + [ + 63, + 23, + 18, + 17, + 38, + 9 + ], + [ + 22, + 62, + 33, + 36, + 6, + 2 + ], + [ + 12, + 17, + 59, + 7, + 26, + 49 + ], + [ + 47, + 27, + 3, + 6, + 24, + 22 + ], + [ + 59, + 57, + 46, + 54, + 2, + 10 + ], + [ + 27, + 45, + 57, + 54, + 34, + 61 + ], + [ + 34, + 60, + 55, + 31, + 58, + 43 + ], + [ + 48, + 42, + 32, + 39, + 2, + 38 + ], + [ + 42, + 49, + 32, + 44, + 12, + 61 + ], + [ + 57, + 36, + 39, + 1, + 30, + 59 + ], + [ + 48, + 7, + 21, + 53, + 17, + 29 + ], + [ + 8, + 60, + 58, + 35, + 14, + 46 + ], + [ + 51, + 15, + 28, + 47, + 33, + 30 + ], + [ + 11, + 19, + 4, + 8, + 58, + 52 + ], + [ + 24, + 5, + 41, + 60, + 40, + 54 + ], + [ + 61, + 3, + 21, + 58, + 19, + 8 + ], + [ + 60, + 35, + 54, + 49, + 1, + 0 + ] + ], + [ + [ + 6, + 24, + 63, + 25, + 26, + 45 + ], + [ + 47, + 13, + 49, + 44, + 20, + 19 + ], + [ + 23, + 32, + 49, + 20, + 24, + 2 + ], + [ + 43, + 21, + 8, + 40, + 39, + 45 + ], + [ + 39, + 29, + 3, + 5, + 41, + 12 + ], + [ + 61, + 33, + 48, + 40, + 29, + 62 + ], + [ + 29, + 28, + 25, + 33, + 44, + 31 + ], + [ + 50, + 33, + 13, + 11, + 30, + 54 + ], + [ + 48, + 26, + 17, + 35, + 55, + 6 + ], + [ + 43, + 23, + 12, + 9, + 25, + 62 + ], + [ + 63, + 35, + 18, + 53, + 38, + 17 + ], + [ + 19, + 48, + 5, + 36, + 59, + 17 + ], + [ + 7, + 26, + 59, + 17, + 12, + 30 + ], + [ + 47, + 22, + 24, + 62, + 25, + 42 + ], + [ + 59, + 46, + 35, + 39, + 57, + 54 + ], + [ + 27, + 9, + 20, + 0, + 57, + 52 + ], + [ + 38, + 58, + 60, + 34, + 43, + 29 + ], + [ + 42, + 32, + 39, + 54, + 38, + 57 + ], + [ + 59, + 32, + 42, + 6, + 21, + 37 + ], + [ + 24, + 36, + 57, + 4, + 30, + 60 + ], + [ + 9, + 48, + 7, + 53, + 21, + 58 + ], + [ + 49, + 31, + 14, + 8, + 22, + 19 + ], + [ + 15, + 33, + 44, + 8, + 14, + 3 + ], + [ + 11, + 4, + 21, + 28, + 41, + 23 + ], + [ + 37, + 27, + 24, + 12, + 9, + 20 + ], + [ + 58, + 9, + 19, + 3, + 12, + 48 + ], + [ + 24, + 14, + 60, + 47, + 25, + 13 + ] + ], + [ + [ + 26, + 62, + 58, + 18, + 38, + 5 + ], + [ + 19, + 12, + 40, + 39, + 31, + 57 + ], + [ + 27, + 38, + 9, + 22, + 23, + 61 + ], + [ + 42, + 20, + 63, + 39, + 45, + 43 + ], + [ + 7, + 36, + 60, + 29, + 57, + 24 + ], + [ + 61, + 34, + 3, + 1, + 44, + 51 + ], + [ + 29, + 42, + 25, + 47, + 30, + 17 + ], + [ + 33, + 38, + 39, + 50, + 62, + 0 + ], + [ + 17, + 26, + 13, + 50, + 41, + 11 + ], + [ + 12, + 43, + 35, + 16, + 55, + 60 + ], + [ + 3, + 18, + 39, + 63, + 35, + 62 + ], + [ + 19, + 48, + 39, + 60, + 54, + 2 + ], + [ + 7, + 42, + 56, + 59, + 37, + 24 + ], + [ + 56, + 47, + 54, + 59, + 3, + 6 + ], + [ + 35, + 39, + 43, + 59, + 2, + 13 + ], + [ + 20, + 27, + 9, + 56, + 0, + 54 + ], + [ + 17, + 60, + 29, + 18, + 58, + 5 + ], + [ + 57, + 42, + 45, + 0, + 24, + 29 + ], + [ + 49, + 28, + 52, + 32, + 50, + 3 + ], + [ + 24, + 2, + 36, + 30, + 59, + 46 + ], + [ + 11, + 7, + 48, + 54, + 53, + 21 + ], + [ + 33, + 14, + 42, + 6, + 8, + 5 + ], + [ + 15, + 36, + 33, + 51, + 17, + 44 + ], + [ + 41, + 11, + 2, + 4, + 8, + 29 + ], + [ + 30, + 15, + 60, + 46, + 5, + 9 + ], + [ + 3, + 21, + 14, + 19, + 61, + 58 + ], + [ + 24, + 60, + 1, + 14, + 35, + 53 + ] + ], + [ + [ + 8, + 56, + 54, + 4, + 37, + 7 + ], + [ + 36, + 9, + 24, + 8, + 1, + 2 + ], + [ + 16, + 57, + 29, + 32, + 58, + 5 + ], + [ + 32, + 63, + 42, + 52, + 4, + 20 + ], + [ + 27, + 24, + 44, + 39, + 20, + 63 + ], + [ + 24, + 11, + 28, + 6, + 15, + 31 + ], + [ + 30, + 41, + 42, + 25, + 52, + 2 + ], + [ + 26, + 36, + 62, + 18, + 50, + 21 + ], + [ + 48, + 50, + 32, + 51, + 27, + 26 + ], + [ + 13, + 43, + 3, + 26, + 12, + 41 + ], + [ + 3, + 32, + 18, + 53, + 39, + 17 + ], + [ + 21, + 19, + 9, + 48, + 36, + 1 + ], + [ + 35, + 59, + 42, + 54, + 63, + 17 + ], + [ + 15, + 12, + 38, + 9, + 51, + 54 + ], + [ + 3, + 2, + 48, + 59, + 57, + 13 + ], + [ + 27, + 9, + 4, + 20, + 22, + 3 + ], + [ + 60, + 40, + 22, + 27, + 46, + 32 + ], + [ + 30, + 21, + 25, + 17, + 0, + 42 + ], + [ + 33, + 27, + 34, + 28, + 38, + 44 + ], + [ + 53, + 32, + 33, + 35, + 31, + 56 + ], + [ + 40, + 11, + 20, + 47, + 48, + 24 + ], + [ + 45, + 62, + 33, + 53, + 17, + 34 + ], + [ + 41, + 9, + 39, + 25, + 17, + 32 + ], + [ + 44, + 30, + 2, + 31, + 0, + 47 + ], + [ + 6, + 62, + 37, + 52, + 55, + 33 + ], + [ + 10, + 24, + 57, + 9, + 49, + 13 + ], + [ + 45, + 24, + 27, + 47, + 19, + 26 + ] + ], + [ + [ + 4, + 59, + 16, + 13, + 3, + 22 + ], + [ + 16, + 23, + 9, + 59, + 13, + 50 + ], + [ + 11, + 35, + 21, + 7, + 59, + 9 + ], + [ + 44, + 1, + 25, + 26, + 15, + 20 + ], + [ + 35, + 57, + 52, + 31, + 24, + 5 + ], + [ + 35, + 15, + 38, + 53, + 12, + 0 + ], + [ + 36, + 20, + 63, + 28, + 60, + 33 + ], + [ + 49, + 50, + 12, + 13, + 8, + 16 + ], + [ + 42, + 48, + 6, + 16, + 35, + 41 + ], + [ + 23, + 9, + 54, + 34, + 30, + 13 + ], + [ + 23, + 63, + 51, + 53, + 55, + 3 + ], + [ + 33, + 36, + 62, + 19, + 59, + 57 + ], + [ + 59, + 38, + 12, + 32, + 17, + 53 + ], + [ + 48, + 0, + 19, + 24, + 61, + 22 + ], + [ + 55, + 6, + 39, + 7, + 60, + 18 + ], + [ + 27, + 57, + 19, + 1, + 60, + 47 + ], + [ + 30, + 40, + 10, + 17, + 36, + 60 + ], + [ + 20, + 13, + 1, + 31, + 17, + 54 + ], + [ + 50, + 32, + 23, + 7, + 33, + 30 + ], + [ + 14, + 19, + 23, + 13, + 10, + 7 + ], + [ + 53, + 23, + 19, + 13, + 5, + 7 + ], + [ + 30, + 18, + 46, + 14, + 1, + 15 + ], + [ + 15, + 18, + 52, + 0, + 51, + 27 + ], + [ + 4, + 21, + 53, + 41, + 10, + 14 + ], + [ + 4, + 9, + 39, + 8, + 16, + 54 + ], + [ + 3, + 19, + 16, + 47, + 30, + 27 + ], + [ + 33, + 8, + 46, + 40, + 29, + 14 + ] + ], + [ + [ + 6, + 26, + 3, + 24, + 11, + 38 + ], + [ + 49, + 19, + 16, + 57, + 0, + 18 + ], + [ + 16, + 60, + 57, + 0, + 22, + 30 + ], + [ + 35, + 46, + 49, + 44, + 26, + 17 + ], + [ + 25, + 54, + 9, + 43, + 45, + 5 + ], + [ + 55, + 33, + 30, + 6, + 28, + 57 + ], + [ + 55, + 31, + 49, + 52, + 25, + 15 + ], + [ + 11, + 36, + 15, + 29, + 30, + 62 + ], + [ + 55, + 42, + 48, + 54, + 6, + 46 + ], + [ + 10, + 3, + 43, + 21, + 62, + 54 + ], + [ + 32, + 56, + 6, + 3, + 18, + 55 + ], + [ + 50, + 21, + 19, + 36, + 4, + 42 + ], + [ + 38, + 35, + 53, + 39, + 41, + 32 + ], + [ + 28, + 24, + 38, + 41, + 15, + 12 + ], + [ + 2, + 40, + 18, + 60, + 55, + 13 + ], + [ + 27, + 57, + 22, + 8, + 54, + 37 + ], + [ + 40, + 27, + 60, + 46, + 44, + 50 + ], + [ + 17, + 21, + 30, + 12, + 29, + 26 + ], + [ + 33, + 38, + 23, + 2, + 13, + 27 + ], + [ + 32, + 56, + 9, + 44, + 31, + 52 + ], + [ + 53, + 12, + 22, + 40, + 5, + 41 + ], + [ + 44, + 45, + 49, + 17, + 14, + 56 + ], + [ + 25, + 39, + 41, + 4, + 9, + 53 + ], + [ + 44, + 1, + 17, + 0, + 13, + 58 + ], + [ + 62, + 37, + 52, + 54, + 42, + 36 + ], + [ + 10, + 9, + 28, + 55, + 2, + 34 + ], + [ + 57, + 30, + 27, + 45, + 47, + 16 + ] + ], + [ + [ + 11, + 16, + 31, + 46, + 0, + 5 + ], + [ + 13, + 49, + 34, + 31, + 16, + 8 + ], + [ + 36, + 13, + 27, + 18, + 34, + 58 + ], + [ + 24, + 61, + 15, + 46, + 63, + 25 + ], + [ + 17, + 2, + 50, + 35, + 58, + 44 + ], + [ + 8, + 23, + 7, + 49, + 26, + 47 + ], + [ + 61, + 38, + 29, + 58, + 36, + 0 + ], + [ + 42, + 20, + 9, + 4, + 11, + 27 + ], + [ + 1, + 34, + 30, + 48, + 26, + 7 + ], + [ + 27, + 10, + 28, + 62, + 13, + 54 + ], + [ + 60, + 1, + 30, + 59, + 6, + 3 + ], + [ + 31, + 62, + 25, + 42, + 21, + 36 + ], + [ + 34, + 32, + 19, + 2, + 14, + 62 + ], + [ + 6, + 40, + 24, + 31, + 57, + 13 + ], + [ + 44, + 2, + 40, + 34, + 8, + 45 + ], + [ + 8, + 27, + 7, + 57, + 50, + 25 + ], + [ + 13, + 60, + 45, + 22, + 52, + 25 + ], + [ + 4, + 23, + 33, + 46, + 58, + 19 + ], + [ + 8, + 33, + 56, + 51, + 45, + 4 + ], + [ + 38, + 53, + 39, + 35, + 31, + 0 + ], + [ + 41, + 11, + 53, + 46, + 8, + 59 + ], + [ + 11, + 44, + 33, + 52, + 55, + 53 + ], + [ + 31, + 1, + 16, + 9, + 15, + 41 + ], + [ + 2, + 31, + 14, + 10, + 44, + 4 + ], + [ + 45, + 18, + 6, + 24, + 62, + 12 + ], + [ + 10, + 50, + 42, + 54, + 24, + 3 + ], + [ + 17, + 31, + 37, + 5, + 10, + 41 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 59 + ], + [ + 8, + 59, + 10, + 60, + 43, + 55 + ], + [ + 45, + 51, + 28, + 59, + 34, + 31 + ], + [ + 14, + 31, + 22, + 2, + 19, + 44 + ], + [ + 59, + 50, + 4, + 31, + 9, + 44 + ], + [ + 58, + 23, + 47, + 9, + 53, + 30 + ], + [ + 54, + 13, + 51, + 22, + 29, + 61 + ], + [ + 47, + 42, + 32, + 20, + 24, + 37 + ], + [ + 33, + 12, + 47, + 10, + 30, + 44 + ], + [ + 30, + 27, + 10, + 28, + 33, + 13 + ], + [ + 61, + 23, + 14, + 20, + 1, + 29 + ], + [ + 12, + 14, + 32, + 62, + 31, + 52 + ], + [ + 5, + 36, + 19, + 20, + 2, + 32 + ], + [ + 46, + 24, + 1, + 40, + 17, + 33 + ], + [ + 44, + 8, + 16, + 30, + 2, + 6 + ], + [ + 6, + 46, + 15, + 54, + 21, + 57 + ], + [ + 2, + 30, + 60, + 61, + 10, + 18 + ], + [ + 6, + 17, + 33, + 27, + 58, + 4 + ], + [ + 56, + 46, + 8, + 60, + 16, + 22 + ], + [ + 5, + 13, + 14, + 36, + 35, + 52 + ], + [ + 15, + 53, + 23, + 59, + 6, + 46 + ], + [ + 36, + 30, + 53, + 33, + 60, + 38 + ], + [ + 10, + 15, + 16, + 52, + 38, + 4 + ], + [ + 31, + 1, + 10, + 4, + 41, + 59 + ], + [ + 32, + 9, + 8, + 59, + 18, + 33 + ], + [ + 33, + 19, + 6, + 3, + 45, + 50 + ], + [ + 55, + 4, + 43, + 5, + 25, + 34 + ] + ], + [ + [ + 25, + 14, + 18, + 49, + 51, + 63 + ], + [ + 42, + 21, + 30, + 43, + 24, + 7 + ], + [ + 54, + 39, + 9, + 59, + 28, + 45 + ], + [ + 1, + 23, + 55, + 8, + 16, + 7 + ], + [ + 59, + 37, + 30, + 41, + 16, + 8 + ], + [ + 4, + 31, + 41, + 56, + 45, + 18 + ], + [ + 28, + 55, + 44, + 54, + 9, + 51 + ], + [ + 24, + 47, + 12, + 63, + 32, + 61 + ], + [ + 33, + 47, + 39, + 44, + 38, + 56 + ], + [ + 40, + 30, + 59, + 33, + 58, + 10 + ], + [ + 19, + 8, + 23, + 4, + 25, + 34 + ], + [ + 45, + 14, + 12, + 42, + 62, + 52 + ], + [ + 58, + 45, + 59, + 29, + 25, + 12 + ], + [ + 8, + 59, + 1, + 47, + 33, + 32 + ], + [ + 24, + 30, + 56, + 38, + 4, + 7 + ], + [ + 46, + 22, + 44, + 57, + 12, + 40 + ], + [ + 34, + 60, + 2, + 17, + 18, + 29 + ], + [ + 6, + 61, + 51, + 53, + 39, + 32 + ], + [ + 49, + 5, + 47, + 60, + 21, + 59 + ], + [ + 57, + 4, + 23, + 39, + 45, + 60 + ], + [ + 7, + 36, + 45, + 53, + 21, + 56 + ], + [ + 60, + 3, + 39, + 18, + 53, + 8 + ], + [ + 57, + 33, + 15, + 47, + 38, + 9 + ], + [ + 20, + 11, + 19, + 2, + 4, + 8 + ], + [ + 36, + 34, + 60, + 47, + 1, + 40 + ], + [ + 33, + 14, + 3, + 21, + 19, + 35 + ], + [ + 60, + 24, + 22, + 52, + 35, + 53 + ] + ], + [ + [ + 23, + 54, + 53, + 58, + 11, + 8 + ], + [ + 11, + 30, + 15, + 59, + 55, + 63 + ], + [ + 20, + 58, + 29, + 17, + 42, + 30 + ], + [ + 18, + 1, + 43, + 15, + 8, + 3 + ], + [ + 59, + 55, + 13, + 28, + 26, + 63 + ], + [ + 33, + 45, + 27, + 53, + 63, + 19 + ], + [ + 28, + 57, + 51, + 54, + 34, + 53 + ], + [ + 24, + 13, + 3, + 47, + 45, + 50 + ], + [ + 47, + 59, + 33, + 57, + 37, + 35 + ], + [ + 23, + 42, + 33, + 41, + 48, + 30 + ], + [ + 42, + 63, + 23, + 25, + 17, + 34 + ], + [ + 14, + 62, + 2, + 19, + 45, + 43 + ], + [ + 12, + 6, + 20, + 30, + 29, + 17 + ], + [ + 6, + 3, + 17, + 8, + 27, + 31 + ], + [ + 19, + 59, + 2, + 10, + 8, + 55 + ], + [ + 45, + 61, + 19, + 42, + 57, + 17 + ], + [ + 55, + 60, + 41, + 34, + 35, + 53 + ], + [ + 36, + 2, + 8, + 31, + 14, + 5 + ], + [ + 36, + 45, + 16, + 38, + 51, + 21 + ], + [ + 57, + 39, + 48, + 6, + 19, + 60 + ], + [ + 39, + 37, + 59, + 48, + 42, + 14 + ], + [ + 52, + 46, + 33, + 28, + 18, + 55 + ], + [ + 1, + 15, + 28, + 13, + 52, + 18 + ], + [ + 10, + 21, + 4, + 23, + 2, + 31 + ], + [ + 45, + 9, + 41, + 18, + 54, + 3 + ], + [ + 42, + 61, + 36, + 3, + 19, + 27 + ], + [ + 41, + 46, + 33, + 3, + 10, + 15 + ] + ], + [ + [ + 49, + 52, + 60, + 63, + 21, + 0 + ], + [ + 14, + 7, + 25, + 52, + 58, + 36 + ], + [ + 46, + 57, + 28, + 24, + 49, + 12 + ], + [ + 10, + 21, + 39, + 1, + 11, + 17 + ], + [ + 53, + 3, + 2, + 16, + 46, + 44 + ], + [ + 25, + 40, + 39, + 15, + 34, + 30 + ], + [ + 62, + 24, + 47, + 52, + 17, + 49 + ], + [ + 34, + 41, + 39, + 11, + 33, + 15 + ], + [ + 36, + 26, + 48, + 39, + 15, + 56 + ], + [ + 23, + 43, + 36, + 20, + 21, + 10 + ], + [ + 48, + 63, + 51, + 10, + 62, + 18 + ], + [ + 10, + 42, + 36, + 39, + 49, + 19 + ], + [ + 22, + 38, + 51, + 56, + 50, + 7 + ], + [ + 4, + 31, + 19, + 22, + 17, + 7 + ], + [ + 41, + 7, + 39, + 2, + 57, + 26 + ], + [ + 33, + 27, + 58, + 28, + 20, + 45 + ], + [ + 58, + 8, + 37, + 18, + 34, + 62 + ], + [ + 24, + 34, + 43, + 39, + 17, + 19 + ], + [ + 23, + 50, + 12, + 32, + 43, + 31 + ], + [ + 9, + 7, + 51, + 16, + 30, + 60 + ], + [ + 25, + 62, + 14, + 53, + 58, + 0 + ], + [ + 53, + 4, + 18, + 22, + 14, + 29 + ], + [ + 6, + 14, + 15, + 48, + 58, + 0 + ], + [ + 35, + 4, + 45, + 3, + 59, + 39 + ], + [ + 17, + 57, + 23, + 16, + 40, + 37 + ], + [ + 9, + 19, + 3, + 36, + 11, + 47 + ], + [ + 39, + 14, + 22, + 33, + 62, + 42 + ] + ], + [ + [ + 14, + 17, + 2, + 39, + 47, + 63 + ], + [ + 23, + 58, + 2, + 25, + 5, + 18 + ], + [ + 27, + 33, + 0, + 56, + 6, + 7 + ], + [ + 23, + 5, + 30, + 7, + 21, + 39 + ], + [ + 39, + 53, + 27, + 38, + 54, + 41 + ], + [ + 37, + 25, + 50, + 40, + 33, + 36 + ], + [ + 49, + 37, + 24, + 55, + 29, + 28 + ], + [ + 34, + 15, + 36, + 16, + 30, + 11 + ], + [ + 5, + 36, + 2, + 26, + 16, + 48 + ], + [ + 56, + 46, + 10, + 23, + 39, + 45 + ], + [ + 10, + 56, + 62, + 45, + 49, + 3 + ], + [ + 35, + 6, + 36, + 50, + 11, + 1 + ], + [ + 50, + 46, + 39, + 51, + 38, + 41 + ], + [ + 41, + 19, + 31, + 6, + 48, + 24 + ], + [ + 2, + 7, + 57, + 18, + 47, + 63 + ], + [ + 27, + 58, + 22, + 8, + 57, + 45 + ], + [ + 37, + 59, + 51, + 44, + 55, + 18 + ], + [ + 52, + 21, + 25, + 12, + 34, + 39 + ], + [ + 23, + 34, + 60, + 13, + 27, + 32 + ], + [ + 32, + 56, + 26, + 24, + 44, + 30 + ], + [ + 41, + 12, + 53, + 2, + 5, + 56 + ], + [ + 44, + 18, + 39, + 33, + 38, + 46 + ], + [ + 25, + 0, + 59, + 18, + 11, + 53 + ], + [ + 1, + 7, + 17, + 30, + 4, + 37 + ], + [ + 62, + 48, + 19, + 54, + 37, + 22 + ], + [ + 10, + 28, + 3, + 16, + 11, + 42 + ], + [ + 26, + 30, + 57, + 42, + 41, + 33 + ] + ], + [ + [ + 36, + 31, + 37, + 16, + 43, + 63 + ], + [ + 26, + 51, + 0, + 48, + 42, + 21 + ], + [ + 18, + 41, + 37, + 34, + 24, + 30 + ], + [ + 58, + 4, + 22, + 44, + 18, + 51 + ], + [ + 18, + 4, + 26, + 61, + 14, + 31 + ], + [ + 9, + 23, + 34, + 29, + 3, + 31 + ], + [ + 12, + 58, + 36, + 46, + 16, + 25 + ], + [ + 4, + 56, + 9, + 33, + 60, + 32 + ], + [ + 14, + 26, + 43, + 30, + 46, + 35 + ], + [ + 2, + 0, + 10, + 43, + 28, + 27 + ], + [ + 61, + 18, + 19, + 17, + 2, + 33 + ], + [ + 60, + 19, + 23, + 62, + 37, + 43 + ], + [ + 56, + 32, + 8, + 61, + 58, + 38 + ], + [ + 25, + 28, + 22, + 0, + 26, + 29 + ], + [ + 22, + 49, + 7, + 60, + 23, + 55 + ], + [ + 46, + 57, + 19, + 37, + 12, + 27 + ], + [ + 22, + 60, + 11, + 30, + 19, + 20 + ], + [ + 36, + 20, + 17, + 2, + 37, + 55 + ], + [ + 13, + 16, + 32, + 58, + 41, + 10 + ], + [ + 50, + 55, + 13, + 25, + 14, + 41 + ], + [ + 14, + 3, + 44, + 48, + 60, + 6 + ], + [ + 36, + 18, + 57, + 22, + 1, + 43 + ], + [ + 10, + 22, + 15, + 52, + 13, + 18 + ], + [ + 4, + 56, + 10, + 31, + 14, + 52 + ], + [ + 9, + 4, + 16, + 54, + 8, + 59 + ], + [ + 3, + 43, + 27, + 45, + 6, + 30 + ], + [ + 33, + 4, + 43, + 40, + 46, + 18 + ] + ], + [ + [ + 48, + 42, + 38, + 63, + 47, + 7 + ], + [ + 3, + 62, + 10, + 6, + 26, + 2 + ], + [ + 39, + 44, + 6, + 7, + 45, + 40 + ], + [ + 56, + 33, + 2, + 41, + 48, + 62 + ], + [ + 6, + 18, + 37, + 41, + 30, + 27 + ], + [ + 59, + 45, + 23, + 31, + 53, + 9 + ], + [ + 0, + 13, + 12, + 44, + 33, + 16 + ], + [ + 12, + 56, + 50, + 60, + 28, + 63 + ], + [ + 14, + 33, + 26, + 24, + 32, + 16 + ], + [ + 43, + 2, + 0, + 50, + 28, + 58 + ], + [ + 61, + 9, + 63, + 8, + 17, + 39 + ], + [ + 60, + 19, + 59, + 23, + 24, + 30 + ], + [ + 56, + 59, + 53, + 32, + 7, + 38 + ], + [ + 28, + 25, + 22, + 24, + 59, + 42 + ], + [ + 22, + 49, + 46, + 23, + 60, + 7 + ], + [ + 46, + 34, + 27, + 29, + 12, + 0 + ], + [ + 17, + 22, + 5, + 31, + 20, + 27 + ], + [ + 39, + 48, + 57, + 37, + 36, + 17 + ], + [ + 42, + 49, + 32, + 7, + 16, + 12 + ], + [ + 1, + 7, + 23, + 14, + 36, + 29 + ], + [ + 21, + 7, + 48, + 60, + 35, + 14 + ], + [ + 8, + 18, + 24, + 60, + 1, + 0 + ], + [ + 51, + 15, + 33, + 36, + 5, + 30 + ], + [ + 52, + 4, + 29, + 8, + 9, + 26 + ], + [ + 4, + 9, + 16, + 40, + 58, + 60 + ], + [ + 3, + 17, + 21, + 61, + 19, + 8 + ], + [ + 35, + 54, + 1, + 53, + 60, + 0 + ] + ], + [ + [ + 19, + 1, + 31, + 52, + 49, + 63 + ], + [ + 7, + 47, + 5, + 60, + 22, + 46 + ], + [ + 59, + 30, + 3, + 11, + 0, + 19 + ], + [ + 43, + 42, + 19, + 62, + 8, + 56 + ], + [ + 61, + 15, + 25, + 18, + 39, + 27 + ], + [ + 61, + 50, + 36, + 45, + 33, + 44 + ], + [ + 8, + 37, + 52, + 1, + 2, + 41 + ], + [ + 36, + 30, + 53, + 11, + 16, + 29 + ], + [ + 14, + 58, + 46, + 49, + 3, + 26 + ], + [ + 62, + 43, + 0, + 45, + 22, + 46 + ], + [ + 6, + 56, + 45, + 18, + 10, + 41 + ], + [ + 60, + 21, + 50, + 47, + 30, + 35 + ], + [ + 11, + 53, + 28, + 56, + 41, + 39 + ], + [ + 23, + 9, + 33, + 28, + 22, + 26 + ], + [ + 23, + 56, + 34, + 27, + 2, + 63 + ], + [ + 22, + 9, + 44, + 41, + 37, + 47 + ], + [ + 1, + 11, + 46, + 27, + 3, + 52 + ], + [ + 51, + 37, + 17, + 21, + 61, + 30 + ], + [ + 13, + 19, + 32, + 5, + 2, + 9 + ], + [ + 41, + 32, + 6, + 47, + 29, + 56 + ], + [ + 53, + 33, + 9, + 35, + 38, + 12 + ], + [ + 40, + 19, + 51, + 7, + 26, + 22 + ], + [ + 37, + 5, + 25, + 46, + 34, + 53 + ], + [ + 16, + 55, + 20, + 24, + 44, + 52 + ], + [ + 21, + 46, + 11, + 0, + 36, + 7 + ], + [ + 18, + 60, + 32, + 3, + 34, + 28 + ], + [ + 52, + 9, + 36, + 48, + 11, + 41 + ] + ], + [ + [ + 53, + 15, + 34, + 0, + 36, + 56 + ], + [ + 8, + 12, + 41, + 11, + 19, + 55 + ], + [ + 56, + 13, + 31, + 36, + 23, + 47 + ], + [ + 36, + 51, + 30, + 7, + 26, + 54 + ], + [ + 58, + 13, + 50, + 2, + 53, + 34 + ], + [ + 49, + 52, + 23, + 32, + 7, + 26 + ], + [ + 61, + 38, + 23, + 0, + 4, + 28 + ], + [ + 42, + 27, + 17, + 9, + 18, + 20 + ], + [ + 1, + 34, + 45, + 4, + 12, + 7 + ], + [ + 54, + 27, + 57, + 38, + 44, + 6 + ], + [ + 59, + 40, + 1, + 60, + 48, + 30 + ], + [ + 25, + 31, + 32, + 51, + 62, + 8 + ], + [ + 62, + 19, + 14, + 2, + 37, + 15 + ], + [ + 40, + 57, + 37, + 35, + 22, + 61 + ], + [ + 45, + 16, + 34, + 42, + 37, + 48 + ], + [ + 50, + 16, + 62, + 33, + 25, + 37 + ], + [ + 42, + 13, + 39, + 47, + 3, + 63 + ], + [ + 46, + 23, + 28, + 27, + 4, + 15 + ], + [ + 56, + 62, + 31, + 35, + 59, + 45 + ], + [ + 15, + 38, + 13, + 4, + 63, + 48 + ], + [ + 34, + 15, + 57, + 38, + 13, + 24 + ], + [ + 62, + 36, + 41, + 54, + 46, + 29 + ], + [ + 22, + 53, + 46, + 34, + 30, + 27 + ], + [ + 24, + 10, + 4, + 47, + 18, + 36 + ], + [ + 50, + 57, + 51, + 11, + 49, + 3 + ], + [ + 51, + 7, + 0, + 18, + 11, + 44 + ], + [ + 39, + 37, + 9, + 42, + 40, + 44 + ] + ], + [ + [ + 36, + 25, + 57, + 55, + 47, + 63 + ], + [ + 0, + 2, + 46, + 3, + 51, + 34 + ], + [ + 24, + 2, + 46, + 15, + 33, + 43 + ], + [ + 22, + 31, + 17, + 19, + 10, + 55 + ], + [ + 58, + 59, + 3, + 9, + 40, + 57 + ], + [ + 23, + 31, + 43, + 57, + 2, + 38 + ], + [ + 9, + 62, + 13, + 42, + 47, + 52 + ], + [ + 24, + 7, + 14, + 10, + 46, + 59 + ], + [ + 47, + 18, + 4, + 37, + 0, + 13 + ], + [ + 54, + 4, + 25, + 47, + 36, + 38 + ], + [ + 15, + 47, + 1, + 24, + 58, + 14 + ], + [ + 54, + 5, + 16, + 63, + 14, + 7 + ], + [ + 49, + 3, + 33, + 13, + 46, + 10 + ], + [ + 18, + 10, + 11, + 13, + 63, + 39 + ], + [ + 19, + 62, + 32, + 58, + 10, + 43 + ], + [ + 43, + 48, + 63, + 5, + 55, + 53 + ], + [ + 24, + 51, + 47, + 15, + 59, + 32 + ], + [ + 44, + 0, + 34, + 43, + 3, + 6 + ], + [ + 58, + 38, + 54, + 47, + 11, + 59 + ], + [ + 55, + 27, + 15, + 9, + 42, + 31 + ], + [ + 43, + 41, + 1, + 51, + 5, + 29 + ], + [ + 49, + 27, + 20, + 6, + 4, + 13 + ], + [ + 11, + 25, + 2, + 27, + 54, + 50 + ], + [ + 38, + 44, + 40, + 54, + 33, + 14 + ], + [ + 13, + 63, + 52, + 2, + 8, + 29 + ], + [ + 23, + 41, + 59, + 57, + 38, + 15 + ], + [ + 23, + 6, + 62, + 50, + 51, + 34 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 23 + ], + [ + 51, + 41, + 5, + 15, + 40, + 21 + ], + [ + 43, + 1, + 29, + 55, + 21, + 35 + ], + [ + 24, + 53, + 25, + 51, + 32, + 29 + ], + [ + 41, + 31, + 49, + 57, + 60, + 34 + ], + [ + 17, + 4, + 35, + 30, + 10, + 38 + ], + [ + 34, + 7, + 21, + 9, + 48, + 31 + ], + [ + 14, + 24, + 7, + 46, + 25, + 27 + ], + [ + 47, + 0, + 12, + 6, + 37, + 60 + ], + [ + 4, + 30, + 25, + 47, + 36, + 54 + ], + [ + 24, + 61, + 15, + 47, + 46, + 1 + ], + [ + 5, + 14, + 24, + 16, + 57, + 63 + ], + [ + 49, + 3, + 17, + 26, + 36, + 44 + ], + [ + 13, + 10, + 61, + 0, + 11, + 22 + ], + [ + 10, + 6, + 2, + 49, + 58, + 46 + ], + [ + 19, + 11, + 2, + 25, + 54, + 18 + ], + [ + 10, + 5, + 52, + 24, + 18, + 17 + ], + [ + 44, + 13, + 55, + 31, + 63, + 38 + ], + [ + 58, + 7, + 25, + 32, + 38, + 14 + ], + [ + 49, + 55, + 34, + 13, + 16, + 40 + ], + [ + 51, + 23, + 59, + 35, + 5, + 4 + ], + [ + 57, + 15, + 46, + 27, + 42, + 32 + ], + [ + 33, + 15, + 23, + 52, + 24, + 27 + ], + [ + 4, + 41, + 33, + 10, + 26, + 40 + ], + [ + 8, + 13, + 59, + 4, + 9, + 54 + ], + [ + 19, + 3, + 31, + 27, + 43, + 2 + ], + [ + 46, + 61, + 25, + 8, + 29, + 50 + ] + ], + [ + [ + 48, + 38, + 62, + 63, + 42, + 50 + ], + [ + 3, + 2, + 26, + 17, + 10, + 6 + ], + [ + 39, + 44, + 6, + 45, + 7, + 40 + ], + [ + 5, + 62, + 53, + 50, + 41, + 3 + ], + [ + 6, + 41, + 49, + 37, + 30, + 23 + ], + [ + 59, + 60, + 46, + 4, + 53, + 29 + ], + [ + 44, + 34, + 7, + 15, + 13, + 43 + ], + [ + 12, + 14, + 24, + 25, + 58, + 7 + ], + [ + 33, + 35, + 4, + 37, + 8, + 36 + ], + [ + 40, + 30, + 25, + 36, + 20, + 54 + ], + [ + 9, + 8, + 24, + 25, + 5, + 63 + ], + [ + 59, + 34, + 5, + 24, + 6, + 57 + ], + [ + 17, + 49, + 44, + 26, + 55, + 7 + ], + [ + 47, + 13, + 59, + 27, + 22, + 26 + ], + [ + 49, + 2, + 44, + 10, + 46, + 54 + ], + [ + 34, + 2, + 54, + 55, + 57, + 53 + ], + [ + 5, + 17, + 29, + 31, + 43, + 52 + ], + [ + 48, + 57, + 38, + 63, + 39, + 43 + ], + [ + 42, + 38, + 49, + 32, + 7, + 40 + ], + [ + 59, + 1, + 16, + 23, + 60, + 10 + ], + [ + 21, + 7, + 35, + 53, + 48, + 31 + ], + [ + 8, + 60, + 24, + 42, + 14, + 35 + ], + [ + 33, + 51, + 15, + 28, + 23, + 5 + ], + [ + 4, + 41, + 9, + 11, + 8, + 51 + ], + [ + 60, + 24, + 19, + 48, + 9, + 4 + ], + [ + 19, + 3, + 26, + 58, + 12, + 61 + ], + [ + 35, + 54, + 1, + 60, + 53, + 49 + ] + ], + [ + [ + 21, + 7, + 53, + 56, + 63, + 33 + ], + [ + 3, + 34, + 57, + 16, + 20, + 51 + ], + [ + 55, + 11, + 16, + 60, + 0, + 13 + ], + [ + 62, + 43, + 5, + 50, + 8, + 53 + ], + [ + 35, + 52, + 9, + 43, + 27, + 0 + ], + [ + 16, + 60, + 29, + 61, + 28, + 58 + ], + [ + 31, + 34, + 43, + 30, + 2, + 18 + ], + [ + 36, + 25, + 53, + 24, + 35, + 48 + ], + [ + 48, + 4, + 35, + 32, + 8, + 60 + ], + [ + 25, + 54, + 30, + 10, + 4, + 41 + ], + [ + 9, + 3, + 19, + 20, + 61, + 24 + ], + [ + 30, + 5, + 17, + 45, + 18, + 49 + ], + [ + 49, + 21, + 17, + 26, + 37, + 42 + ], + [ + 38, + 25, + 10, + 51, + 54, + 13 + ], + [ + 2, + 27, + 26, + 10, + 58, + 30 + ], + [ + 9, + 2, + 54, + 21, + 25, + 13 + ], + [ + 22, + 23, + 33, + 27, + 51, + 52 + ], + [ + 30, + 21, + 35, + 55, + 5, + 17 + ], + [ + 38, + 27, + 24, + 56, + 21, + 35 + ], + [ + 53, + 45, + 22, + 28, + 0, + 32 + ], + [ + 40, + 42, + 48, + 37, + 52, + 8 + ], + [ + 46, + 45, + 57, + 36, + 32, + 51 + ], + [ + 62, + 10, + 38, + 42, + 41, + 54 + ], + [ + 60, + 1, + 10, + 31, + 44, + 36 + ], + [ + 29, + 6, + 9, + 8, + 56, + 7 + ], + [ + 9, + 59, + 22, + 0, + 35, + 57 + ], + [ + 45, + 27, + 62, + 47, + 3, + 28 + ] + ], + [ + [ + 27, + 13, + 18, + 8, + 63, + 55 + ], + [ + 36, + 21, + 57, + 8, + 46, + 55 + ], + [ + 43, + 61, + 10, + 13, + 41, + 37 + ], + [ + 43, + 16, + 24, + 6, + 26, + 61 + ], + [ + 60, + 29, + 35, + 31, + 23, + 16 + ], + [ + 9, + 58, + 0, + 60, + 17, + 38 + ], + [ + 63, + 16, + 7, + 13, + 31, + 18 + ], + [ + 16, + 25, + 24, + 2, + 47, + 58 + ], + [ + 8, + 35, + 6, + 14, + 48, + 4 + ], + [ + 30, + 25, + 54, + 4, + 10, + 34 + ], + [ + 20, + 23, + 19, + 7, + 38, + 61 + ], + [ + 5, + 3, + 24, + 32, + 12, + 42 + ], + [ + 49, + 17, + 10, + 26, + 32, + 60 + ], + [ + 13, + 10, + 29, + 22, + 58, + 54 + ], + [ + 44, + 8, + 2, + 58, + 6, + 5 + ], + [ + 25, + 6, + 2, + 54, + 19, + 53 + ], + [ + 61, + 5, + 30, + 17, + 1, + 27 + ], + [ + 22, + 55, + 63, + 57, + 19, + 33 + ], + [ + 35, + 46, + 6, + 32, + 14, + 7 + ], + [ + 45, + 13, + 51, + 14, + 7, + 5 + ], + [ + 15, + 23, + 50, + 51, + 13, + 59 + ], + [ + 57, + 36, + 53, + 61, + 6, + 14 + ], + [ + 10, + 23, + 15, + 33, + 27, + 38 + ], + [ + 4, + 10, + 33, + 41, + 26, + 36 + ], + [ + 8, + 4, + 33, + 9, + 47, + 48 + ], + [ + 3, + 43, + 6, + 19, + 2, + 30 + ], + [ + 55, + 4, + 46, + 40, + 18, + 20 + ] + ], + [ + [ + 48, + 38, + 63, + 37, + 7, + 42 + ], + [ + 3, + 26, + 10, + 35, + 6, + 2 + ], + [ + 39, + 44, + 7, + 45, + 6, + 40 + ], + [ + 56, + 23, + 61, + 16, + 33, + 41 + ], + [ + 6, + 37, + 44, + 16, + 30, + 26 + ], + [ + 59, + 17, + 60, + 19, + 5, + 52 + ], + [ + 44, + 55, + 33, + 63, + 13, + 31 + ], + [ + 12, + 25, + 63, + 2, + 24, + 54 + ], + [ + 33, + 8, + 42, + 4, + 35, + 41 + ], + [ + 30, + 40, + 25, + 21, + 48, + 51 + ], + [ + 8, + 23, + 34, + 5, + 19, + 22 + ], + [ + 34, + 59, + 42, + 11, + 5, + 3 + ], + [ + 10, + 26, + 50, + 55, + 53, + 59 + ], + [ + 59, + 54, + 13, + 10, + 47, + 25 + ], + [ + 24, + 2, + 49, + 46, + 38, + 8 + ], + [ + 34, + 2, + 53, + 54, + 40, + 57 + ], + [ + 17, + 29, + 5, + 27, + 18, + 43 + ], + [ + 48, + 57, + 63, + 39, + 38, + 32 + ], + [ + 42, + 49, + 7, + 32, + 38, + 61 + ], + [ + 1, + 59, + 23, + 14, + 16, + 10 + ], + [ + 21, + 7, + 35, + 53, + 13, + 11 + ], + [ + 8, + 60, + 24, + 25, + 42, + 14 + ], + [ + 33, + 15, + 5, + 51, + 28, + 23 + ], + [ + 4, + 9, + 51, + 8, + 41, + 39 + ], + [ + 60, + 4, + 19, + 48, + 9, + 24 + ], + [ + 19, + 3, + 26, + 12, + 21, + 53 + ], + [ + 53, + 35, + 1, + 54, + 60, + 20 + ] + ], + [ + [ + 17, + 37, + 31, + 32, + 63, + 50 + ], + [ + 12, + 2, + 9, + 32, + 47, + 17 + ], + [ + 3, + 57, + 56, + 50, + 33, + 38 + ], + [ + 43, + 42, + 19, + 52, + 8, + 17 + ], + [ + 61, + 39, + 27, + 12, + 15, + 57 + ], + [ + 6, + 33, + 36, + 44, + 29, + 61 + ], + [ + 2, + 41, + 42, + 15, + 52, + 5 + ], + [ + 36, + 37, + 15, + 53, + 18, + 62 + ], + [ + 50, + 58, + 3, + 5, + 16, + 4 + ], + [ + 16, + 22, + 55, + 32, + 41, + 26 + ], + [ + 3, + 41, + 56, + 45, + 38, + 10 + ], + [ + 21, + 53, + 50, + 34, + 38, + 35 + ], + [ + 35, + 11, + 37, + 16, + 53, + 42 + ], + [ + 9, + 15, + 54, + 38, + 12, + 52 + ], + [ + 56, + 2, + 23, + 24, + 9, + 5 + ], + [ + 2, + 44, + 9, + 53, + 35, + 54 + ], + [ + 27, + 46, + 1, + 5, + 60, + 50 + ], + [ + 51, + 61, + 50, + 54, + 33, + 44 + ], + [ + 2, + 38, + 19, + 9, + 5, + 32 + ], + [ + 32, + 28, + 6, + 15, + 0, + 33 + ], + [ + 26, + 53, + 27, + 13, + 5, + 4 + ], + [ + 34, + 40, + 45, + 55, + 62, + 53 + ], + [ + 41, + 5, + 46, + 4, + 45, + 3 + ], + [ + 20, + 4, + 22, + 47, + 59, + 58 + ], + [ + 62, + 36, + 25, + 28, + 53, + 33 + ], + [ + 22, + 33, + 7, + 3, + 54, + 0 + ], + [ + 58, + 27, + 52, + 48, + 45, + 17 + ] + ], + [ + [ + 3, + 43, + 17, + 42, + 4, + 35 + ], + [ + 0, + 42, + 36, + 34, + 32, + 24 + ], + [ + 43, + 12, + 19, + 20, + 2, + 29 + ], + [ + 28, + 4, + 54, + 22, + 58, + 23 + ], + [ + 12, + 60, + 52, + 57, + 46, + 7 + ], + [ + 43, + 2, + 1, + 63, + 20, + 11 + ], + [ + 42, + 19, + 57, + 31, + 15, + 21 + ], + [ + 35, + 6, + 46, + 25, + 36, + 33 + ], + [ + 11, + 4, + 35, + 50, + 32, + 52 + ], + [ + 21, + 25, + 38, + 47, + 31, + 58 + ], + [ + 3, + 2, + 22, + 45, + 42, + 15 + ], + [ + 2, + 21, + 5, + 7, + 43, + 38 + ], + [ + 33, + 10, + 41, + 42, + 60, + 16 + ], + [ + 44, + 56, + 30, + 54, + 9, + 43 + ], + [ + 9, + 5, + 33, + 28, + 58, + 49 + ], + [ + 8, + 53, + 59, + 54, + 31, + 2 + ], + [ + 47, + 0, + 48, + 27, + 24, + 51 + ], + [ + 0, + 60, + 21, + 12, + 56, + 28 + ], + [ + 38, + 13, + 53, + 51, + 9, + 7 + ], + [ + 19, + 28, + 56, + 17, + 21, + 26 + ], + [ + 12, + 43, + 26, + 4, + 0, + 5 + ], + [ + 43, + 1, + 12, + 55, + 4, + 40 + ], + [ + 41, + 5, + 21, + 45, + 25, + 12 + ], + [ + 4, + 20, + 1, + 22, + 30, + 37 + ], + [ + 5, + 62, + 25, + 54, + 48, + 47 + ], + [ + 10, + 28, + 55, + 23, + 0, + 53 + ], + [ + 30, + 57, + 8, + 5, + 17, + 29 + ] + ], + [ + [ + 11, + 16, + 31, + 0, + 35, + 46 + ], + [ + 13, + 49, + 31, + 50, + 16, + 19 + ], + [ + 36, + 13, + 53, + 27, + 4, + 18 + ], + [ + 24, + 7, + 29, + 26, + 12, + 32 + ], + [ + 17, + 35, + 2, + 44, + 10, + 48 + ], + [ + 8, + 7, + 23, + 5, + 51, + 26 + ], + [ + 58, + 15, + 61, + 29, + 38, + 62 + ], + [ + 20, + 9, + 42, + 35, + 3, + 6 + ], + [ + 4, + 47, + 25, + 11, + 1, + 52 + ], + [ + 54, + 25, + 55, + 38, + 21, + 27 + ], + [ + 1, + 60, + 14, + 59, + 22, + 30 + ], + [ + 51, + 31, + 5, + 25, + 14, + 52 + ], + [ + 34, + 2, + 10, + 26, + 52, + 47 + ], + [ + 40, + 57, + 13, + 54, + 9, + 6 + ], + [ + 8, + 44, + 58, + 5, + 16, + 1 + ], + [ + 8, + 53, + 59, + 25, + 24, + 52 + ], + [ + 13, + 47, + 45, + 0, + 42, + 8 + ], + [ + 23, + 44, + 55, + 33, + 38, + 7 + ], + [ + 53, + 38, + 11, + 1, + 8, + 24 + ], + [ + 15, + 38, + 14, + 28, + 0, + 19 + ], + [ + 41, + 27, + 8, + 42, + 57, + 40 + ], + [ + 11, + 62, + 55, + 16, + 10, + 41 + ], + [ + 31, + 20, + 46, + 37, + 34, + 41 + ], + [ + 47, + 4, + 55, + 33, + 49, + 22 + ], + [ + 11, + 45, + 47, + 48, + 54, + 36 + ], + [ + 50, + 10, + 23, + 51, + 18, + 2 + ], + [ + 17, + 31, + 4, + 5, + 36, + 20 + ] + ], + [ + [ + 22, + 6, + 39, + 57, + 29, + 28 + ], + [ + 27, + 6, + 14, + 17, + 51, + 55 + ], + [ + 1, + 11, + 29, + 26, + 47, + 4 + ], + [ + 14, + 38, + 31, + 22, + 29, + 6 + ], + [ + 14, + 59, + 61, + 16, + 1, + 19 + ], + [ + 30, + 8, + 23, + 21, + 47, + 1 + ], + [ + 58, + 4, + 15, + 61, + 27, + 31 + ], + [ + 20, + 42, + 3, + 9, + 35, + 6 + ], + [ + 47, + 4, + 25, + 8, + 36, + 0 + ], + [ + 54, + 55, + 21, + 19, + 33, + 25 + ], + [ + 14, + 4, + 60, + 20, + 24, + 40 + ], + [ + 51, + 0, + 5, + 32, + 52, + 3 + ], + [ + 2, + 36, + 10, + 52, + 26, + 32 + ], + [ + 40, + 13, + 54, + 36, + 57, + 46 + ], + [ + 44, + 8, + 5, + 37, + 58, + 2 + ], + [ + 6, + 53, + 24, + 2, + 54, + 37 + ], + [ + 13, + 47, + 61, + 5, + 19, + 17 + ], + [ + 58, + 55, + 44, + 38, + 63, + 6 + ], + [ + 35, + 46, + 31, + 1, + 19, + 32 + ], + [ + 15, + 13, + 63, + 45, + 9, + 55 + ], + [ + 27, + 15, + 23, + 6, + 35, + 63 + ], + [ + 36, + 62, + 57, + 41, + 10, + 16 + ], + [ + 10, + 33, + 41, + 20, + 5, + 46 + ], + [ + 47, + 4, + 26, + 55, + 10, + 49 + ], + [ + 4, + 11, + 8, + 48, + 36, + 33 + ], + [ + 18, + 51, + 43, + 33, + 50, + 3 + ], + [ + 4, + 55, + 9, + 36, + 43, + 5 + ] + ], + [ + [ + 48, + 38, + 47, + 63, + 7, + 42 + ], + [ + 3, + 26, + 10, + 6, + 2, + 12 + ], + [ + 39, + 44, + 6, + 7, + 45, + 8 + ], + [ + 60, + 31, + 22, + 0, + 54, + 45 + ], + [ + 6, + 59, + 14, + 16, + 37, + 19 + ], + [ + 59, + 30, + 8, + 47, + 60, + 17 + ], + [ + 44, + 56, + 4, + 13, + 15, + 9 + ], + [ + 12, + 24, + 20, + 58, + 61, + 28 + ], + [ + 47, + 33, + 8, + 4, + 36, + 42 + ], + [ + 40, + 54, + 55, + 33, + 21, + 51 + ], + [ + 8, + 14, + 22, + 39, + 31, + 7 + ], + [ + 34, + 59, + 14, + 5, + 52, + 0 + ], + [ + 36, + 10, + 52, + 26, + 44, + 59 + ], + [ + 59, + 44, + 13, + 1, + 36, + 22 + ], + [ + 24, + 5, + 46, + 44, + 2, + 38 + ], + [ + 34, + 53, + 6, + 40, + 30, + 2 + ], + [ + 17, + 29, + 50, + 5, + 47, + 27 + ], + [ + 48, + 38, + 57, + 63, + 32, + 60 + ], + [ + 42, + 7, + 49, + 46, + 32, + 1 + ], + [ + 59, + 1, + 23, + 14, + 10, + 16 + ], + [ + 21, + 35, + 7, + 53, + 6, + 17 + ], + [ + 8, + 24, + 60, + 13, + 55, + 14 + ], + [ + 33, + 5, + 51, + 15, + 3, + 23 + ], + [ + 4, + 9, + 51, + 26, + 41, + 28 + ], + [ + 60, + 4, + 19, + 47, + 40, + 48 + ], + [ + 19, + 3, + 26, + 21, + 53, + 8 + ], + [ + 53, + 1, + 35, + 54, + 60, + 20 + ] + ], + [ + [ + 37, + 46, + 39, + 54, + 27, + 55 + ], + [ + 34, + 5, + 16, + 47, + 6, + 42 + ], + [ + 32, + 38, + 16, + 42, + 3, + 20 + ], + [ + 43, + 19, + 60, + 42, + 52, + 11 + ], + [ + 7, + 61, + 39, + 57, + 12, + 46 + ], + [ + 50, + 36, + 44, + 24, + 28, + 31 + ], + [ + 15, + 42, + 41, + 2, + 40, + 32 + ], + [ + 36, + 35, + 38, + 53, + 58, + 51 + ], + [ + 50, + 4, + 16, + 3, + 36, + 58 + ], + [ + 59, + 16, + 32, + 38, + 26, + 56 + ], + [ + 3, + 45, + 37, + 41, + 21, + 33 + ], + [ + 21, + 53, + 34, + 35, + 5, + 39 + ], + [ + 35, + 42, + 63, + 16, + 11, + 46 + ], + [ + 23, + 9, + 51, + 54, + 15, + 38 + ], + [ + 2, + 5, + 20, + 43, + 24, + 29 + ], + [ + 53, + 44, + 9, + 2, + 54, + 20 + ], + [ + 27, + 46, + 1, + 47, + 50, + 5 + ], + [ + 61, + 54, + 57, + 44, + 51, + 43 + ], + [ + 19, + 9, + 33, + 61, + 38, + 37 + ], + [ + 0, + 33, + 15, + 32, + 6, + 9 + ], + [ + 26, + 27, + 53, + 5, + 47, + 54 + ], + [ + 55, + 34, + 12, + 62, + 3, + 4 + ], + [ + 41, + 5, + 46, + 40, + 4, + 32 + ], + [ + 55, + 20, + 44, + 26, + 4, + 40 + ], + [ + 62, + 12, + 28, + 34, + 23, + 33 + ], + [ + 18, + 7, + 22, + 3, + 54, + 14 + ], + [ + 36, + 9, + 27, + 52, + 48, + 11 + ] + ], + [ + [ + 46, + 37, + 61, + 18, + 36, + 63 + ], + [ + 22, + 34, + 28, + 59, + 24, + 56 + ], + [ + 32, + 15, + 17, + 60, + 38, + 20 + ], + [ + 28, + 4, + 58, + 16, + 30, + 35 + ], + [ + 7, + 36, + 9, + 57, + 33, + 23 + ], + [ + 43, + 63, + 2, + 30, + 11, + 19 + ], + [ + 19, + 42, + 57, + 15, + 3, + 22 + ], + [ + 35, + 46, + 47, + 6, + 58, + 8 + ], + [ + 32, + 4, + 37, + 36, + 35, + 57 + ], + [ + 21, + 47, + 55, + 58, + 38, + 54 + ], + [ + 3, + 42, + 2, + 38, + 5, + 33 + ], + [ + 43, + 18, + 2, + 21, + 5, + 49 + ], + [ + 10, + 49, + 24, + 32, + 25, + 36 + ], + [ + 30, + 21, + 63, + 49, + 16, + 51 + ], + [ + 62, + 2, + 13, + 5, + 29, + 56 + ], + [ + 53, + 2, + 9, + 63, + 46, + 54 + ], + [ + 27, + 0, + 63, + 47, + 5, + 14 + ], + [ + 60, + 57, + 43, + 44, + 0, + 12 + ], + [ + 38, + 9, + 61, + 33, + 60, + 13 + ], + [ + 25, + 0, + 15, + 34, + 35, + 11 + ], + [ + 26, + 54, + 28, + 47, + 53, + 37 + ], + [ + 55, + 1, + 34, + 4, + 21, + 50 + ], + [ + 41, + 5, + 40, + 21, + 17, + 23 + ], + [ + 25, + 43, + 52, + 26, + 4, + 55 + ], + [ + 52, + 29, + 35, + 17, + 45, + 60 + ], + [ + 38, + 10, + 15, + 7, + 50, + 3 + ], + [ + 23, + 6, + 19, + 56, + 41, + 15 + ] + ], + [ + [ + 44, + 14, + 20, + 47, + 19, + 56 + ], + [ + 28, + 34, + 2, + 56, + 0, + 11 + ], + [ + 46, + 15, + 61, + 14, + 22, + 60 + ], + [ + 15, + 50, + 36, + 47, + 25, + 21 + ], + [ + 25, + 12, + 13, + 36, + 23, + 57 + ], + [ + 22, + 43, + 1, + 37, + 36, + 30 + ], + [ + 28, + 42, + 19, + 31, + 14, + 21 + ], + [ + 35, + 51, + 47, + 33, + 7, + 46 + ], + [ + 32, + 4, + 16, + 11, + 19, + 35 + ], + [ + 21, + 38, + 16, + 47, + 62, + 15 + ], + [ + 3, + 45, + 39, + 50, + 21, + 2 + ], + [ + 21, + 5, + 56, + 43, + 53, + 52 + ], + [ + 10, + 41, + 33, + 11, + 63, + 37 + ], + [ + 56, + 16, + 9, + 63, + 33, + 45 + ], + [ + 58, + 52, + 2, + 5, + 30, + 56 + ], + [ + 53, + 8, + 59, + 55, + 2, + 50 + ], + [ + 0, + 47, + 51, + 45, + 37, + 14 + ], + [ + 0, + 43, + 21, + 12, + 60, + 62 + ], + [ + 38, + 53, + 60, + 36, + 34, + 9 + ], + [ + 27, + 9, + 56, + 0, + 53, + 11 + ], + [ + 28, + 12, + 43, + 54, + 62, + 5 + ], + [ + 50, + 55, + 34, + 16, + 4, + 21 + ], + [ + 4, + 21, + 5, + 40, + 32, + 54 + ], + [ + 25, + 57, + 49, + 1, + 44, + 43 + ], + [ + 5, + 35, + 25, + 42, + 2, + 22 + ], + [ + 28, + 15, + 38, + 55, + 35, + 37 + ], + [ + 19, + 30, + 6, + 57, + 39, + 33 + ] + ], + [ + [ + 11, + 46, + 0, + 49, + 31, + 16 + ], + [ + 13, + 49, + 50, + 16, + 31, + 19 + ], + [ + 36, + 13, + 27, + 34, + 4, + 52 + ], + [ + 24, + 32, + 7, + 59, + 13, + 15 + ], + [ + 17, + 35, + 2, + 44, + 10, + 63 + ], + [ + 8, + 7, + 23, + 26, + 56, + 42 + ], + [ + 58, + 61, + 29, + 38, + 62, + 50 + ], + [ + 20, + 42, + 3, + 35, + 61, + 47 + ], + [ + 4, + 10, + 7, + 47, + 25, + 34 + ], + [ + 54, + 55, + 18, + 38, + 27, + 28 + ], + [ + 60, + 1, + 14, + 59, + 3, + 30 + ], + [ + 51, + 31, + 5, + 21, + 25, + 52 + ], + [ + 34, + 10, + 2, + 26, + 11, + 47 + ], + [ + 16, + 40, + 23, + 34, + 9, + 48 + ], + [ + 34, + 8, + 1, + 31, + 5, + 32 + ], + [ + 25, + 59, + 8, + 52, + 53, + 27 + ], + [ + 13, + 47, + 45, + 48, + 42, + 0 + ], + [ + 4, + 23, + 53, + 10, + 3, + 25 + ], + [ + 53, + 8, + 38, + 11, + 47, + 24 + ], + [ + 38, + 15, + 0, + 21, + 11, + 8 + ], + [ + 27, + 23, + 41, + 62, + 8, + 42 + ], + [ + 55, + 11, + 10, + 41, + 16, + 21 + ], + [ + 31, + 21, + 41, + 54, + 34, + 9 + ], + [ + 55, + 25, + 22, + 33, + 44, + 47 + ], + [ + 47, + 45, + 20, + 35, + 12, + 42 + ], + [ + 50, + 10, + 47, + 35, + 53, + 57 + ], + [ + 17, + 31, + 5, + 37, + 36, + 20 + ] + ], + [ + [ + 22, + 6, + 29, + 39, + 44, + 57 + ], + [ + 27, + 6, + 17, + 14, + 55, + 57 + ], + [ + 11, + 1, + 29, + 26, + 47, + 4 + ], + [ + 14, + 38, + 31, + 22, + 36, + 29 + ], + [ + 14, + 61, + 59, + 44, + 16, + 63 + ], + [ + 30, + 8, + 47, + 23, + 21, + 0 + ], + [ + 58, + 4, + 54, + 61, + 15, + 62 + ], + [ + 20, + 42, + 58, + 3, + 35, + 0 + ], + [ + 10, + 47, + 4, + 8, + 41, + 19 + ], + [ + 54, + 55, + 33, + 11, + 38, + 21 + ], + [ + 14, + 1, + 4, + 60, + 20, + 40 + ], + [ + 51, + 5, + 14, + 0, + 32, + 62 + ], + [ + 2, + 10, + 36, + 49, + 32, + 52 + ], + [ + 13, + 40, + 54, + 36, + 22, + 11 + ], + [ + 2, + 44, + 37, + 58, + 5, + 8 + ], + [ + 6, + 24, + 2, + 53, + 19, + 52 + ], + [ + 13, + 47, + 61, + 50, + 5, + 30 + ], + [ + 55, + 58, + 4, + 63, + 22, + 38 + ], + [ + 35, + 32, + 46, + 14, + 7, + 58 + ], + [ + 15, + 13, + 45, + 0, + 51, + 60 + ], + [ + 15, + 23, + 27, + 50, + 6, + 17 + ], + [ + 57, + 36, + 10, + 55, + 14, + 16 + ], + [ + 10, + 23, + 41, + 33, + 30, + 54 + ], + [ + 33, + 26, + 4, + 10, + 49, + 55 + ], + [ + 4, + 8, + 33, + 9, + 20, + 12 + ], + [ + 43, + 6, + 3, + 50, + 18, + 38 + ], + [ + 4, + 55, + 43, + 36, + 46, + 5 + ] + ], + [ + [ + 48, + 38, + 47, + 63, + 7, + 42 + ], + [ + 3, + 26, + 10, + 2, + 6, + 35 + ], + [ + 39, + 44, + 6, + 7, + 45, + 8 + ], + [ + 60, + 31, + 22, + 0, + 54, + 27 + ], + [ + 6, + 59, + 16, + 14, + 37, + 44 + ], + [ + 59, + 8, + 30, + 47, + 48, + 3 + ], + [ + 44, + 54, + 4, + 56, + 15, + 13 + ], + [ + 12, + 24, + 20, + 58, + 31, + 61 + ], + [ + 47, + 33, + 10, + 8, + 4, + 36 + ], + [ + 54, + 40, + 55, + 33, + 11, + 51 + ], + [ + 8, + 14, + 23, + 29, + 22, + 31 + ], + [ + 34, + 59, + 14, + 5, + 57, + 50 + ], + [ + 44, + 36, + 10, + 59, + 52, + 9 + ], + [ + 59, + 44, + 13, + 9, + 1, + 54 + ], + [ + 24, + 2, + 46, + 44, + 38, + 47 + ], + [ + 34, + 53, + 6, + 2, + 57, + 40 + ], + [ + 17, + 29, + 47, + 5, + 50, + 34 + ], + [ + 48, + 63, + 57, + 38, + 60, + 32 + ], + [ + 42, + 7, + 49, + 32, + 46, + 1 + ], + [ + 1, + 59, + 23, + 14, + 16, + 10 + ], + [ + 21, + 35, + 7, + 53, + 6, + 17 + ], + [ + 8, + 24, + 60, + 14, + 13, + 55 + ], + [ + 33, + 5, + 51, + 15, + 23, + 3 + ], + [ + 9, + 4, + 51, + 26, + 41, + 8 + ], + [ + 4, + 60, + 19, + 47, + 40, + 9 + ], + [ + 19, + 3, + 26, + 21, + 17, + 8 + ], + [ + 1, + 35, + 53, + 54, + 60, + 20 + ] + ], + [ + [ + 12, + 41, + 14, + 62, + 24, + 10 + ], + [ + 10, + 53, + 39, + 35, + 41, + 58 + ], + [ + 33, + 32, + 50, + 31, + 3, + 34 + ], + [ + 43, + 10, + 42, + 11, + 17, + 47 + ], + [ + 42, + 12, + 11, + 19, + 58, + 54 + ], + [ + 36, + 50, + 55, + 61, + 25, + 56 + ], + [ + 41, + 22, + 16, + 52, + 2, + 15 + ], + [ + 26, + 36, + 62, + 53, + 15, + 51 + ], + [ + 58, + 16, + 5, + 53, + 3, + 49 + ], + [ + 32, + 46, + 26, + 45, + 16, + 62 + ], + [ + 41, + 45, + 56, + 49, + 11, + 3 + ], + [ + 52, + 34, + 35, + 50, + 21, + 53 + ], + [ + 59, + 53, + 46, + 30, + 39, + 37 + ], + [ + 20, + 9, + 52, + 2, + 7, + 33 + ], + [ + 20, + 50, + 24, + 29, + 23, + 2 + ], + [ + 53, + 2, + 44, + 41, + 9, + 13 + ], + [ + 47, + 27, + 1, + 5, + 45, + 46 + ], + [ + 61, + 7, + 51, + 30, + 35, + 9 + ], + [ + 19, + 34, + 32, + 17, + 2, + 14 + ], + [ + 15, + 32, + 6, + 45, + 9, + 11 + ], + [ + 35, + 27, + 6, + 53, + 13, + 60 + ], + [ + 26, + 62, + 16, + 28, + 41, + 3 + ], + [ + 5, + 20, + 46, + 37, + 11, + 55 + ], + [ + 55, + 47, + 4, + 16, + 14, + 27 + ], + [ + 36, + 11, + 27, + 62, + 33, + 7 + ], + [ + 33, + 7, + 54, + 3, + 32, + 12 + ], + [ + 58, + 52, + 27, + 26, + 48, + 38 + ] + ], + [ + [ + 6, + 52, + 19, + 63, + 38, + 46 + ], + [ + 8, + 42, + 4, + 47, + 57, + 56 + ], + [ + 31, + 46, + 32, + 4, + 14, + 10 + ], + [ + 28, + 27, + 4, + 37, + 58, + 20 + ], + [ + 57, + 59, + 60, + 62, + 22, + 14 + ], + [ + 30, + 9, + 2, + 57, + 11, + 13 + ], + [ + 20, + 19, + 57, + 42, + 51, + 27 + ], + [ + 46, + 35, + 47, + 32, + 7, + 0 + ], + [ + 12, + 4, + 35, + 10, + 47, + 50 + ], + [ + 15, + 47, + 54, + 25, + 38, + 51 + ], + [ + 18, + 42, + 5, + 15, + 38, + 61 + ], + [ + 18, + 22, + 8, + 16, + 5, + 7 + ], + [ + 5, + 3, + 24, + 35, + 4, + 30 + ], + [ + 17, + 13, + 55, + 20, + 36, + 22 + ], + [ + 22, + 10, + 21, + 54, + 6, + 47 + ], + [ + 30, + 53, + 6, + 19, + 2, + 54 + ], + [ + 22, + 26, + 5, + 7, + 47, + 21 + ], + [ + 4, + 41, + 13, + 46, + 55, + 43 + ], + [ + 17, + 56, + 32, + 45, + 14, + 6 + ], + [ + 5, + 49, + 53, + 28, + 34, + 60 + ], + [ + 42, + 55, + 57, + 17, + 28, + 22 + ], + [ + 40, + 23, + 28, + 57, + 21, + 16 + ], + [ + 51, + 21, + 35, + 24, + 44, + 10 + ], + [ + 38, + 25, + 4, + 14, + 62, + 31 + ], + [ + 31, + 50, + 13, + 56, + 39, + 53 + ], + [ + 6, + 51, + 55, + 8, + 0, + 21 + ], + [ + 29, + 46, + 18, + 55, + 37, + 50 + ] + ], + [ + [ + 46, + 31, + 35, + 49, + 14, + 21 + ], + [ + 13, + 16, + 50, + 31, + 33, + 8 + ], + [ + 13, + 36, + 27, + 52, + 3, + 19 + ], + [ + 9, + 24, + 29, + 12, + 55, + 61 + ], + [ + 35, + 17, + 2, + 10, + 41, + 48 + ], + [ + 51, + 23, + 30, + 57, + 52, + 17 + ], + [ + 34, + 48, + 0, + 38, + 27, + 51 + ], + [ + 47, + 32, + 2, + 35, + 58, + 55 + ], + [ + 15, + 12, + 35, + 45, + 52, + 4 + ], + [ + 36, + 15, + 4, + 59, + 11, + 54 + ], + [ + 60, + 5, + 59, + 29, + 50, + 18 + ], + [ + 18, + 8, + 31, + 23, + 24, + 3 + ], + [ + 34, + 4, + 5, + 58, + 14, + 27 + ], + [ + 17, + 57, + 36, + 55, + 51, + 6 + ], + [ + 10, + 45, + 8, + 54, + 22, + 47 + ], + [ + 30, + 50, + 14, + 6, + 16, + 37 + ], + [ + 26, + 0, + 22, + 5, + 13, + 4 + ], + [ + 4, + 46, + 23, + 43, + 60, + 41 + ], + [ + 45, + 8, + 56, + 62, + 17, + 51 + ], + [ + 4, + 38, + 53, + 15, + 58, + 5 + ], + [ + 57, + 24, + 34, + 15, + 55, + 42 + ], + [ + 23, + 37, + 40, + 41, + 11, + 54 + ], + [ + 48, + 29, + 51, + 31, + 34, + 9 + ], + [ + 62, + 14, + 18, + 31, + 4, + 59 + ], + [ + 50, + 31, + 49, + 45, + 56, + 57 + ], + [ + 51, + 50, + 56, + 8, + 12, + 52 + ], + [ + 17, + 37, + 5, + 39, + 44, + 20 + ] + ], + [ + [ + 45, + 13, + 63, + 37, + 38, + 56 + ], + [ + 63, + 6, + 12, + 18, + 27, + 51 + ], + [ + 3, + 21, + 4, + 48, + 17, + 27 + ], + [ + 14, + 55, + 9, + 37, + 29, + 26 + ], + [ + 35, + 57, + 2, + 13, + 41, + 10 + ], + [ + 39, + 30, + 57, + 13, + 53, + 23 + ], + [ + 34, + 48, + 38, + 27, + 56, + 51 + ], + [ + 47, + 35, + 46, + 2, + 32, + 7 + ], + [ + 12, + 56, + 4, + 35, + 52, + 50 + ], + [ + 15, + 36, + 14, + 30, + 47, + 48 + ], + [ + 5, + 60, + 59, + 22, + 15, + 46 + ], + [ + 8, + 22, + 23, + 47, + 5, + 26 + ], + [ + 4, + 5, + 30, + 58, + 26, + 8 + ], + [ + 17, + 29, + 13, + 50, + 14, + 6 + ], + [ + 54, + 10, + 41, + 14, + 21, + 5 + ], + [ + 50, + 6, + 54, + 30, + 16, + 56 + ], + [ + 0, + 47, + 7, + 4, + 5, + 34 + ], + [ + 4, + 46, + 23, + 41, + 59, + 55 + ], + [ + 62, + 45, + 1, + 47, + 57, + 32 + ], + [ + 4, + 53, + 15, + 50, + 60, + 1 + ], + [ + 34, + 57, + 24, + 31, + 15, + 53 + ], + [ + 23, + 41, + 45, + 12, + 57, + 14 + ], + [ + 51, + 29, + 49, + 48, + 18, + 33 + ], + [ + 14, + 4, + 62, + 26, + 18, + 28 + ], + [ + 50, + 49, + 7, + 24, + 9, + 48 + ], + [ + 56, + 20, + 51, + 3, + 14, + 26 + ], + [ + 55, + 37, + 50, + 14, + 42, + 20 + ] + ], + [ + [ + 51, + 43, + 27, + 30, + 5, + 12 + ], + [ + 24, + 16, + 48, + 15, + 7, + 30 + ], + [ + 26, + 21, + 50, + 52, + 4, + 56 + ], + [ + 19, + 17, + 2, + 14, + 57, + 22 + ], + [ + 3, + 35, + 37, + 45, + 1, + 6 + ], + [ + 13, + 30, + 23, + 39, + 57, + 60 + ], + [ + 9, + 34, + 48, + 17, + 27, + 26 + ], + [ + 47, + 32, + 59, + 2, + 28, + 57 + ], + [ + 18, + 12, + 31, + 20, + 52, + 4 + ], + [ + 14, + 15, + 36, + 30, + 58, + 31 + ], + [ + 5, + 4, + 26, + 19, + 22, + 59 + ], + [ + 22, + 8, + 34, + 26, + 42, + 52 + ], + [ + 20, + 5, + 52, + 4, + 30, + 33 + ], + [ + 34, + 17, + 2, + 5, + 39, + 20 + ], + [ + 54, + 61, + 30, + 14, + 20, + 25 + ], + [ + 56, + 41, + 2, + 5, + 54, + 37 + ], + [ + 12, + 34, + 47, + 7, + 54, + 59 + ], + [ + 9, + 11, + 2, + 43, + 33, + 50 + ], + [ + 8, + 10, + 19, + 20, + 32, + 18 + ], + [ + 17, + 38, + 6, + 29, + 49, + 41 + ], + [ + 38, + 52, + 45, + 57, + 63, + 27 + ], + [ + 26, + 28, + 3, + 40, + 5, + 47 + ], + [ + 60, + 49, + 35, + 34, + 38, + 3 + ], + [ + 61, + 24, + 4, + 55, + 45, + 16 + ], + [ + 46, + 11, + 27, + 0, + 56, + 48 + ], + [ + 60, + 45, + 44, + 25, + 32, + 3 + ], + [ + 11, + 38, + 52, + 48, + 9, + 21 + ] + ], + [ + [ + 22, + 19, + 46, + 31, + 3, + 23 + ], + [ + 32, + 62, + 15, + 54, + 10, + 55 + ], + [ + 47, + 30, + 38, + 5, + 7, + 60 + ], + [ + 15, + 13, + 1, + 8, + 25, + 43 + ], + [ + 13, + 59, + 5, + 6, + 62, + 52 + ], + [ + 27, + 63, + 62, + 45, + 12, + 56 + ], + [ + 9, + 50, + 8, + 51, + 48, + 18 + ], + [ + 59, + 57, + 2, + 28, + 61, + 6 + ], + [ + 18, + 59, + 6, + 52, + 39, + 57 + ], + [ + 14, + 23, + 11, + 36, + 15, + 32 + ], + [ + 26, + 5, + 42, + 25, + 22, + 23 + ], + [ + 33, + 22, + 55, + 28, + 16, + 24 + ], + [ + 6, + 20, + 33, + 14, + 15, + 52 + ], + [ + 17, + 6, + 5, + 39, + 2, + 34 + ], + [ + 54, + 62, + 25, + 61, + 21, + 14 + ], + [ + 41, + 45, + 14, + 5, + 2, + 54 + ], + [ + 47, + 34, + 10, + 31, + 5, + 41 + ], + [ + 9, + 2, + 36, + 6, + 43, + 38 + ], + [ + 57, + 20, + 41, + 10, + 32, + 18 + ], + [ + 47, + 6, + 49, + 15, + 34, + 7 + ], + [ + 3, + 57, + 44, + 38, + 50, + 53 + ], + [ + 47, + 28, + 57, + 26, + 19, + 22 + ], + [ + 11, + 1, + 18, + 5, + 46, + 22 + ], + [ + 55, + 9, + 4, + 3, + 25, + 10 + ], + [ + 45, + 11, + 0, + 63, + 48, + 57 + ], + [ + 42, + 10, + 3, + 43, + 49, + 0 + ], + [ + 38, + 15, + 17, + 41, + 10, + 3 + ] + ], + [ + [ + 19, + 18, + 51, + 25, + 60, + 55 + ], + [ + 56, + 27, + 61, + 42, + 55, + 23 + ], + [ + 32, + 39, + 37, + 46, + 20, + 52 + ], + [ + 41, + 21, + 37, + 13, + 57, + 2 + ], + [ + 10, + 9, + 3, + 46, + 58, + 32 + ], + [ + 62, + 57, + 27, + 43, + 2, + 53 + ], + [ + 50, + 21, + 19, + 48, + 15, + 6 + ], + [ + 14, + 21, + 54, + 8, + 57, + 28 + ], + [ + 6, + 18, + 4, + 52, + 39, + 24 + ], + [ + 32, + 38, + 23, + 53, + 25, + 17 + ], + [ + 26, + 22, + 25, + 5, + 42, + 33 + ], + [ + 5, + 22, + 16, + 9, + 61, + 55 + ], + [ + 49, + 33, + 30, + 25, + 20, + 22 + ], + [ + 18, + 17, + 25, + 63, + 39, + 11 + ], + [ + 62, + 54, + 10, + 5, + 58, + 37 + ], + [ + 42, + 41, + 5, + 24, + 37, + 54 + ], + [ + 47, + 34, + 27, + 10, + 22, + 55 + ], + [ + 60, + 43, + 44, + 7, + 52, + 37 + ], + [ + 12, + 32, + 39, + 38, + 1, + 20 + ], + [ + 34, + 52, + 49, + 15, + 28, + 40 + ], + [ + 28, + 39, + 3, + 26, + 30, + 22 + ], + [ + 29, + 1, + 40, + 22, + 19, + 63 + ], + [ + 41, + 5, + 53, + 33, + 26, + 39 + ], + [ + 25, + 38, + 4, + 34, + 49, + 51 + ], + [ + 29, + 52, + 48, + 47, + 20, + 33 + ], + [ + 50, + 3, + 16, + 38, + 53, + 15 + ], + [ + 19, + 62, + 6, + 23, + 10, + 36 + ] + ], + [ + [ + 5, + 14, + 17, + 57, + 10, + 27 + ], + [ + 43, + 9, + 56, + 1, + 14, + 33 + ], + [ + 63, + 35, + 43, + 1, + 10, + 27 + ], + [ + 51, + 50, + 57, + 41, + 20, + 54 + ], + [ + 11, + 43, + 40, + 33, + 30, + 54 + ], + [ + 27, + 53, + 63, + 30, + 15, + 58 + ], + [ + 51, + 48, + 21, + 57, + 9, + 50 + ], + [ + 21, + 14, + 59, + 52, + 28, + 8 + ], + [ + 6, + 18, + 4, + 0, + 12, + 59 + ], + [ + 14, + 53, + 30, + 4, + 42, + 36 + ], + [ + 26, + 42, + 5, + 58, + 22, + 19 + ], + [ + 22, + 5, + 34, + 38, + 52, + 29 + ], + [ + 49, + 33, + 0, + 20, + 26, + 8 + ], + [ + 17, + 2, + 39, + 13, + 44, + 63 + ], + [ + 10, + 54, + 20, + 37, + 5, + 59 + ], + [ + 41, + 43, + 24, + 5, + 53, + 37 + ], + [ + 12, + 47, + 34, + 57, + 27, + 5 + ], + [ + 9, + 33, + 7, + 38, + 43, + 31 + ], + [ + 19, + 10, + 20, + 32, + 18, + 1 + ], + [ + 36, + 63, + 15, + 49, + 40, + 50 + ], + [ + 39, + 3, + 38, + 27, + 36, + 35 + ], + [ + 29, + 19, + 40, + 47, + 62, + 14 + ], + [ + 5, + 46, + 33, + 53, + 49, + 21 + ], + [ + 55, + 4, + 27, + 61, + 51, + 34 + ], + [ + 11, + 0, + 59, + 48, + 14, + 27 + ], + [ + 7, + 32, + 3, + 51, + 18, + 14 + ], + [ + 11, + 9, + 36, + 48, + 0, + 46 + ] + ], + [ + [ + 63, + 62, + 60, + 19, + 23, + 56 + ], + [ + 48, + 32, + 1, + 35, + 5, + 21 + ], + [ + 22, + 24, + 46, + 58, + 59, + 60 + ], + [ + 27, + 37, + 50, + 28, + 61, + 6 + ], + [ + 10, + 12, + 15, + 58, + 35, + 23 + ], + [ + 2, + 43, + 57, + 36, + 30, + 20 + ], + [ + 21, + 14, + 19, + 63, + 41, + 42 + ], + [ + 46, + 7, + 35, + 43, + 21, + 36 + ], + [ + 6, + 18, + 0, + 35, + 40, + 4 + ], + [ + 47, + 53, + 32, + 38, + 1, + 58 + ], + [ + 26, + 18, + 22, + 15, + 5, + 46 + ], + [ + 7, + 22, + 24, + 63, + 53, + 5 + ], + [ + 11, + 3, + 10, + 37, + 18, + 24 + ], + [ + 63, + 18, + 13, + 39, + 0, + 17 + ], + [ + 62, + 54, + 5, + 18, + 6, + 14 + ], + [ + 19, + 43, + 31, + 30, + 57, + 42 + ], + [ + 24, + 5, + 51, + 34, + 47, + 55 + ], + [ + 59, + 13, + 33, + 44, + 11, + 41 + ], + [ + 58, + 32, + 1, + 7, + 39, + 38 + ], + [ + 49, + 13, + 28, + 9, + 34, + 7 + ], + [ + 32, + 51, + 3, + 53, + 48, + 13 + ], + [ + 40, + 15, + 37, + 57, + 1, + 23 + ], + [ + 5, + 53, + 35, + 51, + 22, + 38 + ], + [ + 4, + 44, + 14, + 41, + 11, + 38 + ], + [ + 47, + 48, + 8, + 61, + 13, + 9 + ], + [ + 23, + 3, + 27, + 43, + 0, + 61 + ], + [ + 46, + 23, + 59, + 62, + 18, + 50 + ] + ], + [ + [ + 62, + 0, + 9, + 32, + 26, + 41 + ], + [ + 45, + 29, + 35, + 7, + 30, + 62 + ], + [ + 56, + 31, + 23, + 53, + 28, + 2 + ], + [ + 36, + 2, + 5, + 4, + 48, + 41 + ], + [ + 18, + 0, + 15, + 23, + 16, + 11 + ], + [ + 4, + 36, + 57, + 56, + 14, + 5 + ], + [ + 59, + 56, + 21, + 8, + 7, + 33 + ], + [ + 12, + 23, + 7, + 2, + 28, + 34 + ], + [ + 17, + 6, + 23, + 19, + 62, + 27 + ], + [ + 53, + 32, + 51, + 38, + 41, + 58 + ], + [ + 18, + 63, + 19, + 22, + 26, + 5 + ], + [ + 22, + 7, + 60, + 11, + 12, + 19 + ], + [ + 24, + 7, + 11, + 17, + 30, + 37 + ], + [ + 47, + 13, + 63, + 50, + 39, + 45 + ], + [ + 62, + 54, + 18, + 14, + 29, + 56 + ], + [ + 56, + 30, + 0, + 55, + 51, + 10 + ], + [ + 12, + 5, + 21, + 34, + 63, + 29 + ], + [ + 42, + 8, + 38, + 57, + 41, + 44 + ], + [ + 32, + 54, + 1, + 37, + 49, + 40 + ], + [ + 5, + 57, + 49, + 28, + 34, + 10 + ], + [ + 46, + 32, + 13, + 7, + 61, + 48 + ], + [ + 40, + 3, + 30, + 60, + 39, + 1 + ], + [ + 5, + 44, + 33, + 36, + 28, + 31 + ], + [ + 4, + 11, + 28, + 41, + 51, + 5 + ], + [ + 48, + 47, + 61, + 28, + 60, + 27 + ], + [ + 14, + 3, + 26, + 12, + 53, + 61 + ], + [ + 54, + 60, + 49, + 35, + 10, + 62 + ] + ], + [ + [ + 27, + 62, + 63, + 23, + 47, + 56 + ], + [ + 7, + 4, + 2, + 35, + 10, + 36 + ], + [ + 3, + 0, + 27, + 62, + 50, + 60 + ], + [ + 36, + 3, + 42, + 18, + 2, + 48 + ], + [ + 61, + 12, + 27, + 10, + 15, + 14 + ], + [ + 36, + 33, + 50, + 6, + 29, + 16 + ], + [ + 2, + 41, + 8, + 43, + 40, + 59 + ], + [ + 36, + 53, + 15, + 7, + 37, + 2 + ], + [ + 3, + 58, + 4, + 19, + 5, + 27 + ], + [ + 22, + 26, + 53, + 45, + 25, + 38 + ], + [ + 18, + 41, + 22, + 3, + 34, + 27 + ], + [ + 21, + 53, + 5, + 22, + 9, + 28 + ], + [ + 11, + 24, + 28, + 16, + 35, + 8 + ], + [ + 38, + 9, + 21, + 52, + 41, + 39 + ], + [ + 27, + 54, + 14, + 5, + 8, + 63 + ], + [ + 56, + 55, + 43, + 30, + 35, + 51 + ], + [ + 51, + 3, + 14, + 5, + 60, + 26 + ], + [ + 62, + 30, + 8, + 21, + 44, + 35 + ], + [ + 38, + 34, + 27, + 32, + 22, + 8 + ], + [ + 22, + 53, + 28, + 60, + 49, + 12 + ], + [ + 49, + 32, + 41, + 53, + 12, + 33 + ], + [ + 40, + 45, + 22, + 44, + 1, + 34 + ], + [ + 5, + 4, + 53, + 25, + 41, + 63 + ], + [ + 4, + 11, + 25, + 43, + 31, + 14 + ], + [ + 31, + 52, + 5, + 48, + 58, + 43 + ], + [ + 23, + 1, + 28, + 3, + 0, + 55 + ], + [ + 27, + 45, + 32, + 4, + 30, + 6 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 32 + ], + [ + 51, + 5, + 41, + 40, + 21, + 44 + ], + [ + 43, + 1, + 29, + 55, + 21, + 35 + ], + [ + 36, + 58, + 25, + 3, + 18, + 54 + ], + [ + 31, + 60, + 24, + 12, + 61, + 41 + ], + [ + 4, + 10, + 35, + 36, + 0, + 43 + ], + [ + 45, + 43, + 63, + 35, + 36, + 48 + ], + [ + 3, + 9, + 8, + 7, + 43, + 27 + ], + [ + 0, + 43, + 4, + 40, + 18, + 44 + ], + [ + 34, + 15, + 50, + 38, + 53, + 25 + ], + [ + 24, + 46, + 26, + 13, + 2, + 18 + ], + [ + 57, + 37, + 5, + 7, + 22, + 53 + ], + [ + 3, + 9, + 38, + 26, + 37, + 23 + ], + [ + 60, + 26, + 0, + 39, + 13, + 55 + ], + [ + 54, + 21, + 58, + 6, + 55, + 14 + ], + [ + 11, + 19, + 43, + 57, + 56, + 37 + ], + [ + 10, + 24, + 60, + 21, + 5, + 58 + ], + [ + 31, + 20, + 63, + 44, + 55, + 33 + ], + [ + 7, + 25, + 40, + 32, + 54, + 58 + ], + [ + 49, + 30, + 50, + 13, + 55, + 28 + ], + [ + 30, + 3, + 51, + 61, + 0, + 10 + ], + [ + 57, + 46, + 43, + 40, + 30, + 22 + ], + [ + 52, + 33, + 51, + 19, + 27, + 29 + ], + [ + 14, + 4, + 41, + 40, + 0, + 3 + ], + [ + 4, + 13, + 8, + 61, + 47, + 9 + ], + [ + 31, + 27, + 43, + 1, + 3, + 0 + ], + [ + 8, + 4, + 46, + 9, + 3, + 50 + ] + ] ] } } \ No newline at end of file From e0d3dd52d09ad1d781de4a855aa031ae4c535b09 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:54:00 -0800 Subject: [PATCH 22/24] add routing indices to metrics --- .../model_config.yaml | 1 + .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml index d33a854e49e..5ab6d37f17e 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml @@ -89,3 +89,4 @@ MODEL_ARGS: METRICS: - "generated_tokens" - "logprobs" + - "routing_indices" diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index b9d78097bbd..a4fd86c4219 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -39,7 +39,7 @@ spec: ARGUMENTS=( "CHECKPOINT_LOAD_PATH=/mnt/artifacts" "CHECKPOINT_SAVE_PATH=/tmp/checkpoints" - "DATA_PATH=null" + "DATA_PATH=/mnt/artifacts/" "DATA_CACHE_PATH=/workspace/data/cache" "TRAINING_SCRIPT_PATH=examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py" "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" From dda0b6a29dac10364474ebeccfdffa0b2560abae Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 11 Feb 2026 04:45:35 -0800 Subject: [PATCH 23/24] move test to github --- .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index a4fd86c4219..4ce11808bdc 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -63,6 +63,6 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [mr] + scope: [mr-github] platforms: [dgx_h100] From b632151e9f75dc8dd635c6f5ba95f21de4628651 Mon Sep 17 00:00:00 2001 From: Siddharth Singh <136645615+sidsingh-nvidia@users.noreply.github.com> Date: Wed, 11 Feb 2026 05:27:23 -0800 Subject: [PATCH 24/24] Update test_inference_regular_pipeline.py --- .../python_test_utils/test_inference_regular_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py index 00a4890af45..93bfa6e1299 100644 --- a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py @@ -172,7 +172,7 @@ def test_inference_pipeline( f"\nCurrent (truncated to {min_len} chars): {generated_text_current[:min_len]}" ) - if "routing_indices" in groundtruth_results: + if "routing_indices" in groundtruth_results and "routing_indices" in metrics: at_least_one_test_loop = True routing_indices_groundtruth = groundtruth_results["routing_indices"] routing_indices_current = current_results["routing_indices"]