From cbd57c1792db69dc3d4790659c7c45128d33824b Mon Sep 17 00:00:00 2001 From: litianjian Date: Mon, 3 Nov 2025 17:45:10 +0800 Subject: [PATCH 01/56] feat: add router replay --- megatron/core/transformer/moe/moe_utils.py | 147 +++++++++++++++++- megatron/core/transformer/moe/router.py | 6 + .../core/transformer/transformer_config.py | 3 + 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index dc857129834..638a6c48cce 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1,6 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import math +from enum import Enum from typing import List, Optional, Union import torch @@ -519,6 +520,102 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map +class RoutingMode(Enum): + NONE = "none" + RECORD = "record" + REPLAY_FORWARD = "replay_forward" + REPLAY_BACKWARD = "replay_backward" + FALLTHROUGH = "fallthrough" + + +class RouterReplay: + """ + A class to manage the recording and replaying of MoE routing decisions. + It holds all router instances and provides static methods to globally + control recording and replaying. + """ + + # Static variable to hold all router instances, one per MoE layer. + router_instances = [] + + @staticmethod + def set_replay_data(all_layers_topk_indices: list): + """ + Distributes the topk indices for all layers to their respective RouterReplay instances. + :param all_layers_topk_indices: A list of tensors, where each tensor contains the + topk indices for a specific layer. The order + must match the instantiation order of the routers. + """ + if len(all_layers_topk_indices) != len(RouterReplay.router_instances): + raise ValueError( + f"The number of replay tensors ({len(all_layers_topk_indices)}) " + f"does not match the number of router instances ({len(RouterReplay.router_instances)})." + ) + for i, router_instance in enumerate(RouterReplay.router_instances): + router_instance.set_target_indices(all_layers_topk_indices[i]) + + @staticmethod + def get_recorded_data() -> list: + """ + Collects the recorded topk indices from all RouterReplay instances. + :return: A list of tensors, each containing the recorded topk indices for a layer. + """ + return [router.get_recorded_indices() for router in RouterReplay.router_instances] + + @staticmethod + def clear_global_indices(): + """Clears the recorded and target topk indices in all instances.""" + for router in RouterReplay.router_instances: + router.clear_indices() + + def __init__(self): + """Initializes a RouterReplay instance for a specific layer.""" + self.target_topk_idx = None # For replay + self.recorded_topk_idx = None # For recording + self.routing_mode = None # Routing mode for this layer + self.replay_backward_list = [] # List of tensors for backward pass replay + RouterReplay.router_instances.append(self) + + def set_target_indices(self, topk_indices: torch.Tensor): + """Sets the target topk indices for replay.""" + self.target_topk_idx = topk_indices + self.replay_backward_list.append(topk_indices) + + def get_recorded_indices(self) -> Optional[torch.Tensor]: + """Returns the recorded topk indices.""" + return self.recorded_topk_idx + + def record_indices(self, topk_indices: torch.Tensor): + """Records the topk indices.""" + self.recorded_topk_idx = topk_indices + + def clear_indices(self): + """Clears the recorded and target topk indices.""" + self.recorded_topk_idx = None + self.target_topk_idx = None + self.replay_backward_list = [] + + def set_routing_mode(self, routing_mode: RoutingMode): + """Sets the routing mode for this layer.""" + self.routing_mode = routing_mode + + def clear_routing_mode(self): + """Clears the routing mode for this layer.""" + self.routing_mode = None + + @staticmethod + def set_global_routing_mode(routing_mode: RoutingMode): + """Sets the routing mode for all router instances.""" + for router in RouterReplay.router_instances: + router.set_routing_mode(routing_mode) + + @staticmethod + def clear_global_routing_mode(): + """Clears the routing mode for all router instances.""" + for router in RouterReplay.router_instances: + router.clear_routing_mode() + + def topk_routing_with_score_function( logits: torch.Tensor, topk: int, @@ -529,6 +626,7 @@ def topk_routing_with_score_function( score_function: str = "softmax", expert_bias: Optional[torch.Tensor] = None, fused: bool = False, + router_replay: Optional['RouterReplay'] = None, ): """Compute the routing probabilities and map for top-k selection with score function. Args: @@ -540,6 +638,10 @@ def topk_routing_with_score_function( scaling_factor (float): Scaling factor of routing score in top-k selection. score_function (str): The score function to use. Can be either "softmax" or "sigmoid". expert_bias (torch.Tensor): The bias added to logits for expert routing. + router_replay (Optional['RouterReplay']): For debugging and development, allows for + deterministic routing by replaying a previously + recorded routing sequence. + Returns: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - routing_probs (torch.Tensor): A tensor of shape [num_tokens, num_experts] containing @@ -566,7 +668,7 @@ def topk_routing_with_score_function( expert_bias=expert_bias, ) - def compute_topk(scores, topk, num_groups=None, group_topk=None): + def _compute_topk(scores, topk, num_groups=None, group_topk=None): if group_topk: return group_limited_topk( scores=scores, @@ -579,6 +681,49 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): else: return torch.topk(scores, k=topk, dim=1) + def compute_topk(scores, topk, num_groups=None, group_topk=None): + # Default behavior if no replay is active + + routing_action = router_replay.routing_mode.value if router_replay is not None else None + + if routing_action is None or routing_action == "fallthrough": + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + if routing_action == "record": + probs, top_indices = _compute_topk( + scores, topk, num_groups=num_groups, group_topk=group_topk + ) + if router_replay is not None: + router_replay.record_indices(top_indices) + return probs, top_indices + + elif routing_action == "forward_replay": + if router_replay is None or router_replay.target_topk_idx is None: + # Fallback if replay data is not available + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + # Use the provided indices for replay + top_indices = router_replay.target_topk_idx + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + elif routing_action == "backward_replay": + if router_replay is None or not router_replay.replay_backward_list: + # Fallback if replay data is not available + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + + # Use the last recorded indices for backward replay + top_indices = router_replay.replay_backward_list.pop() + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + else: # Unknown action, fallback + return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) + if score_function == "softmax": if use_pre_softmax: scores = torch.softmax(logits, dim=-1, dtype=torch.float32).type_as(logits) diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 068d680c798..dee7ffa4711 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -10,6 +10,7 @@ from megatron.core.transformer.moe.moe_utils import ( MoEAuxLossAutoScaler, ProcessGroupCollection, + RouterReplay, apply_random_logits, apply_router_token_dropping, compute_routing_scores_for_aux_loss, @@ -198,6 +199,10 @@ def __init__( self.global_tokens_per_expert = None self.ga_steps = None + self.router_replay = None + if self.config.enable_routing_replay: + self.router_replay = RouterReplay() + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. @@ -497,6 +502,7 @@ def routing(self, logits: torch.Tensor): score_function=self.score_function, expert_bias=self.expert_bias, fused=self.config.moe_router_fusion, + router_replay=self.router_replay, ) # Apply token dropping to probs and routing_map. diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 147c5b23b3d..a129c9faecb 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -464,6 +464,9 @@ class TransformerConfig(ModelParallelConfig): moe_router_topk: int = 2 """Number of experts to route to for each token.""" + enable_routing_replay: bool = False + """If True, enable the routing replay feature for MoE layers.""" + moe_router_topk_limited_devices: Optional[int] = None """Number of EP ranks to consider for each token in group-limited routing, DEPRECATED and replaced by moe_router_num_groups and moe_router_group_topk. From bd32db8dee9a13e32c84d62cb8c3bf29d0e8110f Mon Sep 17 00:00:00 2001 From: litianjian Date: Mon, 24 Nov 2025 14:18:14 +0800 Subject: [PATCH 02/56] refactor(router): rename RouterMode to RouterReplayAction --- megatron/core/transformer/moe/moe_utils.py | 40 ++++++++++------------ 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index edeec5bcdb6..941932530f5 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -524,12 +524,10 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map -class RoutingMode(Enum): - NONE = "none" +class RouterReplayAction(Enum): RECORD = "record" REPLAY_FORWARD = "replay_forward" REPLAY_BACKWARD = "replay_backward" - FALLTHROUGH = "fallthrough" class RouterReplay: @@ -576,7 +574,7 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx = None # For replay self.recorded_topk_idx = None # For recording - self.routing_mode = None # Routing mode for this layer + self.router_replay_action = None # Router replay action for this layer self.replay_backward_list = [] # List of tensors for backward pass replay RouterReplay.router_instances.append(self) @@ -599,25 +597,25 @@ def clear_indices(self): self.target_topk_idx = None self.replay_backward_list = [] - def set_routing_mode(self, routing_mode: RoutingMode): - """Sets the routing mode for this layer.""" - self.routing_mode = routing_mode + def set_router_replay_action(self, router_replay_action: RouterReplayAction): + """Sets the router replay action for this layer.""" + self.router_replay_action = router_replay_action - def clear_routing_mode(self): - """Clears the routing mode for this layer.""" - self.routing_mode = None + def clear_router_replay_action(self): + """Clears the router replay action for this layer.""" + self.router_replay_action = None @staticmethod - def set_global_routing_mode(routing_mode: RoutingMode): - """Sets the routing mode for all router instances.""" + def set_global_router_replay_action(router_replay_action: RouterReplayAction): + """Sets the router replay action for all router instances.""" for router in RouterReplay.router_instances: - router.set_routing_mode(routing_mode) + router.set_router_replay_action(router_replay_action) @staticmethod - def clear_global_routing_mode(): - """Clears the routing mode for all router instances.""" + def clear_global_router_replay_action(): + """Clears the router replay action for all router instances.""" for router in RouterReplay.router_instances: - router.clear_routing_mode() + router.clear_router_replay_action() def topk_routing_with_score_function( @@ -688,12 +686,12 @@ def _compute_topk(scores, topk, num_groups=None, group_topk=None): def compute_topk(scores, topk, num_groups=None, group_topk=None): # Default behavior if no replay is active - routing_action = router_replay.routing_mode.value if router_replay is not None else None + routing_action = router_replay.router_replay_action if router_replay is not None else None - if routing_action is None or routing_action == "fallthrough": + if routing_action is None: return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - if routing_action == "record": + if routing_action == RouterReplayAction.RECORD: probs, top_indices = _compute_topk( scores, topk, num_groups=num_groups, group_topk=group_topk ) @@ -701,7 +699,7 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): router_replay.record_indices(top_indices) return probs, top_indices - elif routing_action == "forward_replay": + elif routing_action == RouterReplayAction.REPLAY_FORWARD: if router_replay is None or router_replay.target_topk_idx is None: # Fallback if replay data is not available return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) @@ -713,7 +711,7 @@ def compute_topk(scores, topk, num_groups=None, group_topk=None): # Gather the scores for the replayed indices to get the probabilities probs = scores.gather(1, top_indices) return probs, top_indices - elif routing_action == "backward_replay": + elif routing_action == RouterReplayAction.REPLAY_BACKWARD: if router_replay is None or not router_replay.replay_backward_list: # Fallback if replay data is not available return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) From 1aec041b418e2feb6bca9e415a74dc30173ef901 Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 16 Dec 2025 13:10:54 +0800 Subject: [PATCH 03/56] simplify compute topk function --- megatron/core/transformer/moe/moe_utils.py | 130 ++++++++++----------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index d81e93b291c..a4581b4d051 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -525,9 +525,9 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso class RouterReplayAction(Enum): - RECORD = "record" - REPLAY_FORWARD = "replay_forward" - REPLAY_BACKWARD = "replay_backward" + RECORD = "record" # Record the topk indices for replay + REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass + REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass class RouterReplay: @@ -538,45 +538,57 @@ class RouterReplay: """ # Static variable to hold all router instances, one per MoE layer. - router_instances = [] + global_router_replay_instances: List['RouterReplay'] = [] @staticmethod - def set_replay_data(all_layers_topk_indices: list): + def set_replay_data(all_layers_topk_indices: List[torch.Tensor]): """ Distributes the topk indices for all layers to their respective RouterReplay instances. :param all_layers_topk_indices: A list of tensors, where each tensor contains the topk indices for a specific layer. The order must match the instantiation order of the routers. """ - if len(all_layers_topk_indices) != len(RouterReplay.router_instances): + if len(all_layers_topk_indices) != len(RouterReplay.global_router_replay_instances): raise ValueError( f"The number of replay tensors ({len(all_layers_topk_indices)}) " - f"does not match the number of router instances ({len(RouterReplay.router_instances)})." + f"does not match router instances ({len(RouterReplay.global_router_replay_instances)})." ) - for i, router_instance in enumerate(RouterReplay.router_instances): + for i, router_instance in enumerate(RouterReplay.global_router_replay_instances): router_instance.set_target_indices(all_layers_topk_indices[i]) @staticmethod - def get_recorded_data() -> list: + def get_recorded_data() -> List[torch.Tensor]: """ Collects the recorded topk indices from all RouterReplay instances. :return: A list of tensors, each containing the recorded topk indices for a layer. """ - return [router.get_recorded_indices() for router in RouterReplay.router_instances] + return [router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances] @staticmethod def clear_global_indices(): """Clears the recorded and target topk indices in all instances.""" - for router in RouterReplay.router_instances: + for router in RouterReplay.global_router_replay_instances: router.clear_indices() + @staticmethod + def set_global_router_replay_action(router_replay_action: RouterReplayAction): + """Sets the router replay action for all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.set_router_replay_action(router_replay_action) + + @staticmethod + def clear_global_router_replay_action(): + """Clears the router replay action for all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_router_replay_action() + def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" - self.target_topk_idx = None # For replay - self.recorded_topk_idx = None # For recording - self.router_replay_action = None # Router replay action for this layer - self.replay_backward_list = [] # List of tensors for backward pass replay - RouterReplay.router_instances.append(self) + self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay + self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay + self.router_replay_action: Optional[RouterReplayAction] = None # Router replay action for this layer + self.replay_backward_list: List[torch.Tensor] = [] # List of tensors for backward pass replay + RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): """Sets the target topk indices for replay.""" @@ -605,18 +617,39 @@ def clear_router_replay_action(self): """Clears the router replay action for this layer.""" self.router_replay_action = None - @staticmethod - def set_global_router_replay_action(router_replay_action: RouterReplayAction): - """Sets the router replay action for all router instances.""" - for router in RouterReplay.router_instances: - router.set_router_replay_action(router_replay_action) - - @staticmethod - def clear_global_router_replay_action(): - """Clears the router replay action for all router instances.""" - for router in RouterReplay.router_instances: - router.clear_router_replay_action() - + def get_replay_topk( + self, + scores: torch.Tensor, + topk: int, + num_groups: Optional[int] = None, + group_topk: Optional[int] = None, + default_compute_topk: Callable[[torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor] = None, + ) -> torch.Tensor: + """Returns the target topk indices for replay.""" + if self.router_replay_action == RouterReplayAction.RECORD: + probs, top_indices = default_compute_topk( + scores, topk, num_groups=num_groups, group_topk=group_topk + ) + self.record_indices(top_indices) + return probs, top_indices + elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: + # Use the provided indices for replay + top_indices = router_replay.target_topk_idx + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: + # Use the last recorded indices for backward replay + top_indices = router_replay.replay_backward_list.pop(0) + # Ensure indices are on the correct device + top_indices = top_indices.to(scores.device) + # Gather the scores for the replayed indices to get the probabilities + probs = scores.gather(1, top_indices) + return probs, top_indices + else: + return default_compute_topk(scores, topk, num_groups, group_topk) def topk_routing_with_score_function( logits: torch.Tensor, @@ -686,45 +719,12 @@ def _compute_topk(scores, topk, num_groups=None, group_topk=None): def compute_topk(scores, topk, num_groups=None, group_topk=None): # Default behavior if no replay is active - routing_action = router_replay.router_replay_action if router_replay is not None else None - - if routing_action is None: + if router_replay is None: return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - if routing_action == RouterReplayAction.RECORD: - probs, top_indices = _compute_topk( - scores, topk, num_groups=num_groups, group_topk=group_topk + else: + return router_replay.get_replay_topk( + scores, topk, num_groups, group_topk, _compute_topk ) - if router_replay is not None: - router_replay.record_indices(top_indices) - return probs, top_indices - - elif routing_action == RouterReplayAction.REPLAY_FORWARD: - if router_replay is None or router_replay.target_topk_idx is None: - # Fallback if replay data is not available - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - # Use the provided indices for replay - top_indices = router_replay.target_topk_idx - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - elif routing_action == RouterReplayAction.REPLAY_BACKWARD: - if router_replay is None or not router_replay.replay_backward_list: - # Fallback if replay data is not available - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) - - # Use the last recorded indices for backward replay - top_indices = router_replay.replay_backward_list.pop() - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - else: # Unknown action, fallback - return _compute_topk(scores, topk, num_groups=num_groups, group_topk=group_topk) if score_function == "softmax": if use_pre_softmax: From 39fd47ae621d769ee649cfaed8ae77fc45facf23 Mon Sep 17 00:00:00 2001 From: litianjian Date: Wed, 17 Dec 2025 15:45:19 +0800 Subject: [PATCH 04/56] update router replay --- megatron/core/transformer/moe/moe_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index a4581b4d051..0ece25f1b45 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -633,16 +633,14 @@ def get_replay_topk( self.record_indices(top_indices) return probs, top_indices elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: - # Use the provided indices for replay - top_indices = router_replay.target_topk_idx + top_indices = self.target_topk_idx # Ensure indices are on the correct device top_indices = top_indices.to(scores.device) # Gather the scores for the replayed indices to get the probabilities probs = scores.gather(1, top_indices) return probs, top_indices elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: - # Use the last recorded indices for backward replay - top_indices = router_replay.replay_backward_list.pop(0) + top_indices = self.replay_backward_list.pop(0) # Ensure indices are on the correct device top_indices = top_indices.to(scores.device) # Gather the scores for the replayed indices to get the probabilities From 49da256654a25554fa72154165181d1fb44772bb Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 23 Dec 2025 17:22:42 +0800 Subject: [PATCH 05/56] add unit test and doc --- docs/api-guide/router_replay.md | 174 ++++++++++++++++++ megatron/core/transformer/moe/moe_utils.py | 25 ++- .../transformer/moe/test_router_replay.py | 110 +++++++++++ 3 files changed, 301 insertions(+), 8 deletions(-) create mode 100644 docs/api-guide/router_replay.md create mode 100644 tests/unit_tests/transformer/moe/test_router_replay.py diff --git a/docs/api-guide/router_replay.md b/docs/api-guide/router_replay.md new file mode 100644 index 00000000000..3772997530f --- /dev/null +++ b/docs/api-guide/router_replay.md @@ -0,0 +1,174 @@ +# Design Document: MoE Router Replay Feature + +### 1. Overview + +This document provides a detailed description of the "Router Replay" feature implemented within the Megatron-LM Core for Mixture-of-Experts (MoE) models. + +This feature is designed to enhance determinism and analyzability in MoE model training and inference. It enables the model to load routing decisions from a predefined file and enforce their use during the forward pass, thereby bypassing the real-time routing computation. + +### 2. Motivation + +* **Determinism & Reproducibility**: In distributed training, MoE routing decisions can exhibit minor variations due to factors like floating-point precision. By replaying a fixed routing table, the MoE computation path is guaranteed to be identical across runs, which facilitates debugging and reproducing experimental results. +* **Performance Profiling**: The router's own computation (e.g., logits calculation, top-k selection) incurs overhead. In replay mode, this part of the computation can be completely skipped, allowing for more precise isolation and profiling of performance bottlenecks within the Expert Layers themselves. +* **Debugging Aid**: When issues arise in the model, fixing the routing decisions helps to isolate variables, making it easier to determine whether the problem lies with the routing mechanism or the expert computations. + +### 3. Design and Architecture + +The design follows the principles of being non-intrusive and on-demand, with the core idea of activating the replay logic only when explicitly requested by the user. + +* **Core Components**: + * `RouterReplay` (located in `megatron/core/transformer/moe/moe_utils.py`): A utility class for replaying MoE routing decisions. When enabled via the `enable_routing_replay` flag, a separate instance of `RouterReplay` is created for each MoE layer's router. Each instance is responsible for loading routing data and providing the deterministic routing decisions for its corresponding layer during the forward pass. + * `enable_routing_replay` (located in `megatron/core/transformer/transformer_config.py`): A boolean global configuration flag that serves as the sole entry point for enabling this feature. + +* **Workflow**: + The feature supports different modes, such as recording and replaying, controlled by a `RouterReplayAction`. + + 1. **Enabling the Feature**: The user sets `enable_routing_replay` to `True` in the model configuration. + 2. **Initialization**: When `enable_routing_replay` is true, each `TopKRouter` creates its own `RouterReplay` instance. + 3. **Mode Configuration**: The user must programmatically set the desired router replay action (e.g., `record`, `forward_replay`, `backward_replay`) on the `RouterReplay` instances. + 4. **Execution Flow (within a mini-batch)**: + * **Forward Pass**: + * For each micro-batch, the `topk_routing_with_score_function` checks the `router_replay_action`. + * **In `record` mode**: The dynamically computed `top-k` expert indices are captured and stored. + * **In `forward_replay` mode**: The function retrieves pre-loaded expert indices from `target_topk_idx`. These indices are used for the forward computation and are also appended to the `replay_backward_list` to prepare for the backward pass. + * **Backward Pass**: + * For each micro-batch (processed in reverse order in pipeline parallelism), the `router_replay_action` is checked again. + * **In `backward_replay` mode**: The function retrieves the expert indices for the corresponding micro-batch by popping them from the `replay_backward_list`. This mode is intended for training recomputation (e.g., activation checkpointing and pipeline recompute) so the same routing decisions are used during recompute/backward as in forward, ensuring determinism and correctness. + +### 4. Implementation Details + +The implementation cleanly separates the replay logic from the router's core computation. + +* **`megatron/core/transformer/transformer_config.py`**: + * Adds the configuration option `enable_routing_replay: bool = False`. + +* **`megatron/core/transformer/moe/moe_utils.py`**: + * Introduces the `RouterReplay` class to manage the state for recording and replaying routing decisions for a single MoE layer. + * `target_topk_idx`: An attribute holding the expert indices for the current micro-batch during forward replay mode. + * `recorded_topk_idx`: An attribute for storing the computed expert indices when in record mode. + * `replay_backward_list`: A list that accumulates the `top-k` indices used during the forward passes of a mini-batch. This list is consumed in FIFO order during the backward pass to ensure correctness under pipeline parallelism. + * `set_target_indices()`: A method to load the replay indices into `target_topk_idx` for the forward pass. + * `record_indices()`: A method to save the computed indices. + * The `topk_routing_with_score_function` is modified to contain the core logic. It checks the `router_replay_action` on the `router_replay` instance and accordingly performs one of the following actions: computes and records indices, replays indices from `target_topk_idx` (for forward), replays indices from `replay_backward_list` (for backward), or falls through to the default dynamic routing. + +#### Training recompute usage +- During forward replay, `set_target_indices()` prepares `replay_backward_list` so each micro-batch’s indices are available for recomputation. +- During recompute/backward, set action to `REPLAY_BACKWARD` so indices are consumed in FIFO order to mirror the forward sequence. + +### 5. Usage Guide + +1. **Enable & Instantiate** + - Create one `RouterReplay` instance per MoE router layer when building the model. + - Optionally use the global helpers to set/clear actions across all layers. +2. **Record Routing Decisions** + - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD)`. + - Run the model; retrieve per-layer indices via `RouterReplay.get_recorded_data()` and persist. +3. **Forward Replay** + - Load indices and distribute: `RouterReplay.set_replay_data(list_of_tensors)`. + - Set action: `RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD)`. + - Run the model; dynamic top‑k is bypassed and target indices are used. +4. **Backward Replay** + - For training recomputation (activation checkpointing or pipeline recompute), set action: `REPLAY_BACKWARD` during recomputation. + - Per micro‑batch indices are consumed from `replay_backward_list` in FIFO order. +5. **Cleanup** + - Use `RouterReplay.clear_global_indices()` and `RouterReplay.clear_global_router_replay_action()` to restore default behavior. + +#### Quick usage with `topk_routing_with_score_function` + +```python +import torch +from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, RouterReplayAction, topk_routing_with_score_function, +) + +rr = RouterReplay() + +# Record +RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) +logits = torch.randn(8, 16) +probs_rec, routing_map_rec = topk_routing_with_score_function( + logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr, +) +recorded = rr.get_recorded_indices() +torch.save(recorded, "/tmp/replay.pt") + +# Forward replay +rr.clear_router_replay_action() +rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) +target = torch.load("/tmp/replay.pt") +rr.set_target_indices(target) +probs_rep, routing_map_rep = topk_routing_with_score_function( + logits=logits, topk=2, use_pre_softmax=False, score_function="softmax", router_replay=rr, +) + +RouterReplay.clear_global_router_replay_action() +RouterReplay.clear_global_indices() +``` + +### 6. Minimal Demo + +Here is a minimal code example showing how to use RouterReplay for recording and replaying: + +```python +import torch +import torch.distributed as dist +from megatron.core.config import TransformerConfig +from megatron.core.transformer.moe.router import TopKRouter +from megatron.core.transformer.moe.moe_utils import RouterReplayAction, RouterReplay + + +# Initialize distributed training +if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + +# Create a transformer config with RouterReplay enabled +config = TransformerConfig( + num_experts=8, + expert_model_parallel_size=1, + num_top_k=2, + enable_routing_replay=True +) + +# Create a TopKRouter instance +router = TopKRouter(config) + +# Generate sample input (batch_size, sequence_length, hidden_size) +logits = torch.randn(16, 32, 8).to(torch.cuda.current_device()) + +# ----------------- +# 1. Recording Mode +# ----------------- +print("=== Recording Mode ===") +# Set global router replay action to RECORD +RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + +# Perform routing +routing_output = router.forward(logits) +print(f"Recorded top-k indices shape: {routing_output.top_k_idx.shape}") + +# ----------------- +# 2. Forward Replay Mode +# ----------------- +print("\n=== Forward Replay Mode ===") +# Save recorded indices to a file +torch.save(routing_output.top_k_idx, "/tmp/replay.pt") + +# Load indices from file and set as target for replay +replay_indices = torch.load("/tmp/replay.pt") +for router_instance in RouterReplay.router_instances: + router_instance.target_topk_idx = replay_indices + +# Set global router replay action to REPLAY_FORWARD +RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + +# Perform routing again - this will use the replayed indices +replay_routing_output = router.forward(logits) +print(f"Replayed top-k indices shape: {replay_routing_output.top_k_idx.shape}") +print(f"Are indices the same? {torch.equal(routing_output.top_k_idx, replay_routing_output.top_k_idx)}") + + +# Clean up +RouterReplay.clear_global_router_replay_action() +if dist.is_initialized(): + dist.destroy_process_group() +``` diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 0ece25f1b45..360cef9789d 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -2,7 +2,7 @@ import math from enum import Enum -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union import torch @@ -525,9 +525,9 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso class RouterReplayAction(Enum): - RECORD = "record" # Record the topk indices for replay - REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass - REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass + RECORD = "record" # Record the topk indices for replay + REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass + REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass class RouterReplay: @@ -562,7 +562,9 @@ def get_recorded_data() -> List[torch.Tensor]: Collects the recorded topk indices from all RouterReplay instances. :return: A list of tensors, each containing the recorded topk indices for a layer. """ - return [router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances] + return [ + router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances + ] @staticmethod def clear_global_indices(): @@ -586,8 +588,12 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay - self.router_replay_action: Optional[RouterReplayAction] = None # Router replay action for this layer - self.replay_backward_list: List[torch.Tensor] = [] # List of tensors for backward pass replay + self.router_replay_action: Optional[RouterReplayAction] = ( + None # Router replay action for this layer + ) + self.replay_backward_list: List[torch.Tensor] = ( + [] + ) # List of tensors for backward pass replay RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): @@ -623,7 +629,9 @@ def get_replay_topk( topk: int, num_groups: Optional[int] = None, group_topk: Optional[int] = None, - default_compute_topk: Callable[[torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor] = None, + default_compute_topk: Callable[ + [torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor + ] = None, ) -> torch.Tensor: """Returns the target topk indices for replay.""" if self.router_replay_action == RouterReplayAction.RECORD: @@ -649,6 +657,7 @@ def get_replay_topk( else: return default_compute_topk(scores, topk, num_groups, group_topk) + def topk_routing_with_score_function( logits: torch.Tensor, topk: int, diff --git a/tests/unit_tests/transformer/moe/test_router_replay.py b/tests/unit_tests/transformer/moe/test_router_replay.py new file mode 100644 index 00000000000..00732a63ee1 --- /dev/null +++ b/tests/unit_tests/transformer/moe/test_router_replay.py @@ -0,0 +1,110 @@ +import torch +import pytest + +from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, + RouterReplayAction, + topk_routing_with_score_function, +) + + +def setup_function(): + RouterReplay.global_router_replay_instances.clear() + + +def teardown_function(): + RouterReplay.global_router_replay_instances.clear() + + +def test_record_mode_with_topk_routing_softmax_post(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.RECORD) + logits = torch.randn(4, 6) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=False, + router_replay=rr, + score_function="softmax", + ) + recorded = rr.get_recorded_indices() + expected_idx = torch.topk(logits, k=2, dim=1).indices + assert recorded is not None + assert torch.equal(recorded, expected_idx) + assert probs.shape == (4, 6) + assert routing_map.shape == (4, 6) + assert routing_map.sum(dim=1).eq(2).all() + + +def test_replay_forward_with_topk_routing_softmax_pre(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + logits = torch.randn(3, 5) + target = torch.tensor([[1, 2], [0, 3], [2, 4]], dtype=torch.long) + rr.set_target_indices(target) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=True, + router_replay=rr, + score_function="softmax", + ) + assert routing_map.sum(dim=1).eq(2).all() + scores = torch.softmax(logits, dim=-1) + assert torch.equal(probs.gather(1, target), scores.gather(1, target)) + + +def test_replay_forward_with_topk_routing_softmax_post(): + rr = RouterReplay() + rr.set_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + logits = torch.randn(3, 6) + target = torch.tensor([[1, 2], [0, 5], [3, 4]], dtype=torch.long) + rr.set_target_indices(target) + probs, routing_map = topk_routing_with_score_function( + logits=logits, + topk=2, + use_pre_softmax=False, + router_replay=rr, + score_function="softmax", + ) + selected = torch.softmax(logits.gather(1, target), dim=-1) + assert torch.equal(probs.gather(1, target), selected) + assert routing_map.sum(dim=1).eq(2).all() + + +def test_global_set_get_clear_indices(): + r1 = RouterReplay() + r2 = RouterReplay() + t1 = torch.tensor([[0, 1]], dtype=torch.long) + t2 = torch.tensor([[1, 0]], dtype=torch.long) + RouterReplay.set_replay_data([t1, t2]) + assert torch.equal(r1.target_topk_idx, t1) + assert torch.equal(r2.target_topk_idx, t2) + r1.record_indices(t1) + r2.record_indices(t2) + rec = RouterReplay.get_recorded_data() + assert len(rec) == 2 + assert torch.equal(rec[0], t1) + assert torch.equal(rec[1], t2) + RouterReplay.clear_global_indices() + assert r1.target_topk_idx is None and r2.target_topk_idx is None + assert r1.get_recorded_indices() is None and r2.get_recorded_indices() is None + + +def test_global_action_set_and_clear(): + r1 = RouterReplay() + r2 = RouterReplay() + RouterReplay.set_global_router_replay_action(RouterReplayAction.REPLAY_FORWARD) + assert r1.router_replay_action == RouterReplayAction.REPLAY_FORWARD + assert r2.router_replay_action == RouterReplayAction.REPLAY_FORWARD + RouterReplay.clear_global_router_replay_action() + assert r1.router_replay_action is None and r2.router_replay_action is None + + +def test_set_replay_data_length_mismatch(): + _ = RouterReplay() + with pytest.raises(ValueError): + RouterReplay.set_replay_data([ + torch.tensor([[0, 1]], dtype=torch.long), + torch.tensor([[1, 0]], dtype=torch.long), + ]) From 590ce52924228d27ef013f90a73433488ece3497 Mon Sep 17 00:00:00 2001 From: litianjian Date: Tue, 23 Dec 2025 17:27:56 +0800 Subject: [PATCH 06/56] format code --- .../transformer/moe/test_router_replay.py | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/tests/unit_tests/transformer/moe/test_router_replay.py b/tests/unit_tests/transformer/moe/test_router_replay.py index 00732a63ee1..fec1d290b2d 100644 --- a/tests/unit_tests/transformer/moe/test_router_replay.py +++ b/tests/unit_tests/transformer/moe/test_router_replay.py @@ -21,11 +21,7 @@ def test_record_mode_with_topk_routing_softmax_post(): rr.set_router_replay_action(RouterReplayAction.RECORD) logits = torch.randn(4, 6) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=False, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax" ) recorded = rr.get_recorded_indices() expected_idx = torch.topk(logits, k=2, dim=1).indices @@ -43,11 +39,7 @@ def test_replay_forward_with_topk_routing_softmax_pre(): target = torch.tensor([[1, 2], [0, 3], [2, 4]], dtype=torch.long) rr.set_target_indices(target) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=True, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=True, router_replay=rr, score_function="softmax" ) assert routing_map.sum(dim=1).eq(2).all() scores = torch.softmax(logits, dim=-1) @@ -61,11 +53,7 @@ def test_replay_forward_with_topk_routing_softmax_post(): target = torch.tensor([[1, 2], [0, 5], [3, 4]], dtype=torch.long) rr.set_target_indices(target) probs, routing_map = topk_routing_with_score_function( - logits=logits, - topk=2, - use_pre_softmax=False, - router_replay=rr, - score_function="softmax", + logits=logits, topk=2, use_pre_softmax=False, router_replay=rr, score_function="softmax" ) selected = torch.softmax(logits.gather(1, target), dim=-1) assert torch.equal(probs.gather(1, target), selected) @@ -104,7 +92,6 @@ def test_global_action_set_and_clear(): def test_set_replay_data_length_mismatch(): _ = RouterReplay() with pytest.raises(ValueError): - RouterReplay.set_replay_data([ - torch.tensor([[0, 1]], dtype=torch.long), - torch.tensor([[1, 0]], dtype=torch.long), - ]) + RouterReplay.set_replay_data( + [torch.tensor([[0, 1]], dtype=torch.long), torch.tensor([[1, 0]], dtype=torch.long)] + ) From 15395b8e567ba9317313da9543e8e445fd32f700 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 20 Jan 2026 15:57:16 -0800 Subject: [PATCH 07/56] first attempt --- .../core/inference/engines/dynamic_engine.py | 33 +++++++++++++++++++ megatron/core/inference/inference_request.py | 4 +++ .../text_generation_controller.py | 13 ++++++++ megatron/training/arguments.py | 5 +++ 4 files changed, 55 insertions(+) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index ca386a13c7e..d1c1821273a 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -789,6 +789,7 @@ def post_process_requests( sample: torch.Tensor, log_probs: torch.Tensor, top_n_logprobs: Optional[Dict[int, List[Tuple[torch.Tensor, torch.Tensor]]]] = None, + routing_indices: Optional[List[torch.Tensor]] = None, ) -> Tuple[List[DynamicInferenceRequest], List[DynamicInferenceRequest]]: """ Handles post-processing for requests after a step. @@ -801,6 +802,8 @@ def post_process_requests( log_probs: (List): Log probs for each request top_n_logprobs: (Dict): Top-n log probs for each request. Maps request_idx to list of (top_n_logprobs, top_n_indices) tuples. + routing_indices: (List[Tensor]): MoE routing indices per layer. Each tensor has + shape [total_tokens, topk]. Will be split per request. Returns: A list of active requests and completed requests as `DynamicInferenceRequest` objects @@ -812,6 +815,26 @@ def post_process_requests( log_probs_iter = log_probs if log_probs else repeat(None) + # Split routing indices per request if available + routing_indices_per_request = None + if routing_indices is not None: + # Get the query lengths for splitting per request + active_request_slice = slice( + self.context.paused_request_count, self.context.total_request_count + ) + active_query_lengths = self.context.request_query_lengths[active_request_slice].tolist() + + # Split each layer's routing tensor by request query lengths + routing_indices_per_request = {} + for req_idx in range(len(active_query_lengths)): + routing_indices_per_request[req_idx] = [] + + for layer_routing in routing_indices: + # layer_routing has shape [total_tokens, topk] + routing_splits = layer_routing.split(active_query_lengths, dim=0) + for req_idx, routing_split in enumerate(routing_splits): + routing_indices_per_request[req_idx].append(routing_split) + for req_idx, (request_id, token, request_log_probs) in enumerate( zip(request_ids.tolist(), sample.tolist(), log_probs_iter) ): @@ -916,6 +939,14 @@ def post_process_requests( else: request.generated_top_n_logprobs.append(logit_dict) + # Process routing indices if available + if routing_indices_per_request is not None and req_idx in routing_indices_per_request: + # Initialize routing_indices list if it doesn't exist + if request.routing_indices is None: + request.routing_indices = [] + # Append this step's routing data (list of tensors, one per MoE layer) + request.routing_indices.append(routing_indices_per_request[req_idx]) + return active_request_ids, finished_request_records def schedule_waiting_requests(self): @@ -1101,6 +1132,7 @@ async def async_bookkeep( sample = step_result["sample"] log_probs = step_result["log_probs"] top_n_logprobs = step_result.get("top_n_logprobs", None) + routing_indices = step_result.get("routing_indices", None) cuda_graph_request_count = step_result["cuda_graph_request_count"] # Add paused events. @@ -1118,6 +1150,7 @@ async def async_bookkeep( sample, log_probs, top_n_logprobs, + routing_indices, ) else: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 458fbad387f..f411aa1100a 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -246,6 +246,10 @@ class DynamicInferenceRequest(InferenceRequest): remaining_prompt_tokens: Optional[torch.Tensor] = None latency: Optional[float] = None finished_chunk_token_count = 0 + # routing_indices stores MoE routing decisions per layer for each token. + # Structure: List[List[torch.Tensor]] - outer list is per-step, inner list is per-layer, + # each tensor has shape [num_tokens_this_step, topk] + routing_indices: Optional[List[List[torch.Tensor]]] = None def __post_init__(self): self.sampling_params = copy.deepcopy(self.sampling_params) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 532cdbccdf0..dabdcd153a4 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,6 +27,7 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.transformer.moe.moe_layer import BaseMoELayer +from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model @@ -855,10 +856,22 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() + # Collect routing indices if routing replay is enabled + routing_indices = None + config = self.inference_wrapped_model.model.config + if getattr(config, 'enable_routing_replay', False): + routing_indices = RouterReplay.get_recorded_data() + # Filter out None entries (non-MoE layers) + if routing_indices: + routing_indices = [r for r in routing_indices if r is not None] + if not routing_indices: + routing_indices = None + ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, "top_n_logprobs": top_n_logprobs, + "routing_indices": routing_indices, "cuda_graph_request_count": cuda_graph_request_count, } ret.update(request_bookkeeping) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index c311e6af9dc..9b504c43f1b 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -3256,6 +3256,11 @@ def _add_moe_args(parser): help="some MoE routers have a D2H sync that will break cuda graphs. If this flag is set the router will switch" \ " to dropping and padding during decode time which does not have a D2H sync. The capacity factor is set to the" \ " max that an expert could see during inference so no tokens are actually dropped.") + group.add_argument('--enable-routing-replay', action='store_true', + help='Enable the routing replay feature for MoE layers. When enabled, each TopKRouter ' + 'creates a RouterReplay instance that can record or replay routing decisions. ' + 'Use RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) to record ' + 'routing decisions, and RouterReplay.get_recorded_data() to retrieve them.') return parser def _add_mla_args(parser): From 14f134712eaeb32b83ebc11815a45caffd434173 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 11:09:07 -0800 Subject: [PATCH 08/56] non cudagraphable implementation tested --- .../core/inference/engines/dynamic_engine.py | 66 ++++++++-------- megatron/core/inference/inference_request.py | 24 +++++- .../text_generation_controller.py | 79 ++++++++++++++++--- 3 files changed, 123 insertions(+), 46 deletions(-) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index e43e11f5227..ecf808f679f 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -804,7 +804,7 @@ def post_process_requests( sample: torch.Tensor, log_probs: torch.Tensor, top_n_logprobs: Optional[Dict[int, List[Tuple[torch.Tensor, torch.Tensor]]]] = None, - routing_indices: Optional[List[torch.Tensor]] = None, + routing_indices_per_request: Optional[Dict[int, torch.Tensor]] = None, ) -> Tuple[List[DynamicInferenceRequest], List[DynamicInferenceRequest]]: """ Handles post-processing for requests after a step. @@ -818,8 +818,9 @@ def post_process_requests( log_probs: (List): Log probs for each request top_n_logprobs: (Dict): Top-n log probs for each request. Maps request_idx to list of (top_n_logprobs, top_n_indices) tuples. - routing_indices: (List[Tensor]): MoE routing indices per layer. Each tensor has - shape [total_tokens, topk]. Will be split per request. + routing_indices_per_request: (Dict[int, Tensor]): MoE routing indices + pre-mapped by request_id. Each value is a tensor of shape + [num_tokens_this_step, num_layers, topk]. Returns: A list of active requests and completed requests as `DynamicInferenceRequest` objects @@ -833,26 +834,6 @@ def post_process_requests( log_probs_iter = log_probs if log_probs else repeat(None) - # Split routing indices per request if available - routing_indices_per_request = None - if routing_indices is not None: - # Get the query lengths for splitting per request - active_request_slice = slice( - self.context.paused_request_count, self.context.total_request_count - ) - active_query_lengths = self.context.request_query_lengths[active_request_slice].tolist() - - # Split each layer's routing tensor by request query lengths - routing_indices_per_request = {} - for req_idx in range(len(active_query_lengths)): - routing_indices_per_request[req_idx] = [] - - for layer_routing in routing_indices: - # layer_routing has shape [total_tokens, topk] - routing_splits = layer_routing.split(active_query_lengths, dim=0) - for req_idx, routing_split in enumerate(routing_splits): - routing_indices_per_request[req_idx].append(routing_split) - for req_idx, (request_id, token, request_log_probs) in enumerate( zip(request_ids.tolist(), sample.tolist(), log_probs_iter) ): @@ -969,13 +950,36 @@ def post_process_requests( else: request.generated_top_n_logprobs.append(logit_dict) - # Process routing indices if available - if routing_indices_per_request is not None and req_idx in routing_indices_per_request: - # Initialize routing_indices list if it doesn't exist + # Process routing indices if available (keyed by request_id) + # Each step's routing is a tensor of shape [num_tokens_this_step, num_layers, topk] + # We concatenate along dim=0 to accumulate: [total_tokens, num_layers, topk] + if routing_indices_per_request is not None and request_id in routing_indices_per_request: + step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] if request.routing_indices is None: - request.routing_indices = [] - # Append this step's routing data (list of tensors, one per MoE layer) - request.routing_indices.append(routing_indices_per_request[req_idx]) + request.routing_indices = step_routing + else: + request.routing_indices = torch.cat( + [request.routing_indices, step_routing], dim=0 + ) + + # Sanity check logging for routing indices (only log for first request, first few steps) + if req_idx == 0 and request.routing_indices.shape[0] <= 15: # first 15 tokens + rank = torch.distributed.get_rank() + logging.info( + f"[ROUTING DEBUG] rank={rank}, request_id={request_id}, " + f"total_tokens={request.routing_indices.shape[0]}, " + f"num_layers={request.routing_indices.shape[1]}, " + f"topk={request.routing_indices.shape[2]}, " + f"tokens_this_step={step_routing.shape[0]}" + ) + # Log first layer's routing for first few tokens + if step_routing.numel() > 0: + num_tokens_to_log = min(3, step_routing.shape[0]) + # step_routing is [num_tokens, num_layers, topk], get layer 0 + logging.info( + f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " + f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" + ) # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: @@ -1234,7 +1238,7 @@ async def async_bookkeep( sample = step_result["sample"] log_probs = step_result["log_probs"] top_n_logprobs = step_result.get("top_n_logprobs", None) - routing_indices = step_result.get("routing_indices", None) + routing_indices_per_request = step_result.get("routing_indices_per_request", None) cuda_graph_request_count = step_result["cuda_graph_request_count"] # Add paused events. @@ -1253,7 +1257,7 @@ async def async_bookkeep( sample, log_probs, top_n_logprobs, - routing_indices, + routing_indices_per_request, ) else: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 0a23accbaf1..b71a489c48b 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -248,10 +248,9 @@ class DynamicInferenceRequest(InferenceRequest): remaining_prompt_tokens: Optional[torch.Tensor] = None latency: Optional[float] = None finished_chunk_token_count = 0 - # routing_indices stores MoE routing decisions per layer for each token. - # Structure: List[List[torch.Tensor]] - outer list is per-step, inner list is per-layer, - # each tensor has shape [num_tokens_this_step, topk] - routing_indices: Optional[List[List[torch.Tensor]]] = None + # routing_indices stores MoE routing decisions for all tokens generated so far. + # Shape: [total_tokens, num_layers, topk] - accumulated across all generation steps + routing_indices: Optional[torch.Tensor] = None stop_word_ids: Optional[List[List[int]]] = None # Tokenized stop words (populated internally) def __post_init__(self): @@ -288,6 +287,18 @@ def serialize(self): """ obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] + + # Serialize routing_indices: Tensor [total_tokens - 1, num_layers, topk] -> 3D list + if self.routing_indices is not None: + total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) + # the last generated token does not undergo a forward pass + # hence we expect routing indices for total_tokens - 1 + assert self.routing_indices.shape[0] == total_tokens-1, ( + f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " + f"total tokens {total_tokens-1}." + ) + obj["routing_indices"] = self.routing_indices.tolist() + return obj @classmethod @@ -302,6 +313,11 @@ def deserialize(cls, obj: dict) -> "DynamicInferenceRequest": """ request = super().deserialize(obj) request.events = [DynamicInferenceEvent.deserialize(e) for e in obj["events"]] + + # Deserialize routing_indices: 3D list -> Tensor [total_tokens - 1, num_layers, topk] + if obj.get("routing_indices") is not None: + request.routing_indices = torch.tensor(obj["routing_indices"], dtype=torch.int64) + return request @property diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 93702a3422b..58424e78fe9 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -677,6 +677,63 @@ def _dynamic_step_log_probs_bookkeeping(self) -> Tuple[bool, bool]: return return_log_probs.any(), top_n_log_probs.any() + def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: + """Collect and map routing indices per request for MoE router recording. + + This method retrieves recorded routing decisions from RouterReplay and maps + them to individual requests using the context's request_ids and query_lengths. + Must be called while context attributes are still valid (before request transitions). + + Returns: + Optional[Dict[int, Tensor]]: A dictionary mapping request_id to a tensor of + shape [num_tokens, num_layers, topk]. Returns None if routing replay is + disabled or no routing data was recorded. + """ + config = self.inference_wrapped_model.model.config + if not getattr(config, 'enable_routing_replay', False): + return None + + raw_routing_indices = RouterReplay.get_recorded_data() + if not raw_routing_indices: + return None + + # Filter out None entries (non-MoE layers) + raw_routing_indices = [r for r in raw_routing_indices if r is not None] + if not raw_routing_indices: + return None + + # Get active request info from context + context = self.inference_wrapped_model.inference_context + active_request_slice = slice(context.paused_request_count, context.total_request_count) + active_request_ids = context.request_ids[active_request_slice].tolist() + active_query_lengths = context.request_query_lengths[active_request_slice].tolist() + + # When using CUDA graphs, tokens are padded to padded_active_token_count. + # The router records routing for all padded tokens, but we only want the + # real tokens (active_token_count). Slice down before splitting by request. + active_token_count = context.active_token_count + + # Slice each layer's routing to real tokens and split by request + # Each layer_routing has shape [padded_token_count, topk] + # After slicing: [active_token_count, topk] + # After splitting: list of [num_tokens_for_request, topk] per request + per_request_per_layer = {req_id: [] for req_id in active_request_ids} + for layer_routing in raw_routing_indices: + layer_routing = layer_routing[:active_token_count] + routing_splits = layer_routing.split(active_query_lengths, dim=0) + for req_id, routing_split in zip(active_request_ids, routing_splits): + per_request_per_layer[req_id].append(routing_split) + + # Stack layers to get [num_tokens, num_layers, topk] per request + routing_indices_per_request = {} + for req_id in active_request_ids: + layer_tensors = per_request_per_layer[req_id] # List of [num_tokens, topk] + # Stack along dim=1 to get [num_tokens, num_layers, topk] + stacked = torch.stack(layer_tensors, dim=1) + routing_indices_per_request[req_id] = stacked + + return routing_indices_per_request + def _dynamic_step_calculate_log_probs(self, logits: Tensor) -> Optional[Tensor]: """Calculate log probs from logits.""" context = self.inference_wrapped_model.inference_context @@ -891,8 +948,17 @@ async def async_generate_output_tokens_dynamic_batch( context.padded_active_request_count if context.is_decode_only() else None ) + # Enable routing recording before forward pass if routing replay is enabled + config = self.inference_wrapped_model.model.config + if getattr(config, 'enable_routing_replay', False): + RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + logits = self._dynamic_step_forward_logits(input_ids, position_ids) + # Collect routing indices per request (must be done before context transitions) + routing_indices_per_request = self._router_record_bookkeeping() + + # This is the best place to yield control back to event loop. # At this point we have enqueued FW pass GPU kernels asynchronously. # While they are running, we can do other useful CPU work. @@ -920,22 +986,13 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() - # Collect routing indices if routing replay is enabled - routing_indices = None - config = self.inference_wrapped_model.model.config - if getattr(config, 'enable_routing_replay', False): - routing_indices = RouterReplay.get_recorded_data() - # Filter out None entries (non-MoE layers) - if routing_indices: - routing_indices = [r for r in routing_indices if r is not None] - if not routing_indices: - routing_indices = None + ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, "top_n_logprobs": top_n_logprobs, - "routing_indices": routing_indices, + "routing_indices_per_request": routing_indices_per_request, "cuda_graph_request_count": cuda_graph_request_count, } ret.update(request_bookkeeping) From 4b804576a6ea97033caaddeabdd638f03d505885 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 12:02:58 -0800 Subject: [PATCH 09/56] make this work with sequence parallel + multiple prompts --- .../text_generation_controller.py | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 58424e78fe9..7b5dbdd088e 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,10 +27,13 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.transformer.enums import CudaGraphScope +from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region from megatron.core.transformer.moe.moe_layer import BaseMoELayer from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel +from megatron.core import parallel_state from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model +from megatron.core.utils import get_pg_size try: import transformer_engine as te # pylint: disable=unused-import @@ -702,35 +705,46 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: if not raw_routing_indices: return None + # raw_routing_indices is a list of length num_layers + # each entry is a Tensor of shape [local_token_count, topk] + # Now we will try to map these to requests + # Get active request info from context context = self.inference_wrapped_model.inference_context active_request_slice = slice(context.paused_request_count, context.total_request_count) active_request_ids = context.request_ids[active_request_slice].tolist() active_query_lengths = context.request_query_lengths[active_request_slice].tolist() - - # When using CUDA graphs, tokens are padded to padded_active_token_count. - # The router records routing for all padded tokens, but we only want the - # real tokens (active_token_count). Slice down before splitting by request. active_token_count = context.active_token_count - # Slice each layer's routing to real tokens and split by request - # Each layer_routing has shape [padded_token_count, topk] - # After slicing: [active_token_count, topk] - # After splitting: list of [num_tokens_for_request, topk] per request - per_request_per_layer = {req_id: [] for req_id in active_request_ids} - for layer_routing in raw_routing_indices: - layer_routing = layer_routing[:active_token_count] - routing_splits = layer_routing.split(active_query_lengths, dim=0) - for req_id, routing_split in zip(active_request_ids, routing_splits): - per_request_per_layer[req_id].append(routing_split) - - # Stack layers to get [num_tokens, num_layers, topk] per request + # Get TP group for all-gather if using sequence parallelism + # With sequence parallelism, each TP rank only sees a portion of the tokens, + # so we need to gather routing indices across all TP ranks. + tp_group = self.inference_wrapped_model.tp_group + tp_size = get_pg_size(tp_group) + + # Stack all layers first to do a single all-gather instead of per-layer all-gathers + # Each layer_routing has shape [local_token_count, topk] + # After stacking: [local_token_count, num_layers, topk] + stacked_routing = torch.stack(raw_routing_indices, dim=1) + + # All-gather across TP group if using sequence parallelism (tp_size > 1) + if tp_size > 1 and get_model_config(self.inference_wrapped_model.model).sequence_parallel: + # gather_from_sequence_parallel_region gathers along dim 0 + # [local_token_count, num_layers, topk] -> [global_token_count, num_layers, topk] + stacked_routing = gather_from_sequence_parallel_region(stacked_routing, group=tp_group) + + # Slice to real tokens (remove CUDA padding) + stacked_routing = stacked_routing[:active_token_count] + + # Split by request along token dimension + # stacked_routing has shape [active_token_count, num_layers, topk] + routing_splits = stacked_routing.split(active_query_lengths, dim=0) + + # Map to request IDs routing_indices_per_request = {} - for req_id in active_request_ids: - layer_tensors = per_request_per_layer[req_id] # List of [num_tokens, topk] - # Stack along dim=1 to get [num_tokens, num_layers, topk] - stacked = torch.stack(layer_tensors, dim=1) - routing_indices_per_request[req_id] = stacked + for req_id, routing_split in zip(active_request_ids, routing_splits): + # routing_split has shape [num_tokens_for_request, num_layers, topk] + routing_indices_per_request[req_id] = routing_split return routing_indices_per_request From 36e850a1f1c0e1555dea7b62dcb20d7243bbe65a Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 12:55:46 -0800 Subject: [PATCH 10/56] extract number of moe layers --- .../inference/gpt/gpt_dynamic_inference.py | 12 +++++ .../inference/contexts/dynamic_context.py | 6 +++ megatron/core/transformer/moe/moe_utils.py | 50 +++++++++++++------ 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference.py b/examples/inference/gpt/gpt_dynamic_inference.py index 679dd78b42b..9e966e57d99 100644 --- a/examples/inference/gpt/gpt_dynamic_inference.py +++ b/examples/inference/gpt/gpt_dynamic_inference.py @@ -44,6 +44,7 @@ TextGenerationController, ) from megatron.core.tokenizers.text.utils.build_tokenizer import build_tokenizer +from megatron.core.transformer.moe.moe_utils import get_num_moe_layers from megatron.core.transformer.module import MegatronModule from megatron.core.utils import get_mamba_inference_state_config_from_model @@ -159,6 +160,15 @@ def get_inference_context( if args.inference_logging_step_interval > 0 and args.inference_wandb_logging: metrics_writer = get_wandb_writer() + # Calculate number of MoE layers from moe_layer_freq. + num_moe_layers = None + moe_router_topk = None + if args.num_experts is not None: + num_moe_layers = get_num_moe_layers( + args.num_layers, args.moe_layer_freq + ) // args.pipeline_model_parallel_size + moe_router_topk = args.moe_router_topk + # Inference context. context = DynamicInferenceContext( params_dtype=args.params_dtype, @@ -191,6 +201,8 @@ def get_inference_context( cuda_graph_max_tokens=args.inference_dynamic_batching_cuda_graph_max_tokens, cuda_graph_mixed_prefill_count=args.inference_dynamic_batching_cuda_graph_mixed_prefill_count, metrics_writer=metrics_writer, + num_moe_layers=num_moe_layers, + moe_router_topk=moe_router_topk, ) return context diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 4267f9d0952..0b0d87e8001 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -286,9 +286,15 @@ def __init__( metrics_writer: Optional['WandbModule'] = None, request_metadata_types: Optional[List[Tuple[str, torch.dtype, bool]]] = None, persist_cuda_graphs: Optional[bool] = False, + num_moe_layers: Optional[int] = None, + moe_router_topk: Optional[int] = None, ): super().__init__(materialize_only_last_token_logits=materialize_only_last_token_logits) + # MoE routing replay parameters. + self.num_moe_layers = num_moe_layers + self.moe_router_topk = moe_router_topk + self.cache_mla_latent = cache_mla_latent if self.cache_mla_latent: assert ( diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 0d77d558b08..27b8db07cea 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -958,6 +958,41 @@ def reduce_aux_losses_tracker_across_ranks( ) +def get_num_moe_layers( + num_layers: int, + moe_layer_freq: Optional[Union[int, List[int]]] = None, + mtp_num_layers: Optional[int] = None, +) -> int: + """Calculate the number of MoE layers based on moe_layer_freq. + + Args: + num_layers (int): Total number of transformer layers. + moe_layer_freq (Optional[Union[int, List[int]]]): Frequency of MoE layers. + If int, every moe_layer_freq-th layer is an MoE layer. + If list, it's a binary pattern indicating which layers are MoE. + If None, all layers are assumed to be MoE layers. + mtp_num_layers (Optional[int]): Number of MTP (multi-token prediction) layers to add. + + Returns: + int: The number of MoE layers. + """ + if moe_layer_freq is None: + num_moe_layers = num_layers + elif isinstance(moe_layer_freq, int): + assert isinstance(num_layers, int) + moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] + num_moe_layers = sum(moe_layer_pattern) + elif isinstance(moe_layer_freq, list): + num_moe_layers = sum(moe_layer_freq) + else: + raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") + + if mtp_num_layers is not None: + num_moe_layers += mtp_num_layers + + return num_moe_layers + + def track_moe_metrics( loss_scale: float, iteration: int, @@ -987,20 +1022,7 @@ def track_moe_metrics( tracker[key]["reduce_group_has_dp"] = False reduce_aux_losses_tracker_across_ranks(track_names, pg_collection=pg_collection) - # Get number of MoE layers - if moe_layer_freq is None: - num_moe_layers = num_layers - elif isinstance(moe_layer_freq, int): - assert isinstance(num_layers, int) - moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] - num_moe_layers = sum(moe_layer_pattern) - elif isinstance(moe_layer_freq, list): - num_moe_layers = sum(moe_layer_freq) - else: - raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") - - if mtp_num_layers is not None: - num_moe_layers += mtp_num_layers + num_moe_layers = get_num_moe_layers(num_layers, moe_layer_freq, mtp_num_layers) aux_losses = {k: v['values'].float() * loss_scale for k, v in tracker.items()} for name, loss_list in aux_losses.items(): From c3e785451a5ca7252cd07f0eb84959ea5aede26b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 14:39:51 -0800 Subject: [PATCH 11/56] cuda graphability --- .../inference/contexts/dynamic_context.py | 17 ++++ .../inference/contexts/routing_metadata.py | 88 +++++++++++++++++++ .../text_generation_controller.py | 27 +++--- megatron/core/transformer/moe/moe_utils.py | 50 ++++++++++- 4 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 megatron/core/inference/contexts/routing_metadata.py diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 0b0d87e8001..ffa8029ec31 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -702,6 +702,17 @@ def allocate_mamba_states(): allocate_memory_buffer() allocate_mamba_states() + # Allocate routing metadata for MoE models. + if self.moe_router_topk is not None: + from megatron.core.inference.contexts.routing_metadata import RoutingMetadata + + self.routing_metadata = RoutingMetadata( + context=self, + moe_router_topk=self.moe_router_topk, + ) + else: + self.routing_metadata = None + # Reset attention and Mamba state. self.reset_attention_state() self.reset_mamba_state() @@ -1413,6 +1424,12 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) + if self.routing_metadata is not None: + if self.using_cuda_graph_this_step(): + self.routing_metadata.enable_static_buffer_recording() + else: + self.routing_metadata.disable_static_buffer_recording() + def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py new file mode 100644 index 00000000000..dbbdc760eb9 --- /dev/null +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -0,0 +1,88 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +from typing import TYPE_CHECKING, Optional + +import torch + +if TYPE_CHECKING: + from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext + + +class RoutingMetadata: + """Manages routing indices metadata for MoE layers during inference. + + This class provides static buffers for CUDA graph compatibility when + recording routing decisions. It holds a reference to the inference context + to automatically determine whether to use static buffers based on CUDA graph state. + + Args: + context (DynamicInferenceContext): The inference context. + moe_router_topk (int): Number of experts selected per token. + """ + + def __init__( + self, + context: 'DynamicInferenceContext', + moe_router_topk: int, + ): + from megatron.core.transformer.moe.moe_utils import RouterReplay + + self.context = context + self.max_tokens = context.max_tokens + # Get actual number of MoE layers from RouterReplay instances on this rank. + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) + self.moe_router_topk = moe_router_topk + self.device = torch.cuda.current_device() + + # Static buffer for CUDA graph compatibility. + # Shape: [max_tokens, num_moe_layers, moe_router_topk] + self.routing_indices_buffer = torch.empty( + (self.max_tokens, self.num_moe_layers, self.moe_router_topk), + dtype=torch.int32, + device=self.device, + ) + + def get_routing_indices(self) -> Optional[torch.Tensor]: + """Get the recorded routing indices. + + Automatically uses the static buffer when CUDA graphs are active, + otherwise retrieves from RouterReplay utility. + + Returns: + Tensor of shape [num_tokens, num_moe_layers, topk] or None if not available. + """ + if self.context.using_cuda_graph_this_step(): + # Return view of static buffer up to current token count. + return self.routing_indices_buffer[:self.context.active_token_count] + else: + # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. + from megatron.core.transformer.moe.moe_utils import RouterReplay + + recorded_data = RouterReplay.get_recorded_data() + if recorded_data is None or len(recorded_data) == 0: + return None + if recorded_data[0] is None: + return None + # Stack: list of [num_tokens, topk] -> [num_tokens, num_layers, topk] + return torch.stack(recorded_data, dim=1) + + def enable_static_buffer_recording(self) -> None: + """Enable recording into the static buffer for CUDA graph compatibility. + + This sets up RouterReplay instances to copy routing indices into our + pre-allocated static buffer instead of creating new tensors. + """ + from megatron.core.transformer.moe.moe_utils import RouterReplay + + RouterReplay.set_global_static_buffers(self.routing_indices_buffer) + + def disable_static_buffer_recording(self) -> None: + """Disable static buffer recording, reverting to normal tensor assignment.""" + from megatron.core.transformer.moe.moe_utils import RouterReplay + + RouterReplay.clear_global_static_buffers() + + def reset(self) -> None: + """Reset the routing metadata state.""" + pass + self.current_token_count = 0 diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 7b5dbdd088e..6e496a883a8 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -683,8 +683,9 @@ def _dynamic_step_log_probs_bookkeeping(self) -> Tuple[bool, bool]: def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: """Collect and map routing indices per request for MoE router recording. - This method retrieves recorded routing decisions from RouterReplay and maps - them to individual requests using the context's request_ids and query_lengths. + This method retrieves recorded routing decisions and maps them to individual + requests using the context's request_ids and query_lengths. Uses the context's + routing_metadata when available (which handles CUDA graph static buffers automatically). Must be called while context attributes are still valid (before request transitions). Returns: @@ -696,37 +697,29 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: if not getattr(config, 'enable_routing_replay', False): return None - raw_routing_indices = RouterReplay.get_recorded_data() - if not raw_routing_indices: + # Get routing indices - use routing_metadata if available (handles CUDA graph static buffers) + context = self.inference_wrapped_model.inference_context + if context.routing_metadata is None: return None + + stacked_routing = context.routing_metadata.get_routing_indices() - # Filter out None entries (non-MoE layers) - raw_routing_indices = [r for r in raw_routing_indices if r is not None] - if not raw_routing_indices: + if stacked_routing is None: return None - # raw_routing_indices is a list of length num_layers - # each entry is a Tensor of shape [local_token_count, topk] - # Now we will try to map these to requests - # Get active request info from context - context = self.inference_wrapped_model.inference_context active_request_slice = slice(context.paused_request_count, context.total_request_count) active_request_ids = context.request_ids[active_request_slice].tolist() active_query_lengths = context.request_query_lengths[active_request_slice].tolist() active_token_count = context.active_token_count + # Get TP group for all-gather if using sequence parallelism # With sequence parallelism, each TP rank only sees a portion of the tokens, # so we need to gather routing indices across all TP ranks. tp_group = self.inference_wrapped_model.tp_group tp_size = get_pg_size(tp_group) - # Stack all layers first to do a single all-gather instead of per-layer all-gathers - # Each layer_routing has shape [local_token_count, topk] - # After stacking: [local_token_count, num_layers, topk] - stacked_routing = torch.stack(raw_routing_indices, dim=1) - # All-gather across TP group if using sequence parallelism (tp_size > 1) if tp_size > 1 and get_model_config(self.inference_wrapped_model.model).sequence_parallel: # gather_from_sequence_parallel_region gathers along dim 0 diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 27b8db07cea..a7bf666698a 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -578,6 +578,29 @@ def clear_global_indices(): for router in RouterReplay.global_router_replay_instances: router.clear_indices() + @staticmethod + def set_global_static_buffers(static_buffer: torch.Tensor): + """Sets static buffers for all router instances from a combined buffer. + + Args: + static_buffer: Tensor of shape [max_tokens, num_layers, topk]. + Each layer's RouterReplay gets a slice [:, layer_idx, :]. + """ + num_layers = len(RouterReplay.global_router_replay_instances) + assert static_buffer.shape[1] == num_layers, ( + f"Buffer has {static_buffer.shape[1]} layers but there are " + f"{num_layers} RouterReplay instances." + ) + for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): + # Each layer gets a view of shape [max_tokens, topk] + router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) + + @staticmethod + def clear_global_static_buffers(): + """Clears static buffers from all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_static_buffer() + @staticmethod def set_global_router_replay_action(router_replay_action: RouterReplayAction): """Sets the router replay action for all router instances.""" @@ -594,6 +617,7 @@ def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay + self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph recording self.router_replay_action: Optional[RouterReplayAction] = ( None # Router replay action for this layer ) @@ -602,6 +626,18 @@ def __init__(self): ) # List of tensors for backward pass replay RouterReplay.global_router_replay_instances.append(self) + def set_static_buffer(self, buffer: torch.Tensor): + """Sets a static buffer for CUDA graph compatible recording. + + Args: + buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. + """ + self.static_buffer = buffer + + def clear_static_buffer(self): + """Clears the static buffer.""" + self.static_buffer = None + def set_target_indices(self, topk_indices: torch.Tensor): """Sets the target topk indices for replay.""" self.target_topk_idx = topk_indices @@ -612,8 +648,18 @@ def get_recorded_indices(self) -> Optional[torch.Tensor]: return self.recorded_topk_idx def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices.""" - self.recorded_topk_idx = topk_indices + """Records the topk indices. + + If a static buffer is set (for CUDA graph compatibility), copies into it. + Otherwise, just stores the tensor reference. + """ + if self.static_buffer is not None: + # Copy into static buffer for CUDA graph compatibility. + num_tokens = topk_indices.shape[0] + self.static_buffer[:num_tokens].copy_(topk_indices) + self.recorded_topk_idx = self.static_buffer[:num_tokens] + else: + self.recorded_topk_idx = topk_indices def clear_indices(self): """Clears the recorded and target topk indices.""" From 0a72c80ee36bbfe11207000ef259305930ea1e21 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 21 Jan 2026 15:55:28 -0800 Subject: [PATCH 12/56] make this work with cuda graphs --- .../gpt_dynamic_inference_with_coordinator.py | 3 ++ .../inference/contexts/routing_metadata.py | 34 +++++++++++++++---- .../core/inference/engines/dynamic_engine.py | 14 +++++++- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index f354b122a7e..c3a4f521b83 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -137,6 +137,9 @@ async def main( # While we wait for the requests to complete, the engine runs in the background. results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) + for i, res in enumerate(results): + routing_indices = res.requests[0].routing_indices + logging.info(f"Result {i} has routing indices with shape {routing_indices}.") if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index dbbdc760eb9..aa87e448992 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -25,15 +25,33 @@ def __init__( context: 'DynamicInferenceContext', moe_router_topk: int, ): - from megatron.core.transformer.moe.moe_utils import RouterReplay - self.context = context self.max_tokens = context.max_tokens - # Get actual number of MoE layers from RouterReplay instances on this rank. - self.num_moe_layers = len(RouterReplay.global_router_replay_instances) self.moe_router_topk = moe_router_topk self.device = torch.cuda.current_device() + # Static buffer allocated lazily in _ensure_buffer_allocated(). + # We defer allocation because RouterReplay instances don't exist yet at init time. + self.routing_indices_buffer: Optional[torch.Tensor] = None + self.num_moe_layers: Optional[int] = None + + def _ensure_buffer_allocated(self) -> None: + """Allocate the static buffer if not already allocated. + + Gets the actual number of MoE layers from RouterReplay instances. + """ + if self.routing_indices_buffer is not None: + return + + from megatron.core.transformer.moe.moe_utils import RouterReplay + + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) + print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " + f"(from {len(RouterReplay.global_router_replay_instances)} RouterReplay instances)") + + if self.num_moe_layers == 0: + return + # Static buffer for CUDA graph compatibility. # Shape: [max_tokens, num_moe_layers, moe_router_topk] self.routing_indices_buffer = torch.empty( @@ -53,6 +71,8 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: """ if self.context.using_cuda_graph_this_step(): # Return view of static buffer up to current token count. + if self.routing_indices_buffer is None: + return None return self.routing_indices_buffer[:self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. @@ -71,10 +91,13 @@ def enable_static_buffer_recording(self) -> None: This sets up RouterReplay instances to copy routing indices into our pre-allocated static buffer instead of creating new tensors. + Allocates the buffer lazily on first call. """ from megatron.core.transformer.moe.moe_utils import RouterReplay - RouterReplay.set_global_static_buffers(self.routing_indices_buffer) + self._ensure_buffer_allocated() + if self.routing_indices_buffer is not None: + RouterReplay.set_global_static_buffers(self.routing_indices_buffer) def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" @@ -85,4 +108,3 @@ def disable_static_buffer_recording(self) -> None: def reset(self) -> None: """Reset the routing metadata state.""" pass - self.current_token_count = 0 diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index ecf808f679f..5ba66148300 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -308,6 +308,16 @@ def create_cuda_graphs(self, reset_context: bool = True): f"{tbar_idx}/{len(context.cuda_graph_batch_dimensions_list)}. {tbar_str}" ) + # Enable routing recording during warmup if routing replay is enabled. + # This ensures the record_indices copy operation is captured in the CUDA graph. + model_config = controller.inference_wrapped_model.model.config + if getattr(model_config, 'enable_routing_replay', False): + from megatron.core.transformer.moe.moe_utils import ( + RouterReplay, + RouterReplayAction, + ) + RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) + # Forward pass -> logits. controller._dynamic_step_forward_logits(input_ids, position_ids) @@ -956,7 +966,7 @@ def post_process_requests( if routing_indices_per_request is not None and request_id in routing_indices_per_request: step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] if request.routing_indices is None: - request.routing_indices = step_routing + request.routing_indices = step_routing.clone() else: request.routing_indices = torch.cat( [request.routing_indices, step_routing], dim=0 @@ -980,6 +990,8 @@ def post_process_requests( f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" ) + + # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: From d62038d1446b722796ce607290b72465f592bc5d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 23 Jan 2026 08:18:19 -0800 Subject: [PATCH 13/56] save router routing in functional tests and correctly handle inference request record merge --- .../inference/gpt/gpt_dynamic_inference_with_coordinator.py | 4 +++- megatron/core/inference/inference_request.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index c3a4f521b83..7ec826c74a8 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -139,7 +139,7 @@ async def main( results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) for i, res in enumerate(results): routing_indices = res.requests[0].routing_indices - logging.info(f"Result {i} has routing indices with shape {routing_indices}.") + logging.info(f"Result {i} has routing indices with shape {routing_indices.shape}.") if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. @@ -159,6 +159,8 @@ async def main( result_dict["logprobs"] = req.prompt_log_probs + req.generated_log_probs throughput = len(req.generated_tokens) / req.latency throughputs.append(throughput) + if req.routing_indices is not None: + result_dict["routing_indices"] = req.routing_indices.tolist() json_results[req.request_id] = result_dict throughput_dict = {"throughput": throughputs} if args.throughput_check_only: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index b71a489c48b..de75123866c 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -481,7 +481,7 @@ def checkpoint(self, tokenizer: MegatronTokenizer | None = None): new_request = DynamicInferenceRequest( request_id=old_request.request_id, prompt_tokens=new_prompt_tokens, - sampling_params=new_sampling_params, + sampling_params=new_sampling_params ) self.requests.append(new_request) @@ -503,6 +503,8 @@ def merge_lists(key): prompt_tokens = self.requests[0].prompt_tokens prompt_text = self.requests[0].prompt + if self.requests[0].routing_indices is not None: + routing_indices = torch.cat([r.routing_indices for r in self.requests]) generated_tokens = merge_lists("generated_tokens") try: generated_text = "".join(r.generated_text for r in self.requests) @@ -526,6 +528,7 @@ def merge_lists(key): status=self.requests[-1].status, latency=self.latency, events=merge_lists("events"), + routing_indices=routing_indices, ) return request From 351c00ec83f6191ff366c02f4628db9a4153270c Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 27 Jan 2026 11:08:15 -0600 Subject: [PATCH 14/56] Move to using the Inference OpenAI API server; Remove all conversation templating code from rl; Move RL to only use messages for prompts --- .../environments/countdown/countdown_agent.py | 1 + examples/rl/environments/math/math_agent.py | 1 - examples/rl/model_configs/common.sh | 2 +- examples/rl/model_configs/qwen_2p5_3b.sh | 4 +- .../core/inference/engines/dynamic_engine.py | 4 + megatron/core/inference/inference_client.py | 2 + .../endpoints/chat_completions.py | 14 ++- .../dynamic_text_gen_server/flask_server.py | 20 ++-- megatron/rl/__init__.py | 1 + megatron/rl/agent/api.py | 18 --- megatron/rl/agent/pass_at_evaluation_agent.py | 2 +- megatron/rl/agent/reward_only_agent.py | 33 +----- megatron/rl/inference/api.py | 19 +--- megatron/rl/inference/chat_templates.py | 51 --------- megatron/rl/inference/inference_interface.py | 105 ++---------------- megatron/rl/inference/megatron.py | 79 ++++++------- megatron/rl/rl_utils.py | 42 +++---- .../inference/inference_interface_server.py | 12 +- megatron/training/arguments.py | 3 - train_rl.py | 1 - 20 files changed, 106 insertions(+), 308 deletions(-) delete mode 100644 megatron/rl/inference/chat_templates.py diff --git a/examples/rl/environments/countdown/countdown_agent.py b/examples/rl/environments/countdown/countdown_agent.py index bd9413a19d0..85d94039125 100644 --- a/examples/rl/environments/countdown/countdown_agent.py +++ b/examples/rl/environments/countdown/countdown_agent.py @@ -10,6 +10,7 @@ class CountdownAgent(RewardOnlyAgent, HFDatasetAgent): + chat_mode: bool = True def make_prefix(self, target, nums) -> str: if self.chat_mode: diff --git a/examples/rl/environments/math/math_agent.py b/examples/rl/environments/math/math_agent.py index 67feb3b4adb..486c8477b92 100644 --- a/examples/rl/environments/math/math_agent.py +++ b/examples/rl/environments/math/math_agent.py @@ -3,7 +3,6 @@ import re import traceback -from megatron.rl.agent.pass_at_evaluation_agent import PassAtEvaluationAgent from megatron.rl.agent.reward_only_agent import RewardOnlyAgent try: diff --git a/examples/rl/model_configs/common.sh b/examples/rl/model_configs/common.sh index 4f6ca0e18cf..c37d88fb4df 100644 --- a/examples/rl/model_configs/common.sh +++ b/examples/rl/model_configs/common.sh @@ -22,7 +22,7 @@ COMMON_OPTIONS="\ --attention-backend flash \ --timing-log-level 1 \ --log-timers-to-tensorboard \ - --save-retain-interval 120 \ + --save-retain-interval 160 \ --inference-dynamic-batching-num-cuda-graphs 1 \ --inference-dynamic-batching-unified-memory-level 1 \ --adam-beta1 0.9 \ diff --git a/examples/rl/model_configs/qwen_2p5_3b.sh b/examples/rl/model_configs/qwen_2p5_3b.sh index f3250f39ecc..647023d3050 100644 --- a/examples/rl/model_configs/qwen_2p5_3b.sh +++ b/examples/rl/model_configs/qwen_2p5_3b.sh @@ -22,7 +22,7 @@ if [ "$(basename "$ENV_CONFIG")" = "dapo.yaml" ]; then GRPO_KL_BETA=${GRPO_KL_BETA:-"0.0"} ENTROPY_WEIGHT=${ENTROPY_WEIGHT:-"0.0"} TRAINING_BATCH_SIZE=${TRAINING_BATCH_SIZE:-1024} - MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-2} + MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} MAX_SEQ_LENGTH=${MAX_SEQ_LENGTH:-8192} EXIT_INTERVAL=${EXIT_INTERVAL:-16} CHKPT_SAVE_INTERVAL=${CHKPT_SAVE_INTERVAL:-16} @@ -38,7 +38,7 @@ else GRPO_KL_BETA=${GRPO_KL_BETA:-"0.0"} ENTROPY_WEIGHT=${ENTROPY_WEIGHT:-"0.0"} TRAINING_BATCH_SIZE=${TRAINING_BATCH_SIZE:-512} - MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-2} + MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} MAX_SEQ_LENGTH=${MAX_SEQ_LENGTH:-8192} EXIT_INTERVAL=${EXIT_INTERVAL:-16} CHKPT_SAVE_INTERVAL=${CHKPT_SAVE_INTERVAL:-16} diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 134ce3b124d..baa23038a76 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -104,6 +104,9 @@ from torch_memory_saver import torch_memory_saver +logger = logging.getLogger(__name__) + + class EngineSuspendedError(Exception): """Engine is currently suspended and not performing steps.""" @@ -767,6 +770,7 @@ def _add_request( if request.status != Status.FAILED: self.waiting_request_ids.append(request_id) else: + logger.error(f"Request {request_id} failed to add to engine. {request.events}") self.failed_request_ids.append(request_id) if self.rank == 0: warnings.warn( diff --git a/megatron/core/inference/inference_client.py b/megatron/core/inference/inference_client.py index a927a393b8c..d92a0a9625e 100644 --- a/megatron/core/inference/inference_client.py +++ b/megatron/core/inference/inference_client.py @@ -11,6 +11,8 @@ from .headers import Headers +logger = logging.getLogger(__name__) + try: import zmq diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 0c3379bc53f..7d97a87b7a4 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -34,10 +34,8 @@ async def chat_completions(): messages, tokenize=True, add_generation_prompt=True ) except AttributeError: - return ( - "Tokenizer does not support 'apply_chat_template'. " - "Chat completions requires a tokenizer with a configured chat template." - ), 500 + logger.warning("Tokenizer does not support 'apply_chat_template'. Using tokenize instead.") + prompt_tokens = tokenizer.tokenize("\n".join([message["content"] for message in messages])) except Exception as e: return f"Error processing 'messages': {e}", 500 @@ -62,7 +60,7 @@ async def chat_completions(): top_p=top_p, return_log_probs=return_log_probs, top_n_logprobs=top_n_logprobs, - num_tokens_to_generate=int(req.get("max_tokens", 16)), + num_tokens_to_generate=int(max_tokens) if ( (max_tokens := req.get("max_tokens", None)) is not None ) else None, ) except ValueError as e: return f"Invalid sampling parameter: {e}", 400 @@ -78,6 +76,7 @@ async def chat_completions(): return_log_probs=sampling_params.return_log_probs, top_n_logprobs=sampling_params.top_n_logprobs, num_tokens_to_generate=sampling_params.num_tokens_to_generate, + skip_prompt_log_probs=True, ) tasks.append(client.add_request(prompt_tokens, per_req_params)) @@ -85,6 +84,7 @@ async def chat_completions(): try: batch_results = await asyncio.gather(*tasks) except Exception as e: + logger.error(f"Error during inference: {e}") return f"Error during inference: {e}", 500 logger.info( @@ -136,6 +136,10 @@ async def chat_completions(): choice_data = { "index": 0, "message": {"role": "assistant", "content": text_output}, + "prompt_token_ids": prompt_tokens, + "generation_token_ids": result.generated_tokens, + "generation_log_probs": result.generated_log_probs, + "raw_text": result.prompt + result.generated_text, # 'logprobs' in chat API is an object containing 'content' "logprobs": {"content": logprobs_content} if logprobs_content else None, "finish_reason": "length", # Original code hardcoded this. diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index 1701ff63c36..a0936582f60 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -33,7 +33,7 @@ def temp_log_level(level, logger=None): @trace_async_exceptions -async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int): +async def run_flask_server_on_client(client: InferenceClient, tokenizer, rank: int, flask_port: int): """Initializes and runs the async Flask server.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -44,14 +44,10 @@ async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_po logger.warning(f"Could not get hostname: {e}") hostname = "0.0.0.0" - inference_client = InferenceClient(coordinator_addr) - await inference_client.start() - logger.info(f"Rank {rank}: InferenceClient connected.") - app = Flask(__name__) # Store client and tokenizer in app config for Blueprints to use - app.config['client'] = inference_client + app.config['client'] = client app.config['tokenizer'] = tokenizer # Register all blueprints from the 'endpoints' package @@ -69,8 +65,16 @@ def health_check(): with temp_log_level(logging.INFO, logger): logger.info(f"Starting Flask server on http://{hostname}:{flask_port}") + await serve(app, config) + + +@trace_async_exceptions +async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int): + inference_client = InferenceClient(coordinator_addr) + await inference_client.start() + logger.info(f"Rank {rank}: InferenceClient connected.") try: - await serve(app, config) + await run_flask_server_on_client(inference_client, tokenizer, rank, flask_port) finally: await inference_client.stop() - logger.info(f"Rank {rank}: Flask server and client shut down.") + logger.info(f"Rank {rank}: Flask server and client shut down.") \ No newline at end of file diff --git a/megatron/rl/__init__.py b/megatron/rl/__init__.py index 08ae226bfe4..b2a9eeb954d 100644 --- a/megatron/rl/__init__.py +++ b/megatron/rl/__init__.py @@ -65,6 +65,7 @@ class GenericGenerationArgs(BaseModel): top_k: int | None = None top_p: float | None = None max_tokens: int | None = None + n: int | None = None def add(self, generation_args: 'GenericGenerationArgs') -> 'GenericGenerationArgs': return GenericGenerationArgs.model_validate( diff --git a/megatron/rl/agent/api.py b/megatron/rl/agent/api.py index 9568db3a54d..643c43197b6 100644 --- a/megatron/rl/agent/api.py +++ b/megatron/rl/agent/api.py @@ -12,10 +12,7 @@ from ..__init__ import Request, TypeLookupable from ..inference import ( - ChatInferenceInterface, - ChatInferenceRequest, InferenceInterface, - InferenceRequest, LLMChatMessage, ReturnsRaw, ) @@ -124,11 +121,6 @@ async def get_reward_rollouts(self, request: RolloutRequest) -> list[Rollout]: request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - return await asyncio.gather( *[self.rollout(request=request) for _ in range(request.num_rollouts)] ) @@ -158,11 +150,6 @@ async def get_reward_rollouts(self, request: RolloutRequest) -> list[TokenRollou request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - return await asyncio.gather( *[self.rollout(request=request) for _ in range(request.num_rollouts)] ) @@ -187,11 +174,6 @@ async def get_grouped_rollouts(self, request: GroupedRolloutRequest): request.inference_interface, ReturnsRaw ), "InferenceInterface must support raw_text return to provide rollouts." - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - # If num_groups is -1, we generate a stream of groups. # The buffer size is used to create backpressure for each agent in order to balance group generation in a multi-task setting. grouped_rollouts: asyncio.Queue[list[Rollout]] = asyncio.Queue( diff --git a/megatron/rl/agent/pass_at_evaluation_agent.py b/megatron/rl/agent/pass_at_evaluation_agent.py index b10e3b897c8..c04e1c2772f 100644 --- a/megatron/rl/agent/pass_at_evaluation_agent.py +++ b/megatron/rl/agent/pass_at_evaluation_agent.py @@ -7,7 +7,7 @@ import numpy as np from ..__init__ import GenericGenerationArgs -from ..inference import ChatInferenceResponse, LLMChatMessage +from ..inference import LLMChatMessage from .api import EvaluationAgent, EvaluationRequest, EvaluationResponse, RewardEvaluationResult diff --git a/megatron/rl/agent/reward_only_agent.py b/megatron/rl/agent/reward_only_agent.py index 53b1f7407b2..8f74d2b8a39 100644 --- a/megatron/rl/agent/reward_only_agent.py +++ b/megatron/rl/agent/reward_only_agent.py @@ -7,8 +7,6 @@ from tqdm.asyncio import tqdm from ..inference import ( - ChatInferenceInterface, - ChatInferenceResponse, InferenceResponse, LLMChatMessage, ReturnsRaw, @@ -91,11 +89,7 @@ async def rollout_from_response( ), "InferenceInterface must support raw_text return to provide rollouts." raw_text = response.raw_text - response_text = ( - response.response.content - if isinstance(response, ChatInferenceResponse) - else response.response - ) + response_text = response.response.content if isinstance(request.inference_interface, ReturnsTokens): logprobs = response.logprobs @@ -144,19 +138,9 @@ async def group_rollout(self, request: GroupedRolloutRequest) -> list[Rollout]: inference_request = request.inference_interface.prepare_request( [prompt], request.generation_args ) - inference_request.n = request.rollouts_per_group - - groups = await request.inference_interface.agenerate(inference_request) - assert ( - len(groups) == 1 - ), "get_grouped_rollouts only requested a single group but got multiple groups" - responses = groups[0].responses - - rollouts = await asyncio.gather( - *[self.rollout_from_response(request, response, golden) for response in responses] - ) - return rollouts + responses = await asyncio.gather(*[request.inference_interface.agenerate(inference_request) for _ in range(request.rollouts_per_group)]) + return [await self.rollout_from_response(request, response[0], golden) for response in responses] async def _evaluation( self, prompt: str, golden: Any, request: EvaluationRequest @@ -172,11 +156,7 @@ async def _evaluation( ), "evaluation only requested a single response but got multiple responses" response = responses[0] - response_text = ( - response.response.content - if isinstance(response, ChatInferenceResponse) - else response.response - ) + response_text = response.response.content result = RewardEvaluationResult( env_id=self.env_id, @@ -190,11 +170,6 @@ async def _evaluation( async def run_evaluation(self, request: EvaluationRequest): - if isinstance(request.inference_interface, ChatInferenceInterface): - self.chat_mode = True - else: - self.chat_mode = False - # Get all prompts first all_prompts = list( await self.evaluation_prompts( diff --git a/megatron/rl/inference/api.py b/megatron/rl/inference/api.py index ae19380842e..fa647633621 100644 --- a/megatron/rl/inference/api.py +++ b/megatron/rl/inference/api.py @@ -11,11 +11,6 @@ class LLMChatMessage(BaseModel): class InferenceRequest(Request): - prompt: list[str] - n: int | None = None - - -class ChatInferenceRequest(InferenceRequest): prompt: list[list[LLMChatMessage]] tools: list[dict] | None = None @@ -27,7 +22,7 @@ class GroupedInferenceRequest(InferenceRequest): class InferenceResponse(BaseModel): """The minimum required response for an inference interface.""" - response: str + response: LLMChatMessage raw_text: str | None = None token_ids: list[int] | None = None prompt_length: int | None = None @@ -38,15 +33,3 @@ class GroupedInferenceResponse(BaseModel): """An inference response which includes a list of responses.""" responses: list[InferenceResponse] - - -class ChatInferenceResponse(InferenceResponse): - """The minimum required response for a chat inference interface.""" - - response: LLMChatMessage - - -class GroupedChatInferenceResponse(GroupedInferenceResponse): - """A chat inference response which includes a list of responses.""" - - responses: list[ChatInferenceResponse] diff --git a/megatron/rl/inference/chat_templates.py b/megatron/rl/inference/chat_templates.py deleted file mode 100644 index 3e464842859..00000000000 --- a/megatron/rl/inference/chat_templates.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -import warnings - -from pydantic import BaseModel, ConfigDict, Field -from transformers import AutoTokenizer -from transformers.tokenization_utils import PreTrainedTokenizer -from transformers.tokenization_utils_fast import PreTrainedTokenizerFast - -from .api import InferenceResponse, LLMChatMessage - - -class ConversationTemplate(BaseModel): - """Transformers tokenizer based template.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast = Field(repr=False) - stop_words: list[str] = [] - - def format(self, messages: list[LLMChatMessage], tools: list[dict] | None = None) -> str: - return self.tokenizer.apply_chat_template( - messages, add_generation_prompt=True, tokenize=False, tools=tools - ) - - def parse_response(self, responses: list[InferenceResponse]) -> list[LLMChatMessage]: - return [ - LLMChatMessage(role="assistant", content=response.response) for response in responses - ] - - @classmethod - def from_string(cls, tokenizer_name: str) -> 'ConversationTemplate': - if tokenizer_name == "null": - warnings.warn( - "Using NullConversationTemplate. This provides no chat templating to Chat requests." - ) - return NullConversationTemplate() - return cls(tokenizer=AutoTokenizer.from_pretrained(tokenizer_name)) - - -class NullConversationTemplate(ConversationTemplate): - - tokenizer: None = None - - def format(self, messages: list[LLMChatMessage], tools: list[dict] | None = None) -> str: - return "\n".join([f"{message.content}" for message in messages]) + "\n" - - def parse_response(self, responses: list[InferenceResponse]) -> list[LLMChatMessage]: - return [ - LLMChatMessage(role="assistant", content=response.response) for response in responses - ] diff --git a/megatron/rl/inference/inference_interface.py b/megatron/rl/inference/inference_interface.py index e950792e72b..5558ff9c8bc 100644 --- a/megatron/rl/inference/inference_interface.py +++ b/megatron/rl/inference/inference_interface.py @@ -9,71 +9,34 @@ from ..__init__ import GenericGenerationArgs from ..inference.api import ( - ChatInferenceRequest, - ChatInferenceResponse, - GroupedChatInferenceResponse, GroupedInferenceResponse, InferenceRequest, InferenceResponse, LLMChatMessage, ) -from ..inference.chat_templates import ConversationTemplate - - -# Used when generating n resposnes for a single prompt -def grouper(iterable, n, fillvalue=None): - """Fold an iterable into a list of lists of size n.""" - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) class InferenceInterface(BaseModel): - """Inference interface that for base language models.""" + """Inference interface for chat models.""" class Config: arbitrary_types_allowed = True - supports_n: ClassVar[bool] = False - def prepare_request( - self, prompts: list[str], generation_args: GenericGenerationArgs + self, prompts: list[str | list[LLMChatMessage]], generation_args: GenericGenerationArgs ) -> InferenceRequest: - assert all(isinstance(p, str) for p in prompts), "Prompt must be a list of strings" - return InferenceRequest(prompt=prompts, generation_args=generation_args) + prompt = [ + [LLMChatMessage(role='user', content=p)] if isinstance(p, str) else p for p in prompts + ] + return InferenceRequest(prompt=prompt, generation_args=generation_args) async def base_generate(self, request: InferenceRequest) -> list[InferenceResponse]: - raise NotImplementedError( - "Direct Inference Classes must implement the base_generate method." - ) - - def duplicate_requests(self, request: InferenceRequest, n: int) -> list[InferenceRequest]: - return request.model_copy(update={'prompt': request.prompt * n}) - - def fold_responses( - self, responses: list[InferenceResponse], n: int - ) -> list[GroupedInferenceResponse]: - return [GroupedInferenceResponse(responses=x) for x in list(grouper(responses, n))] + assert NotImplementedError("Direct Inference Classes must implement the base_generate method.") async def agenerate( self, request: InferenceRequest ) -> list[InferenceResponse] | list[GroupedInferenceResponse]: - if not self.supports_n and request.n is not None: - request = self.duplicate_requests(request, request.n) - - generations = await self.base_generate(request) - - if request.n is not None: - if self.supports_n: - assert ( - len(generations) == len(request.prompt) * request.n - ), f"Number of generations ({len(generations)}) does not match number of prompts ({len(request.prompt)} * {request.n})." - else: - assert len(generations) == len( - request.prompt - ), f"Number of generations ({len(generations)}) does not match number of prompts ({len(request.prompt)})." - generations = self.fold_responses(generations, request.n) - - return generations + return await self.base_generate(request) def generate( self, request: InferenceRequest @@ -85,58 +48,6 @@ def generate( else: return loop.run_until_complete(self.agenerate(request)) - -def ensure_template(value: Any) -> ConversationTemplate: - if isinstance(value, ConversationTemplate): - return value - elif isinstance(value, str): - return ConversationTemplate.from_string(value) - else: - raise ValueError(f"Invalid conversation template: {value}") - - -class ChatInferenceInterface(InferenceInterface): - """Inference interface for chat models.""" - - conversation_template: Annotated[ConversationTemplate, BeforeValidator(ensure_template)] - - def prepare_request( - self, prompts: list[str | list[LLMChatMessage]], generation_args: GenericGenerationArgs - ) -> ChatInferenceRequest: - prompt = [ - [LLMChatMessage(role='user', content=p)] if isinstance(p, str) else p for p in prompts - ] - return ChatInferenceRequest(prompt=prompt, generation_args=generation_args) - - async def base_generate(self, request: ChatInferenceRequest) -> list[ChatInferenceResponse]: - base_generate_results = await super().base_generate( - InferenceRequest( - prompt=[ - self.conversation_template.format(messages, request.tools) - for messages in request.prompt - ], - generation_args=request.generation_args, - ) - ) - chat_message_results = self.conversation_template.parse_response(base_generate_results) - return [ - ChatInferenceResponse( - response=chat_message, **response.model_dump(exclude={'response'}) - ) - for chat_message, response in zip(chat_message_results, base_generate_results) - ] - - def generate( - self, request: ChatInferenceRequest - ) -> list[ChatInferenceResponse] | list[GroupedChatInferenceResponse]: - return super().generate(request) - - async def agenerate( - self, request: ChatInferenceRequest - ) -> list[ChatInferenceResponse] | list[GroupedChatInferenceResponse]: - return await super().agenerate(request) - - class ReturnsRaw(InferenceInterface): """Mix-In for interface that supports returning complete string fed to the LLM.""" diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 602ff4f7450..7242a697a0d 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -26,7 +26,6 @@ from megatron.training.global_vars import get_args, get_tokenizer from ..inference.inference_interface import ( - ChatInferenceInterface, InferenceRequest, InferenceResponse, LLMChatMessage, @@ -72,51 +71,45 @@ def get_static_inference_engine(args: Namespace, model: MegatronModule) -> Abstr class MegatronLocal(InferenceServer, ReturnsTokens, ReturnsRaw): """Interface to use MCoreEngine directly as an inference engine.""" + host: str + port: int + + _server_task: asyncio.Task = PrivateAttr(None) _client: InferenceClient = PrivateAttr(None) _inference_engine: DynamicInferenceEngine = PrivateAttr(None) async def base_generate(self, request: InferenceRequest): - if any(isinstance(p, LLMChatMessage) for p in request.prompt): - raise ValueError( - "MegatronLocal does not support chat requests." - "Use MegatronChatLocal to apply chat templating." - ) - assert all( - isinstance(p, str) for p in request.prompt - ), "MegatronLocal only supports string prompts." + assert self._server_task is not None, "Infernce server is not initialized" - assert self._client is not None, "Client is not initialized" + from openai import AsyncOpenAI + client = AsyncOpenAI(base_url=f"http://{self.host}:{self.port}", api_key="NONE") - tokenizer = get_tokenizer() - args = get_args() - - sampling_params = SamplingParams( - num_tokens_to_generate=None, - num_tokens_total=request.generation_args.max_tokens, + # Things that may be problematic when doign this switch + # - Add BOS token + # - Skip prompt logprobs + generations = [ client.chat.completions.create( + model="", + messages=[message.model_dump() for message in prompt], temperature=request.generation_args.temperature or 1.0, - top_k=request.generation_args.top_k or 0, top_p=request.generation_args.top_p or 0.0, - termination_id=self._inference_engine.controller.tokenizer.eod, - return_log_probs=True, - skip_prompt_log_probs=True, - add_BOS=(not args.rl_skip_bos_token and tokenizer.bos is not None), - ) - requests = [ - self._client.add_request(prompt=prompt, sampling_params=sampling_params) - for prompt in request.prompt - ] - records = await asyncio.gather(*requests) - responses = [record[-1] for record in records] + n=request.generation_args.n or 1, + logprobs=True, + ) for prompt in request.prompt ] + + responses = await asyncio.gather(*generations) + + assert all(len(response.choices) == 1 for response in responses), "Still need to properly support requests with n > 1" + return [ InferenceResponse( - response=r.generated_text, - raw_text=p + r.generated_text, - token_ids=r.prompt_tokens.tolist() + r.generated_tokens, - logprobs=r.generated_log_probs, - prompt_length=len(r.prompt_tokens), + response=LLMChatMessage(**choice.message.model_dump(include={'role', 'content'})), + raw_text=choice.raw_text, + token_ids=choice.prompt_token_ids + choice.generation_token_ids, + logprobs=choice.generation_log_probs, + prompt_length=len(choice.prompt_token_ids), ) - for p, r in zip(request.prompt, responses) + for response in responses for choice in response.choices ] @classmethod @@ -138,14 +131,25 @@ async def launch(cls, model: GPTModel, **kwargs): dp_addr = await inference_engine.start_listening_to_data_parallel_coordinator( inference_coordinator_port=41521, launch_inference_coordinator=True, ) + if dist.get_rank() == 0: - # TODO: We have to do this only on the rank 0 process, should be fixed in the future when we have support for multiple inference clients. !2278 - client = InferenceClient(inference_coordinator_address=dp_addr) + from megatron.core.inference.text_generation_server.dynamic_text_gen_server.flask_server import run_flask_server_on_client + loop = asyncio.get_event_loop() + client = InferenceClient(inference_coordinator_addr=dp_addr) await client.start() + server_task = loop.create_task(run_flask_server_on_client( + client=client, + tokenizer=inference_engine.controller.tokenizer, + rank=dist.get_rank(), + flask_port=8294, + )) else: client = None + server_task = None + launched_server = cls(**kwargs) launched_server._client = client + launched_server._server_task = server_task launched_server._inference_engine = inference_engine return launched_server @@ -164,6 +168,3 @@ async def resume(self): if dist.get_rank() == 0: self._client.unpause_engines() await self._inference_engine.running.wait() - - -class MegatronChatLocal(ChatInferenceInterface, MegatronLocal): ... diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 3ea43103215..7bc73ca79ff 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -8,6 +8,7 @@ import itertools import math import logging +import os import pickle from collections import Counter, defaultdict from contextlib import contextmanager, nullcontext @@ -60,7 +61,7 @@ TokenRollout, ) from megatron.rl.agent.weighted_multi_task import WeightedMultiTask -from megatron.rl.inference.megatron import MegatronChatLocal, MegatronLocal +from megatron.rl.inference.megatron import MegatronLocal from megatron.rl.logging import LOG_DIR as lang_rl_log_dir from megatron.rl.logging import log as lang_rl_log from megatron.rl.server.inference.inference_interface_server import InferenceInterfaceServer @@ -433,34 +434,19 @@ def get_inference_interface(args, loop, model): if _INFERENCE_INTERFACE is None: rank = torch.distributed.get_rank() if rank == 0 and args.langrl_external_server: - if args.langrl_inference_server_type == 'inplace_megatron': - _INFERENCE_INTERFACE = loop.run_until_complete( - InferenceInterfaceServer.launch(MegatronLocal, model=model[0]) - ) - elif args.langrl_inference_server_type == 'inplace_megatron_chat': - _INFERENCE_INTERFACE = loop.run_until_complete( - InferenceInterfaceServer.launch( - MegatronChatLocal, - model=model[0], - conversation_template=args.langrl_inference_server_conversation_template, - ) - ) - else: - raise ValueError(f"Unknown inference_server_type {args.inference_server_type}") + _INFERENCE_INTERFACE = loop.run_until_complete( + InferenceInterfaceServer.launch(MegatronLocal, + model=model[0], + host='0.0.0.0', + port=os.getenv('MEGATRON_RL_INFERENCE_SERVER_PORT', 8294)) + ) else: - if args.langrl_inference_server_type == 'inplace_megatron': - _INFERENCE_INTERFACE = loop.run_until_complete(MegatronLocal.launch(model[0])) - elif args.langrl_inference_server_type == 'inplace_megatron_chat': - _INFERENCE_INTERFACE = loop.run_until_complete( - MegatronChatLocal.launch( - model[0], - conversation_template=args.langrl_inference_server_conversation_template, - ) - ) - else: - raise ValueError( - f"Unknown inference_server_type {args.langrl_inference_server_type}" - ) + _INFERENCE_INTERFACE = loop.run_until_complete( + MegatronLocal.launch( + model[0], + host='0.0.0.0', + port=os.getenv('MEGATRON_RL_INFERENCE_SERVER_PORT', 8294)) + ) return _INFERENCE_INTERFACE diff --git a/megatron/rl/server/inference/inference_interface_server.py b/megatron/rl/server/inference/inference_interface_server.py index ba595c3ca0e..d502c39fdc1 100644 --- a/megatron/rl/server/inference/inference_interface_server.py +++ b/megatron/rl/server/inference/inference_interface_server.py @@ -11,10 +11,10 @@ from typing_extensions import Self from uvicorn import Config, Server -from ...inference.api import ChatInferenceRequest, ChatInferenceResponse from ...inference.inference_interface import ( - ChatInferenceInterface, InferenceInterface, + InferenceRequest, + InferenceResponse, ReturnsRaw, ReturnsTokens, ) @@ -22,18 +22,18 @@ @InferenceServer.register_subclass -class InferenceInterfaceClient(ChatInferenceInterface, InferenceServer): +class InferenceInterfaceClient(InferenceServer): type_name: str = Field(default='InferenceInterfaceClient', frozen=True) env_server_host_port: str conversation_template: None = None - async def base_generate(self, request: ChatInferenceRequest) -> list[ChatInferenceResponse]: + async def base_generate(self, request: InferenceRequest) -> list[InferenceResponse]: async with httpx.AsyncClient(timeout=None) as client: response = await client.post( f"http://{self.env_server_host_port}/base_generate/", json=request.model_dump() ) return [ - ChatInferenceResponse.model_validate(inference_response) + InferenceResponse.model_validate(inference_response) for inference_response in response.json() ] @@ -69,7 +69,7 @@ async def launch(cls, interface_cls: type[InferenceInterface], **kwargs) -> Self server_ref = weakref.ref(launched_server) @app.post("/base_generate/") - async def base_generate(request: ChatInferenceRequest): + async def base_generate(request: InferenceRequest): server = server_ref() if server is None: raise RuntimeError("Server has been garbage collected") diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 9951203f18f..f71c83e5f30 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1891,9 +1891,6 @@ def _add_rl_args(parser): help="Entropy term weight in GRPO loss.") group.add_argument('--grpo-filter-groups-with-same-reward', action='store_true', help="Filter groups with same reward.") - group.add_argument('--langrl-inference-server-type', type=str, - choices=['inplace_megatron', 'inplace_megatron_chat'], default='inplace_megatron', - help="Type of inference server to use.") group.add_argument('--langrl-inference-server-conversation-template', type=str, default=None, help="Conversation template, if using a chat server.") group.add_argument('--langrl-external-server', action=argparse.BooleanOptionalAction, required=False, default=False) diff --git a/train_rl.py b/train_rl.py index 4b5cec5fcc8..645e78ba986 100644 --- a/train_rl.py +++ b/train_rl.py @@ -33,7 +33,6 @@ logging.basicConfig(level=logging.INFO, force=True) - def _gpt_builder(args, pre_process, post_process, vp_stage=None, config=None, pg_collection=None): # TODO(Peter): This is a hack to get around the fact that we are activation recomputation for training but not # for inference with cuda graphs. Without out this the post checks in the transformer config will assert error. From c195e2444969087177a487b2a16b9153747b8559 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 27 Jan 2026 12:22:24 -0600 Subject: [PATCH 15/56] Remove base model prompting special cases --- .../rl/environment_configs/gsm8k_nanov3.yaml | 2 -- .../environments/countdown/countdown_agent.py | 10 +--------- examples/rl/environments/math/gsm8k_agent.py | 4 ---- examples/rl/environments/math/math_agent.py | 17 ++--------------- 4 files changed, 3 insertions(+), 30 deletions(-) diff --git a/examples/rl/environment_configs/gsm8k_nanov3.yaml b/examples/rl/environment_configs/gsm8k_nanov3.yaml index 30403ed052b..b759423ee5b 100644 --- a/examples/rl/environment_configs/gsm8k_nanov3.yaml +++ b/examples/rl/environment_configs/gsm8k_nanov3.yaml @@ -2,8 +2,6 @@ agent_args: answer_format: "boxed" format_reward: 0.5 - assistant_suffix: "Assistant: " - chat_mode: true negative_reward: 0.0 partial_end_reward: 0.75 weight: 1.0 diff --git a/examples/rl/environments/countdown/countdown_agent.py b/examples/rl/environments/countdown/countdown_agent.py index 85d94039125..e995602b0a2 100644 --- a/examples/rl/environments/countdown/countdown_agent.py +++ b/examples/rl/environments/countdown/countdown_agent.py @@ -10,18 +10,10 @@ class CountdownAgent(RewardOnlyAgent, HFDatasetAgent): - chat_mode: bool = True def make_prefix(self, target, nums) -> str: - if self.chat_mode: - prefix = f"""Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. + prefix = f"""Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. Return the final answer in tags, for example (1 + 2) / 3 . Do not include an = sign.""" - else: - prefix = f"""A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. - User: Using the numbers {nums}, create an equation that equals {target}. You can use basic arithmetic operations (+, -, *, /) and each number can only be used once. Show your work in tags. - And return the final answer in tags, for example (1 + 2) / 3 . Do not include an = sign. - Assistant: Let me solve this step by step. - """ return prefix def get_dataset(self, validation: bool = False): diff --git a/examples/rl/environments/math/gsm8k_agent.py b/examples/rl/environments/math/gsm8k_agent.py index 3bb39bc09f9..6cdfb4f926e 100644 --- a/examples/rl/environments/math/gsm8k_agent.py +++ b/examples/rl/environments/math/gsm8k_agent.py @@ -25,16 +25,12 @@ class GSM8KAgent(MathAgent): def __init__(self, answer_format: str = "boxed", - chat_mode: bool = False, - assistant_suffix: str = "Assistant: Let me solve this step by step.\n", format_reward: float = 0.0, negative_reward: float = 0.0, partial_end_reward: float = 0.0, **kwargs): super().__init__( answer_format=answer_format, - chat_mode=chat_mode, - assistant_suffix=assistant_suffix, format_reward=format_reward, negative_reward=negative_reward, partial_end_reward=partial_end_reward, diff --git a/examples/rl/environments/math/math_agent.py b/examples/rl/environments/math/math_agent.py index 486c8477b92..005791a39c0 100644 --- a/examples/rl/environments/math/math_agent.py +++ b/examples/rl/environments/math/math_agent.py @@ -24,8 +24,6 @@ class MathAgent(RewardOnlyAgent): def __init__(self, format_reward: float = 0.0, answer_format: str = "tagged", - assistant_suffix: str = "Assistant: Let me solve this step by step.\n", - chat_mode: bool = False, negative_reward: float = 0.0, partial_end_reward: float = 0.0, **kwargs): @@ -35,9 +33,6 @@ def __init__(self, even if the answer is incorrect or is missing the end-of-text token. answer_format (str): Which answer format is expected: "tagged" for tags, or "boxed" for \boxed{} LaTeX formatting. - assistant_suffix (str): The suffix string included in the assistant's response, typically to - guide the assistant's output format and "persona". For example, "Let me solve this step by step." - chat_mode (bool): If True, agent operates in a chat (conversational) context. negative_reward (float): Reward assigned for a clearly incorrect or unparseable answer. partial_end_reward (float): Reward when the answer is correct but an expected end token is not matched exactly. **kwargs: Additional arguments for the base RewardOnlyAgent. @@ -48,8 +43,6 @@ def __init__(self, self.format_reward = format_reward self.answer_format = answer_format - self.assistant_suffix = assistant_suffix - self.chat_mode = chat_mode self.negative_reward = negative_reward self.partial_end_reward = partial_end_reward @@ -133,12 +126,6 @@ def make_prefix(self, problem_key: str = "problem", **kwargs) -> str: else: raise ValueError(f"Invalid answer format: {self.answer_format}") - if self.chat_mode: - prefix = f"""{kwargs[problem_key]}\n{answer_format}""" - else: - prefix = f"""A conversation between User and Assistant. The user asks a question, and the Assistant solves it. The assistant first thinks about the reasoning process in the mind and then provides the user with the answer. - The question will be a word math problem. Show your work in tags. - {answer_format} - User: {kwargs[problem_key]} - {self.assistant_suffix}""" + prefix = f"""{kwargs[problem_key]}\n{answer_format}""" + return prefix From 64896009038b95f1f2f32f95b234db7d410b3c50 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 3 Feb 2026 10:40:23 -0600 Subject: [PATCH 16/56] Add tool and reasoning parsers --- .../endpoints/chat_completions.py | 24 +- .../dynamic_text_gen_server/flask_server.py | 7 +- .../core/tokenizers/text/parsers/__init__.py | 11 + .../tokenizers/text/parsers/base_parser.py | 15 + .../parsers/deepseek_r1_reasoning_parser.py | 24 ++ .../text/parsers/qwen3_coder_tool_parser.py | 295 ++++++++++++++++++ megatron/training/checkpointing.py | 2 +- tools/run_dynamic_text_generation_server.py | 2 + 8 files changed, 373 insertions(+), 7 deletions(-) create mode 100644 megatron/core/tokenizers/text/parsers/__init__.py create mode 100644 megatron/core/tokenizers/text/parsers/base_parser.py create mode 100644 megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py create mode 100644 megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 7d97a87b7a4..1eb63d8cb36 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -19,6 +19,7 @@ async def chat_completions(): """Handles async POST requests for chat completions.""" client = current_app.config['client'] tokenizer = current_app.config['tokenizer'] + parsers = current_app.config['parsers'] req = request.get_json() @@ -31,12 +32,14 @@ async def chat_completions(): try: prompt_tokens = tokenizer.apply_chat_template( - messages, tokenize=True, add_generation_prompt=True + messages, tokenize=True, add_generation_prompt=True, tools=req.get("tools", None) ) except AttributeError: logger.warning("Tokenizer does not support 'apply_chat_template'. Using tokenize instead.") prompt_tokens = tokenizer.tokenize("\n".join([message["content"] for message in messages])) except Exception as e: + import traceback + logger.error(f"{traceback.format_exc()}") return f"Error processing 'messages': {e}", 500 # --- 2. Parse Sampling Params --- @@ -133,16 +136,31 @@ async def chat_completions(): } logprobs_content.append(entry) + from megatron.core.tokenizers.text.parsers import PARSER_MAPPING + metadata = {} + message_text = text_output + for parser in parsers: + if parser not in PARSER_MAPPING: + raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") + message_text, new_info = PARSER_MAPPING[parser].parse(message_text, tools=req.get("tools", None)) + assert not (metadata.keys() & new_info.keys()), "Multiple parsers found the same information." + metadata.update(new_info) + message = {"role": "assistant", "content": message_text} + if "tool_calls" in metadata: + message["tool_calls"] = metadata["tool_calls"] + if "reasoning" in metadata: + message["reasoning"] = metadata["reasoning"] + choice_data = { "index": 0, - "message": {"role": "assistant", "content": text_output}, + "message": message, "prompt_token_ids": prompt_tokens, "generation_token_ids": result.generated_tokens, "generation_log_probs": result.generated_log_probs, "raw_text": result.prompt + result.generated_text, # 'logprobs' in chat API is an object containing 'content' "logprobs": {"content": logprobs_content} if logprobs_content else None, - "finish_reason": "length", # Original code hardcoded this. + "finish_reason": "tool_calls" if metadata.get("tool_calls", []) else "stop", # Original code hardcoded this. } choices.append(choice_data) total_completion_tokens += len(result.generated_tokens) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index a0936582f60..aa95b7f7cdf 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -33,7 +33,7 @@ def temp_log_level(level, logger=None): @trace_async_exceptions -async def run_flask_server_on_client(client: InferenceClient, tokenizer, rank: int, flask_port: int): +async def run_flask_server_on_client(client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = []): """Initializes and runs the async Flask server.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -49,6 +49,7 @@ async def run_flask_server_on_client(client: InferenceClient, tokenizer, rank: i # Store client and tokenizer in app config for Blueprints to use app.config['client'] = client app.config['tokenizer'] = tokenizer + app.config['parsers'] = parsers # Register all blueprints from the 'endpoints' package for endpoint in endpoints.__all__: @@ -69,12 +70,12 @@ def health_check(): @trace_async_exceptions -async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int): +async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = []): inference_client = InferenceClient(coordinator_addr) await inference_client.start() logger.info(f"Rank {rank}: InferenceClient connected.") try: - await run_flask_server_on_client(inference_client, tokenizer, rank, flask_port) + await run_flask_server_on_client(inference_client, tokenizer, flask_port, parsers) finally: await inference_client.stop() logger.info(f"Rank {rank}: Flask server and client shut down.") \ No newline at end of file diff --git a/megatron/core/tokenizers/text/parsers/__init__.py b/megatron/core/tokenizers/text/parsers/__init__.py new file mode 100644 index 00000000000..5baa08eadf0 --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/__init__.py @@ -0,0 +1,11 @@ +from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser +from megatron.core.tokenizers.text.parsers.qwen3_coder_tool_parser import Qwen3CoderToolParser + +PARSER_MAPPING = { + "deepseek-r1-reasoning": DeepSeekR1ReasoningParser, + "qwen3-coder-tool": Qwen3CoderToolParser, +} + +__all__ = [ + "PARSER_MAPPING", +] \ No newline at end of file diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py new file mode 100644 index 00000000000..ce97d5b8ecf --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -0,0 +1,15 @@ +class BaseParser: + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + """ + Parses the text into a tuple containing extracted content + and a dictionary of additional information. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, str]]: A tuple containing the unprocessed text + and a dictionary with the extracted information. + """ + return text, {} \ No newline at end of file diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py new file mode 100644 index 00000000000..e27ec90b6ef --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -0,0 +1,24 @@ +from megatron.core.tokenizers.text.parsers.base_parser import BaseParser + +class DeepSeekR1ReasoningParser(BaseParser): + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + """ + Extracts the reasoning content from the text using ... tags. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, str]]: A tuple containing the unprocessed text + and a dictionary with the extracted reasoning content. + """ + + if "" in text: + if "" in text: + # Strip the prefix (it might not be present if it was part of the prompt) + text = text.split("")[1] + reasoning_content = text.split("")[0] + return text.split("")[0]+text.split("")[-1], {'reasoning': reasoning_content} + else: + return text, {} \ No newline at end of file diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py new file mode 100644 index 00000000000..0e2766af237 --- /dev/null +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -0,0 +1,295 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#### TODO: This parser was copied from vLLM and modified to work with megatron-lm inference + + +import ast +import json +import uuid +from collections.abc import Sequence +from typing import Any + +import regex as re +from megatron.core.tokenizers.text.parsers.base_parser import BaseParser +from megatron.core.tokenizers.text.libraries.huggingface_tokenizer import HuggingFaceTokenizer +import logging + +logger = logging.getLogger(__name__) + +# These map to vLLM types but we just use dictionaries for now +ToolCall = dict[str, Any] +FunctionCall = dict[str, Any] +ChatCompletionToolsParam = dict[str, Any] +ChatCompletionRequest = dict[str, Any] +ExtractedToolCallInformation = dict + +class _Qwen3CoderToolParser: + def __init__(self): + + # Sentinel tokens for streaming mode + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + self.tool_call_prefix: str = "(.*?)", re.DOTALL + ) + self.tool_call_regex = re.compile( + r"(.*?)|(.*?)$", re.DOTALL + ) + self.tool_call_function_regex = re.compile( + r"||(?=)|$)", + re.DOTALL, + ) + + + def _generate_tool_call_id(self) -> str: + """Generate a unique tool call ID.""" + return f"call_{uuid.uuid4().hex[:24]}" + + def _get_arguments_config( + self, func_name: str, tools: list[ChatCompletionToolsParam] | None + ) -> dict: + """Extract argument configuration for a function.""" + if tools is None: + return {} + for config in tools: + if not hasattr(config, "type") or not ( + hasattr(config, "function") and hasattr(config.function, "name") + ): + continue + if config.type == "function" and config.function.name == func_name: + if not hasattr(config.function, "parameters"): + return {} + params = config.function.parameters + if isinstance(params, dict) and "properties" in params: + return params["properties"] + elif isinstance(params, dict): + return params + else: + return {} + logger.debug("Tool '%s' is not defined in the tools list.", func_name) + return {} + + def _convert_param_value( + self, param_value: str, param_name: str, param_config: dict, func_name: str + ) -> Any: + """Convert parameter value based on its type in the schema.""" + # Handle null value for any type + if param_value.lower() == "null": + return None + + if param_name not in param_config: + if param_config != {}: + logger.debug( + "Parsed parameter '%s' is not defined in the tool " + "parameters for tool '%s', directly returning the " + "string value.", + param_name, + func_name, + ) + return param_value + + if ( + isinstance(param_config[param_name], dict) + and "type" in param_config[param_name] + ): + param_type = str(param_config[param_name]["type"]).strip().lower() + else: + param_type = "string" + if param_type in ["string", "str", "text", "varchar", "char", "enum"]: + return param_value + elif ( + param_type.startswith("int") + or param_type.startswith("uint") + or param_type.startswith("long") + or param_type.startswith("short") + or param_type.startswith("unsigned") + ): + try: + return int(param_value) + except (ValueError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' is not an " + "integer in tool '%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + elif param_type.startswith("num") or param_type.startswith("float"): + try: + float_param_value = float(param_value) + return ( + float_param_value + if float_param_value - int(float_param_value) != 0 + else int(float_param_value) + ) + except (ValueError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' is not a float " + "in tool '%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + elif param_type in ["boolean", "bool", "binary"]: + param_value = param_value.lower() + if param_value not in ["true", "false"]: + logger.debug( + "Parsed value '%s' of parameter '%s' is not a boolean " + "(`true` or `false`) in tool '%s', degenerating to " + "false.", + param_value, + param_name, + func_name, + ) + return param_value == "true" + else: + if ( + param_type in ["object", "array", "arr"] + or param_type.startswith("dict") + or param_type.startswith("list") + ): + try: + param_value = json.loads(param_value) + return param_value + except (json.JSONDecodeError, TypeError, ValueError): + logger.debug( + "Parsed value '%s' of parameter '%s' cannot be " + "parsed with json.loads in tool '%s', will try " + "other methods to parse it.", + param_value, + param_name, + func_name, + ) + try: + param_value = ast.literal_eval(param_value) # safer + except (ValueError, SyntaxError, TypeError): + logger.debug( + "Parsed value '%s' of parameter '%s' cannot be " + "converted via Python `ast.literal_eval()` in tool " + "'%s', degenerating to string.", + param_value, + param_name, + func_name, + ) + return param_value + + def _parse_xml_function_call( + self, function_call_str: str, tools: list[ChatCompletionToolsParam] | None + ) -> ToolCall | None: + # Extract function name + end_index = function_call_str.index(">") + function_name = function_call_str[:end_index] + param_config = self._get_arguments_config(function_name, tools) + parameters = function_call_str[end_index + 1 :] + param_dict = {} + for match_text in self.tool_call_parameter_regex.findall(parameters): + idx = match_text.index(">") + param_name = match_text[:idx] + param_value = str(match_text[idx + 1 :]) + # Remove prefix and trailing \n + if param_value.startswith("\n"): + param_value = param_value[1:] + if param_value.endswith("\n"): + param_value = param_value[:-1] + + param_dict[param_name] = self._convert_param_value( + param_value, param_name, param_config, function_name + ) + return ToolCall( + type="function", + id=self._generate_tool_call_id(), + function=FunctionCall( + name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False) + ), + ) + + def _get_function_calls(self, model_output: str) -> list[str]: + # Find all tool calls + matched_ranges = self.tool_call_regex.findall(model_output) + raw_tool_calls = [ + match[0] if match[0] else match[1] for match in matched_ranges + ] + + # Back-off strategy if no tool_call tags found + if len(raw_tool_calls) == 0: + raw_tool_calls = [model_output] + + raw_function_calls = [] + for tool_call in raw_tool_calls: + raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call)) + + function_calls = [ + match[0] if match[0] else match[1] for match in raw_function_calls + ] + return function_calls + + def extract_tool_calls( + self, + model_output: str, + tools: list[ChatCompletionToolsParam] | None, + ) -> ExtractedToolCallInformation: + # Quick check to avoid unnecessary processing + if self.tool_call_prefix not in model_output: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + try: + function_calls = self._get_function_calls(model_output) + if len(function_calls) == 0: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + tool_calls = [ + self._parse_xml_function_call(function_call_str, tools) + for function_call_str in function_calls + ] + + # Extract content before tool calls + content_index = model_output.find(self.tool_call_start_token) + idx = model_output.find(self.tool_call_prefix) + content_index = content_index if content_index >= 0 else idx + content = model_output[:content_index] # .rstrip() + + return ExtractedToolCallInformation( + tools_called=(len(tool_calls) > 0), + tool_calls=tool_calls, + content=content if content else None, + ) + + except Exception: + logger.exception("Error in extracting tool call from response.") + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + +class Qwen3CoderToolParser(BaseParser): + @staticmethod + def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + """ + Extracts the tool calls from the text using ... tags. + Uses the _Qwen3CoderToolParser class (copied from vLLM) to extract the tool calls. + + Args: + text (str): The text to parse. + + Returns: + tuple[str, dict[str, str]]: A tuple containing the unprocessed text + and a dictionary with the extracted tool calls. + """ + + information = _Qwen3CoderToolParser().extract_tool_calls(text, tools=kwargs.get("tools", [])) + if information.get("tools_called", False): + return information.get("content", ""), {"tool_calls": information.get("tool_calls", [])} + else: + return text, {} \ No newline at end of file diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index a3d307f1e30..07a41aef9df 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -1452,8 +1452,8 @@ def _set_arg(arg_name, old_arg_name=None, force=False): _set_arg('moe_latent_size', force=True) # Tokenizer args. - # Using checkpoint version might not always be safe (e.g., if running on different cluster). if args.use_tokenizer_model_from_checkpoint_args: + # Using checkpoint version might not always be safe (e.g., if running on different cluster). _set_arg('tokenizer_model', force=True) _set_arg('tokenizer_type', force=True) _set_arg('tiktoken_pattern', force=True) diff --git a/tools/run_dynamic_text_generation_server.py b/tools/run_dynamic_text_generation_server.py index c09c788ca8e..b6092f36732 100644 --- a/tools/run_dynamic_text_generation_server.py +++ b/tools/run_dynamic_text_generation_server.py @@ -19,6 +19,7 @@ def add_text_generation_server_args(parser: argparse.ArgumentParser): parser = add_modelopt_args(parser) parser = add_inference_args(parser) parser.add_argument("--port", type=int, default=5000, help="Port for Flask server to run on") + parser.add_argument("--parsers", type=str, nargs="+", default=[], help="Parsers to use for parsing the response") return parser @@ -46,6 +47,7 @@ async def run_text_generation_server( run_flask_server( coordinator_addr=coordinator_addr, tokenizer=engine.controller.tokenizer, + parsers=args.parsers, rank=rank, flask_port=flask_port, ) From 09b83eba3120a574b9109373554ea08439709682 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 3 Feb 2026 11:36:26 -0600 Subject: [PATCH 17/56] Fix reasoning parser. Add some logging. --- .../dynamic_text_gen_server/flask_server.py | 2 ++ .../text/parsers/deepseek_r1_reasoning_parser.py | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index aa95b7f7cdf..f46cdf429b9 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -65,6 +65,8 @@ def health_check(): # Force logging level to INFO to ensure that hostname is printed with temp_log_level(logging.INFO, logger): logger.info(f"Starting Flask server on http://{hostname}:{flask_port}") + logger.info(f"Using tokenizer: {type(tokenizer)}") + logger.info(f"Using parsers: {parsers}") await serve(app, config) diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index e27ec90b6ef..5bb4a853733 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -17,8 +17,10 @@ def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: if "" in text: if "" in text: # Strip the prefix (it might not be present if it was part of the prompt) - text = text.split("")[1] - reasoning_content = text.split("")[0] - return text.split("")[0]+text.split("")[-1], {'reasoning': reasoning_content} + pre_text, text = text.split("", maxsplit=1) + else: + pre_text = "" + reasoning_content, remaining_text = text.split("", maxsplit=1) + return pre_text+remaining_text, {'reasoning': reasoning_content} else: return text, {} \ No newline at end of file From f32d2155daaf3c4251ee7c3b23688db2dd744364 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 3 Feb 2026 13:45:10 -0600 Subject: [PATCH 18/56] Fix RL code to match new OpenAI server --- megatron/rl/inference/megatron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 7242a697a0d..149c28d9b7c 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -140,8 +140,8 @@ async def launch(cls, model: GPTModel, **kwargs): server_task = loop.create_task(run_flask_server_on_client( client=client, tokenizer=inference_engine.controller.tokenizer, - rank=dist.get_rank(), flask_port=8294, + parsers=[] )) else: client = None From 055964dcc3d036e5d8570ad9b1f109bef0f3d771 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 3 Feb 2026 21:42:45 -0600 Subject: [PATCH 19/56] Force exit --- megatron/rl/rl_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 7bc73ca79ff..256c5c30ae7 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -1786,6 +1786,12 @@ def rl_inference_interface_shutdown(): else: logger.warning("No inference interface to shutdown. This should not happen.") + # TODO(rkirby): This is a hack to hard exit. There is a bug that is preventing us from using sys.exit(0). + # It seem the Flask server has non-daemon threads that are preventing the program from exiting. + # We need to find a way to gracefully complete all in progress requests and shutdown the Flask server. + import os + os._exit(0) + def get_iteration_sequence_count(args): """Get the total number of sequences processed in this iteration across all ranks.""" From 076fac58b3d3a895dec57aecb2779b5b24408bbf Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Tue, 3 Feb 2026 22:14:43 -0600 Subject: [PATCH 20/56] Update configs, remove more arguments --- .../rl/model_configs/llama3p1_8b_instruct.sh | 3 --- examples/rl/model_configs/qwen3_4b.sh | 2 -- examples/rl/model_configs/qwen3_8b.sh | 2 -- .../rl/model_configs/qwen_2p5_distill_7b.sh | 2 -- megatron/core/inference/inference_client.py | 2 -- megatron/rl/__init__.py | 2 +- megatron/rl/rl_utils.py | 19 +++++-------------- megatron/training/arguments.py | 3 --- 8 files changed, 6 insertions(+), 29 deletions(-) diff --git a/examples/rl/model_configs/llama3p1_8b_instruct.sh b/examples/rl/model_configs/llama3p1_8b_instruct.sh index 5398dad1a4e..325c1d80617 100644 --- a/examples/rl/model_configs/llama3p1_8b_instruct.sh +++ b/examples/rl/model_configs/llama3p1_8b_instruct.sh @@ -101,9 +101,6 @@ MODEL_OPTIONS="\ --max-position-embeddings 131072 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model unsloth/Meta-Llama-3.1-8B-Instruct \ - --legacy-tokenizer \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "unsloth/Meta-Llama-3.1-8B-Instruct" \ --lr 3e-7 \ --make-vocab-size-divisible-by 128 \ --clip-grad 1.0 \ diff --git a/examples/rl/model_configs/qwen3_4b.sh b/examples/rl/model_configs/qwen3_4b.sh index 6f6c6b6bf57..81899f2ea7b 100644 --- a/examples/rl/model_configs/qwen3_4b.sh +++ b/examples/rl/model_configs/qwen3_4b.sh @@ -63,8 +63,6 @@ MODEL_OPTIONS="\ --attention-softmax-in-fp32 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model Qwen/Qwen3-4B \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "Qwen/Qwen3-4B" \ --vocab-size 151936 \ --make-vocab-size-divisible-by 128 \ --optimizer adam \ diff --git a/examples/rl/model_configs/qwen3_8b.sh b/examples/rl/model_configs/qwen3_8b.sh index 54ff7385331..4b5e1103f0e 100644 --- a/examples/rl/model_configs/qwen3_8b.sh +++ b/examples/rl/model_configs/qwen3_8b.sh @@ -64,8 +64,6 @@ MODEL_OPTIONS="\ --attention-softmax-in-fp32 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model Qwen/Qwen3-8B \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "Qwen/Qwen3-8B" \ --vocab-size 151936 \ --make-vocab-size-divisible-by 128 \ --optimizer adam \ diff --git a/examples/rl/model_configs/qwen_2p5_distill_7b.sh b/examples/rl/model_configs/qwen_2p5_distill_7b.sh index 1438bca0726..ed214b3aae9 100644 --- a/examples/rl/model_configs/qwen_2p5_distill_7b.sh +++ b/examples/rl/model_configs/qwen_2p5_distill_7b.sh @@ -70,8 +70,6 @@ MODEL_OPTIONS="\ --max-position-embeddings 131072 \ --tokenizer-type HuggingFaceTokenizer \ --tokenizer-model "unsloth/DeepSeek-R1-Distill-Qwen-7B" \ - --langrl-inference-server-type "inplace_megatron_chat" \ - --langrl-inference-server-conversation-template "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" \ --lr 0.000001 \ --lr-warmup-samples 0 \ --make-vocab-size-divisible-by 128 \ diff --git a/megatron/core/inference/inference_client.py b/megatron/core/inference/inference_client.py index d92a0a9625e..a927a393b8c 100644 --- a/megatron/core/inference/inference_client.py +++ b/megatron/core/inference/inference_client.py @@ -11,8 +11,6 @@ from .headers import Headers -logger = logging.getLogger(__name__) - try: import zmq diff --git a/megatron/rl/__init__.py b/megatron/rl/__init__.py index b2a9eeb954d..6c43678dc0a 100644 --- a/megatron/rl/__init__.py +++ b/megatron/rl/__init__.py @@ -65,7 +65,7 @@ class GenericGenerationArgs(BaseModel): top_k: int | None = None top_p: float | None = None max_tokens: int | None = None - n: int | None = None + n: int | None = None # Number of completions to generate per request def add(self, generation_args: 'GenericGenerationArgs') -> 'GenericGenerationArgs': return GenericGenerationArgs.model_validate( diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 256c5c30ae7..8f22d4a397e 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -432,21 +432,12 @@ def get_agent(args, parallel_generation_tasks: int | None = None): def get_inference_interface(args, loop, model): global _INFERENCE_INTERFACE if _INFERENCE_INTERFACE is None: - rank = torch.distributed.get_rank() - if rank == 0 and args.langrl_external_server: - _INFERENCE_INTERFACE = loop.run_until_complete( - InferenceInterfaceServer.launch(MegatronLocal, - model=model[0], - host='0.0.0.0', + _INFERENCE_INTERFACE = loop.run_until_complete( + MegatronLocal.launch( + model[0], + host='0.0.0.0', port=os.getenv('MEGATRON_RL_INFERENCE_SERVER_PORT', 8294)) - ) - else: - _INFERENCE_INTERFACE = loop.run_until_complete( - MegatronLocal.launch( - model[0], - host='0.0.0.0', - port=os.getenv('MEGATRON_RL_INFERENCE_SERVER_PORT', 8294)) - ) + ) return _INFERENCE_INTERFACE diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index f71c83e5f30..2f3f657855a 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1891,9 +1891,6 @@ def _add_rl_args(parser): help="Entropy term weight in GRPO loss.") group.add_argument('--grpo-filter-groups-with-same-reward', action='store_true', help="Filter groups with same reward.") - group.add_argument('--langrl-inference-server-conversation-template', type=str, default=None, - help="Conversation template, if using a chat server.") - group.add_argument('--langrl-external-server', action=argparse.BooleanOptionalAction, required=False, default=False) group.add_argument('--langrl-env-config', type=str, default=None, help="Path to YAML config file for RL environment configuration.") group.add_argument('--rl-default-temperature', type=float, default=1.0, From 3f0bb1552b4f13df9d62c1b23df84f59a5a3079e Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 11:21:20 -0600 Subject: [PATCH 21/56] Fix review comments. Remove 'n' from RL code altogether --- .../core/inference/engines/dynamic_engine.py | 1 - .../endpoints/chat_completions.py | 31 +++++-------- .../dynamic_text_gen_server/flask_server.py | 4 +- .../tokenizers/text/parsers/base_parser.py | 6 ++- .../parsers/deepseek_r1_reasoning_parser.py | 2 + .../text/parsers/qwen3_coder_tool_parser.py | 44 +++++++++---------- megatron/rl/__init__.py | 1 - megatron/rl/agent/reward_only_agent.py | 21 +++------ megatron/rl/inference/api.py | 12 +---- megatron/rl/inference/inference_interface.py | 18 +++----- megatron/rl/inference/megatron.py | 34 +++++++------- megatron/rl/rl_utils.py | 2 +- .../inference/inference_interface_server.py | 9 ++-- 13 files changed, 73 insertions(+), 112 deletions(-) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index baa23038a76..51ef76ec65e 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -770,7 +770,6 @@ def _add_request( if request.status != Status.FAILED: self.waiting_request_ids.append(request_id) else: - logger.error(f"Request {request_id} failed to add to engine. {request.events}") self.failed_request_ids.append(request_id) if self.rank == 0: warnings.warn( diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 1eb63d8cb36..1e43f406633 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -3,8 +3,10 @@ import asyncio import logging import time +import traceback from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.tokenizers.text.parsers import PARSER_MAPPING logger = logging.getLogger(__name__) @@ -38,7 +40,6 @@ async def chat_completions(): logger.warning("Tokenizer does not support 'apply_chat_template'. Using tokenize instead.") prompt_tokens = tokenizer.tokenize("\n".join([message["content"] for message in messages])) except Exception as e: - import traceback logger.error(f"{traceback.format_exc()}") return f"Error processing 'messages': {e}", 500 @@ -64,6 +65,7 @@ async def chat_completions(): return_log_probs=return_log_probs, top_n_logprobs=top_n_logprobs, num_tokens_to_generate=int(max_tokens) if ( (max_tokens := req.get("max_tokens", None)) is not None ) else None, + skip_prompt_log_probs=True, ) except ValueError as e: return f"Invalid sampling parameter: {e}", 400 @@ -72,16 +74,7 @@ async def chat_completions(): # For chat, we run the *same* prompt 'n' times. tasks = [] for _ in range(n): - per_req_params = SamplingParams( - temperature=sampling_params.temperature, - top_k=sampling_params.top_k, - top_p=sampling_params.top_p, - return_log_probs=sampling_params.return_log_probs, - top_n_logprobs=sampling_params.top_n_logprobs, - num_tokens_to_generate=sampling_params.num_tokens_to_generate, - skip_prompt_log_probs=True, - ) - tasks.append(client.add_request(prompt_tokens, per_req_params)) + tasks.append(client.add_request(prompt_tokens, sampling_params)) start_time = time.perf_counter() try: @@ -136,15 +129,15 @@ async def chat_completions(): } logprobs_content.append(entry) - from megatron.core.tokenizers.text.parsers import PARSER_MAPPING metadata = {} message_text = text_output - for parser in parsers: - if parser not in PARSER_MAPPING: - raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") - message_text, new_info = PARSER_MAPPING[parser].parse(message_text, tools=req.get("tools", None)) - assert not (metadata.keys() & new_info.keys()), "Multiple parsers found the same information." - metadata.update(new_info) + if parsers: + for parser in parsers: + if parser not in PARSER_MAPPING: + raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") + message_text, new_info = PARSER_MAPPING[parser].parse(message_text, tools=req.get("tools", None)) + assert not (metadata.keys() & new_info.keys()), "Multiple parsers found the same information." + metadata.update(new_info) message = {"role": "assistant", "content": message_text} if "tool_calls" in metadata: message["tool_calls"] = metadata["tool_calls"] @@ -152,7 +145,7 @@ async def chat_completions(): message["reasoning"] = metadata["reasoning"] choice_data = { - "index": 0, + "index": request_idx, "message": message, "prompt_token_ids": prompt_tokens, "generation_token_ids": result.generated_tokens, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index f46cdf429b9..7c85c29ade4 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -33,7 +33,7 @@ def temp_log_level(level, logger=None): @trace_async_exceptions -async def run_flask_server_on_client(client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = []): +async def run_flask_server_on_client(client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = None): """Initializes and runs the async Flask server.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -72,7 +72,7 @@ def health_check(): @trace_async_exceptions -async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = []): +async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = None): inference_client = InferenceClient(coordinator_addr) await inference_client.start() logger.info(f"Rank {rank}: InferenceClient connected.") diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index ce97d5b8ecf..98749105624 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -1,6 +1,8 @@ +from typing import Any + class BaseParser: @staticmethod - def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: """ Parses the text into a tuple containing extracted content and a dictionary of additional information. @@ -9,7 +11,7 @@ def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: text (str): The text to parse. Returns: - tuple[str, dict[str, str]]: A tuple containing the unprocessed text + tuple[str, dict[str, Any]]: A tuple containing the unprocessed text and a dictionary with the extracted information. """ return text, {} \ No newline at end of file diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index 5bb4a853733..bb763642874 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -5,6 +5,8 @@ class DeepSeekR1ReasoningParser(BaseParser): def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: """ Extracts the reasoning content from the text using ... tags. + Only extracts the first set of think tags. + If an initial tag is not present but a tag is, it will infer a tag at the beginning of the text. Args: text (str): The text to parse. diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 0e2766af237..38ba07f8dc5 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -3,16 +3,14 @@ #### TODO: This parser was copied from vLLM and modified to work with megatron-lm inference - import ast import json import uuid -from collections.abc import Sequence +from types import SimpleNamespace from typing import Any import regex as re from megatron.core.tokenizers.text.parsers.base_parser import BaseParser -from megatron.core.tokenizers.text.libraries.huggingface_tokenizer import HuggingFaceTokenizer import logging logger = logging.getLogger(__name__) @@ -25,27 +23,26 @@ ExtractedToolCallInformation = dict class _Qwen3CoderToolParser: - def __init__(self): - # Sentinel tokens for streaming mode - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - self.tool_call_prefix: str = "(.*?)", re.DOTALL - ) - self.tool_call_regex = re.compile( - r"(.*?)|(.*?)$", re.DOTALL - ) - self.tool_call_function_regex = re.compile( - r"||(?=)|$)", - re.DOTALL, - ) + # Regex patterns + tool_call_complete_regex = re.compile( + r"(.*?)", re.DOTALL + ) + tool_call_regex = re.compile( + r"(.*?)|(.*?)$", re.DOTALL + ) + tool_call_function_regex = re.compile( + r"||(?=)|$)", + re.DOTALL, + ) def _generate_tool_call_id(self) -> str: @@ -59,6 +56,7 @@ def _get_arguments_config( if tools is None: return {} for config in tools: + config = SimpleNamespace(**config) # Convert to SimpleNamespace for ease of access if not hasattr(config, "type") or not ( hasattr(config, "function") and hasattr(config.function, "name") ): @@ -275,7 +273,7 @@ def extract_tool_calls( class Qwen3CoderToolParser(BaseParser): @staticmethod - def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: + def parse(text: str, **kwargs) -> tuple[str, dict[str, list[dict]]]: """ Extracts the tool calls from the text using ... tags. Uses the _Qwen3CoderToolParser class (copied from vLLM) to extract the tool calls. diff --git a/megatron/rl/__init__.py b/megatron/rl/__init__.py index 6c43678dc0a..08ae226bfe4 100644 --- a/megatron/rl/__init__.py +++ b/megatron/rl/__init__.py @@ -65,7 +65,6 @@ class GenericGenerationArgs(BaseModel): top_k: int | None = None top_p: float | None = None max_tokens: int | None = None - n: int | None = None # Number of completions to generate per request def add(self, generation_args: 'GenericGenerationArgs') -> 'GenericGenerationArgs': return GenericGenerationArgs.model_validate( diff --git a/megatron/rl/agent/reward_only_agent.py b/megatron/rl/agent/reward_only_agent.py index 8f74d2b8a39..4099406a98a 100644 --- a/megatron/rl/agent/reward_only_agent.py +++ b/megatron/rl/agent/reward_only_agent.py @@ -120,14 +120,10 @@ async def rollout(self, request: RolloutRequest) -> Rollout: prompt, golden = await self.get_prompt(validation=request.validation) inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args + prompt, request.generation_args ) - responses = await request.inference_interface.agenerate(inference_request) - assert ( - len(responses) == 1 - ), "get_reward_rollouts only requested a single response but got multiple responses" - response = responses[0] + response = await request.inference_interface.agenerate(inference_request) return await self.rollout_from_response(request, response, golden) @@ -136,26 +132,21 @@ async def group_rollout(self, request: GroupedRolloutRequest) -> list[Rollout]: prompt, golden = await self.get_prompt(validation=request.validation) inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args + prompt, request.generation_args ) responses = await asyncio.gather(*[request.inference_interface.agenerate(inference_request) for _ in range(request.rollouts_per_group)]) - return [await self.rollout_from_response(request, response[0], golden) for response in responses] + return [await self.rollout_from_response(request, response, golden) for response in responses] async def _evaluation( self, prompt: str, golden: Any, request: EvaluationRequest ) -> RewardOnlyEvaluationResponse: inference_request = request.inference_interface.prepare_request( - [prompt], request.generation_args + prompt, request.generation_args ) - responses = await request.inference_interface.agenerate(inference_request) - assert ( - len(responses) == 1 - ), "evaluation only requested a single response but got multiple responses" - response = responses[0] - + response = await request.inference_interface.agenerate(inference_request) response_text = response.response.content result = RewardEvaluationResult( diff --git a/megatron/rl/inference/api.py b/megatron/rl/inference/api.py index fa647633621..87f6b87f908 100644 --- a/megatron/rl/inference/api.py +++ b/megatron/rl/inference/api.py @@ -11,14 +11,10 @@ class LLMChatMessage(BaseModel): class InferenceRequest(Request): - prompt: list[list[LLMChatMessage]] + prompt: list[LLMChatMessage] tools: list[dict] | None = None -class GroupedInferenceRequest(InferenceRequest): - group_size: int = 1 - - class InferenceResponse(BaseModel): """The minimum required response for an inference interface.""" @@ -27,9 +23,3 @@ class InferenceResponse(BaseModel): token_ids: list[int] | None = None prompt_length: int | None = None logprobs: list[float] | None = None - - -class GroupedInferenceResponse(BaseModel): - """An inference response which includes a list of responses.""" - - responses: list[InferenceResponse] diff --git a/megatron/rl/inference/inference_interface.py b/megatron/rl/inference/inference_interface.py index 5558ff9c8bc..715f5ada9d4 100644 --- a/megatron/rl/inference/inference_interface.py +++ b/megatron/rl/inference/inference_interface.py @@ -1,15 +1,11 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import asyncio -from abc import abstractmethod -from itertools import zip_longest -from typing import Annotated, Any, ClassVar -from pydantic import BaseModel, BeforeValidator, ValidationError +from pydantic import BaseModel from ..__init__ import GenericGenerationArgs from ..inference.api import ( - GroupedInferenceResponse, InferenceRequest, InferenceResponse, LLMChatMessage, @@ -23,24 +19,22 @@ class Config: arbitrary_types_allowed = True def prepare_request( - self, prompts: list[str | list[LLMChatMessage]], generation_args: GenericGenerationArgs + self, prompt: str | list[LLMChatMessage], generation_args: GenericGenerationArgs ) -> InferenceRequest: - prompt = [ - [LLMChatMessage(role='user', content=p)] if isinstance(p, str) else p for p in prompts - ] + prompt = [LLMChatMessage(role='user', content=prompt)] if isinstance(prompt, str) else prompt return InferenceRequest(prompt=prompt, generation_args=generation_args) - async def base_generate(self, request: InferenceRequest) -> list[InferenceResponse]: + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: assert NotImplementedError("Direct Inference Classes must implement the base_generate method.") async def agenerate( self, request: InferenceRequest - ) -> list[InferenceResponse] | list[GroupedInferenceResponse]: + ) -> InferenceResponse: return await self.base_generate(request) def generate( self, request: InferenceRequest - ) -> list[InferenceResponse] | list[GroupedInferenceResponse]: + ) -> InferenceResponse: try: loop = asyncio.get_running_loop() except RuntimeError: diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 149c28d9b7c..c9219e9aebb 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -78,9 +78,9 @@ class MegatronLocal(InferenceServer, ReturnsTokens, ReturnsRaw): _client: InferenceClient = PrivateAttr(None) _inference_engine: DynamicInferenceEngine = PrivateAttr(None) - async def base_generate(self, request: InferenceRequest): + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: - assert self._server_task is not None, "Infernce server is not initialized" + assert self._server_task is not None, "Inference server is not initialized" from openai import AsyncOpenAI client = AsyncOpenAI(base_url=f"http://{self.host}:{self.port}", api_key="NONE") @@ -88,29 +88,25 @@ async def base_generate(self, request: InferenceRequest): # Things that may be problematic when doign this switch # - Add BOS token # - Skip prompt logprobs - generations = [ client.chat.completions.create( + response = await client.chat.completions.create( model="", messages=[message.model_dump() for message in prompt], temperature=request.generation_args.temperature or 1.0, top_p=request.generation_args.top_p or 0.0, - n=request.generation_args.n or 1, + n=1, logprobs=True, - ) for prompt in request.prompt ] - - responses = await asyncio.gather(*generations) + ) - assert all(len(response.choices) == 1 for response in responses), "Still need to properly support requests with n > 1" + choice = response.choices[0] - return [ - InferenceResponse( - response=LLMChatMessage(**choice.message.model_dump(include={'role', 'content'})), - raw_text=choice.raw_text, - token_ids=choice.prompt_token_ids + choice.generation_token_ids, - logprobs=choice.generation_log_probs, - prompt_length=len(choice.prompt_token_ids), - ) - for response in responses for choice in response.choices - ] + return InferenceResponse( + # TODO: Handle tool calls and reasoning in LLMChatMessage + response=LLMChatMessage(**choice.message.model_dump(include={'role', 'content'})), + raw_text=choice.raw_text, + token_ids=choice.prompt_token_ids + choice.generation_token_ids, + logprobs=choice.generation_log_probs, + prompt_length=len(choice.prompt_token_ids), + ) @classmethod async def launch(cls, model: GPTModel, **kwargs): @@ -140,7 +136,7 @@ async def launch(cls, model: GPTModel, **kwargs): server_task = loop.create_task(run_flask_server_on_client( client=client, tokenizer=inference_engine.controller.tokenizer, - flask_port=8294, + flask_port=kwargs.get('port', 8294), parsers=[] )) else: diff --git a/megatron/rl/rl_utils.py b/megatron/rl/rl_utils.py index 8f22d4a397e..d53878d82ab 100644 --- a/megatron/rl/rl_utils.py +++ b/megatron/rl/rl_utils.py @@ -436,7 +436,7 @@ def get_inference_interface(args, loop, model): MegatronLocal.launch( model[0], host='0.0.0.0', - port=os.getenv('MEGATRON_RL_INFERENCE_SERVER_PORT', 8294)) + port=8294) ) return _INFERENCE_INTERFACE diff --git a/megatron/rl/server/inference/inference_interface_server.py b/megatron/rl/server/inference/inference_interface_server.py index d502c39fdc1..ceac4a6cab7 100644 --- a/megatron/rl/server/inference/inference_interface_server.py +++ b/megatron/rl/server/inference/inference_interface_server.py @@ -27,15 +27,12 @@ class InferenceInterfaceClient(InferenceServer): env_server_host_port: str conversation_template: None = None - async def base_generate(self, request: InferenceRequest) -> list[InferenceResponse]: + async def base_generate(self, request: InferenceRequest) -> InferenceResponse: async with httpx.AsyncClient(timeout=None) as client: response = await client.post( f"http://{self.env_server_host_port}/base_generate/", json=request.model_dump() ) - return [ - InferenceResponse.model_validate(inference_response) - for inference_response in response.json() - ] + return InferenceResponse.model_validate(response.json()) @InferenceServer.register_subclass @@ -69,7 +66,7 @@ async def launch(cls, interface_cls: type[InferenceInterface], **kwargs) -> Self server_ref = weakref.ref(launched_server) @app.post("/base_generate/") - async def base_generate(request: InferenceRequest): + async def base_generate(request: InferenceRequest) -> InferenceResponse: server = server_ref() if server is None: raise RuntimeError("Server has been garbage collected") From ebf542c2d5a4e16ad286100c4fc57c5a9c546560 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 11:36:48 -0600 Subject: [PATCH 22/56] remove extra logging --- megatron/core/inference/engines/dynamic_engine.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 51ef76ec65e..134ce3b124d 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -104,9 +104,6 @@ from torch_memory_saver import torch_memory_saver -logger = logging.getLogger(__name__) - - class EngineSuspendedError(Exception): """Engine is currently suspended and not performing steps.""" From 9a5207b772dc517ea102eb3e254e7f8c8e092e73 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 12:05:01 -0600 Subject: [PATCH 23/56] Remove unused code, Fix typo --- megatron/rl/inference/megatron.py | 48 ++----------------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index c9219e9aebb..88df6950a58 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -2,27 +2,14 @@ import asyncio import logging -from argparse import Namespace import torch.distributed as dist from pydantic import PrivateAttr -from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext -from megatron.core.inference.engines.abstract_engine import AbstractEngine from megatron.core.inference.engines.dynamic_engine import DynamicInferenceEngine -from megatron.core.inference.engines.mcore_engine import MCoreEngine from megatron.core.inference.inference_client import InferenceClient -from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( - GPTInferenceWrapper, -) -from megatron.core.inference.sampling_params import SamplingParams -from megatron.core.inference.text_generation_controllers.text_generation_controller import ( - TextGenerationController, -) from megatron.core.models.gpt.gpt_model import GPTModel -from megatron.core.transformer.module import MegatronModule -from megatron.core.utils import get_attr_wrapped_model, log_single_rank -from megatron.training import get_wandb_writer +from megatron.core.utils import log_single_rank from megatron.training.global_vars import get_args, get_tokenizer from ..inference.inference_interface import ( @@ -37,37 +24,6 @@ logger = logging.getLogger(__name__) -## This code is copied from tools/run_text_generation_server.py -def get_static_inference_engine(args: Namespace, model: MegatronModule) -> AbstractEngine: - """Get the relevant backend for running inference. - - This function will automatically choose the TRTLLMBackend when possible, - and default to Mcore backend if the user does not specify any backends. - TRTLLMBackend is not implmented yet. - - Args: - args (Namespace): The user arguments parsed from command line - model (MegatronModule): The megatron model. - - Returns: - AbstractBackend: The chosen backend - """ - tokenizer = get_tokenizer() - - inference_wrapped_model = GPTInferenceWrapper(model) - pg_collection = get_attr_wrapped_model(model, "pg_collection") - pp_group = pg_collection.pp - text_generation_controller = TextGenerationController( - inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer, pp_group=pp_group - ) - return MCoreEngine( - text_generation_controller=text_generation_controller, - max_batch_size=( - args.inference_max_requests if args.inference_max_requests is not None else 1 - ), - ) - - class MegatronLocal(InferenceServer, ReturnsTokens, ReturnsRaw): """Interface to use MCoreEngine directly as an inference engine.""" @@ -90,7 +46,7 @@ async def base_generate(self, request: InferenceRequest) -> InferenceResponse: # - Skip prompt logprobs response = await client.chat.completions.create( model="", - messages=[message.model_dump() for message in prompt], + messages=[message.model_dump() for message in request.prompt], temperature=request.generation_args.temperature or 1.0, top_p=request.generation_args.top_p or 0.0, n=1, From fc0307ddb38d6f123529e1add2f670cd1ea09269 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 13:11:14 -0600 Subject: [PATCH 24/56] formatter --- .../endpoints/chat_completions.py | 26 ++++++++--- .../dynamic_text_gen_server/flask_server.py | 10 ++-- .../core/tokenizers/text/parsers/__init__.py | 8 ++-- .../tokenizers/text/parsers/base_parser.py | 3 +- .../parsers/deepseek_r1_reasoning_parser.py | 5 +- .../text/parsers/qwen3_coder_tool_parser.py | 46 +++++++------------ megatron/rl/inference/megatron.py | 2 +- 7 files changed, 54 insertions(+), 46 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 1e43f406633..60a6ccedab7 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -37,8 +37,12 @@ async def chat_completions(): messages, tokenize=True, add_generation_prompt=True, tools=req.get("tools", None) ) except AttributeError: - logger.warning("Tokenizer does not support 'apply_chat_template'. Using tokenize instead.") - prompt_tokens = tokenizer.tokenize("\n".join([message["content"] for message in messages])) + logger.warning( + "Tokenizer does not support 'apply_chat_template'. Using tokenize instead." + ) + prompt_tokens = tokenizer.tokenize( + "\n".join([message["content"] for message in messages]) + ) except Exception as e: logger.error(f"{traceback.format_exc()}") return f"Error processing 'messages': {e}", 500 @@ -64,7 +68,11 @@ async def chat_completions(): top_p=top_p, return_log_probs=return_log_probs, top_n_logprobs=top_n_logprobs, - num_tokens_to_generate=int(max_tokens) if ( (max_tokens := req.get("max_tokens", None)) is not None ) else None, + num_tokens_to_generate=( + int(max_tokens) + if ((max_tokens := req.get("max_tokens", None)) is not None) + else None + ), skip_prompt_log_probs=True, ) except ValueError as e: @@ -135,8 +143,12 @@ async def chat_completions(): for parser in parsers: if parser not in PARSER_MAPPING: raise ValueError(f"Parser {parser} not found in PARSER_MAPPING") - message_text, new_info = PARSER_MAPPING[parser].parse(message_text, tools=req.get("tools", None)) - assert not (metadata.keys() & new_info.keys()), "Multiple parsers found the same information." + message_text, new_info = PARSER_MAPPING[parser].parse( + message_text, tools=req.get("tools", None) + ) + assert not ( + metadata.keys() & new_info.keys() + ), "Multiple parsers found the same information." metadata.update(new_info) message = {"role": "assistant", "content": message_text} if "tool_calls" in metadata: @@ -153,7 +165,9 @@ async def chat_completions(): "raw_text": result.prompt + result.generated_text, # 'logprobs' in chat API is an object containing 'content' "logprobs": {"content": logprobs_content} if logprobs_content else None, - "finish_reason": "tool_calls" if metadata.get("tool_calls", []) else "stop", # Original code hardcoded this. + "finish_reason": ( + "tool_calls" if metadata.get("tool_calls", []) else "stop" + ), # Original code hardcoded this. } choices.append(choice_data) total_completion_tokens += len(result.generated_tokens) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index 7c85c29ade4..a96a984672f 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -33,7 +33,9 @@ def temp_log_level(level, logger=None): @trace_async_exceptions -async def run_flask_server_on_client(client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = None): +async def run_flask_server_on_client( + client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = None +): """Initializes and runs the async Flask server.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -72,7 +74,9 @@ def health_check(): @trace_async_exceptions -async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = None): +async def run_flask_server( + coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = None +): inference_client = InferenceClient(coordinator_addr) await inference_client.start() logger.info(f"Rank {rank}: InferenceClient connected.") @@ -80,4 +84,4 @@ async def run_flask_server(coordinator_addr: str, tokenizer, rank: int, flask_po await run_flask_server_on_client(inference_client, tokenizer, flask_port, parsers) finally: await inference_client.stop() - logger.info(f"Rank {rank}: Flask server and client shut down.") \ No newline at end of file + logger.info(f"Rank {rank}: Flask server and client shut down.") diff --git a/megatron/core/tokenizers/text/parsers/__init__.py b/megatron/core/tokenizers/text/parsers/__init__.py index 5baa08eadf0..a7ce407f605 100644 --- a/megatron/core/tokenizers/text/parsers/__init__.py +++ b/megatron/core/tokenizers/text/parsers/__init__.py @@ -1,4 +1,6 @@ -from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser +from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import ( + DeepSeekR1ReasoningParser, +) from megatron.core.tokenizers.text.parsers.qwen3_coder_tool_parser import Qwen3CoderToolParser PARSER_MAPPING = { @@ -6,6 +8,4 @@ "qwen3-coder-tool": Qwen3CoderToolParser, } -__all__ = [ - "PARSER_MAPPING", -] \ No newline at end of file +__all__ = ["PARSER_MAPPING"] diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index 98749105624..fb2d91a3760 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -1,5 +1,6 @@ from typing import Any + class BaseParser: @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: @@ -14,4 +15,4 @@ def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: tuple[str, dict[str, Any]]: A tuple containing the unprocessed text and a dictionary with the extracted information. """ - return text, {} \ No newline at end of file + return text, {} diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index bb763642874..b8eb990ac5a 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -1,5 +1,6 @@ from megatron.core.tokenizers.text.parsers.base_parser import BaseParser + class DeepSeekR1ReasoningParser(BaseParser): @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: @@ -23,6 +24,6 @@ def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: else: pre_text = "" reasoning_content, remaining_text = text.split("", maxsplit=1) - return pre_text+remaining_text, {'reasoning': reasoning_content} + return pre_text + remaining_text, {'reasoning': reasoning_content} else: - return text, {} \ No newline at end of file + return text, {} diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 38ba07f8dc5..8b92969edf4 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -5,13 +5,14 @@ import ast import json +import logging import uuid from types import SimpleNamespace from typing import Any import regex as re + from megatron.core.tokenizers.text.parsers.base_parser import BaseParser -import logging logger = logging.getLogger(__name__) @@ -22,6 +23,7 @@ ChatCompletionRequest = dict[str, Any] ExtractedToolCallInformation = dict + class _Qwen3CoderToolParser: # Sentinel tokens for streaming mode @@ -30,21 +32,13 @@ class _Qwen3CoderToolParser: tool_call_prefix: str = "(.*?)", re.DOTALL - ) - tool_call_regex = re.compile( - r"(.*?)|(.*?)$", re.DOTALL - ) - tool_call_function_regex = re.compile( - r"|(.*?)", re.DOTALL) + tool_call_regex = re.compile(r"(.*?)|(.*?)$", re.DOTALL) + tool_call_function_regex = re.compile(r"||(?=)|$)", - re.DOTALL, + r"|(?=)|$)", re.DOTALL ) - def _generate_tool_call_id(self) -> str: """Generate a unique tool call ID.""" return f"call_{uuid.uuid4().hex[:24]}" @@ -56,7 +50,7 @@ def _get_arguments_config( if tools is None: return {} for config in tools: - config = SimpleNamespace(**config) # Convert to SimpleNamespace for ease of access + config = SimpleNamespace(**config) # Convert to SimpleNamespace for ease of access if not hasattr(config, "type") or not ( hasattr(config, "function") and hasattr(config.function, "name") ): @@ -93,10 +87,7 @@ def _convert_param_value( ) return param_value - if ( - isinstance(param_config[param_name], dict) - and "type" in param_config[param_name] - ): + if isinstance(param_config[param_name], dict) and "type" in param_config[param_name]: param_type = str(param_config[param_name]["type"]).strip().lower() else: param_type = "string" @@ -213,9 +204,7 @@ def _parse_xml_function_call( def _get_function_calls(self, model_output: str) -> list[str]: # Find all tool calls matched_ranges = self.tool_call_regex.findall(model_output) - raw_tool_calls = [ - match[0] if match[0] else match[1] for match in matched_ranges - ] + raw_tool_calls = [match[0] if match[0] else match[1] for match in matched_ranges] # Back-off strategy if no tool_call tags found if len(raw_tool_calls) == 0: @@ -225,15 +214,11 @@ def _get_function_calls(self, model_output: str) -> list[str]: for tool_call in raw_tool_calls: raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call)) - function_calls = [ - match[0] if match[0] else match[1] for match in raw_function_calls - ] + function_calls = [match[0] if match[0] else match[1] for match in raw_function_calls] return function_calls def extract_tool_calls( - self, - model_output: str, - tools: list[ChatCompletionToolsParam] | None, + self, model_output: str, tools: list[ChatCompletionToolsParam] | None ) -> ExtractedToolCallInformation: # Quick check to avoid unnecessary processing if self.tool_call_prefix not in model_output: @@ -271,6 +256,7 @@ def extract_tool_calls( tools_called=False, tool_calls=[], content=model_output ) + class Qwen3CoderToolParser(BaseParser): @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, list[dict]]]: @@ -286,8 +272,10 @@ def parse(text: str, **kwargs) -> tuple[str, dict[str, list[dict]]]: and a dictionary with the extracted tool calls. """ - information = _Qwen3CoderToolParser().extract_tool_calls(text, tools=kwargs.get("tools", [])) + information = _Qwen3CoderToolParser().extract_tool_calls( + text, tools=kwargs.get("tools", []) + ) if information.get("tools_called", False): return information.get("content", ""), {"tool_calls": information.get("tool_calls", [])} else: - return text, {} \ No newline at end of file + return text, {} diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 88df6950a58..2fa64ed6988 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -87,7 +87,7 @@ async def launch(cls, model: GPTModel, **kwargs): if dist.get_rank() == 0: from megatron.core.inference.text_generation_server.dynamic_text_gen_server.flask_server import run_flask_server_on_client loop = asyncio.get_event_loop() - client = InferenceClient(inference_coordinator_addr=dp_addr) + client = InferenceClient(inference_coordinator_address=dp_addr) await client.start() server_task = loop.create_task(run_flask_server_on_client( client=client, From d321cc5b7f9c6e23a4e2e9253feeca425525e102 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 13:43:39 -0600 Subject: [PATCH 25/56] Fix lint --- .../dynamic_text_gen_server/flask_server.py | 4 +++- megatron/core/tokenizers/text/parsers/base_parser.py | 2 ++ .../tokenizers/text/parsers/deepseek_r1_reasoning_parser.py | 5 ++++- .../core/tokenizers/text/parsers/qwen3_coder_tool_parser.py | 3 +++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py index a96a984672f..b4bcfb3513d 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/flask_server.py @@ -36,7 +36,7 @@ def temp_log_level(level, logger=None): async def run_flask_server_on_client( client: InferenceClient, tokenizer, flask_port: int, parsers: list[str] = None ): - """Initializes and runs the async Flask server.""" + """Initializes and runs the async Flask server using the provided InferenceClient.""" if not HAS_FLASK: raise RuntimeError(f"Flask not available") @@ -77,6 +77,8 @@ def health_check(): async def run_flask_server( coordinator_addr: str, tokenizer, rank: int, flask_port: int, parsers: list[str] = None ): + """Initializes and runs the async Flask server + starting an InferenceClient with the provided coordinator address.""" inference_client = InferenceClient(coordinator_addr) await inference_client.start() logger.info(f"Rank {rank}: InferenceClient connected.") diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index fb2d91a3760..4fdd644f1df 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -2,6 +2,8 @@ class BaseParser: + """Base class for text parsers.""" + @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: """ diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index b8eb990ac5a..541dda43e72 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -2,12 +2,15 @@ class DeepSeekR1ReasoningParser(BaseParser): + """Parser for DeepSeek R1 style reasoning output.""" + @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: """ Extracts the reasoning content from the text using ... tags. Only extracts the first set of think tags. - If an initial tag is not present but a tag is, it will infer a tag at the beginning of the text. + If an initial tag is not present but a tag is, + it will infer a tag at the beginning of the text. Args: text (str): The text to parse. diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 8b92969edf4..9903af4d645 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -220,6 +220,7 @@ def _get_function_calls(self, model_output: str) -> list[str]: def extract_tool_calls( self, model_output: str, tools: list[ChatCompletionToolsParam] | None ) -> ExtractedToolCallInformation: + """Extracts the tool calls from the text using ... tags.""" # Quick check to avoid unnecessary processing if self.tool_call_prefix not in model_output: return ExtractedToolCallInformation( @@ -258,6 +259,8 @@ def extract_tool_calls( class Qwen3CoderToolParser(BaseParser): + """Parser for Qwen3 Coder style tool calls.""" + @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, list[dict]]]: """ From d69730f0687e693c5ed36878eb0d8ed0ea7052e4 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 13:47:24 -0600 Subject: [PATCH 26/56] Fix test args --- .../model_config.yaml | 1 - .../model_config.yaml | 1 - .../model_config.yaml | 1 - .../gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml | 2 -- .../gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml | 2 -- .../model_config.yaml | 1 - 6 files changed, 8 deletions(-) diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml index b12911358f0..4f9be214289 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest/model_config.yaml @@ -43,7 +43,6 @@ MODEL_ARGS: --straggler-minmax-count: 16 --tensorboard-log-interval: 1 --empty-unused-memory-level: 2 - --langrl-inference-server-type: inplace_megatron --seed: 42 --calculate-per-token-loss: true --rl-use-sequence-packing: true diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml index bff55aea7fe..686bee61031 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp1tp2_pp1_dp8_583m_throughputtest_github/model_config.yaml @@ -43,7 +43,6 @@ MODEL_ARGS: --straggler-minmax-count: 16 --tensorboard-log-interval: 1 --empty-unused-memory-level: 2 - --langrl-inference-server-type: inplace_megatron --seed: 42 --calculate-per-token-loss: true --rl-use-sequence-packing: true diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp2tp1_pp4pp2_dp8_583m_throughputtest/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp2tp1_pp4pp2_dp8_583m_throughputtest/model_config.yaml index b5788d64049..69e51a801a8 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp2tp1_pp4pp2_dp8_583m_throughputtest/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp2tp1_pp4pp2_dp8_583m_throughputtest/model_config.yaml @@ -44,7 +44,6 @@ MODEL_ARGS: --straggler-minmax-count: 16 --tensorboard-log-interval: 1 --empty-unused-memory-level: 2 - --langrl-inference-server-type: inplace_megatron --seed: 42 --calculate-per-token-loss: true --rl-use-sequence-packing: true diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml index ada0350b876..80664dcdc59 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml index 4490ced3988..cc25f3ab90e 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_throughput_github/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam diff --git a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml index c7dcfa594d8..139c5a82e57 100644 --- a/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_grpo_tp8tp4_pp1_ep8ep2_dp8_throughputtest/model_config.yaml @@ -90,7 +90,6 @@ MODEL_ARGS: --mock-data: true --max-tokens-to-oom: 3600000 --inference-max-seq-length: 256 - --langrl-inference-server-type: inplace_megatron --calculate-per-token-loss: true --rl-use-sequence-packing: true --rl-sequence-packing-algo: fifo From d5edaaa58cb095480865a7632b6f6e2793b5f238 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 13:47:24 -0600 Subject: [PATCH 27/56] Fix test args --- .../dynamic_text_gen_server/endpoints/chat_completions.py | 2 +- megatron/core/tokenizers/text/parsers/base_parser.py | 2 +- .../tokenizers/text/parsers/deepseek_r1_reasoning_parser.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 60a6ccedab7..58394a440eb 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -36,7 +36,7 @@ async def chat_completions(): prompt_tokens = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, tools=req.get("tools", None) ) - except AttributeError: + except (AttributeError, AssertionError): logger.warning( "Tokenizer does not support 'apply_chat_template'. Using tokenize instead." ) diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index 4fdd644f1df..dee87593951 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -3,7 +3,7 @@ class BaseParser: """Base class for text parsers.""" - + @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, Any]]: """ diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index 541dda43e72..baf9390c45e 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -3,13 +3,13 @@ class DeepSeekR1ReasoningParser(BaseParser): """Parser for DeepSeek R1 style reasoning output.""" - + @staticmethod def parse(text: str, **kwargs) -> tuple[str, dict[str, str]]: """ Extracts the reasoning content from the text using ... tags. Only extracts the first set of think tags. - If an initial tag is not present but a tag is, + If an initial tag is not present but a tag is, it will infer a tag at the beginning of the text. Args: From 24120443176ad4fa417d137943c14241af5745f2 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 14:06:50 -0600 Subject: [PATCH 28/56] Add copyright --- megatron/core/tokenizers/text/parsers/__init__.py | 1 + megatron/core/tokenizers/text/parsers/base_parser.py | 2 +- .../tokenizers/text/parsers/deepseek_r1_reasoning_parser.py | 1 + .../core/tokenizers/text/parsers/qwen3_coder_tool_parser.py | 2 -- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/megatron/core/tokenizers/text/parsers/__init__.py b/megatron/core/tokenizers/text/parsers/__init__.py index a7ce407f605..dc27763f905 100644 --- a/megatron/core/tokenizers/text/parsers/__init__.py +++ b/megatron/core/tokenizers/text/parsers/__init__.py @@ -1,3 +1,4 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from megatron.core.tokenizers.text.parsers.deepseek_r1_reasoning_parser import ( DeepSeekR1ReasoningParser, ) diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index dee87593951..a4f7d7286c6 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -1,6 +1,6 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from typing import Any - class BaseParser: """Base class for text parsers.""" diff --git a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py index baf9390c45e..17952c61daf 100644 --- a/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py +++ b/megatron/core/tokenizers/text/parsers/deepseek_r1_reasoning_parser.py @@ -1,3 +1,4 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from megatron.core.tokenizers.text.parsers.base_parser import BaseParser diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 9903af4d645..695e6e255c2 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -#### TODO: This parser was copied from vLLM and modified to work with megatron-lm inference - import ast import json import logging From dc4fcbc7c9bdc704416b261b513de66535ad4b9f Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 14:18:53 -0600 Subject: [PATCH 29/56] Really important newline. --- megatron/core/tokenizers/text/parsers/base_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/megatron/core/tokenizers/text/parsers/base_parser.py b/megatron/core/tokenizers/text/parsers/base_parser.py index a4f7d7286c6..afd847d1bfc 100644 --- a/megatron/core/tokenizers/text/parsers/base_parser.py +++ b/megatron/core/tokenizers/text/parsers/base_parser.py @@ -1,6 +1,7 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from typing import Any + class BaseParser: """Base class for text parsers.""" From 441e49a9edd229b7fc78256830d2bf9aa1da799e Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 14:23:44 -0600 Subject: [PATCH 30/56] More copyright --- megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 695e6e255c2..0e1b5c6cb7f 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -1,3 +1,4 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project From 81c78f9d1cdd21ddc8b1939e0b8967433d5d84fc Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 14:41:52 -0600 Subject: [PATCH 31/56] Update regex library --- .../core/tokenizers/text/parsers/qwen3_coder_tool_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 0e1b5c6cb7f..36f7e093388 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -9,7 +9,7 @@ from types import SimpleNamespace from typing import Any -import regex as re +import re from megatron.core.tokenizers.text.parsers.base_parser import BaseParser From fc27fa3480ee67e8c95aa2a6239f6914d9009362 Mon Sep 17 00:00:00 2001 From: Robert Kirby Date: Wed, 4 Feb 2026 14:52:32 -0600 Subject: [PATCH 32/56] More linting --- .../core/tokenizers/text/parsers/qwen3_coder_tool_parser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py index 36f7e093388..1d1f20a3a5c 100644 --- a/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py +++ b/megatron/core/tokenizers/text/parsers/qwen3_coder_tool_parser.py @@ -5,12 +5,11 @@ import ast import json import logging +import re import uuid from types import SimpleNamespace from typing import Any -import re - from megatron.core.tokenizers.text.parsers.base_parser import BaseParser logger = logging.getLogger(__name__) From 0666c2ef9eb26460df4fa6c4f850e3d1640d4a33 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 14:47:58 -0800 Subject: [PATCH 33/56] make it work with merge --- .../gpt_dynamic_inference_with_coordinator.py | 5 +- .../inference/contexts/dynamic_context.py | 26 +- .../inference/contexts/routing_metadata.py | 10 +- .../core/inference/engines/dynamic_engine.py | 12 +- megatron/core/inference/inference_request.py | 5 +- .../text_generation_controller.py | 10 +- megatron/core/transformer/moe/moe_utils.py | 234 ++---------------- megatron/core/transformer/moe/router.py | 4 +- .../core/transformer/moe/router_replay.py | 51 ++++ .../model_config.yaml | 1 + 10 files changed, 99 insertions(+), 259 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index a0d2e1cfdf3..771b3eca519 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -125,10 +125,7 @@ async def main( await asyncio.sleep(0) # While we wait for the requests to complete, the engine runs in the background. - results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) - for i, res in enumerate(results): - routing_indices = res.requests[0].routing_indices - logging.info(f"Result {i} has routing indices with shape {routing_indices.shape}.") + results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 07e2e53b74f..4e3dbb78043 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -35,6 +35,7 @@ from .attention_context.mha_metadata import GraphedMHAMetadata, NonGraphedMHAMetadata from .base_context import BaseInferenceContext from .dynamic_block_allocator import BlockAllocator +from .routing_metadata import RoutingMetadata try: from .fused_kv_append_kernel import triton_append_key_value_cache @@ -469,6 +470,14 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC max_seqlen=self.max_sequence_length, ) + self.moe_enable_routing_replay = model_config.moe_enable_routing_replay + if self.moe_enable_routing_replay: + assert model_config.num_moe_experts is not None, "Router recording/replay requested but no MoE experts specified!" + self.moe_routing_metadata = RoutingMetadata( + self, model_config.moe_router_topk + ) + + # CUDA graph config list self.use_cuda_graphs_for_non_decode_steps = ( inference_config.use_cuda_graphs_for_non_decode_steps @@ -659,17 +668,6 @@ def allocate_mamba_states(): allocate_memory_buffer() allocate_mamba_states() - # Allocate routing metadata for MoE models. - if self.moe_router_topk is not None: - from megatron.core.inference.contexts.routing_metadata import RoutingMetadata - - self.routing_metadata = RoutingMetadata( - context=self, - moe_router_topk=self.moe_router_topk, - ) - else: - self.routing_metadata = None - # Reset attention and Mamba state. self.reset_attention_state() self.reset_mamba_state() @@ -1305,11 +1303,11 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) - if self.routing_metadata is not None: + if self.moe_enable_routing_replay is not None: if self.using_cuda_graph_this_step(): - self.routing_metadata.enable_static_buffer_recording() + self.moe_routing_metadata.enable_static_buffer_recording() else: - self.routing_metadata.disable_static_buffer_recording() + self.moe_routing_metadata.disable_static_buffer_recording() def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index aa87e448992..faa184c24b7 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext +from megatron.core.transformer.moe.router_replay import RouterReplay class RoutingMetadata: """Manages routing indices metadata for MoE layers during inference. @@ -43,7 +44,6 @@ def _ensure_buffer_allocated(self) -> None: if self.routing_indices_buffer is not None: return - from megatron.core.transformer.moe.moe_utils import RouterReplay self.num_moe_layers = len(RouterReplay.global_router_replay_instances) print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " @@ -73,11 +73,11 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: # Return view of static buffer up to current token count. if self.routing_indices_buffer is None: return None + # Only return up to active token count, to skip entries + # for padding tokens. return self.routing_indices_buffer[:self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. - from megatron.core.transformer.moe.moe_utils import RouterReplay - recorded_data = RouterReplay.get_recorded_data() if recorded_data is None or len(recorded_data) == 0: return None @@ -93,16 +93,12 @@ def enable_static_buffer_recording(self) -> None: pre-allocated static buffer instead of creating new tensors. Allocates the buffer lazily on first call. """ - from megatron.core.transformer.moe.moe_utils import RouterReplay - self._ensure_buffer_allocated() if self.routing_indices_buffer is not None: RouterReplay.set_global_static_buffers(self.routing_indices_buffer) def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" - from megatron.core.transformer.moe.moe_utils import RouterReplay - RouterReplay.clear_global_static_buffers() def reset(self) -> None: diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 2cc98481e69..9faff001071 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -38,6 +38,10 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.transformer.moe.router_replay import ( + RouterReplay, + RouterReplayAction, + ) from megatron.core.inference.utils import Counter, await_process_call from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.cuda_graphs import delete_cuda_graphs @@ -52,6 +56,8 @@ trace_async_exceptions, ) + + from .async_zmq_communicator import AsyncZMQCommunicator try: @@ -300,11 +306,7 @@ def create_cuda_graphs(self, reset_context: bool = True): # Enable routing recording during warmup if routing replay is enabled. # This ensures the record_indices copy operation is captured in the CUDA graph. model_config = controller.inference_wrapped_model.model.config - if getattr(model_config, 'enable_routing_replay', False): - from megatron.core.transformer.moe.moe_utils import ( - RouterReplay, - RouterReplayAction, - ) + if model_config.moe_enable_routing_replay: RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) # Forward pass -> logits. diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 4356eb33bf9..3fd66633742 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -313,7 +313,7 @@ def serialize(self) -> dict: obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] - # Serialize routing_indices: Tensor [total_tokens - 1, num_layers, topk] -> 3D list + # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] if self.routing_indices is not None: total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) # the last generated token does not undergo a forward pass @@ -322,8 +322,7 @@ def serialize(self) -> dict: f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " f"total tokens {total_tokens-1}." ) - obj["routing_indices"] = self.routing_indices.tolist() - + torch.cuda.nvtx.range_pop() return obj diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index b58fd749b41..8e70055a88f 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -30,7 +30,7 @@ from megatron.core.transformer.enums import CudaGraphScope from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region from megatron.core.transformer.moe.moe_layer import BaseMoELayer -from megatron.core.transformer.moe.moe_utils import RouterReplay, RouterReplayAction +from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel from megatron.core import parallel_state from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model @@ -693,15 +693,15 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: disabled or no routing data was recorded. """ config = self.inference_wrapped_model.model.config - if not getattr(config, 'enable_routing_replay', False): + if not config.moe_enable_routing_replay: return None # Get routing indices - use routing_metadata if available (handles CUDA graph static buffers) context = self.inference_wrapped_model.inference_context - if context.routing_metadata is None: + if context.moe_routing_metadata is None: return None - stacked_routing = context.routing_metadata.get_routing_indices() + stacked_routing = context.moe_routing_metadata.get_routing_indices() if stacked_routing is None: return None @@ -956,7 +956,7 @@ async def async_generate_output_tokens_dynamic_batch( # Enable routing recording before forward pass if routing replay is enabled config = self.inference_wrapped_model.model.config - if getattr(config, 'enable_routing_replay', False): + if config.moe_enable_routing_replay: RouterReplay.set_global_router_replay_action(RouterReplayAction.RECORD) logits = self._dynamic_step_forward_logits(input_ids, position_ids) diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index c0d41993daf..666d6e16c38 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -2,8 +2,6 @@ import functools import math -from enum import Enum -from typing import Callable, List, Optional, Union from dataclasses import dataclass from typing import List, Optional, Tuple, Union @@ -655,186 +653,6 @@ def pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tenso return routing_map -class RouterReplayAction(Enum): - RECORD = "record" # Record the topk indices for replay - REPLAY_FORWARD = "replay_forward" # Replay the recorded topk indices for forward pass - REPLAY_BACKWARD = "replay_backward" # Replay topk indices for re-compute during backward pass - - -class RouterReplay: - """ - A class to manage the recording and replaying of MoE routing decisions. - It holds all router instances and provides static methods to globally - control recording and replaying. - """ - - # Static variable to hold all router instances, one per MoE layer. - global_router_replay_instances: List['RouterReplay'] = [] - - @staticmethod - def set_replay_data(all_layers_topk_indices: List[torch.Tensor]): - """ - Distributes the topk indices for all layers to their respective RouterReplay instances. - :param all_layers_topk_indices: A list of tensors, where each tensor contains the - topk indices for a specific layer. The order - must match the instantiation order of the routers. - """ - if len(all_layers_topk_indices) != len(RouterReplay.global_router_replay_instances): - raise ValueError( - f"The number of replay tensors ({len(all_layers_topk_indices)}) " - f"does not match router instances ({len(RouterReplay.global_router_replay_instances)})." - ) - for i, router_instance in enumerate(RouterReplay.global_router_replay_instances): - router_instance.set_target_indices(all_layers_topk_indices[i]) - - @staticmethod - def get_recorded_data() -> List[torch.Tensor]: - """ - Collects the recorded topk indices from all RouterReplay instances. - :return: A list of tensors, each containing the recorded topk indices for a layer. - """ - return [ - router.get_recorded_indices() for router in RouterReplay.global_router_replay_instances - ] - - @staticmethod - def clear_global_indices(): - """Clears the recorded and target topk indices in all instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_indices() - - @staticmethod - def set_global_static_buffers(static_buffer: torch.Tensor): - """Sets static buffers for all router instances from a combined buffer. - - Args: - static_buffer: Tensor of shape [max_tokens, num_layers, topk]. - Each layer's RouterReplay gets a slice [:, layer_idx, :]. - """ - num_layers = len(RouterReplay.global_router_replay_instances) - assert static_buffer.shape[1] == num_layers, ( - f"Buffer has {static_buffer.shape[1]} layers but there are " - f"{num_layers} RouterReplay instances." - ) - for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): - # Each layer gets a view of shape [max_tokens, topk] - router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) - - @staticmethod - def clear_global_static_buffers(): - """Clears static buffers from all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_static_buffer() - - @staticmethod - def set_global_router_replay_action(router_replay_action: RouterReplayAction): - """Sets the router replay action for all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.set_router_replay_action(router_replay_action) - - @staticmethod - def clear_global_router_replay_action(): - """Clears the router replay action for all router instances.""" - for router in RouterReplay.global_router_replay_instances: - router.clear_router_replay_action() - - def __init__(self): - """Initializes a RouterReplay instance for a specific layer.""" - self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay - self.recorded_topk_idx: Optional[torch.Tensor] = None # Recorded topk indices for replay - self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph recording - self.router_replay_action: Optional[RouterReplayAction] = ( - None # Router replay action for this layer - ) - self.replay_backward_list: List[torch.Tensor] = ( - [] - ) # List of tensors for backward pass replay - RouterReplay.global_router_replay_instances.append(self) - - def set_static_buffer(self, buffer: torch.Tensor): - """Sets a static buffer for CUDA graph compatible recording. - - Args: - buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. - """ - self.static_buffer = buffer - - def clear_static_buffer(self): - """Clears the static buffer.""" - self.static_buffer = None - - def set_target_indices(self, topk_indices: torch.Tensor): - """Sets the target topk indices for replay.""" - self.target_topk_idx = topk_indices - self.replay_backward_list.append(topk_indices) - - def get_recorded_indices(self) -> Optional[torch.Tensor]: - """Returns the recorded topk indices.""" - return self.recorded_topk_idx - - def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices. - - If a static buffer is set (for CUDA graph compatibility), copies into it. - Otherwise, just stores the tensor reference. - """ - if self.static_buffer is not None: - # Copy into static buffer for CUDA graph compatibility. - num_tokens = topk_indices.shape[0] - self.static_buffer[:num_tokens].copy_(topk_indices) - self.recorded_topk_idx = self.static_buffer[:num_tokens] - else: - self.recorded_topk_idx = topk_indices - - def clear_indices(self): - """Clears the recorded and target topk indices.""" - self.recorded_topk_idx = None - self.target_topk_idx = None - self.replay_backward_list = [] - - def set_router_replay_action(self, router_replay_action: RouterReplayAction): - """Sets the router replay action for this layer.""" - self.router_replay_action = router_replay_action - - def clear_router_replay_action(self): - """Clears the router replay action for this layer.""" - self.router_replay_action = None - - def get_replay_topk( - self, - scores: torch.Tensor, - topk: int, - num_groups: Optional[int] = None, - group_topk: Optional[int] = None, - default_compute_topk: Callable[ - [torch.Tensor, int, Optional[int], Optional[int]], torch.Tensor - ] = None, - ) -> torch.Tensor: - """Returns the target topk indices for replay.""" - if self.router_replay_action == RouterReplayAction.RECORD: - probs, top_indices = default_compute_topk( - scores, topk, num_groups=num_groups, group_topk=group_topk - ) - self.record_indices(top_indices) - return probs, top_indices - elif self.router_replay_action == RouterReplayAction.REPLAY_FORWARD: - top_indices = self.target_topk_idx - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - elif self.router_replay_action == RouterReplayAction.REPLAY_BACKWARD: - top_indices = self.replay_backward_list.pop(0) - # Ensure indices are on the correct device - top_indices = top_indices.to(scores.device) - # Gather the scores for the replayed indices to get the probabilities - probs = scores.gather(1, top_indices) - return probs, top_indices - else: - return default_compute_topk(scores, topk, num_groups, group_topk) - - def topk_routing_with_score_function( logits: torch.Tensor, topk: int, @@ -1182,41 +1000,6 @@ def reduce_aux_losses_tracker_across_ranks( ) -def get_num_moe_layers( - num_layers: int, - moe_layer_freq: Optional[Union[int, List[int]]] = None, - mtp_num_layers: Optional[int] = None, -) -> int: - """Calculate the number of MoE layers based on moe_layer_freq. - - Args: - num_layers (int): Total number of transformer layers. - moe_layer_freq (Optional[Union[int, List[int]]]): Frequency of MoE layers. - If int, every moe_layer_freq-th layer is an MoE layer. - If list, it's a binary pattern indicating which layers are MoE. - If None, all layers are assumed to be MoE layers. - mtp_num_layers (Optional[int]): Number of MTP (multi-token prediction) layers to add. - - Returns: - int: The number of MoE layers. - """ - if moe_layer_freq is None: - num_moe_layers = num_layers - elif isinstance(moe_layer_freq, int): - assert isinstance(num_layers, int) - moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] - num_moe_layers = sum(moe_layer_pattern) - elif isinstance(moe_layer_freq, list): - num_moe_layers = sum(moe_layer_freq) - else: - raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") - - if mtp_num_layers is not None: - num_moe_layers += mtp_num_layers - - return num_moe_layers - - def track_moe_metrics( loss_scale: float, iteration: int, @@ -1266,7 +1049,20 @@ def track_moe_metrics( tracker[key]["reduce_group_has_dp"] = False reduce_aux_losses_tracker_across_ranks(track_names, pg_collection=pg_collection) - num_moe_layers = get_num_moe_layers(num_layers, moe_layer_freq, mtp_num_layers) + # Get number of MoE layers + if moe_layer_freq is None: + num_moe_layers = num_layers + elif isinstance(moe_layer_freq, int): + assert isinstance(num_layers, int) + moe_layer_pattern = [1 if (i % moe_layer_freq == 0) else 0 for i in range(num_layers)] + num_moe_layers = sum(moe_layer_pattern) + elif isinstance(moe_layer_freq, list): + num_moe_layers = sum(moe_layer_freq) + else: + raise ValueError(f"Invalid moe_layer_freq: {moe_layer_freq}") + + if mtp_num_layers is not None: + num_moe_layers += mtp_num_layers aux_losses = {k: v['values'].float() * loss_scale for k, v in tracker.items()} for name, loss_list in aux_losses.items(): @@ -1777,4 +1573,4 @@ def wrapped_func(moe_layer, *args, **kwargs): return wrapped_func - return decorator + return decorator \ No newline at end of file diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 6ee98bbfd3a..9d1b557d2e4 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -10,7 +10,6 @@ from megatron.core.transformer.moe.moe_utils import ( MoEAuxLossAutoScaler, ProcessGroupCollection, - RouterReplay, apply_random_logits, apply_router_token_dropping, compute_routing_scores_for_aux_loss, @@ -213,9 +212,10 @@ def __init__( self.ga_steps = None self.router_replay = None + if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() - + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index b6b8e26a0a6..75fbb9009a3 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -74,6 +74,29 @@ def clear_global_router_replay_instances(): """Clear the global list of router replay instances to prevent memory leaks.""" RouterReplay.global_router_replay_instances.clear() + @staticmethod + def set_global_static_buffers(static_buffer: torch.Tensor): + """Sets static buffers for all router instances from a combined buffer. + + Args: + static_buffer: Tensor of shape [max_tokens, num_layers, topk]. + Each layer's RouterReplay gets a slice [:, layer_idx, :]. + """ + num_layers = len(RouterReplay.global_router_replay_instances) + assert static_buffer.shape[1] == num_layers, ( + f"Buffer has {static_buffer.shape[1]} layers but there are " + f"{num_layers} RouterReplay instances." + ) + for layer_idx, router_instance in enumerate(RouterReplay.global_router_replay_instances): + # Each layer gets a view of shape [max_tokens, topk] + router_instance.set_static_buffer(static_buffer[:, layer_idx, :]) + + @staticmethod + def clear_global_static_buffers(): + """Clears static buffers from all router instances.""" + for router in RouterReplay.global_router_replay_instances: + router.clear_static_buffer() + def __init__(self): """Initializes a RouterReplay instance for a specific layer.""" self.target_topk_idx: Optional[torch.Tensor] = None # Target topk indices for replay @@ -84,6 +107,7 @@ def __init__(self): self.replay_backward_list: List[torch.Tensor] = ( [] ) # List of tensors for backward pass replay + self.static_buffer: Optional[torch.Tensor] = None # Static buffer for CUDA graph RouterReplay.global_router_replay_instances.append(self) def set_target_indices(self, topk_indices: torch.Tensor): @@ -159,3 +183,30 @@ def get_replay_topk( return probs, top_indices else: return default_compute_topk(scores, topk, num_groups, group_topk) + + + def set_static_buffer(self, buffer: torch.Tensor): + """Sets a static buffer for CUDA graph compatible recording. + + Args: + buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. + """ + self.static_buffer = buffer + + def clear_static_buffer(self): + """Clears the static buffer.""" + self.static_buffer = None + + def record_indices(self, topk_indices: torch.Tensor): + """Records the topk indices. + + If a static buffer is set (for CUDA graph compatibility), copies into it. + Otherwise, just stores the tensor reference. + """ + if self.static_buffer is not None: + # Copy into static buffer for CUDA graph compatibility. + num_tokens = topk_indices.shape[0] + self.static_buffer[:num_tokens].copy_(topk_indices) + self.recorded_topk_idx = self.static_buffer[:num_tokens] + else: + self.recorded_topk_idx = topk_indices diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml index edc5fc2eb32..d33a854e49e 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml @@ -84,6 +84,7 @@ MODEL_ARGS: --inference-dynamic-batching-max-requests: 512 --inference-logging-step-interval: 1 --sequence-parallel: true + --moe-enable-routing-replay: true METRICS: - "generated_tokens" From 9109ca196a310243d15d50ba48b56112f918ee58 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:42:04 -0800 Subject: [PATCH 34/56] hook upto openAI API --- megatron/core/inference/inference_request.py | 1 + .../endpoints/chat_completions.py | 100 ++++++----- .../endpoints/completions.py | 170 +++++++++--------- 3 files changed, 147 insertions(+), 124 deletions(-) diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index 3fd66633742..d67e8570033 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -513,6 +513,7 @@ def merge_lists(key): prompt_tokens = self.requests[0].prompt_tokens prompt_text = self.requests[0].prompt + routing_indices = None if self.requests[0].routing_indices is not None: routing_indices = torch.cat([r.routing_indices for r in self.requests]) generated_tokens = merge_lists("generated_tokens") diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 0c3379bc53f..e9d16e08ca0 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -99,50 +99,62 @@ async def chat_completions(): request_idx = 0 for record in batch_results: - for result in record.requests: - text_output = result.generated_text - - logprobs_content = None - if sampling_params.return_log_probs: - token_logprobs = getattr(result, 'log_probs', []) - tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] - - # Get top_n_logprobs if available - generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) - - logprobs_content = [] - for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): - # Build top_logprobs list for this token position - top_logprobs_list = [] - if generated_top_n_logprobs and i < len(generated_top_n_logprobs): - top_n_dict = generated_top_n_logprobs[i] - for token_str, logprob in top_n_dict.items(): - top_logprobs_list.append( - { - "token": token_str, - "logprob": logprob, - "bytes": list(token_str.encode("utf-8")), - } - ) - - entry = { - "token": tok, - "logprob": lp, - "bytes": list(tok.encode("utf-8")), - "top_logprobs": top_logprobs_list, - } - logprobs_content.append(entry) - - choice_data = { - "index": 0, - "message": {"role": "assistant", "content": text_output}, - # 'logprobs' in chat API is an object containing 'content' - "logprobs": {"content": logprobs_content} if logprobs_content else None, - "finish_reason": "length", # Original code hardcoded this. - } - choices.append(choice_data) - total_completion_tokens += len(result.generated_tokens) - request_idx += 0 + assert len(record.requests) == 1, "Each record should contain one request result." + result = record.merge() + text_output = result.generated_text + + logprobs_content = None + if sampling_params.return_log_probs: + token_logprobs = getattr(result, 'log_probs', []) + tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] + + # Get top_n_logprobs if available + generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) + + logprobs_content = [] + for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): + # Build top_logprobs list for this token position + top_logprobs_list = [] + if generated_top_n_logprobs and i < len(generated_top_n_logprobs): + top_n_dict = generated_top_n_logprobs[i] + for token_str, logprob in top_n_dict.items(): + top_logprobs_list.append( + { + "token": token_str, + "logprob": logprob, + "bytes": list(token_str.encode("utf-8")), + } + ) + + entry = { + "token": tok, + "logprob": lp, + "bytes": list(tok.encode("utf-8")), + "top_logprobs": top_logprobs_list, + } + logprobs_content.append(entry) + + choice_data = { + "index": 0, + "message": {"role": "assistant", "content": text_output}, + # 'logprobs' in chat API is an object containing 'content' + "logprobs": {"content": logprobs_content} if logprobs_content else None, + "finish_reason": "length", # Original code hardcoded this. + } + logging.info(result) + if result.routing_indices is not None: + choice_data["moe_topk_indices"] = result.routing_indices.tolist() + prompt_length = ( + len(result.prompt_tokens) + if result.prompt_tokens is not None + else 0 + ) + if prompt_length: + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices.append(choice_data) + total_completion_tokens += len(result.generated_tokens) + request_idx += 0 + response = { "choices": choices, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py index b749205cdfd..cb0adc718d3 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py @@ -125,88 +125,98 @@ async def completions(): request_idx = 0 for record in batch_results: - for result in record.requests: - full_text = result.generated_text or "" - text_output = (prompts_as_strings[request_idx] + full_text) if echo else full_text - - logprobs_data = None - if sampling_params.return_log_probs: - # Get prompt tokens and logprobs - prompt_tokens_list = [] - if result.prompt_tokens is not None: - if hasattr(result.prompt_tokens, 'tolist'): - prompt_tokens_list = result.prompt_tokens.tolist() - else: - prompt_tokens_list = list(result.prompt_tokens) - - prompt_log_probs = getattr(result, 'prompt_log_probs', None) or [] - prompt_top_n_logprobs = getattr(result, 'prompt_top_n_logprobs', None) or [] - - # Get generated tokens and logprobs - generated_tokens_list = ( - list(result.generated_tokens) if result.generated_tokens else [] - ) - generated_log_probs = getattr(result, 'generated_log_probs', None) or [] - generated_top_n_logprobs = ( - getattr(result, 'generated_top_n_logprobs', None) or [] - ) - - if echo: - # When echo=True, include prompt tokens and their logprobs - # Prompt logprobs are for tokens [1:] (first token has no logprob) - all_token_ids = prompt_tokens_list + generated_tokens_list - tokens = [tokenizer.detokenize([tok]) for tok in all_token_ids] - - # Build token_logprobs: [None] for first token, then prompt logprobs, - # then generated logprobs - token_logprobs = [None] + list(prompt_log_probs) + list(generated_log_probs) - - # Build top_logprobs: [None] for first token, then prompt top_n, - # then generated top_n - top_logprobs = None - if prompt_top_n_logprobs or generated_top_n_logprobs: - top_logprobs = ( - [None] - + list(prompt_top_n_logprobs) - + list(generated_top_n_logprobs) - ) - - # Calculate text_offset: cumulative character positions starting from 0 - text_offset = [] - current_offset = 0 - for tok_str in tokens: - text_offset.append(current_offset) - current_offset += len(tok_str) + result = record.merge() + full_text = result.generated_text or "" + text_output = (prompts_as_strings[request_idx] + full_text) if echo else full_text + + logprobs_data = None + if sampling_params.return_log_probs: + # Get prompt tokens and logprobs + prompt_tokens_list = [] + if result.prompt_tokens is not None: + if hasattr(result.prompt_tokens, 'tolist'): + prompt_tokens_list = result.prompt_tokens.tolist() else: - # When echo=False, only return generated tokens and their logprobs - tokens = [tokenizer.detokenize([tok]) for tok in generated_tokens_list] - - # Prepend [None] to match OpenAI format - token_logprobs = [None] + list(generated_log_probs) - - # Build top_logprobs - top_logprobs = None - if generated_top_n_logprobs: - top_logprobs = [None] + list(generated_top_n_logprobs) - - # Calculate text_offset for generated tokens only - text_offset = [] - current_offset = 0 - for tok_str in tokens: - text_offset.append(current_offset) - current_offset += len(tok_str) - - logprobs_data = { - "token_logprobs": token_logprobs, - "tokens": tokens, - "text_offset": text_offset, - "top_logprobs": top_logprobs, - } - - choices.append( - {"index": request_idx, "text": text_output, "logprobs": logprobs_data} + prompt_tokens_list = list(result.prompt_tokens) + + prompt_log_probs = getattr(result, 'prompt_log_probs', None) or [] + prompt_top_n_logprobs = getattr(result, 'prompt_top_n_logprobs', None) or [] + + # Get generated tokens and logprobs + generated_tokens_list = ( + list(result.generated_tokens) if result.generated_tokens else [] + ) + generated_log_probs = getattr(result, 'generated_log_probs', None) or [] + generated_top_n_logprobs = ( + getattr(result, 'generated_top_n_logprobs', None) or [] ) - request_idx += 1 + + if echo: + # When echo=True, include prompt tokens and their logprobs + # Prompt logprobs are for tokens [1:] (first token has no logprob) + all_token_ids = prompt_tokens_list + generated_tokens_list + tokens = [tokenizer.detokenize([tok]) for tok in all_token_ids] + + # Build token_logprobs: [None] for first token, then prompt logprobs, + # then generated logprobs + token_logprobs = [None] + list(prompt_log_probs) + list(generated_log_probs) + + # Build top_logprobs: [None] for first token, then prompt top_n, + # then generated top_n + top_logprobs = None + if prompt_top_n_logprobs or generated_top_n_logprobs: + top_logprobs = ( + [None] + + list(prompt_top_n_logprobs) + + list(generated_top_n_logprobs) + ) + + # Calculate text_offset: cumulative character positions starting from 0 + text_offset = [] + current_offset = 0 + for tok_str in tokens: + text_offset.append(current_offset) + current_offset += len(tok_str) + else: + # When echo=False, only return generated tokens and their logprobs + tokens = [tokenizer.detokenize([tok]) for tok in generated_tokens_list] + + # Prepend [None] to match OpenAI format + token_logprobs = [None] + list(generated_log_probs) + + # Build top_logprobs + top_logprobs = None + if generated_top_n_logprobs: + top_logprobs = [None] + list(generated_top_n_logprobs) + + # Calculate text_offset for generated tokens only + text_offset = [] + current_offset = 0 + for tok_str in tokens: + text_offset.append(current_offset) + current_offset += len(tok_str) + + logprobs_data = { + "token_logprobs": token_logprobs, + "tokens": tokens, + "text_offset": text_offset, + "top_logprobs": top_logprobs, + } + + choices.append( + {"index": request_idx, "text": text_output, "logprobs": logprobs_data} + ) + if result.routing_indices is not None: + choices[-1]["moe_topk_indices"] = result.routing_indices.tolist() + prompt_length = ( + len(result.prompt_tokens) + if result.prompt_tokens is not None + else 0 + ) + if prompt_length: + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + + request_idx += 1 return jsonify({"choices": choices}) From 7feee53d11ae6c9ba3c1fff52c3a5764ee4ef4bd Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:53:55 -0800 Subject: [PATCH 35/56] format --- .../inference/contexts/dynamic_context.py | 11 +++--- .../inference/contexts/routing_metadata.py | 34 ++++++++---------- .../core/inference/engines/dynamic_engine.py | 36 +++++-------------- megatron/core/inference/inference_request.py | 12 +++---- .../text_generation_controller.py | 14 +++----- .../endpoints/chat_completions.py | 11 +++--- .../endpoints/completions.py | 22 ++++-------- megatron/core/transformer/moe/moe_utils.py | 2 +- megatron/core/transformer/moe/router.py | 2 +- .../core/transformer/moe/router_replay.py | 7 ++-- 10 files changed, 53 insertions(+), 98 deletions(-) diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 4e3dbb78043..47b0bb50748 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -472,11 +472,10 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC self.moe_enable_routing_replay = model_config.moe_enable_routing_replay if self.moe_enable_routing_replay: - assert model_config.num_moe_experts is not None, "Router recording/replay requested but no MoE experts specified!" - self.moe_routing_metadata = RoutingMetadata( - self, model_config.moe_router_topk - ) - + assert ( + model_config.num_moe_experts is not None + ), "Router recording/replay requested but no MoE experts specified!" + self.moe_routing_metadata = RoutingMetadata(self, model_config.moe_router_topk) # CUDA graph config list self.use_cuda_graphs_for_non_decode_steps = ( @@ -1308,7 +1307,7 @@ def initialize_attention_state( self.moe_routing_metadata.enable_static_buffer_recording() else: self.moe_routing_metadata.disable_static_buffer_recording() - + def reset(self) -> None: """Reset entire context. diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index faa184c24b7..07abe0c3469 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -9,23 +9,20 @@ from megatron.core.transformer.moe.router_replay import RouterReplay + class RoutingMetadata: """Manages routing indices metadata for MoE layers during inference. - + This class provides static buffers for CUDA graph compatibility when recording routing decisions. It holds a reference to the inference context to automatically determine whether to use static buffers based on CUDA graph state. - + Args: context (DynamicInferenceContext): The inference context. moe_router_topk (int): Number of experts selected per token. """ - def __init__( - self, - context: 'DynamicInferenceContext', - moe_router_topk: int, - ): + def __init__(self, context: 'DynamicInferenceContext', moe_router_topk: int): self.context = context self.max_tokens = context.max_tokens self.moe_router_topk = moe_router_topk @@ -38,20 +35,17 @@ def __init__( def _ensure_buffer_allocated(self) -> None: """Allocate the static buffer if not already allocated. - + Gets the actual number of MoE layers from RouterReplay instances. """ if self.routing_indices_buffer is not None: return - - + self.num_moe_layers = len(RouterReplay.global_router_replay_instances) - print(f"[RoutingMetadata] Allocating buffer with num_moe_layers={self.num_moe_layers} " - f"(from {len(RouterReplay.global_router_replay_instances)} RouterReplay instances)") - + if self.num_moe_layers == 0: return - + # Static buffer for CUDA graph compatibility. # Shape: [max_tokens, num_moe_layers, moe_router_topk] self.routing_indices_buffer = torch.empty( @@ -62,10 +56,10 @@ def _ensure_buffer_allocated(self) -> None: def get_routing_indices(self) -> Optional[torch.Tensor]: """Get the recorded routing indices. - + Automatically uses the static buffer when CUDA graphs are active, otherwise retrieves from RouterReplay utility. - + Returns: Tensor of shape [num_tokens, num_moe_layers, topk] or None if not available. """ @@ -73,9 +67,9 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: # Return view of static buffer up to current token count. if self.routing_indices_buffer is None: return None - # Only return up to active token count, to skip entries - # for padding tokens. - return self.routing_indices_buffer[:self.context.active_token_count] + # Only return up to active token count, to skip entries + # for padding tokens. + return self.routing_indices_buffer[: self.context.active_token_count] else: # Get from RouterReplay and stack into [num_tokens, num_layers, topk]. recorded_data = RouterReplay.get_recorded_data() @@ -88,7 +82,7 @@ def get_routing_indices(self) -> Optional[torch.Tensor]: def enable_static_buffer_recording(self) -> None: """Enable recording into the static buffer for CUDA graph compatibility. - + This sets up RouterReplay instances to copy routing indices into our pre-allocated static buffer instead of creating new tensors. Allocates the buffer lazily on first call. diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index 9faff001071..729588506d4 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -38,13 +38,10 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) -from megatron.core.transformer.moe.router_replay import ( - RouterReplay, - RouterReplayAction, - ) from megatron.core.inference.utils import Counter, await_process_call from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.cuda_graphs import delete_cuda_graphs +from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.utils import ( deprecate_args, experimental_api, @@ -56,8 +53,6 @@ trace_async_exceptions, ) - - from .async_zmq_communicator import AsyncZMQCommunicator try: @@ -998,8 +993,13 @@ def post_process_requests( # Process routing indices if available (keyed by request_id) # Each step's routing is a tensor of shape [num_tokens_this_step, num_layers, topk] # We concatenate along dim=0 to accumulate: [total_tokens, num_layers, topk] - if routing_indices_per_request is not None and request_id in routing_indices_per_request: - step_routing = routing_indices_per_request[request_id] # [num_tokens, num_layers, topk] + if ( + routing_indices_per_request is not None + and request_id in routing_indices_per_request + ): + step_routing = routing_indices_per_request[ + request_id + ] # [num_tokens, num_layers, topk] if request.routing_indices is None: request.routing_indices = step_routing.clone() else: @@ -1007,26 +1007,6 @@ def post_process_requests( [request.routing_indices, step_routing], dim=0 ) - # Sanity check logging for routing indices (only log for first request, first few steps) - if req_idx == 0 and request.routing_indices.shape[0] <= 15: # first 15 tokens - rank = torch.distributed.get_rank() - logging.info( - f"[ROUTING DEBUG] rank={rank}, request_id={request_id}, " - f"total_tokens={request.routing_indices.shape[0]}, " - f"num_layers={request.routing_indices.shape[1]}, " - f"topk={request.routing_indices.shape[2]}, " - f"tokens_this_step={step_routing.shape[0]}" - ) - # Log first layer's routing for first few tokens - if step_routing.numel() > 0: - num_tokens_to_log = min(3, step_routing.shape[0]) - # step_routing is [num_tokens, num_layers, topk], get layer 0 - logging.info( - f"[ROUTING DEBUG] rank={rank}, Layer 0 routing (first {num_tokens_to_log} tokens): " - f"{step_routing[:num_tokens_to_log, 0, :].tolist()}" - ) - - # Handle evicted requests. if evict_request_ids is not None and evict_request_ids.numel() > 0: diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index d67e8570033..d8179614a3c 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -312,17 +312,17 @@ def serialize(self) -> dict: torch.cuda.nvtx.range_push("DynamicInferenceRequest.serialize") obj = super().serialize() obj["events"] = [e.serialize() for e in self.events] - - # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] + + # Sanity check routing_indices: Tensor [total_tokens - 1, num_layers, topk] if self.routing_indices is not None: total_tokens = len(self.prompt_tokens) + len(self.generated_tokens) - # the last generated token does not undergo a forward pass + # the last generated token does not undergo a forward pass # hence we expect routing indices for total_tokens - 1 - assert self.routing_indices.shape[0] == total_tokens-1, ( + assert self.routing_indices.shape[0] == total_tokens - 1, ( f"routing_indices first dimension {self.routing_indices.shape[0]} does not match " f"total tokens {total_tokens-1}." ) - + torch.cuda.nvtx.range_pop() return obj @@ -491,7 +491,7 @@ def checkpoint(self, tokenizer: MegatronTokenizer | None = None): new_request = DynamicInferenceRequest( request_id=old_request.request_id, prompt_tokens=new_prompt_tokens, - sampling_params=new_sampling_params + sampling_params=new_sampling_params, ) self.requests.append(new_request) diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 8e70055a88f..f56e5b1c761 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -27,14 +27,12 @@ from megatron.core.inference.sampling_params import SamplingParams from megatron.core.inference.utils import get_attention_mask, set_decode_expert_padding from megatron.core.models.multimodal.llava_model import LLaVAModel -from megatron.core.transformer.enums import CudaGraphScope from megatron.core.tensor_parallel.mappings import gather_from_sequence_parallel_region +from megatron.core.transformer.enums import CudaGraphScope from megatron.core.transformer.moe.moe_layer import BaseMoELayer from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.transformer.utils import set_model_to_sequence_parallel -from megatron.core import parallel_state -from megatron.core.utils import get_asyncio_loop, get_model_config, unwrap_model -from megatron.core.utils import get_pg_size +from megatron.core.utils import get_asyncio_loop, get_model_config, get_pg_size, unwrap_model try: import transformer_engine as te # pylint: disable=unused-import @@ -700,7 +698,7 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: context = self.inference_wrapped_model.inference_context if context.moe_routing_metadata is None: return None - + stacked_routing = context.moe_routing_metadata.get_routing_indices() if stacked_routing is None: @@ -712,7 +710,6 @@ def _router_record_bookkeeping(self) -> Optional[Dict[int, Tensor]]: active_query_lengths = context.request_query_lengths[active_request_slice].tolist() active_token_count = context.active_token_count - # Get TP group for all-gather if using sequence parallelism # With sequence parallelism, each TP rank only sees a portion of the tokens, # so we need to gather routing indices across all TP ranks. @@ -961,10 +958,9 @@ async def async_generate_output_tokens_dynamic_batch( logits = self._dynamic_step_forward_logits(input_ids, position_ids) - # Collect routing indices per request (must be done before context transitions) + # Collect routing indices per request (must be done before context transitions) routing_indices_per_request = self._router_record_bookkeeping() - # This is the best place to yield control back to event loop. # At this point we have enqueued FW pass GPU kernels asynchronously. # While they are running, we can do other useful CPU work. @@ -992,8 +988,6 @@ async def async_generate_output_tokens_dynamic_batch( else: request_bookkeeping = self._dynamic_step_context_bookkeeping() - - ret = { "sample": self._sampled_tokens_cuda[:active_request_count], "log_probs": log_probs, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index e9d16e08ca0..34c3b954074 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -144,17 +144,14 @@ async def chat_completions(): logging.info(result) if result.routing_indices is not None: choice_data["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = ( - len(result.prompt_tokens) - if result.prompt_tokens is not None - else 0 - ) + prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 if prompt_length: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ + :prompt_length + ].tolist() choices.append(choice_data) total_completion_tokens += len(result.generated_tokens) request_idx += 0 - response = { "choices": choices, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py index cb0adc718d3..072848e3507 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/completions.py @@ -147,9 +147,7 @@ async def completions(): list(result.generated_tokens) if result.generated_tokens else [] ) generated_log_probs = getattr(result, 'generated_log_probs', None) or [] - generated_top_n_logprobs = ( - getattr(result, 'generated_top_n_logprobs', None) or [] - ) + generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) or [] if echo: # When echo=True, include prompt tokens and their logprobs @@ -166,9 +164,7 @@ async def completions(): top_logprobs = None if prompt_top_n_logprobs or generated_top_n_logprobs: top_logprobs = ( - [None] - + list(prompt_top_n_logprobs) - + list(generated_top_n_logprobs) + [None] + list(prompt_top_n_logprobs) + list(generated_top_n_logprobs) ) # Calculate text_offset: cumulative character positions starting from 0 @@ -203,18 +199,14 @@ async def completions(): "top_logprobs": top_logprobs, } - choices.append( - {"index": request_idx, "text": text_output, "logprobs": logprobs_data} - ) + choices.append({"index": request_idx, "text": text_output, "logprobs": logprobs_data}) if result.routing_indices is not None: choices[-1]["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = ( - len(result.prompt_tokens) - if result.prompt_tokens is not None - else 0 - ) + prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 if prompt_length: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[:prompt_length].tolist() + choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ + :prompt_length + ].tolist() request_idx += 1 diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index 666d6e16c38..47debdd27df 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1573,4 +1573,4 @@ def wrapped_func(moe_layer, *args, **kwargs): return wrapped_func - return decorator \ No newline at end of file + return decorator diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 9d1b557d2e4..ca9372670f5 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -215,7 +215,7 @@ def __init__( if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() - + def _maintain_float32_expert_bias(self): """ Maintain the expert bias in float32. diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index 75fbb9009a3..e3c5d40fa3c 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -77,7 +77,7 @@ def clear_global_router_replay_instances(): @staticmethod def set_global_static_buffers(static_buffer: torch.Tensor): """Sets static buffers for all router instances from a combined buffer. - + Args: static_buffer: Tensor of shape [max_tokens, num_layers, topk]. Each layer's RouterReplay gets a slice [:, layer_idx, :]. @@ -184,10 +184,9 @@ def get_replay_topk( else: return default_compute_topk(scores, topk, num_groups, group_topk) - def set_static_buffer(self, buffer: torch.Tensor): """Sets a static buffer for CUDA graph compatible recording. - + Args: buffer: Tensor of shape [max_tokens, topk] to copy routing indices into. """ @@ -199,7 +198,7 @@ def clear_static_buffer(self): def record_indices(self, topk_indices: torch.Tensor): """Records the topk indices. - + If a static buffer is set (for CUDA graph compatibility), copies into it. Otherwise, just stores the tensor reference. """ From 473a65238a9e05d838004a56f849362daf4baad4 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 9 Feb 2026 16:55:21 -0800 Subject: [PATCH 36/56] minor --- .../inference/gpt/gpt_dynamic_inference_with_coordinator.py | 2 +- megatron/core/transformer/moe/router.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index 771b3eca519..5c58b4d52e3 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -125,7 +125,7 @@ async def main( await asyncio.sleep(0) # While we wait for the requests to complete, the engine runs in the background. - results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) + results: List[DynamicInferenceRequestRecord] = await asyncio.gather(*futures) if dist.get_rank() == 0: # Write results to JSON. Primarily used for functional testing. diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index ca9372670f5..e42fd1ca8aa 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -212,7 +212,6 @@ def __init__( self.ga_steps = None self.router_replay = None - if self.config.moe_enable_routing_replay: self.router_replay = RouterReplay() From 452acb20c2316546de2ad2cfe2363100ef48b75c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 07:06:27 -0800 Subject: [PATCH 37/56] remove unnecessary methods --- megatron/core/inference/contexts/routing_metadata.py | 4 ---- megatron/core/transformer/moe/router_replay.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/megatron/core/inference/contexts/routing_metadata.py b/megatron/core/inference/contexts/routing_metadata.py index 07abe0c3469..6664681a7ce 100644 --- a/megatron/core/inference/contexts/routing_metadata.py +++ b/megatron/core/inference/contexts/routing_metadata.py @@ -94,7 +94,3 @@ def enable_static_buffer_recording(self) -> None: def disable_static_buffer_recording(self) -> None: """Disable static buffer recording, reverting to normal tensor assignment.""" RouterReplay.clear_global_static_buffers() - - def reset(self) -> None: - """Reset the routing metadata state.""" - pass diff --git a/megatron/core/transformer/moe/router_replay.py b/megatron/core/transformer/moe/router_replay.py index e3c5d40fa3c..5430f75568f 100644 --- a/megatron/core/transformer/moe/router_replay.py +++ b/megatron/core/transformer/moe/router_replay.py @@ -119,10 +119,6 @@ def get_recorded_indices(self) -> Optional[torch.Tensor]: """Returns the recorded topk indices.""" return self.recorded_topk_idx - def record_indices(self, topk_indices: torch.Tensor): - """Records the topk indices.""" - self.recorded_topk_idx = topk_indices - def clear_indices(self): """Clears the recorded and target topk indices.""" self.recorded_topk_idx = None From 9b8843a1be8a429fc52efe8dc0f7aa724baccbc3 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Tue, 10 Feb 2026 11:31:11 -0600 Subject: [PATCH 38/56] Add dependencies --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e103a3bccc8..a437aab4ca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,8 @@ dev = [ "fastapi~=0.50", # Forcing a little bit more recent version of fastapi to be compatible with pydantic 2.0 "datasets", "emerging_optimizers", + "flask[async]", + "hypercorn", ] lts = [ From 96dc390b61899f51ce6f46cccca50d7ba04eae3d Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Tue, 10 Feb 2026 14:29:30 -0600 Subject: [PATCH 39/56] Update uv.lock --- uv.lock | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/uv.lock b/uv.lock index 4b51612c3a3..d4c957ba5e1 100644 --- a/uv.lock +++ b/uv.lock @@ -291,6 +291,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "asgiref" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.330676Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039625Z" }, +] + [[package]] name = "apache-tvm-ffi" version = "0.1.8.post2" @@ -1437,6 +1449,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/f9/7f9263c5695f4bd0023734af91bedb2ff8209e8de6ead162f35d8dc762fd/flask-3.1.2-py3-none-any.whl", hash = "sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c", size = 103308, upload-time = "2025-08-19T21:03:19.499Z" }, ] +[package.optional-dependencies] +async = [ + { name = "asgiref" }, +] + [[package]] name = "flask-restful" version = "0.3.10" @@ -1819,6 +1836,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypercorn" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "h11" }, + { name = "h2" }, + { name = "priority" }, + { name = "taskgroup", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/01/39f41a014b83dd5c795217362f2ca9071cf243e6a75bdcd6cd5b944658cc/hypercorn-0.18.0.tar.gz", hash = "sha256:d63267548939c46b0247dc8e5b45a9947590e35e64ee73a23c074aa3cf88e9da", size = 68420, upload-time = "2025-11-08T13:54:04.780563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/35/850277d1b17b206bd10874c8a9a3f52e059452fb49bb0d22cbb908f6038b/hypercorn-0.18.0-py3-none-any.whl", hash = "sha256:225e268f2c1c2f28f6d8f6db8f40cb8c992963610c5725e13ccfcddccb24b1cd", size = 61640, upload-time = "2025-11-08T13:54:03.202784Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -2255,6 +2291,8 @@ dev = [ { name = "fastapi" }, { name = "flash-linear-attention" }, { name = "flashinfer-python" }, + { name = "flask", extra = ["async"] }, + { name = "hypercorn" }, { name = "mamba-ssm" }, { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" }, { name = "multi-storage-client" }, @@ -2369,7 +2407,9 @@ requires-dist = [ { name = "flash-linear-attention", marker = "extra == 'dev'", specifier = "~=0.4.0" }, { name = "flashinfer-python", marker = "extra == 'dev'", specifier = "~=0.5.0" }, { name = "flashinfer-python", marker = "extra == 'lts'", specifier = "~=0.5.0" }, + { name = "flask", extras = ["async"], marker = "extra == 'dev'" }, { name = "flask-restful", marker = "extra == 'mlm'" }, + { name = "hypercorn", marker = "extra == 'dev'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, @@ -3612,6 +3652,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/8c/83087ebc47ab0396ce092363001fa37c17153119ee282700c0713a195853/prettytable-3.17.0-py3-none-any.whl", hash = "sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287", size = 34433, upload-time = "2025-11-14T17:33:19.093Z" }, ] +[[package]] +name = "priority" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/3c/eb7c35f4dcede96fca1842dac5f4f5d15511aa4b52f3a961219e68ae9204/priority-2.0.0.tar.gz", hash = "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0", size = 24792, upload-time = "2021-06-27T10:15:05.487867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/5f/82c8074f7e84978129347c2c6ec8b6c59f3584ff1a20bc3c940a3e061790/priority-2.0.0-py3-none-any.whl", hash = "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", size = 8946, upload-time = "2021-06-27T10:15:03.856590Z" }, +] + [[package]] name = "prometheus-client" version = "0.24.0" @@ -5347,6 +5396,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" }, ] +[[package]] +name = "taskgroup" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/8d/e218e0160cc1b692e6e0e5ba34e8865dbb171efeb5fc9a704544b3020605/taskgroup-0.2.2.tar.gz", hash = "sha256:078483ac3e78f2e3f973e2edbf6941374fbea81b9c5d0a96f51d297717f4752d", size = 11504, upload-time = "2025-01-03T09:24:13.761490Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/b1/74babcc824a57904e919f3af16d86c08b524c0691504baf038ef2d7f655c/taskgroup-0.2.2-py2.py3-none-any.whl", hash = "sha256:e2c53121609f4ae97303e9ea1524304b4de6faf9eb2c9280c7f87976479a52fb", size = 14237, upload-time = "2025-01-03T09:24:11.410239Z" }, +] + [[package]] name = "tensorboard" version = "2.20.0" @@ -6237,6 +6299,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454632Z" }, +] + [[package]] name = "xattr" version = "1.3.0" From bf27426eea2ae0781a90c78a4d25847a16516499 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:14:18 -0800 Subject: [PATCH 40/56] miinor bugfix --- megatron/core/inference/contexts/dynamic_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index 47b0bb50748..e9b324cc0b3 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -1302,7 +1302,7 @@ def initialize_attention_state( padded_batch_dimensions=self.padded_batch_dimensions, ) - if self.moe_enable_routing_replay is not None: + if self.moe_enable_routing_replay: if self.using_cuda_graph_this_step(): self.moe_routing_metadata.enable_static_buffer_recording() else: From 26002fc161c4e78925c290e40881740298c81a43 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:14:48 -0800 Subject: [PATCH 41/56] attempt to reactivate functional test --- .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index 513aa92834b..b9d78097bbd 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -63,6 +63,6 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [flaky] + scope: [mr] platforms: [dgx_h100] From db37b3b27f4f73b0c3cb8af44a504e0788079a26 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:35:09 -0800 Subject: [PATCH 42/56] update functional test to run router recording --- .../gpt_dynamic_inference_with_coordinator.py | 1 + .../test_inference_regular_pipeline.py | 8 + .../golden_values_dev_dgx_h100.json | 25510 +++++++++++++++- 3 files changed, 25518 insertions(+), 1 deletion(-) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py index 5c58b4d52e3..536f533eccd 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py @@ -147,6 +147,7 @@ async def main( throughputs.append(throughput) if req.routing_indices is not None: result_dict["routing_indices"] = req.routing_indices.tolist() + json_results[req.request_id] = result_dict throughput_dict = {"throughput": throughputs} if args.throughput_check_only: diff --git a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py index 346b464b79d..a0d7216edf8 100644 --- a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py @@ -156,5 +156,13 @@ def test_inference_pipeline(golden_values_path: str, test_values_path: str) -> N f"\nCurrent (truncated to {min_len} chars): {generated_text_current[:min_len]}" ) + if "routing_indices" in groundtruth_results: + at_least_one_test_loop = True + routing_indices_groundtruth = groundtruth_results["routing_indices"] + routing_indices_current = current_results["routing_indices"] + assert ( + routing_indices_groundtruth == routing_indices_current + ), f"Routing indices mismatch:\nGround truth: {routing_indices_groundtruth}\nCurrent: {routing_indices_current}" + if not at_least_one_test_loop: raise AssertionError(f"No test performed for output {groundtruth_results}") diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json index b239ac96c3d..ab970730d9b 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/golden_values_dev_dgx_h100.json @@ -34,7 +34,7 @@ 1394, 1636 ], - "latency": 1.974948332994245, + "latency": 1.972374301403761, "logprobs": [ -10.737512588500977, -3.724862575531006, @@ -153,6 +153,25514 @@ -1.2551532983779907, -1.256169080734253, -0.49199968576431274 + ], + "routing_indices": [ + [ + [ + 33, + 50, + 36, + 4, + 25, + 63 + ], + [ + 0, + 16, + 3, + 26, + 9, + 54 + ], + [ + 62, + 60, + 8, + 16, + 58, + 52 + ], + [ + 58, + 10, + 6, + 45, + 16, + 32 + ], + [ + 43, + 49, + 18, + 54, + 55, + 13 + ], + [ + 27, + 19, + 26, + 44, + 12, + 28 + ], + [ + 53, + 42, + 3, + 27, + 26, + 19 + ], + [ + 6, + 47, + 1, + 8, + 19, + 22 + ], + [ + 51, + 27, + 1, + 38, + 16, + 62 + ], + [ + 41, + 49, + 21, + 57, + 16, + 24 + ], + [ + 13, + 28, + 38, + 22, + 49, + 48 + ], + [ + 52, + 58, + 6, + 25, + 29, + 17 + ], + [ + 49, + 50, + 25, + 41, + 54, + 58 + ], + [ + 27, + 41, + 3, + 1, + 26, + 29 + ], + [ + 41, + 18, + 34, + 45, + 1, + 33 + ], + [ + 17, + 26, + 59, + 22, + 19, + 4 + ], + [ + 1, + 43, + 62, + 57, + 61, + 21 + ], + [ + 11, + 46, + 15, + 28, + 61, + 50 + ], + [ + 14, + 30, + 58, + 26, + 38, + 53 + ], + [ + 59, + 20, + 63, + 54, + 47, + 61 + ], + [ + 9, + 5, + 43, + 33, + 15, + 46 + ], + [ + 16, + 52, + 33, + 61, + 49, + 11 + ], + [ + 52, + 35, + 40, + 43, + 29, + 36 + ], + [ + 57, + 34, + 38, + 44, + 20, + 18 + ], + [ + 44, + 51, + 2, + 63, + 7, + 22 + ], + [ + 32, + 47, + 58, + 9, + 54, + 5 + ], + [ + 12, + 59, + 54, + 33, + 50, + 6 + ] + ], + [ + [ + 49, + 43, + 18, + 28, + 23, + 25 + ], + [ + 34, + 24, + 23, + 60, + 2, + 18 + ], + [ + 33, + 40, + 30, + 3, + 59, + 48 + ], + [ + 16, + 58, + 17, + 48, + 6, + 45 + ], + [ + 23, + 58, + 46, + 37, + 34, + 48 + ], + [ + 28, + 26, + 35, + 33, + 22, + 43 + ], + [ + 6, + 31, + 13, + 46, + 41, + 37 + ], + [ + 60, + 29, + 44, + 36, + 39, + 15 + ], + [ + 51, + 18, + 62, + 5, + 27, + 35 + ], + [ + 62, + 45, + 32, + 56, + 25, + 3 + ], + [ + 6, + 20, + 3, + 16, + 49, + 41 + ], + [ + 36, + 41, + 50, + 45, + 35, + 48 + ], + [ + 39, + 46, + 48, + 25, + 21, + 33 + ], + [ + 30, + 38, + 11, + 22, + 15, + 9 + ], + [ + 24, + 2, + 32, + 56, + 63, + 3 + ], + [ + 36, + 55, + 35, + 17, + 32, + 44 + ], + [ + 59, + 46, + 32, + 44, + 24, + 14 + ], + [ + 51, + 15, + 61, + 43, + 30, + 22 + ], + [ + 32, + 2, + 5, + 39, + 11, + 50 + ], + [ + 28, + 42, + 6, + 30, + 57, + 37 + ], + [ + 28, + 55, + 45, + 0, + 7, + 5 + ], + [ + 48, + 40, + 34, + 3, + 49, + 22 + ], + [ + 25, + 6, + 62, + 50, + 18, + 53 + ], + [ + 13, + 22, + 20, + 28, + 25, + 59 + ], + [ + 58, + 39, + 36, + 47, + 29, + 37 + ], + [ + 4, + 33, + 41, + 12, + 3, + 17 + ], + [ + 13, + 22, + 20, + 52, + 24, + 62 + ] + ], + [ + [ + 17, + 10, + 57, + 54, + 6, + 15 + ], + [ + 33, + 43, + 13, + 1, + 16, + 62 + ], + [ + 63, + 1, + 35, + 43, + 27, + 10 + ], + [ + 47, + 4, + 38, + 50, + 51, + 0 + ], + [ + 11, + 51, + 57, + 23, + 14, + 34 + ], + [ + 10, + 43, + 35, + 33, + 20, + 22 + ], + [ + 36, + 48, + 35, + 19, + 21, + 28 + ], + [ + 14, + 8, + 7, + 46, + 35, + 13 + ], + [ + 18, + 44, + 63, + 6, + 4, + 37 + ], + [ + 62, + 29, + 15, + 38, + 39, + 34 + ], + [ + 1, + 6, + 16, + 46, + 22, + 13 + ], + [ + 36, + 8, + 16, + 37, + 10, + 14 + ], + [ + 8, + 0, + 32, + 3, + 43, + 10 + ], + [ + 25, + 0, + 22, + 30, + 60, + 57 + ], + [ + 60, + 58, + 55, + 32, + 2, + 7 + ], + [ + 55, + 11, + 19, + 5, + 24, + 43 + ], + [ + 9, + 47, + 36, + 39, + 5, + 42 + ], + [ + 31, + 20, + 9, + 43, + 18, + 41 + ], + [ + 12, + 5, + 32, + 50, + 3, + 31 + ], + [ + 40, + 18, + 30, + 63, + 7, + 25 + ], + [ + 14, + 7, + 3, + 38, + 59, + 54 + ], + [ + 31, + 20, + 1, + 22, + 47, + 6 + ], + [ + 51, + 15, + 18, + 53, + 40, + 52 + ], + [ + 27, + 54, + 8, + 38, + 3, + 59 + ], + [ + 11, + 16, + 39, + 59, + 9, + 23 + ], + [ + 7, + 37, + 51, + 30, + 18, + 48 + ], + [ + 0, + 36, + 33, + 40, + 46, + 48 + ] + ], + [ + [ + 48, + 33, + 52, + 53, + 24, + 38 + ], + [ + 3, + 40, + 45, + 19, + 22, + 35 + ], + [ + 46, + 15, + 39, + 60, + 25, + 31 + ], + [ + 49, + 62, + 61, + 48, + 55, + 46 + ], + [ + 41, + 35, + 53, + 52, + 13, + 10 + ], + [ + 52, + 15, + 45, + 63, + 46, + 10 + ], + [ + 14, + 15, + 35, + 60, + 49, + 31 + ], + [ + 44, + 3, + 6, + 8, + 23, + 48 + ], + [ + 32, + 15, + 44, + 27, + 4, + 41 + ], + [ + 51, + 25, + 62, + 53, + 48, + 10 + ], + [ + 8, + 50, + 5, + 19, + 16, + 22 + ], + [ + 36, + 49, + 60, + 44, + 15, + 33 + ], + [ + 13, + 27, + 53, + 30, + 56, + 43 + ], + [ + 26, + 22, + 14, + 7, + 32, + 17 + ], + [ + 26, + 60, + 2, + 58, + 54, + 10 + ], + [ + 50, + 55, + 17, + 51, + 47, + 14 + ], + [ + 0, + 12, + 16, + 4, + 23, + 9 + ], + [ + 10, + 23, + 27, + 46, + 56, + 55 + ], + [ + 3, + 37, + 4, + 60, + 16, + 59 + ], + [ + 38, + 3, + 29, + 40, + 25, + 50 + ], + [ + 31, + 16, + 62, + 54, + 42, + 5 + ], + [ + 47, + 35, + 11, + 37, + 46, + 32 + ], + [ + 51, + 27, + 20, + 50, + 16, + 55 + ], + [ + 63, + 55, + 46, + 27, + 48, + 12 + ], + [ + 53, + 50, + 30, + 2, + 39, + 20 + ], + [ + 53, + 44, + 24, + 8, + 51, + 14 + ], + [ + 19, + 49, + 37, + 14, + 44, + 36 + ] + ], + [ + [ + 0, + 52, + 16, + 12, + 54, + 7 + ], + [ + 42, + 25, + 51, + 61, + 35, + 58 + ], + [ + 51, + 42, + 19, + 57, + 28, + 8 + ], + [ + 49, + 62, + 5, + 2, + 46, + 21 + ], + [ + 3, + 41, + 53, + 25, + 39, + 37 + ], + [ + 45, + 15, + 37, + 48, + 19, + 60 + ], + [ + 14, + 15, + 47, + 17, + 35, + 24 + ], + [ + 3, + 52, + 63, + 16, + 28, + 47 + ], + [ + 32, + 27, + 15, + 24, + 62, + 23 + ], + [ + 62, + 25, + 51, + 53, + 0, + 20 + ], + [ + 8, + 19, + 50, + 16, + 32, + 22 + ], + [ + 36, + 13, + 42, + 49, + 60, + 44 + ], + [ + 30, + 27, + 53, + 56, + 0, + 10 + ], + [ + 22, + 7, + 14, + 26, + 32, + 17 + ], + [ + 60, + 7, + 26, + 2, + 58, + 13 + ], + [ + 55, + 51, + 17, + 60, + 62, + 47 + ], + [ + 53, + 16, + 18, + 4, + 50, + 5 + ], + [ + 28, + 39, + 41, + 1, + 55, + 18 + ], + [ + 59, + 49, + 16, + 23, + 42, + 11 + ], + [ + 17, + 30, + 46, + 55, + 25, + 0 + ], + [ + 54, + 9, + 45, + 0, + 6, + 19 + ], + [ + 51, + 9, + 22, + 23, + 16, + 25 + ], + [ + 62, + 50, + 43, + 51, + 55, + 27 + ], + [ + 13, + 43, + 27, + 1, + 14, + 52 + ], + [ + 61, + 26, + 1, + 17, + 32, + 63 + ], + [ + 37, + 46, + 63, + 20, + 24, + 4 + ], + [ + 63, + 11, + 12, + 61, + 31, + 22 + ] + ], + [ + [ + 49, + 54, + 56, + 11, + 38, + 3 + ], + [ + 53, + 46, + 49, + 38, + 57, + 17 + ], + [ + 9, + 48, + 31, + 12, + 56, + 6 + ], + [ + 49, + 62, + 23, + 5, + 12, + 63 + ], + [ + 36, + 26, + 38, + 7, + 20, + 23 + ], + [ + 37, + 33, + 41, + 58, + 57, + 32 + ], + [ + 10, + 14, + 15, + 31, + 8, + 43 + ], + [ + 16, + 52, + 3, + 2, + 34, + 14 + ], + [ + 15, + 32, + 35, + 27, + 62, + 54 + ], + [ + 51, + 62, + 25, + 53, + 10, + 20 + ], + [ + 19, + 8, + 22, + 50, + 1, + 5 + ], + [ + 13, + 49, + 36, + 60, + 42, + 20 + ], + [ + 27, + 30, + 53, + 54, + 26, + 43 + ], + [ + 14, + 32, + 26, + 22, + 7, + 17 + ], + [ + 26, + 60, + 7, + 2, + 52, + 54 + ], + [ + 55, + 17, + 51, + 47, + 62, + 60 + ], + [ + 16, + 18, + 44, + 4, + 53, + 50 + ], + [ + 1, + 47, + 39, + 45, + 28, + 56 + ], + [ + 23, + 16, + 55, + 49, + 8, + 32 + ], + [ + 17, + 55, + 30, + 62, + 31, + 23 + ], + [ + 9, + 54, + 0, + 44, + 14, + 56 + ], + [ + 23, + 9, + 51, + 22, + 31, + 50 + ], + [ + 50, + 27, + 14, + 51, + 8, + 34 + ], + [ + 12, + 13, + 33, + 1, + 5, + 43 + ], + [ + 26, + 32, + 1, + 50, + 37, + 57 + ], + [ + 37, + 47, + 63, + 46, + 4, + 5 + ], + [ + 63, + 11, + 12, + 19, + 33, + 61 + ] + ], + [ + [ + 49, + 54, + 40, + 56, + 3, + 11 + ], + [ + 37, + 15, + 12, + 33, + 59, + 17 + ], + [ + 38, + 49, + 14, + 46, + 35, + 59 + ], + [ + 25, + 20, + 39, + 62, + 49, + 12 + ], + [ + 26, + 51, + 16, + 36, + 18, + 8 + ], + [ + 37, + 51, + 41, + 33, + 32, + 60 + ], + [ + 10, + 14, + 59, + 8, + 15, + 40 + ], + [ + 16, + 52, + 19, + 61, + 32, + 2 + ], + [ + 32, + 15, + 27, + 24, + 62, + 35 + ], + [ + 51, + 25, + 62, + 0, + 49, + 60 + ], + [ + 8, + 50, + 19, + 48, + 16, + 54 + ], + [ + 13, + 49, + 36, + 42, + 11, + 60 + ], + [ + 53, + 27, + 7, + 26, + 30, + 21 + ], + [ + 14, + 22, + 26, + 37, + 32, + 4 + ], + [ + 26, + 40, + 60, + 2, + 52, + 7 + ], + [ + 55, + 51, + 17, + 46, + 13, + 62 + ], + [ + 38, + 16, + 53, + 44, + 4, + 18 + ], + [ + 39, + 1, + 4, + 14, + 56, + 57 + ], + [ + 55, + 23, + 32, + 14, + 13, + 16 + ], + [ + 55, + 17, + 2, + 30, + 62, + 12 + ], + [ + 13, + 54, + 0, + 62, + 61, + 25 + ], + [ + 9, + 51, + 5, + 22, + 19, + 16 + ], + [ + 50, + 51, + 19, + 7, + 48, + 53 + ], + [ + 12, + 1, + 5, + 43, + 61, + 13 + ], + [ + 26, + 32, + 30, + 37, + 34, + 20 + ], + [ + 37, + 63, + 46, + 4, + 47, + 8 + ], + [ + 63, + 12, + 11, + 31, + 33, + 61 + ] + ], + [ + [ + 47, + 34, + 30, + 25, + 31, + 3 + ], + [ + 15, + 24, + 46, + 21, + 8, + 6 + ], + [ + 34, + 21, + 18, + 62, + 28, + 55 + ], + [ + 35, + 32, + 20, + 39, + 59, + 54 + ], + [ + 26, + 27, + 15, + 48, + 60, + 47 + ], + [ + 37, + 8, + 50, + 18, + 54, + 61 + ], + [ + 35, + 31, + 8, + 24, + 14, + 15 + ], + [ + 16, + 52, + 34, + 29, + 48, + 36 + ], + [ + 11, + 32, + 62, + 27, + 46, + 26 + ], + [ + 61, + 62, + 25, + 56, + 46, + 53 + ], + [ + 56, + 50, + 63, + 3, + 45, + 28 + ], + [ + 11, + 36, + 5, + 60, + 35, + 50 + ], + [ + 21, + 26, + 41, + 51, + 46, + 53 + ], + [ + 14, + 22, + 33, + 19, + 41, + 16 + ], + [ + 2, + 52, + 34, + 60, + 21, + 49 + ], + [ + 59, + 55, + 29, + 8, + 61, + 22 + ], + [ + 51, + 44, + 2, + 59, + 47, + 53 + ], + [ + 39, + 25, + 18, + 12, + 51, + 56 + ], + [ + 34, + 53, + 32, + 12, + 9, + 38 + ], + [ + 30, + 53, + 56, + 7, + 40, + 62 + ], + [ + 40, + 49, + 14, + 28, + 23, + 55 + ], + [ + 15, + 48, + 40, + 47, + 9, + 1 + ], + [ + 50, + 41, + 53, + 25, + 18, + 0 + ], + [ + 22, + 1, + 59, + 3, + 55, + 10 + ], + [ + 1, + 32, + 53, + 26, + 47, + 3 + ], + [ + 4, + 33, + 28, + 37, + 55, + 54 + ], + [ + 30, + 22, + 57, + 12, + 33, + 63 + ] + ], + [ + [ + 16, + 11, + 0, + 46, + 31, + 21 + ], + [ + 49, + 13, + 31, + 5, + 11, + 14 + ], + [ + 36, + 13, + 56, + 27, + 46, + 3 + ], + [ + 24, + 44, + 62, + 29, + 15, + 13 + ], + [ + 17, + 2, + 50, + 8, + 45, + 1 + ], + [ + 8, + 7, + 49, + 0, + 62, + 13 + ], + [ + 35, + 61, + 58, + 23, + 36, + 0 + ], + [ + 16, + 48, + 42, + 4, + 32, + 1 + ], + [ + 63, + 18, + 32, + 4, + 34, + 62 + ], + [ + 57, + 62, + 54, + 27, + 25, + 53 + ], + [ + 1, + 59, + 60, + 29, + 22, + 14 + ], + [ + 31, + 36, + 11, + 14, + 20, + 47 + ], + [ + 34, + 2, + 19, + 14, + 8, + 37 + ], + [ + 57, + 22, + 40, + 14, + 62, + 48 + ], + [ + 44, + 60, + 7, + 14, + 45, + 2 + ], + [ + 8, + 6, + 55, + 25, + 50, + 59 + ], + [ + 13, + 47, + 23, + 61, + 42, + 39 + ], + [ + 23, + 25, + 4, + 14, + 46, + 53 + ], + [ + 8, + 45, + 32, + 53, + 10, + 54 + ], + [ + 15, + 38, + 53, + 55, + 7, + 30 + ], + [ + 41, + 14, + 28, + 27, + 5, + 58 + ], + [ + 11, + 41, + 57, + 10, + 1, + 47 + ], + [ + 50, + 0, + 53, + 51, + 34, + 55 + ], + [ + 1, + 14, + 55, + 25, + 8, + 3 + ], + [ + 11, + 49, + 1, + 9, + 0, + 3 + ], + [ + 50, + 51, + 6, + 42, + 4, + 54 + ], + [ + 17, + 37, + 31, + 40, + 5, + 36 + ] + ], + [ + [ + 22, + 53, + 47, + 6, + 57, + 21 + ], + [ + 27, + 11, + 14, + 6, + 57, + 16 + ], + [ + 1, + 11, + 29, + 26, + 41, + 17 + ], + [ + 14, + 22, + 38, + 31, + 29, + 36 + ], + [ + 14, + 59, + 29, + 61, + 45, + 52 + ], + [ + 30, + 8, + 0, + 21, + 47, + 58 + ], + [ + 58, + 35, + 4, + 61, + 23, + 36 + ], + [ + 42, + 20, + 48, + 16, + 9, + 4 + ], + [ + 47, + 29, + 4, + 18, + 63, + 32 + ], + [ + 19, + 54, + 62, + 53, + 57, + 29 + ], + [ + 1, + 60, + 14, + 59, + 29, + 4 + ], + [ + 36, + 0, + 47, + 3, + 31, + 8 + ], + [ + 2, + 19, + 36, + 8, + 20, + 37 + ], + [ + 57, + 22, + 40, + 49, + 31, + 14 + ], + [ + 44, + 37, + 2, + 5, + 60, + 21 + ], + [ + 6, + 43, + 24, + 5, + 2, + 59 + ], + [ + 13, + 61, + 19, + 47, + 50, + 39 + ], + [ + 58, + 14, + 28, + 4, + 22, + 11 + ], + [ + 35, + 32, + 46, + 10, + 31, + 45 + ], + [ + 15, + 13, + 55, + 45, + 18, + 63 + ], + [ + 15, + 27, + 28, + 14, + 5, + 60 + ], + [ + 57, + 41, + 47, + 19, + 36, + 10 + ], + [ + 34, + 10, + 53, + 55, + 22, + 19 + ], + [ + 38, + 55, + 39, + 27, + 3, + 25 + ], + [ + 11, + 39, + 0, + 9, + 3, + 49 + ], + [ + 51, + 6, + 43, + 18, + 50, + 53 + ], + [ + 55, + 43, + 9, + 36, + 40, + 5 + ] + ], + [ + [ + 18, + 9, + 1, + 36, + 61, + 44 + ], + [ + 56, + 34, + 19, + 42, + 3, + 5 + ], + [ + 39, + 20, + 15, + 60, + 46, + 32 + ], + [ + 60, + 22, + 31, + 27, + 14, + 55 + ], + [ + 59, + 58, + 10, + 7, + 46, + 18 + ], + [ + 43, + 2, + 57, + 62, + 11, + 30 + ], + [ + 54, + 19, + 9, + 21, + 48, + 56 + ], + [ + 46, + 24, + 7, + 14, + 3, + 8 + ], + [ + 47, + 0, + 4, + 18, + 31, + 29 + ], + [ + 54, + 62, + 47, + 38, + 4, + 32 + ], + [ + 1, + 14, + 15, + 22, + 59, + 38 + ], + [ + 16, + 36, + 42, + 55, + 15, + 18 + ], + [ + 49, + 8, + 20, + 14, + 0, + 33 + ], + [ + 18, + 39, + 25, + 2, + 62, + 22 + ], + [ + 62, + 5, + 58, + 37, + 7, + 32 + ], + [ + 43, + 5, + 42, + 63, + 55, + 37 + ], + [ + 47, + 33, + 15, + 63, + 50, + 12 + ], + [ + 60, + 0, + 7, + 16, + 32, + 13 + ], + [ + 12, + 39, + 32, + 61, + 16, + 45 + ], + [ + 52, + 34, + 15, + 18, + 62, + 30 + ], + [ + 28, + 26, + 46, + 40, + 6, + 38 + ], + [ + 1, + 19, + 17, + 20, + 4, + 21 + ], + [ + 41, + 40, + 4, + 53, + 55, + 19 + ], + [ + 25, + 38, + 27, + 34, + 52, + 46 + ], + [ + 11, + 29, + 52, + 44, + 13, + 35 + ], + [ + 50, + 51, + 41, + 16, + 4, + 14 + ], + [ + 19, + 6, + 23, + 36, + 60, + 0 + ] + ], + [ + [ + 17, + 10, + 57, + 27, + 5, + 54 + ], + [ + 33, + 9, + 43, + 40, + 56, + 11 + ], + [ + 63, + 1, + 35, + 43, + 10, + 27 + ], + [ + 51, + 47, + 20, + 21, + 28, + 61 + ], + [ + 25, + 11, + 58, + 23, + 55, + 46 + ], + [ + 43, + 10, + 12, + 2, + 30, + 62 + ], + [ + 48, + 21, + 19, + 8, + 7, + 54 + ], + [ + 14, + 7, + 24, + 8, + 46, + 2 + ], + [ + 4, + 47, + 37, + 0, + 44, + 27 + ], + [ + 54, + 38, + 62, + 47, + 15, + 14 + ], + [ + 1, + 46, + 15, + 22, + 51, + 38 + ], + [ + 36, + 16, + 42, + 24, + 55, + 37 + ], + [ + 49, + 10, + 0, + 3, + 43, + 8 + ], + [ + 39, + 58, + 0, + 62, + 22, + 25 + ], + [ + 58, + 38, + 7, + 62, + 55, + 56 + ], + [ + 19, + 42, + 55, + 43, + 11, + 37 + ], + [ + 9, + 47, + 43, + 52, + 18, + 50 + ], + [ + 31, + 41, + 32, + 25, + 20, + 13 + ], + [ + 12, + 32, + 61, + 3, + 21, + 43 + ], + [ + 36, + 13, + 40, + 7, + 62, + 16 + ], + [ + 14, + 53, + 50, + 47, + 51, + 1 + ], + [ + 1, + 38, + 19, + 30, + 18, + 16 + ], + [ + 0, + 19, + 51, + 18, + 52, + 15 + ], + [ + 8, + 52, + 27, + 34, + 38, + 3 + ], + [ + 27, + 53, + 59, + 9, + 40, + 4 + ], + [ + 37, + 3, + 26, + 48, + 8, + 16 + ], + [ + 46, + 18, + 11, + 40, + 33, + 44 + ] + ], + [ + [ + 48, + 62, + 61, + 26, + 0, + 50 + ], + [ + 3, + 45, + 40, + 35, + 29, + 54 + ], + [ + 56, + 31, + 23, + 28, + 2, + 53 + ], + [ + 62, + 49, + 20, + 61, + 6, + 41 + ], + [ + 18, + 25, + 50, + 0, + 14, + 57 + ], + [ + 58, + 4, + 10, + 43, + 56, + 46 + ], + [ + 35, + 15, + 25, + 24, + 3, + 7 + ], + [ + 14, + 23, + 8, + 12, + 7, + 24 + ], + [ + 29, + 17, + 35, + 44, + 24, + 4 + ], + [ + 62, + 15, + 38, + 20, + 58, + 17 + ], + [ + 19, + 1, + 46, + 26, + 22, + 63 + ], + [ + 36, + 60, + 42, + 16, + 11, + 55 + ], + [ + 17, + 7, + 49, + 14, + 16, + 26 + ], + [ + 45, + 47, + 22, + 62, + 0, + 58 + ], + [ + 58, + 38, + 48, + 63, + 2, + 49 + ], + [ + 55, + 0, + 1, + 37, + 30, + 10 + ], + [ + 12, + 43, + 21, + 9, + 23, + 47 + ], + [ + 32, + 57, + 42, + 25, + 63, + 43 + ], + [ + 3, + 32, + 49, + 61, + 21, + 12 + ], + [ + 5, + 36, + 16, + 22, + 62, + 42 + ], + [ + 53, + 7, + 46, + 61, + 14, + 52 + ], + [ + 55, + 30, + 3, + 5, + 31, + 53 + ], + [ + 0, + 44, + 15, + 18, + 19, + 28 + ], + [ + 8, + 52, + 51, + 11, + 28, + 29 + ], + [ + 27, + 40, + 4, + 9, + 35, + 39 + ], + [ + 14, + 26, + 3, + 48, + 16, + 21 + ], + [ + 60, + 54, + 35, + 20, + 53, + 12 + ] + ], + [ + [ + 21, + 50, + 29, + 41, + 34, + 60 + ], + [ + 28, + 51, + 60, + 33, + 14, + 45 + ], + [ + 31, + 2, + 46, + 33, + 24, + 49 + ], + [ + 42, + 3, + 18, + 62, + 39, + 49 + ], + [ + 25, + 15, + 62, + 27, + 12, + 11 + ], + [ + 44, + 50, + 36, + 57, + 55, + 41 + ], + [ + 41, + 37, + 22, + 15, + 2, + 40 + ], + [ + 36, + 62, + 53, + 30, + 14, + 57 + ], + [ + 16, + 58, + 2, + 29, + 4, + 3 + ], + [ + 41, + 38, + 26, + 16, + 45, + 46 + ], + [ + 45, + 46, + 32, + 41, + 56, + 26 + ], + [ + 17, + 53, + 21, + 11, + 36, + 35 + ], + [ + 11, + 16, + 28, + 14, + 51, + 39 + ], + [ + 9, + 35, + 33, + 22, + 52, + 62 + ], + [ + 58, + 50, + 63, + 30, + 7, + 27 + ], + [ + 55, + 3, + 8, + 41, + 63, + 37 + ], + [ + 3, + 51, + 32, + 46, + 15, + 6 + ], + [ + 32, + 12, + 10, + 25, + 5, + 16 + ], + [ + 34, + 2, + 37, + 61, + 63, + 39 + ], + [ + 42, + 22, + 27, + 53, + 11, + 56 + ], + [ + 53, + 12, + 0, + 47, + 61, + 49 + ], + [ + 39, + 45, + 53, + 17, + 48, + 14 + ], + [ + 6, + 0, + 4, + 53, + 25, + 11 + ], + [ + 51, + 11, + 1, + 63, + 54, + 59 + ], + [ + 40, + 56, + 37, + 53, + 35, + 5 + ], + [ + 59, + 28, + 41, + 10, + 1, + 45 + ], + [ + 27, + 30, + 28, + 24, + 32, + 57 + ] + ], + [ + [ + 24, + 56, + 6, + 0, + 19, + 45 + ], + [ + 11, + 57, + 59, + 25, + 46, + 30 + ], + [ + 11, + 26, + 37, + 29, + 14, + 52 + ], + [ + 3, + 32, + 7, + 38, + 36, + 24 + ], + [ + 61, + 2, + 24, + 14, + 51, + 44 + ], + [ + 20, + 47, + 0, + 63, + 30, + 58 + ], + [ + 4, + 36, + 29, + 58, + 16, + 3 + ], + [ + 20, + 0, + 45, + 14, + 28, + 44 + ], + [ + 29, + 56, + 47, + 35, + 16, + 4 + ], + [ + 33, + 61, + 55, + 41, + 51, + 38 + ], + [ + 58, + 1, + 38, + 14, + 4, + 19 + ], + [ + 0, + 36, + 14, + 18, + 52, + 42 + ], + [ + 29, + 36, + 45, + 25, + 8, + 6 + ], + [ + 6, + 57, + 50, + 40, + 58, + 61 + ], + [ + 44, + 58, + 29, + 19, + 61, + 56 + ], + [ + 23, + 18, + 28, + 55, + 5, + 37 + ], + [ + 13, + 9, + 19, + 43, + 37, + 3 + ], + [ + 32, + 22, + 63, + 14, + 57, + 41 + ], + [ + 10, + 61, + 3, + 1, + 19, + 32 + ], + [ + 16, + 55, + 10, + 41, + 59, + 22 + ], + [ + 53, + 7, + 29, + 38, + 27, + 46 + ], + [ + 24, + 47, + 18, + 53, + 39, + 30 + ], + [ + 0, + 33, + 19, + 51, + 5, + 17 + ], + [ + 51, + 8, + 11, + 45, + 44, + 41 + ], + [ + 40, + 4, + 23, + 11, + 27, + 19 + ], + [ + 16, + 18, + 3, + 48, + 51, + 21 + ], + [ + 43, + 46, + 60, + 19, + 53, + 12 + ] + ], + [ + [ + 48, + 62, + 61, + 30, + 0, + 2 + ], + [ + 45, + 3, + 35, + 54, + 29, + 2 + ], + [ + 56, + 31, + 53, + 23, + 49, + 28 + ], + [ + 60, + 14, + 57, + 46, + 41, + 48 + ], + [ + 18, + 61, + 59, + 14, + 44, + 32 + ], + [ + 45, + 58, + 47, + 20, + 4, + 30 + ], + [ + 54, + 13, + 25, + 36, + 26, + 47 + ], + [ + 20, + 12, + 0, + 47, + 30, + 45 + ], + [ + 56, + 29, + 47, + 17, + 35, + 16 + ], + [ + 33, + 61, + 55, + 11, + 38, + 48 + ], + [ + 58, + 19, + 14, + 1, + 38, + 36 + ], + [ + 14, + 36, + 0, + 60, + 11, + 52 + ], + [ + 29, + 44, + 7, + 36, + 16, + 45 + ], + [ + 6, + 47, + 50, + 33, + 42, + 62 + ], + [ + 44, + 58, + 61, + 38, + 29, + 56 + ], + [ + 23, + 55, + 18, + 0, + 57, + 37 + ], + [ + 9, + 12, + 43, + 19, + 13, + 6 + ], + [ + 32, + 22, + 63, + 57, + 42, + 29 + ], + [ + 3, + 61, + 1, + 10, + 49, + 32 + ], + [ + 5, + 16, + 55, + 36, + 22, + 59 + ], + [ + 53, + 7, + 29, + 46, + 9, + 14 + ], + [ + 24, + 30, + 18, + 39, + 55, + 53 + ], + [ + 33, + 0, + 19, + 44, + 51, + 5 + ], + [ + 51, + 8, + 53, + 41, + 11, + 4 + ], + [ + 40, + 4, + 27, + 19, + 23, + 16 + ], + [ + 16, + 14, + 48, + 3, + 21, + 26 + ], + [ + 60, + 54, + 35, + 53, + 12, + 43 + ] + ], + [ + [ + 19, + 41, + 8, + 7, + 13, + 2 + ], + [ + 48, + 46, + 62, + 29, + 5, + 41 + ], + [ + 12, + 5, + 59, + 3, + 58, + 49 + ], + [ + 60, + 3, + 42, + 39, + 14, + 18 + ], + [ + 42, + 12, + 27, + 11, + 25, + 19 + ], + [ + 50, + 36, + 44, + 26, + 33, + 37 + ], + [ + 41, + 54, + 22, + 52, + 37, + 35 + ], + [ + 62, + 30, + 36, + 53, + 10, + 14 + ], + [ + 2, + 16, + 58, + 29, + 7, + 41 + ], + [ + 18, + 32, + 45, + 16, + 22, + 38 + ], + [ + 45, + 56, + 41, + 10, + 3, + 46 + ], + [ + 50, + 21, + 36, + 35, + 53, + 12 + ], + [ + 11, + 28, + 16, + 41, + 39, + 46 + ], + [ + 16, + 9, + 33, + 38, + 28, + 52 + ], + [ + 58, + 50, + 63, + 62, + 27, + 52 + ], + [ + 55, + 59, + 8, + 13, + 41, + 43 + ], + [ + 51, + 3, + 15, + 46, + 57, + 47 + ], + [ + 32, + 10, + 30, + 12, + 25, + 18 + ], + [ + 34, + 2, + 27, + 61, + 53, + 39 + ], + [ + 42, + 22, + 56, + 53, + 44, + 34 + ], + [ + 53, + 12, + 49, + 41, + 44, + 8 + ], + [ + 45, + 1, + 48, + 47, + 16, + 61 + ], + [ + 4, + 0, + 53, + 25, + 24, + 11 + ], + [ + 1, + 44, + 11, + 34, + 45, + 51 + ], + [ + 40, + 5, + 53, + 6, + 22, + 18 + ], + [ + 28, + 10, + 1, + 15, + 3, + 4 + ], + [ + 30, + 27, + 24, + 57, + 32, + 16 + ] + ], + [ + [ + 24, + 56, + 6, + 0, + 19, + 45 + ], + [ + 11, + 57, + 59, + 46, + 25, + 30 + ], + [ + 26, + 11, + 37, + 14, + 29, + 49 + ], + [ + 38, + 36, + 3, + 24, + 18, + 20 + ], + [ + 61, + 51, + 14, + 2, + 24, + 1 + ], + [ + 20, + 0, + 47, + 30, + 8, + 35 + ], + [ + 4, + 58, + 36, + 54, + 29, + 12 + ], + [ + 20, + 58, + 44, + 28, + 45, + 9 + ], + [ + 56, + 47, + 10, + 29, + 35, + 27 + ], + [ + 61, + 33, + 55, + 54, + 4, + 36 + ], + [ + 58, + 1, + 14, + 4, + 38, + 52 + ], + [ + 14, + 0, + 36, + 63, + 15, + 52 + ], + [ + 29, + 36, + 44, + 8, + 16, + 2 + ], + [ + 6, + 40, + 27, + 57, + 50, + 42 + ], + [ + 19, + 44, + 58, + 61, + 37, + 38 + ], + [ + 23, + 18, + 17, + 57, + 13, + 5 + ], + [ + 13, + 9, + 19, + 37, + 50, + 15 + ], + [ + 32, + 14, + 57, + 58, + 29, + 22 + ], + [ + 10, + 61, + 1, + 3, + 14, + 59 + ], + [ + 55, + 16, + 34, + 18, + 22, + 7 + ], + [ + 53, + 27, + 38, + 28, + 23, + 44 + ], + [ + 24, + 47, + 18, + 41, + 30, + 62 + ], + [ + 33, + 51, + 19, + 5, + 0, + 34 + ], + [ + 51, + 8, + 25, + 53, + 27, + 55 + ], + [ + 40, + 11, + 27, + 4, + 23, + 19 + ], + [ + 16, + 18, + 51, + 48, + 3, + 47 + ], + [ + 46, + 43, + 36, + 9, + 5, + 33 + ] + ], + [ + [ + 37, + 10, + 46, + 60, + 61, + 55 + ], + [ + 35, + 53, + 34, + 43, + 19, + 57 + ], + [ + 49, + 56, + 45, + 30, + 6, + 12 + ], + [ + 60, + 27, + 14, + 48, + 46, + 57 + ], + [ + 61, + 59, + 14, + 41, + 16, + 1 + ], + [ + 45, + 4, + 3, + 24, + 58, + 47 + ], + [ + 54, + 13, + 9, + 43, + 16, + 26 + ], + [ + 47, + 23, + 12, + 20, + 63, + 30 + ], + [ + 23, + 44, + 56, + 29, + 47, + 17 + ], + [ + 33, + 60, + 61, + 48, + 41, + 14 + ], + [ + 58, + 63, + 19, + 11, + 38, + 9 + ], + [ + 60, + 63, + 0, + 36, + 15, + 9 + ], + [ + 29, + 36, + 30, + 59, + 11, + 27 + ], + [ + 6, + 7, + 47, + 62, + 50, + 57 + ], + [ + 27, + 58, + 19, + 46, + 29, + 56 + ], + [ + 29, + 60, + 56, + 55, + 23, + 26 + ], + [ + 53, + 59, + 6, + 9, + 16, + 43 + ], + [ + 41, + 32, + 57, + 63, + 18, + 37 + ], + [ + 42, + 61, + 3, + 10, + 34, + 59 + ], + [ + 4, + 43, + 17, + 16, + 52, + 60 + ], + [ + 45, + 53, + 61, + 56, + 16, + 7 + ], + [ + 55, + 9, + 61, + 18, + 60, + 3 + ], + [ + 60, + 47, + 53, + 33, + 12, + 27 + ], + [ + 43, + 51, + 11, + 8, + 45, + 63 + ], + [ + 6, + 40, + 15, + 27, + 26, + 23 + ], + [ + 49, + 14, + 9, + 58, + 21, + 12 + ], + [ + 63, + 24, + 60, + 31, + 12, + 34 + ] + ], + [ + [ + 16, + 13, + 44, + 4, + 23, + 46 + ], + [ + 16, + 50, + 9, + 13, + 23, + 36 + ], + [ + 11, + 35, + 21, + 7, + 59, + 9 + ], + [ + 1, + 3, + 25, + 15, + 60, + 39 + ], + [ + 54, + 61, + 31, + 35, + 55, + 1 + ], + [ + 51, + 52, + 46, + 15, + 4, + 45 + ], + [ + 60, + 54, + 59, + 44, + 10, + 7 + ], + [ + 12, + 22, + 14, + 47, + 0, + 30 + ], + [ + 42, + 29, + 23, + 56, + 33, + 47 + ], + [ + 33, + 61, + 20, + 60, + 0, + 49 + ], + [ + 35, + 58, + 63, + 14, + 51, + 24 + ], + [ + 29, + 33, + 36, + 60, + 49, + 0 + ], + [ + 29, + 17, + 30, + 31, + 12, + 36 + ], + [ + 6, + 0, + 61, + 50, + 48, + 3 + ], + [ + 8, + 6, + 58, + 37, + 29, + 19 + ], + [ + 60, + 39, + 27, + 19, + 1, + 57 + ], + [ + 56, + 9, + 30, + 6, + 43, + 10 + ], + [ + 32, + 20, + 13, + 57, + 63, + 49 + ], + [ + 8, + 42, + 21, + 61, + 37, + 4 + ], + [ + 45, + 49, + 16, + 13, + 2, + 58 + ], + [ + 53, + 29, + 14, + 50, + 61, + 3 + ], + [ + 37, + 57, + 27, + 54, + 46, + 9 + ], + [ + 52, + 19, + 22, + 0, + 18, + 5 + ], + [ + 14, + 49, + 30, + 33, + 53, + 34 + ], + [ + 13, + 9, + 4, + 40, + 23, + 39 + ], + [ + 27, + 43, + 47, + 36, + 49, + 3 + ], + [ + 59, + 43, + 40, + 28, + 0, + 33 + ] + ], + [ + [ + 48, + 50, + 42, + 63, + 34, + 38 + ], + [ + 3, + 40, + 61, + 62, + 2, + 6 + ], + [ + 39, + 7, + 36, + 6, + 45, + 40 + ], + [ + 41, + 35, + 46, + 13, + 63, + 56 + ], + [ + 6, + 1, + 54, + 37, + 38, + 34 + ], + [ + 59, + 46, + 51, + 31, + 4, + 52 + ], + [ + 60, + 44, + 11, + 54, + 4, + 24 + ], + [ + 12, + 0, + 2, + 63, + 50, + 47 + ], + [ + 33, + 42, + 29, + 23, + 16, + 56 + ], + [ + 20, + 61, + 33, + 60, + 53, + 0 + ], + [ + 35, + 58, + 63, + 9, + 8, + 19 + ], + [ + 59, + 48, + 36, + 60, + 10, + 14 + ], + [ + 29, + 44, + 7, + 17, + 36, + 12 + ], + [ + 47, + 27, + 6, + 62, + 42, + 48 + ], + [ + 18, + 58, + 49, + 46, + 42, + 38 + ], + [ + 60, + 34, + 27, + 18, + 23, + 55 + ], + [ + 43, + 40, + 9, + 50, + 18, + 45 + ], + [ + 32, + 57, + 48, + 42, + 29, + 39 + ], + [ + 42, + 61, + 49, + 3, + 32, + 1 + ], + [ + 23, + 37, + 1, + 16, + 36, + 39 + ], + [ + 53, + 21, + 7, + 61, + 50, + 31 + ], + [ + 8, + 60, + 18, + 24, + 9, + 30 + ], + [ + 51, + 33, + 28, + 5, + 44, + 8 + ], + [ + 51, + 52, + 8, + 4, + 41, + 45 + ], + [ + 40, + 4, + 27, + 9, + 60, + 19 + ], + [ + 3, + 61, + 16, + 26, + 48, + 12 + ], + [ + 54, + 61, + 35, + 1, + 53, + 20 + ] + ], + [ + [ + 62, + 28, + 1, + 42, + 8, + 55 + ], + [ + 18, + 12, + 8, + 41, + 40, + 31 + ], + [ + 12, + 6, + 50, + 4, + 23, + 45 + ], + [ + 43, + 35, + 8, + 20, + 42, + 46 + ], + [ + 39, + 41, + 29, + 22, + 3, + 56 + ], + [ + 61, + 45, + 46, + 48, + 28, + 51 + ], + [ + 44, + 4, + 11, + 25, + 54, + 59 + ], + [ + 12, + 33, + 56, + 52, + 30, + 17 + ], + [ + 55, + 29, + 17, + 42, + 23, + 14 + ], + [ + 60, + 12, + 18, + 61, + 33, + 28 + ], + [ + 35, + 58, + 37, + 63, + 6, + 27 + ], + [ + 48, + 59, + 10, + 36, + 58, + 60 + ], + [ + 17, + 7, + 28, + 31, + 29, + 27 + ], + [ + 47, + 42, + 50, + 6, + 8, + 14 + ], + [ + 39, + 58, + 56, + 37, + 18, + 59 + ], + [ + 60, + 18, + 57, + 9, + 55, + 23 + ], + [ + 43, + 63, + 18, + 60, + 22, + 19 + ], + [ + 1, + 32, + 42, + 57, + 35, + 63 + ], + [ + 42, + 61, + 3, + 32, + 1, + 50 + ], + [ + 37, + 36, + 10, + 23, + 16, + 57 + ], + [ + 53, + 61, + 7, + 57, + 21, + 23 + ], + [ + 9, + 39, + 30, + 18, + 14, + 17 + ], + [ + 33, + 44, + 8, + 5, + 0, + 19 + ], + [ + 51, + 53, + 49, + 52, + 4, + 41 + ], + [ + 40, + 4, + 27, + 6, + 9, + 16 + ], + [ + 3, + 16, + 48, + 26, + 12, + 4 + ], + [ + 61, + 14, + 12, + 54, + 20, + 35 + ] + ], + [ + [ + 47, + 23, + 63, + 11, + 61, + 55 + ], + [ + 17, + 44, + 28, + 39, + 47, + 27 + ], + [ + 34, + 53, + 50, + 38, + 29, + 5 + ], + [ + 11, + 10, + 17, + 52, + 47, + 42 + ], + [ + 15, + 41, + 27, + 20, + 12, + 6 + ], + [ + 34, + 44, + 50, + 39, + 36, + 61 + ], + [ + 37, + 41, + 52, + 29, + 46, + 47 + ], + [ + 62, + 36, + 34, + 30, + 39, + 22 + ], + [ + 62, + 16, + 58, + 5, + 2, + 8 + ], + [ + 32, + 41, + 56, + 12, + 46, + 8 + ], + [ + 10, + 35, + 45, + 41, + 3, + 56 + ], + [ + 50, + 48, + 35, + 53, + 36, + 12 + ], + [ + 39, + 11, + 46, + 7, + 23, + 51 + ], + [ + 9, + 47, + 19, + 22, + 52, + 34 + ], + [ + 35, + 18, + 56, + 50, + 3, + 58 + ], + [ + 3, + 60, + 38, + 36, + 9, + 58 + ], + [ + 46, + 28, + 32, + 5, + 43, + 56 + ], + [ + 30, + 32, + 57, + 42, + 52, + 19 + ], + [ + 2, + 34, + 32, + 61, + 14, + 42 + ], + [ + 42, + 37, + 20, + 48, + 50, + 9 + ], + [ + 53, + 7, + 56, + 25, + 60, + 13 + ], + [ + 17, + 39, + 53, + 14, + 30, + 25 + ], + [ + 5, + 40, + 6, + 33, + 29, + 35 + ], + [ + 51, + 4, + 11, + 58, + 57, + 28 + ], + [ + 40, + 37, + 4, + 44, + 8, + 48 + ], + [ + 3, + 48, + 26, + 9, + 12, + 41 + ], + [ + 47, + 61, + 26, + 24, + 20, + 53 + ] + ], + [ + [ + 31, + 43, + 41, + 47, + 11, + 25 + ], + [ + 50, + 25, + 31, + 40, + 24, + 46 + ], + [ + 23, + 9, + 62, + 15, + 20, + 53 + ], + [ + 4, + 47, + 44, + 58, + 48, + 25 + ], + [ + 2, + 19, + 12, + 52, + 0, + 40 + ], + [ + 49, + 15, + 24, + 34, + 60, + 42 + ], + [ + 12, + 46, + 17, + 29, + 41, + 3 + ], + [ + 39, + 60, + 44, + 41, + 33, + 36 + ], + [ + 21, + 60, + 16, + 44, + 51, + 57 + ], + [ + 24, + 41, + 12, + 33, + 13, + 21 + ], + [ + 43, + 62, + 3, + 12, + 28, + 45 + ], + [ + 58, + 19, + 39, + 17, + 49, + 42 + ], + [ + 25, + 54, + 4, + 11, + 7, + 6 + ], + [ + 35, + 34, + 4, + 42, + 62, + 19 + ], + [ + 43, + 41, + 42, + 35, + 40, + 32 + ], + [ + 21, + 63, + 3, + 17, + 20, + 50 + ], + [ + 58, + 46, + 44, + 1, + 25, + 20 + ], + [ + 26, + 32, + 16, + 25, + 46, + 41 + ], + [ + 37, + 63, + 61, + 28, + 56, + 12 + ], + [ + 33, + 42, + 40, + 37, + 48, + 50 + ], + [ + 11, + 53, + 25, + 39, + 4, + 61 + ], + [ + 12, + 54, + 4, + 27, + 50, + 14 + ], + [ + 47, + 19, + 42, + 17, + 35, + 40 + ], + [ + 54, + 40, + 60, + 63, + 45, + 57 + ], + [ + 44, + 56, + 40, + 62, + 37, + 3 + ], + [ + 59, + 41, + 57, + 34, + 48, + 22 + ], + [ + 56, + 13, + 59, + 51, + 58, + 23 + ] + ], + [ + [ + 35, + 32, + 8, + 40, + 51, + 52 + ], + [ + 52, + 5, + 22, + 21, + 6, + 33 + ], + [ + 22, + 58, + 11, + 25, + 3, + 51 + ], + [ + 63, + 2, + 56, + 4, + 23, + 54 + ], + [ + 39, + 12, + 23, + 32, + 30, + 46 + ], + [ + 50, + 34, + 36, + 58, + 26, + 28 + ], + [ + 46, + 41, + 3, + 2, + 22, + 16 + ], + [ + 60, + 36, + 53, + 30, + 54, + 39 + ], + [ + 16, + 51, + 3, + 39, + 2, + 26 + ], + [ + 45, + 26, + 18, + 41, + 32, + 46 + ], + [ + 45, + 3, + 10, + 56, + 36, + 35 + ], + [ + 21, + 36, + 35, + 50, + 11, + 19 + ], + [ + 28, + 11, + 46, + 59, + 41, + 15 + ], + [ + 23, + 16, + 38, + 19, + 15, + 22 + ], + [ + 27, + 7, + 34, + 58, + 3, + 42 + ], + [ + 9, + 22, + 36, + 46, + 26, + 41 + ], + [ + 3, + 51, + 40, + 56, + 46, + 8 + ], + [ + 12, + 25, + 21, + 50, + 17, + 62 + ], + [ + 27, + 34, + 61, + 13, + 60, + 11 + ], + [ + 53, + 12, + 56, + 0, + 42, + 33 + ], + [ + 53, + 37, + 12, + 24, + 25, + 63 + ], + [ + 45, + 55, + 18, + 26, + 17, + 1 + ], + [ + 4, + 25, + 32, + 48, + 1, + 53 + ], + [ + 17, + 27, + 63, + 4, + 62, + 44 + ], + [ + 6, + 52, + 62, + 40, + 46, + 23 + ], + [ + 10, + 42, + 28, + 49, + 3, + 1 + ], + [ + 45, + 27, + 41, + 21, + 16, + 47 + ] + ], + [ + [ + 44, + 24, + 56, + 33, + 15, + 41 + ], + [ + 38, + 26, + 24, + 29, + 19, + 53 + ], + [ + 12, + 15, + 29, + 9, + 1, + 63 + ], + [ + 58, + 38, + 50, + 0, + 43, + 61 + ], + [ + 24, + 51, + 31, + 34, + 60, + 7 + ], + [ + 0, + 7, + 22, + 43, + 35, + 1 + ], + [ + 63, + 36, + 11, + 1, + 16, + 4 + ], + [ + 8, + 50, + 56, + 4, + 30, + 55 + ], + [ + 43, + 16, + 42, + 29, + 60, + 35 + ], + [ + 34, + 0, + 9, + 22, + 18, + 26 + ], + [ + 54, + 51, + 45, + 35, + 2, + 36 + ], + [ + 37, + 36, + 43, + 60, + 11, + 59 + ], + [ + 56, + 38, + 10, + 28, + 14, + 43 + ], + [ + 30, + 0, + 58, + 62, + 22, + 19 + ], + [ + 7, + 55, + 42, + 58, + 30, + 38 + ], + [ + 11, + 33, + 1, + 39, + 19, + 16 + ], + [ + 55, + 20, + 40, + 9, + 18, + 30 + ], + [ + 18, + 20, + 57, + 45, + 32, + 1 + ], + [ + 43, + 61, + 12, + 32, + 31, + 30 + ], + [ + 23, + 25, + 7, + 28, + 40, + 19 + ], + [ + 14, + 51, + 48, + 58, + 53, + 25 + ], + [ + 18, + 30, + 1, + 49, + 41, + 9 + ], + [ + 2, + 51, + 22, + 0, + 52, + 5 + ], + [ + 53, + 4, + 47, + 52, + 51, + 40 + ], + [ + 40, + 16, + 9, + 47, + 23, + 11 + ], + [ + 47, + 3, + 43, + 46, + 26, + 53 + ], + [ + 8, + 40, + 18, + 46, + 33, + 63 + ] + ], + [ + [ + 48, + 38, + 50, + 42, + 63, + 36 + ], + [ + 3, + 26, + 10, + 2, + 6, + 61 + ], + [ + 39, + 44, + 45, + 40, + 6, + 7 + ], + [ + 41, + 5, + 20, + 49, + 56, + 13 + ], + [ + 6, + 1, + 30, + 37, + 28, + 38 + ], + [ + 59, + 46, + 22, + 35, + 61, + 0 + ], + [ + 1, + 63, + 35, + 3, + 60, + 49 + ], + [ + 8, + 12, + 2, + 50, + 5, + 55 + ], + [ + 42, + 33, + 43, + 16, + 32, + 29 + ], + [ + 9, + 34, + 0, + 20, + 41, + 31 + ], + [ + 51, + 54, + 8, + 19, + 63, + 9 + ], + [ + 37, + 56, + 36, + 11, + 59, + 43 + ], + [ + 38, + 10, + 28, + 17, + 56, + 63 + ], + [ + 27, + 30, + 42, + 19, + 0, + 22 + ], + [ + 7, + 55, + 49, + 42, + 58, + 38 + ], + [ + 29, + 34, + 39, + 33, + 47, + 45 + ], + [ + 55, + 40, + 20, + 18, + 7, + 5 + ], + [ + 43, + 57, + 39, + 54, + 48, + 28 + ], + [ + 12, + 43, + 61, + 42, + 32, + 49 + ], + [ + 23, + 36, + 1, + 7, + 59, + 28 + ], + [ + 14, + 53, + 21, + 7, + 57, + 37 + ], + [ + 18, + 1, + 24, + 60, + 30, + 9 + ], + [ + 51, + 0, + 33, + 2, + 44, + 5 + ], + [ + 52, + 29, + 4, + 41, + 54, + 58 + ], + [ + 40, + 19, + 16, + 9, + 46, + 47 + ], + [ + 61, + 3, + 47, + 22, + 21, + 53 + ], + [ + 35, + 60, + 54, + 1, + 5, + 40 + ] + ], + [ + [ + 17, + 18, + 8, + 53, + 25, + 43 + ], + [ + 9, + 38, + 24, + 47, + 25, + 63 + ], + [ + 20, + 24, + 5, + 12, + 54, + 28 + ], + [ + 43, + 10, + 20, + 42, + 11, + 8 + ], + [ + 53, + 61, + 30, + 39, + 29, + 18 + ], + [ + 61, + 56, + 25, + 40, + 5, + 22 + ], + [ + 62, + 17, + 24, + 1, + 47, + 33 + ], + [ + 41, + 16, + 34, + 39, + 29, + 8 + ], + [ + 39, + 16, + 36, + 42, + 29, + 23 + ], + [ + 9, + 11, + 41, + 63, + 56, + 31 + ], + [ + 48, + 51, + 10, + 62, + 63, + 45 + ], + [ + 36, + 11, + 37, + 42, + 58, + 46 + ], + [ + 51, + 38, + 25, + 63, + 29, + 44 + ], + [ + 4, + 56, + 44, + 62, + 58, + 30 + ], + [ + 3, + 7, + 46, + 42, + 33, + 35 + ], + [ + 39, + 9, + 33, + 58, + 60, + 29 + ], + [ + 40, + 37, + 20, + 16, + 55, + 25 + ], + [ + 54, + 19, + 11, + 57, + 0, + 39 + ], + [ + 12, + 43, + 61, + 25, + 49, + 32 + ], + [ + 4, + 23, + 54, + 36, + 28, + 7 + ], + [ + 40, + 25, + 26, + 14, + 2, + 58 + ], + [ + 18, + 24, + 58, + 1, + 22, + 46 + ], + [ + 2, + 63, + 22, + 6, + 44, + 56 + ], + [ + 52, + 29, + 51, + 4, + 32, + 40 + ], + [ + 40, + 17, + 15, + 16, + 46, + 57 + ], + [ + 9, + 61, + 3, + 47, + 24, + 11 + ], + [ + 2, + 39, + 24, + 42, + 0, + 44 + ] + ], + [ + [ + 0, + 10, + 49, + 23, + 62, + 44 + ], + [ + 28, + 0, + 36, + 26, + 47, + 52 + ], + [ + 30, + 4, + 16, + 48, + 40, + 10 + ], + [ + 61, + 32, + 26, + 16, + 33, + 62 + ], + [ + 30, + 39, + 53, + 5, + 57, + 20 + ], + [ + 5, + 37, + 61, + 15, + 25, + 6 + ], + [ + 15, + 17, + 24, + 60, + 49, + 62 + ], + [ + 34, + 39, + 61, + 0, + 58, + 40 + ], + [ + 16, + 39, + 36, + 51, + 2, + 29 + ], + [ + 9, + 11, + 41, + 31, + 56, + 52 + ], + [ + 10, + 48, + 24, + 45, + 62, + 51 + ], + [ + 11, + 38, + 36, + 37, + 6, + 42 + ], + [ + 51, + 50, + 15, + 30, + 25, + 38 + ], + [ + 4, + 19, + 24, + 35, + 31, + 48 + ], + [ + 7, + 46, + 3, + 58, + 30, + 41 + ], + [ + 58, + 9, + 39, + 32, + 29, + 40 + ], + [ + 40, + 37, + 20, + 8, + 25, + 55 + ], + [ + 19, + 0, + 54, + 52, + 17, + 39 + ], + [ + 25, + 43, + 12, + 61, + 11, + 14 + ], + [ + 23, + 4, + 54, + 36, + 28, + 33 + ], + [ + 40, + 2, + 25, + 58, + 36, + 53 + ], + [ + 18, + 46, + 35, + 22, + 53, + 16 + ], + [ + 2, + 6, + 63, + 14, + 42, + 11 + ], + [ + 35, + 7, + 52, + 40, + 29, + 57 + ], + [ + 40, + 15, + 19, + 57, + 17, + 23 + ], + [ + 9, + 11, + 47, + 22, + 49, + 1 + ], + [ + 24, + 39, + 42, + 2, + 16, + 22 + ] + ], + [ + [ + 55, + 39, + 9, + 43, + 21, + 46 + ], + [ + 56, + 0, + 63, + 39, + 30, + 41 + ], + [ + 20, + 1, + 26, + 58, + 34, + 19 + ], + [ + 54, + 24, + 32, + 51, + 26, + 44 + ], + [ + 30, + 53, + 56, + 39, + 34, + 40 + ], + [ + 5, + 37, + 25, + 50, + 6, + 61 + ], + [ + 24, + 49, + 37, + 15, + 6, + 29 + ], + [ + 34, + 16, + 30, + 61, + 10, + 36 + ], + [ + 16, + 29, + 2, + 5, + 51, + 26 + ], + [ + 9, + 56, + 11, + 31, + 46, + 45 + ], + [ + 10, + 45, + 56, + 62, + 25, + 36 + ], + [ + 11, + 6, + 35, + 36, + 1, + 52 + ], + [ + 51, + 50, + 41, + 46, + 38, + 4 + ], + [ + 19, + 33, + 41, + 16, + 31, + 52 + ], + [ + 34, + 7, + 17, + 47, + 63, + 3 + ], + [ + 58, + 9, + 22, + 61, + 59, + 8 + ], + [ + 40, + 37, + 3, + 51, + 22, + 57 + ], + [ + 12, + 52, + 21, + 54, + 25, + 19 + ], + [ + 34, + 53, + 27, + 43, + 14, + 29 + ], + [ + 56, + 53, + 44, + 43, + 60, + 24 + ], + [ + 12, + 53, + 40, + 2, + 49, + 62 + ], + [ + 18, + 39, + 61, + 26, + 23, + 44 + ], + [ + 0, + 4, + 53, + 41, + 25, + 21 + ], + [ + 1, + 7, + 25, + 10, + 40, + 56 + ], + [ + 40, + 22, + 6, + 29, + 19, + 48 + ], + [ + 28, + 10, + 47, + 55, + 42, + 44 + ], + [ + 30, + 27, + 57, + 16, + 50, + 59 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 3 + ], + [ + 8, + 60, + 10, + 59, + 43, + 6 + ], + [ + 51, + 45, + 28, + 59, + 63, + 34 + ], + [ + 4, + 16, + 20, + 58, + 44, + 28 + ], + [ + 50, + 31, + 57, + 24, + 51, + 53 + ], + [ + 58, + 9, + 0, + 61, + 35, + 41 + ], + [ + 16, + 63, + 11, + 61, + 23, + 36 + ], + [ + 4, + 47, + 42, + 53, + 8, + 30 + ], + [ + 44, + 14, + 16, + 33, + 3, + 20 + ], + [ + 34, + 28, + 26, + 57, + 22, + 18 + ], + [ + 20, + 35, + 19, + 59, + 2, + 38 + ], + [ + 12, + 60, + 43, + 63, + 32, + 62 + ], + [ + 28, + 12, + 29, + 11, + 14, + 50 + ], + [ + 23, + 29, + 33, + 22, + 11, + 19 + ], + [ + 23, + 60, + 51, + 50, + 7, + 22 + ], + [ + 44, + 46, + 49, + 7, + 1, + 12 + ], + [ + 2, + 54, + 27, + 61, + 18, + 5 + ], + [ + 17, + 50, + 51, + 32, + 33, + 34 + ], + [ + 5, + 19, + 61, + 27, + 32, + 11 + ], + [ + 6, + 0, + 5, + 13, + 41, + 57 + ], + [ + 27, + 33, + 53, + 45, + 38, + 32 + ], + [ + 26, + 36, + 55, + 59, + 61, + 18 + ], + [ + 47, + 46, + 3, + 37, + 57, + 49 + ], + [ + 20, + 22, + 4, + 16, + 51, + 11 + ], + [ + 62, + 11, + 21, + 34, + 4, + 1 + ], + [ + 34, + 18, + 7, + 60, + 33, + 32 + ], + [ + 45, + 52, + 4, + 36, + 21, + 9 + ] + ], + [ + [ + 18, + 8, + 20, + 49, + 30, + 23 + ], + [ + 1, + 27, + 26, + 22, + 59, + 36 + ], + [ + 43, + 26, + 15, + 58, + 0, + 46 + ], + [ + 55, + 1, + 35, + 28, + 16, + 32 + ], + [ + 59, + 9, + 10, + 53, + 12, + 21 + ], + [ + 9, + 2, + 27, + 11, + 61, + 43 + ], + [ + 16, + 57, + 63, + 23, + 19, + 12 + ], + [ + 46, + 45, + 26, + 4, + 30, + 37 + ], + [ + 43, + 44, + 20, + 16, + 14, + 9 + ], + [ + 34, + 47, + 42, + 43, + 26, + 51 + ], + [ + 2, + 42, + 38, + 45, + 20, + 36 + ], + [ + 18, + 7, + 12, + 2, + 43, + 60 + ], + [ + 1, + 28, + 12, + 3, + 29, + 33 + ], + [ + 25, + 13, + 0, + 63, + 2, + 62 + ], + [ + 18, + 36, + 6, + 29, + 19, + 15 + ], + [ + 1, + 42, + 63, + 41, + 57, + 19 + ], + [ + 57, + 54, + 5, + 27, + 18, + 31 + ], + [ + 50, + 6, + 13, + 32, + 17, + 20 + ], + [ + 17, + 5, + 27, + 32, + 1, + 55 + ], + [ + 49, + 0, + 61, + 10, + 30, + 5 + ], + [ + 29, + 53, + 51, + 13, + 33, + 46 + ], + [ + 29, + 17, + 21, + 30, + 14, + 40 + ], + [ + 5, + 17, + 33, + 32, + 18, + 28 + ], + [ + 4, + 51, + 20, + 54, + 58, + 41 + ], + [ + 47, + 4, + 27, + 48, + 37, + 60 + ], + [ + 3, + 26, + 12, + 59, + 2, + 53 + ], + [ + 46, + 43, + 20, + 18, + 9, + 53 + ] + ], + [ + [ + 45, + 6, + 57, + 43, + 40, + 55 + ], + [ + 38, + 63, + 36, + 27, + 54, + 33 + ], + [ + 37, + 14, + 19, + 41, + 58, + 63 + ], + [ + 9, + 12, + 2, + 55, + 28, + 23 + ], + [ + 39, + 59, + 7, + 13, + 33, + 43 + ], + [ + 45, + 9, + 63, + 27, + 32, + 58 + ], + [ + 16, + 57, + 10, + 63, + 11, + 23 + ], + [ + 51, + 45, + 25, + 4, + 21, + 30 + ], + [ + 21, + 44, + 14, + 16, + 39, + 33 + ], + [ + 42, + 44, + 43, + 5, + 37, + 34 + ], + [ + 42, + 19, + 20, + 2, + 38, + 61 + ], + [ + 4, + 12, + 2, + 62, + 63, + 36 + ], + [ + 32, + 55, + 0, + 11, + 47, + 28 + ], + [ + 43, + 13, + 2, + 44, + 26, + 50 + ], + [ + 49, + 33, + 15, + 28, + 29, + 35 + ], + [ + 44, + 41, + 7, + 2, + 22, + 63 + ], + [ + 48, + 6, + 54, + 20, + 2, + 27 + ], + [ + 50, + 51, + 32, + 3, + 17, + 36 + ], + [ + 5, + 61, + 57, + 48, + 19, + 32 + ], + [ + 21, + 0, + 6, + 31, + 29, + 47 + ], + [ + 33, + 9, + 53, + 27, + 17, + 36 + ], + [ + 29, + 26, + 55, + 19, + 17, + 62 + ], + [ + 12, + 46, + 5, + 37, + 57, + 3 + ], + [ + 20, + 51, + 4, + 22, + 16, + 41 + ], + [ + 21, + 11, + 62, + 46, + 23, + 48 + ], + [ + 32, + 60, + 37, + 18, + 3, + 7 + ], + [ + 9, + 11, + 36, + 48, + 0, + 45 + ] + ], + [ + [ + 49, + 42, + 28, + 23, + 33, + 61 + ], + [ + 4, + 2, + 12, + 8, + 6, + 55 + ], + [ + 12, + 0, + 26, + 41, + 6, + 27 + ], + [ + 9, + 57, + 6, + 23, + 51, + 28 + ], + [ + 40, + 7, + 20, + 33, + 16, + 15 + ], + [ + 45, + 59, + 63, + 62, + 32, + 3 + ], + [ + 10, + 39, + 57, + 13, + 16, + 19 + ], + [ + 45, + 23, + 51, + 33, + 25, + 46 + ], + [ + 28, + 21, + 44, + 11, + 16, + 59 + ], + [ + 5, + 42, + 44, + 24, + 43, + 47 + ], + [ + 42, + 53, + 30, + 18, + 2, + 27 + ], + [ + 2, + 62, + 4, + 43, + 10, + 36 + ], + [ + 0, + 56, + 55, + 47, + 32, + 49 + ], + [ + 43, + 25, + 2, + 5, + 3, + 49 + ], + [ + 4, + 28, + 15, + 8, + 49, + 58 + ], + [ + 2, + 42, + 44, + 41, + 7, + 63 + ], + [ + 48, + 27, + 54, + 20, + 2, + 18 + ], + [ + 50, + 51, + 0, + 36, + 3, + 32 + ], + [ + 57, + 5, + 61, + 19, + 32, + 38 + ], + [ + 21, + 0, + 6, + 63, + 23, + 51 + ], + [ + 33, + 53, + 27, + 36, + 9, + 38 + ], + [ + 29, + 26, + 55, + 62, + 18, + 31 + ], + [ + 46, + 56, + 12, + 53, + 29, + 0 + ], + [ + 20, + 16, + 22, + 4, + 51, + 17 + ], + [ + 21, + 62, + 11, + 31, + 46, + 33 + ], + [ + 37, + 60, + 18, + 7, + 32, + 44 + ], + [ + 11, + 9, + 36, + 0, + 48, + 63 + ] + ], + [ + [ + 41, + 32, + 49, + 39, + 44, + 61 + ], + [ + 47, + 26, + 16, + 21, + 36, + 22 + ], + [ + 4, + 30, + 37, + 42, + 60, + 54 + ], + [ + 9, + 57, + 26, + 32, + 50, + 20 + ], + [ + 56, + 3, + 40, + 33, + 36, + 54 + ], + [ + 11, + 38, + 2, + 32, + 61, + 30 + ], + [ + 39, + 57, + 19, + 10, + 16, + 42 + ], + [ + 46, + 21, + 35, + 39, + 45, + 25 + ], + [ + 21, + 37, + 12, + 20, + 11, + 28 + ], + [ + 5, + 47, + 44, + 10, + 42, + 23 + ], + [ + 18, + 42, + 61, + 2, + 38, + 31 + ], + [ + 54, + 4, + 2, + 7, + 22, + 16 + ], + [ + 5, + 3, + 17, + 56, + 32, + 55 + ], + [ + 55, + 0, + 2, + 25, + 43, + 5 + ], + [ + 22, + 28, + 15, + 6, + 5, + 49 + ], + [ + 2, + 57, + 19, + 54, + 41, + 30 + ], + [ + 7, + 48, + 20, + 54, + 27, + 0 + ], + [ + 3, + 56, + 13, + 37, + 43, + 59 + ], + [ + 45, + 55, + 57, + 61, + 48, + 52 + ], + [ + 21, + 5, + 0, + 16, + 27, + 23 + ], + [ + 25, + 42, + 17, + 54, + 23, + 14 + ], + [ + 21, + 44, + 15, + 20, + 42, + 18 + ], + [ + 35, + 12, + 25, + 53, + 61, + 2 + ], + [ + 38, + 54, + 48, + 53, + 21, + 36 + ], + [ + 13, + 31, + 48, + 33, + 18, + 55 + ], + [ + 38, + 27, + 19, + 6, + 44, + 3 + ], + [ + 29, + 62, + 43, + 59, + 46, + 5 + ] + ], + [ + [ + 57, + 9, + 19, + 51, + 18, + 41 + ], + [ + 28, + 57, + 36, + 8, + 48, + 60 + ], + [ + 2, + 51, + 59, + 5, + 34, + 9 + ], + [ + 9, + 55, + 59, + 26, + 4, + 2 + ], + [ + 49, + 56, + 35, + 42, + 30, + 23 + ], + [ + 18, + 30, + 22, + 29, + 19, + 52 + ], + [ + 39, + 34, + 33, + 51, + 56, + 3 + ], + [ + 32, + 21, + 1, + 7, + 46, + 49 + ], + [ + 33, + 54, + 23, + 21, + 12, + 17 + ], + [ + 5, + 30, + 60, + 47, + 15, + 18 + ], + [ + 4, + 18, + 46, + 27, + 20, + 22 + ], + [ + 22, + 59, + 54, + 48, + 19, + 4 + ], + [ + 17, + 5, + 56, + 31, + 49, + 4 + ], + [ + 29, + 47, + 55, + 2, + 53, + 60 + ], + [ + 8, + 22, + 11, + 44, + 36, + 15 + ], + [ + 60, + 44, + 30, + 57, + 54, + 39 + ], + [ + 7, + 44, + 27, + 20, + 2, + 61 + ], + [ + 48, + 17, + 37, + 21, + 32, + 57 + ], + [ + 48, + 32, + 46, + 6, + 61, + 42 + ], + [ + 4, + 57, + 1, + 36, + 0, + 30 + ], + [ + 7, + 17, + 61, + 53, + 21, + 63 + ], + [ + 60, + 14, + 53, + 35, + 18, + 42 + ], + [ + 10, + 15, + 33, + 51, + 36, + 5 + ], + [ + 11, + 4, + 19, + 51, + 21, + 52 + ], + [ + 47, + 19, + 48, + 43, + 4, + 58 + ], + [ + 3, + 33, + 26, + 21, + 52, + 19 + ], + [ + 24, + 45, + 60, + 35, + 49, + 1 + ] + ], + [ + [ + 16, + 4, + 44, + 23, + 22, + 35 + ], + [ + 16, + 50, + 23, + 9, + 32, + 13 + ], + [ + 11, + 35, + 21, + 7, + 48, + 59 + ], + [ + 55, + 15, + 1, + 11, + 8, + 40 + ], + [ + 35, + 61, + 30, + 59, + 31, + 62 + ], + [ + 51, + 29, + 15, + 52, + 38, + 61 + ], + [ + 60, + 0, + 55, + 34, + 59, + 33 + ], + [ + 12, + 22, + 56, + 63, + 54, + 55 + ], + [ + 42, + 54, + 23, + 33, + 27, + 47 + ], + [ + 30, + 60, + 20, + 5, + 4, + 22 + ], + [ + 4, + 35, + 22, + 46, + 23, + 19 + ], + [ + 29, + 22, + 59, + 49, + 24, + 28 + ], + [ + 17, + 31, + 4, + 56, + 5, + 9 + ], + [ + 61, + 29, + 0, + 48, + 59, + 50 + ], + [ + 8, + 6, + 22, + 60, + 55, + 31 + ], + [ + 60, + 39, + 19, + 57, + 53, + 27 + ], + [ + 56, + 30, + 22, + 10, + 5, + 55 + ], + [ + 20, + 31, + 26, + 1, + 61, + 45 + ], + [ + 8, + 23, + 7, + 46, + 48, + 4 + ], + [ + 45, + 23, + 51, + 17, + 4, + 13 + ], + [ + 17, + 13, + 61, + 14, + 29, + 55 + ], + [ + 14, + 27, + 43, + 21, + 56, + 42 + ], + [ + 52, + 51, + 15, + 58, + 8, + 5 + ], + [ + 4, + 51, + 49, + 14, + 21, + 34 + ], + [ + 4, + 9, + 13, + 47, + 8, + 61 + ], + [ + 27, + 3, + 16, + 43, + 31, + 47 + ], + [ + 59, + 43, + 29, + 61, + 0, + 18 + ] + ], + [ + [ + 48, + 21, + 18, + 49, + 41, + 23 + ], + [ + 36, + 4, + 60, + 8, + 49, + 44 + ], + [ + 20, + 39, + 30, + 59, + 45, + 55 + ], + [ + 35, + 46, + 15, + 48, + 33, + 2 + ], + [ + 61, + 37, + 8, + 15, + 54, + 10 + ], + [ + 46, + 51, + 6, + 29, + 58, + 4 + ], + [ + 28, + 11, + 44, + 60, + 0, + 1 + ], + [ + 63, + 12, + 13, + 27, + 10, + 0 + ], + [ + 33, + 42, + 54, + 44, + 23, + 14 + ], + [ + 18, + 60, + 30, + 22, + 40, + 14 + ], + [ + 35, + 4, + 61, + 9, + 18, + 33 + ], + [ + 59, + 45, + 48, + 28, + 62, + 22 + ], + [ + 17, + 56, + 7, + 53, + 5, + 36 + ], + [ + 8, + 47, + 29, + 59, + 1, + 6 + ], + [ + 9, + 8, + 18, + 22, + 60, + 15 + ], + [ + 46, + 60, + 22, + 44, + 30, + 57 + ], + [ + 61, + 2, + 27, + 34, + 7, + 60 + ], + [ + 48, + 21, + 37, + 17, + 50, + 57 + ], + [ + 44, + 42, + 5, + 2, + 48, + 61 + ], + [ + 37, + 57, + 4, + 36, + 17, + 59 + ], + [ + 7, + 17, + 44, + 61, + 53, + 33 + ], + [ + 60, + 45, + 14, + 42, + 18, + 9 + ], + [ + 10, + 8, + 36, + 33, + 15, + 58 + ], + [ + 19, + 20, + 11, + 4, + 49, + 51 + ], + [ + 61, + 37, + 47, + 23, + 12, + 3 + ], + [ + 17, + 33, + 3, + 40, + 19, + 16 + ], + [ + 61, + 45, + 49, + 1, + 14, + 63 + ] + ], + [ + [ + 13, + 40, + 55, + 63, + 26, + 4 + ], + [ + 5, + 35, + 49, + 40, + 17, + 46 + ], + [ + 38, + 17, + 59, + 49, + 2, + 58 + ], + [ + 40, + 8, + 1, + 16, + 0, + 11 + ], + [ + 37, + 62, + 51, + 10, + 8, + 38 + ], + [ + 9, + 42, + 61, + 29, + 35, + 33 + ], + [ + 63, + 53, + 11, + 16, + 33, + 60 + ], + [ + 63, + 37, + 5, + 13, + 17, + 39 + ], + [ + 44, + 20, + 31, + 54, + 38, + 21 + ], + [ + 43, + 21, + 30, + 34, + 18, + 49 + ], + [ + 2, + 11, + 19, + 35, + 4, + 9 + ], + [ + 22, + 60, + 43, + 2, + 4, + 49 + ], + [ + 29, + 5, + 17, + 22, + 24, + 55 + ], + [ + 2, + 59, + 29, + 5, + 55, + 41 + ], + [ + 23, + 8, + 36, + 15, + 22, + 54 + ], + [ + 12, + 44, + 41, + 45, + 5, + 22 + ], + [ + 54, + 7, + 41, + 11, + 53, + 1 + ], + [ + 6, + 50, + 2, + 9, + 21, + 37 + ], + [ + 13, + 19, + 5, + 10, + 48, + 8 + ], + [ + 41, + 32, + 6, + 21, + 47, + 0 + ], + [ + 38, + 33, + 36, + 53, + 31, + 17 + ], + [ + 3, + 26, + 7, + 62, + 18, + 59 + ], + [ + 56, + 57, + 46, + 12, + 35, + 48 + ], + [ + 20, + 16, + 22, + 24, + 27, + 42 + ], + [ + 36, + 21, + 46, + 34, + 3, + 11 + ], + [ + 33, + 34, + 45, + 60, + 7, + 59 + ], + [ + 56, + 34, + 52, + 58, + 26, + 48 + ] + ], + [ + [ + 54, + 23, + 53, + 11, + 58, + 3 + ], + [ + 11, + 30, + 59, + 58, + 63, + 4 + ], + [ + 20, + 29, + 58, + 17, + 42, + 4 + ], + [ + 1, + 35, + 40, + 45, + 53, + 59 + ], + [ + 40, + 55, + 21, + 33, + 38, + 49 + ], + [ + 45, + 29, + 61, + 27, + 63, + 62 + ], + [ + 33, + 57, + 11, + 28, + 53, + 34 + ], + [ + 11, + 63, + 39, + 10, + 45, + 14 + ], + [ + 30, + 54, + 57, + 59, + 33, + 26 + ], + [ + 43, + 23, + 5, + 18, + 21, + 42 + ], + [ + 11, + 18, + 2, + 9, + 34, + 6 + ], + [ + 22, + 60, + 28, + 63, + 2, + 17 + ], + [ + 5, + 41, + 6, + 17, + 56, + 29 + ], + [ + 55, + 6, + 5, + 2, + 48, + 59 + ], + [ + 23, + 19, + 62, + 22, + 11, + 9 + ], + [ + 12, + 45, + 41, + 27, + 8, + 42 + ], + [ + 11, + 53, + 41, + 44, + 51, + 7 + ], + [ + 21, + 2, + 6, + 36, + 50, + 56 + ], + [ + 13, + 10, + 48, + 53, + 61, + 19 + ], + [ + 47, + 21, + 56, + 44, + 6, + 31 + ], + [ + 44, + 12, + 3, + 55, + 41, + 53 + ], + [ + 44, + 47, + 28, + 43, + 45, + 63 + ], + [ + 1, + 25, + 53, + 11, + 39, + 19 + ], + [ + 1, + 59, + 38, + 3, + 37, + 63 + ], + [ + 45, + 3, + 0, + 21, + 14, + 22 + ], + [ + 10, + 28, + 42, + 49, + 11, + 3 + ], + [ + 30, + 57, + 15, + 16, + 56, + 41 + ] + ], + [ + [ + 53, + 15, + 34, + 0, + 46, + 33 + ], + [ + 8, + 12, + 41, + 19, + 39, + 32 + ], + [ + 56, + 31, + 36, + 13, + 23, + 9 + ], + [ + 36, + 51, + 30, + 21, + 1, + 11 + ], + [ + 13, + 58, + 50, + 2, + 53, + 54 + ], + [ + 49, + 52, + 32, + 7, + 23, + 47 + ], + [ + 61, + 38, + 23, + 39, + 0, + 35 + ], + [ + 42, + 27, + 9, + 20, + 17, + 57 + ], + [ + 34, + 1, + 29, + 4, + 35, + 45 + ], + [ + 54, + 57, + 27, + 19, + 38, + 62 + ], + [ + 59, + 1, + 60, + 26, + 38, + 22 + ], + [ + 25, + 31, + 51, + 36, + 32, + 8 + ], + [ + 14, + 62, + 2, + 19, + 37, + 11 + ], + [ + 57, + 40, + 13, + 22, + 37, + 46 + ], + [ + 45, + 34, + 58, + 44, + 42, + 16 + ], + [ + 50, + 16, + 6, + 5, + 33, + 43 + ], + [ + 42, + 39, + 61, + 13, + 5, + 15 + ], + [ + 46, + 23, + 27, + 28, + 4, + 63 + ], + [ + 62, + 31, + 10, + 45, + 35, + 30 + ], + [ + 15, + 13, + 38, + 63, + 4, + 19 + ], + [ + 34, + 15, + 38, + 57, + 27, + 19 + ], + [ + 41, + 62, + 36, + 57, + 19, + 47 + ], + [ + 34, + 22, + 53, + 10, + 46, + 45 + ], + [ + 24, + 51, + 4, + 47, + 39, + 10 + ], + [ + 11, + 57, + 51, + 50, + 54, + 6 + ], + [ + 51, + 7, + 11, + 50, + 43, + 18 + ], + [ + 39, + 37, + 9, + 42, + 40, + 44 + ] + ], + [ + [ + 57, + 17, + 62, + 42, + 23, + 60 + ], + [ + 18, + 7, + 53, + 43, + 26, + 60 + ], + [ + 60, + 5, + 3, + 53, + 23, + 57 + ], + [ + 17, + 10, + 22, + 19, + 11, + 31 + ], + [ + 10, + 15, + 12, + 27, + 17, + 4 + ], + [ + 2, + 44, + 39, + 36, + 25, + 54 + ], + [ + 52, + 62, + 37, + 21, + 41, + 42 + ], + [ + 62, + 7, + 46, + 30, + 36, + 14 + ], + [ + 50, + 4, + 10, + 58, + 0, + 16 + ], + [ + 32, + 47, + 38, + 54, + 8, + 41 + ], + [ + 41, + 32, + 3, + 18, + 1, + 22 + ], + [ + 16, + 50, + 53, + 7, + 44, + 12 + ], + [ + 11, + 39, + 3, + 35, + 25, + 48 + ], + [ + 18, + 9, + 15, + 13, + 28, + 63 + ], + [ + 62, + 58, + 13, + 5, + 17, + 3 + ], + [ + 3, + 31, + 43, + 53, + 35, + 57 + ], + [ + 24, + 51, + 15, + 46, + 5, + 59 + ], + [ + 13, + 30, + 0, + 32, + 5, + 59 + ], + [ + 2, + 39, + 32, + 38, + 34, + 22 + ], + [ + 42, + 26, + 34, + 28, + 37, + 54 + ], + [ + 28, + 43, + 53, + 41, + 13, + 23 + ], + [ + 14, + 15, + 34, + 1, + 48, + 40 + ], + [ + 5, + 25, + 4, + 33, + 39, + 53 + ], + [ + 58, + 4, + 17, + 57, + 51, + 12 + ], + [ + 47, + 37, + 35, + 31, + 63, + 29 + ], + [ + 15, + 3, + 28, + 33, + 23, + 9 + ], + [ + 23, + 6, + 58, + 47, + 30, + 56 + ] + ], + [ + [ + 47, + 29, + 14, + 6, + 51, + 43 + ], + [ + 30, + 29, + 39, + 7, + 52, + 3 + ], + [ + 63, + 34, + 41, + 2, + 7, + 47 + ], + [ + 4, + 28, + 54, + 45, + 52, + 58 + ], + [ + 29, + 7, + 12, + 15, + 41, + 6 + ], + [ + 34, + 29, + 48, + 3, + 43, + 40 + ], + [ + 30, + 29, + 16, + 47, + 42, + 45 + ], + [ + 33, + 39, + 25, + 60, + 41, + 3 + ], + [ + 50, + 26, + 4, + 25, + 13, + 17 + ], + [ + 5, + 43, + 54, + 16, + 12, + 53 + ], + [ + 18, + 6, + 3, + 35, + 21, + 1 + ], + [ + 56, + 46, + 48, + 10, + 16, + 44 + ], + [ + 9, + 35, + 7, + 24, + 47, + 57 + ], + [ + 53, + 42, + 15, + 56, + 47, + 59 + ], + [ + 39, + 11, + 36, + 32, + 35, + 18 + ], + [ + 46, + 20, + 53, + 38, + 56, + 26 + ], + [ + 58, + 29, + 14, + 26, + 17, + 49 + ], + [ + 24, + 25, + 39, + 16, + 1, + 57 + ], + [ + 24, + 41, + 4, + 1, + 63, + 28 + ], + [ + 42, + 37, + 48, + 34, + 26, + 41 + ], + [ + 11, + 28, + 16, + 7, + 32, + 56 + ], + [ + 14, + 42, + 6, + 16, + 22, + 15 + ], + [ + 33, + 56, + 42, + 8, + 25, + 38 + ], + [ + 4, + 58, + 48, + 33, + 11, + 28 + ], + [ + 37, + 47, + 29, + 48, + 30, + 53 + ], + [ + 12, + 41, + 3, + 4, + 48, + 46 + ], + [ + 14, + 13, + 61, + 6, + 62, + 1 + ] + ], + [ + [ + 45, + 10, + 44, + 43, + 53, + 33 + ], + [ + 32, + 63, + 22, + 27, + 30, + 29 + ], + [ + 54, + 35, + 37, + 32, + 26, + 30 + ], + [ + 24, + 63, + 0, + 17, + 25, + 45 + ], + [ + 40, + 7, + 0, + 57, + 29, + 22 + ], + [ + 10, + 34, + 20, + 22, + 43, + 33 + ], + [ + 42, + 30, + 5, + 25, + 19, + 34 + ], + [ + 33, + 18, + 35, + 51, + 7, + 57 + ], + [ + 50, + 28, + 25, + 4, + 10, + 9 + ], + [ + 5, + 38, + 16, + 43, + 54, + 12 + ], + [ + 18, + 3, + 21, + 6, + 39, + 53 + ], + [ + 56, + 16, + 53, + 39, + 46, + 42 + ], + [ + 9, + 35, + 57, + 11, + 47, + 13 + ], + [ + 56, + 15, + 3, + 59, + 9, + 28 + ], + [ + 4, + 62, + 39, + 58, + 63, + 36 + ], + [ + 20, + 53, + 57, + 8, + 51, + 35 + ], + [ + 51, + 49, + 11, + 26, + 15, + 14 + ], + [ + 0, + 25, + 62, + 59, + 21, + 13 + ], + [ + 39, + 34, + 48, + 53, + 61, + 33 + ], + [ + 47, + 26, + 28, + 34, + 21, + 39 + ], + [ + 28, + 43, + 12, + 53, + 41, + 32 + ], + [ + 14, + 52, + 17, + 1, + 15, + 38 + ], + [ + 25, + 4, + 5, + 11, + 58, + 50 + ], + [ + 58, + 4, + 17, + 10, + 25, + 57 + ], + [ + 29, + 47, + 35, + 31, + 52, + 48 + ], + [ + 55, + 28, + 23, + 15, + 3, + 24 + ], + [ + 6, + 30, + 57, + 32, + 34, + 62 + ] + ], + [ + [ + 39, + 5, + 30, + 17, + 61, + 15 + ], + [ + 11, + 63, + 0, + 23, + 61, + 10 + ], + [ + 61, + 15, + 53, + 22, + 7, + 57 + ], + [ + 50, + 57, + 58, + 63, + 45, + 47 + ], + [ + 55, + 31, + 57, + 24, + 60, + 5 + ], + [ + 22, + 7, + 43, + 1, + 10, + 0 + ], + [ + 14, + 58, + 1, + 34, + 19, + 45 + ], + [ + 50, + 8, + 14, + 7, + 57, + 9 + ], + [ + 43, + 0, + 4, + 10, + 45, + 46 + ], + [ + 38, + 0, + 4, + 55, + 54, + 10 + ], + [ + 51, + 54, + 46, + 39, + 1, + 38 + ], + [ + 16, + 37, + 33, + 36, + 21, + 63 + ], + [ + 10, + 38, + 57, + 58, + 3, + 63 + ], + [ + 30, + 0, + 63, + 13, + 22, + 18 + ], + [ + 55, + 58, + 62, + 38, + 6, + 36 + ], + [ + 53, + 19, + 11, + 1, + 57, + 25 + ], + [ + 11, + 6, + 51, + 16, + 18, + 30 + ], + [ + 20, + 57, + 32, + 36, + 13, + 56 + ], + [ + 3, + 12, + 61, + 26, + 32, + 1 + ], + [ + 25, + 36, + 34, + 51, + 59, + 37 + ], + [ + 13, + 28, + 53, + 16, + 14, + 9 + ], + [ + 14, + 42, + 15, + 55, + 22, + 38 + ], + [ + 62, + 58, + 29, + 33, + 5, + 34 + ], + [ + 21, + 4, + 51, + 49, + 58, + 12 + ], + [ + 61, + 60, + 40, + 35, + 59, + 47 + ], + [ + 37, + 46, + 3, + 48, + 12, + 53 + ], + [ + 63, + 8, + 46, + 33, + 1, + 53 + ] + ], + [ + [ + 62, + 9, + 61, + 0, + 41, + 6 + ], + [ + 45, + 3, + 29, + 35, + 2, + 7 + ], + [ + 56, + 31, + 53, + 28, + 23, + 2 + ], + [ + 13, + 41, + 46, + 49, + 5, + 45 + ], + [ + 18, + 25, + 57, + 55, + 50, + 15 + ], + [ + 58, + 22, + 4, + 46, + 19, + 12 + ], + [ + 1, + 25, + 58, + 22, + 43, + 35 + ], + [ + 23, + 50, + 12, + 8, + 9, + 16 + ], + [ + 29, + 17, + 16, + 43, + 10, + 4 + ], + [ + 38, + 55, + 0, + 40, + 20, + 10 + ], + [ + 51, + 19, + 54, + 39, + 46, + 1 + ], + [ + 60, + 16, + 37, + 11, + 56, + 36 + ], + [ + 10, + 7, + 38, + 57, + 54, + 44 + ], + [ + 45, + 0, + 13, + 30, + 47, + 63 + ], + [ + 58, + 48, + 55, + 38, + 36, + 29 + ], + [ + 29, + 53, + 0, + 1, + 55, + 57 + ], + [ + 11, + 12, + 6, + 29, + 5, + 40 + ], + [ + 57, + 32, + 42, + 8, + 20, + 36 + ], + [ + 3, + 61, + 49, + 32, + 26, + 1 + ], + [ + 42, + 5, + 36, + 16, + 39, + 51 + ], + [ + 9, + 13, + 7, + 53, + 46, + 28 + ], + [ + 14, + 55, + 42, + 24, + 33, + 47 + ], + [ + 33, + 58, + 62, + 15, + 36, + 5 + ], + [ + 51, + 4, + 49, + 58, + 21, + 41 + ], + [ + 40, + 60, + 19, + 35, + 61, + 15 + ], + [ + 14, + 21, + 3, + 12, + 5, + 37 + ], + [ + 35, + 60, + 1, + 10, + 53, + 54 + ] + ], + [ + [ + 40, + 28, + 60, + 31, + 59, + 23 + ], + [ + 42, + 12, + 26, + 34, + 0, + 61 + ], + [ + 16, + 5, + 62, + 3, + 32, + 0 + ], + [ + 13, + 3, + 18, + 39, + 42, + 52 + ], + [ + 25, + 12, + 39, + 0, + 57, + 15 + ], + [ + 33, + 50, + 58, + 36, + 6, + 26 + ], + [ + 41, + 43, + 1, + 15, + 2, + 25 + ], + [ + 36, + 16, + 53, + 14, + 30, + 58 + ], + [ + 16, + 29, + 3, + 2, + 10, + 5 + ], + [ + 38, + 16, + 32, + 26, + 45, + 11 + ], + [ + 45, + 19, + 3, + 46, + 56, + 53 + ], + [ + 21, + 36, + 11, + 9, + 16, + 41 + ], + [ + 11, + 16, + 10, + 50, + 8, + 63 + ], + [ + 35, + 38, + 33, + 51, + 25, + 22 + ], + [ + 58, + 48, + 27, + 50, + 29, + 26 + ], + [ + 29, + 57, + 53, + 3, + 54, + 4 + ], + [ + 3, + 23, + 6, + 46, + 51, + 11 + ], + [ + 32, + 5, + 57, + 54, + 52, + 30 + ], + [ + 34, + 61, + 40, + 0, + 3, + 48 + ], + [ + 22, + 53, + 12, + 33, + 39, + 60 + ], + [ + 20, + 53, + 47, + 40, + 12, + 8 + ], + [ + 55, + 17, + 52, + 14, + 45, + 56 + ], + [ + 1, + 4, + 13, + 11, + 39, + 33 + ], + [ + 17, + 10, + 51, + 44, + 55, + 4 + ], + [ + 35, + 6, + 45, + 21, + 52, + 37 + ], + [ + 10, + 42, + 9, + 1, + 53, + 24 + ], + [ + 27, + 41, + 32, + 45, + 10, + 47 + ] + ], + [ + [ + 11, + 34, + 44, + 51, + 41, + 12 + ], + [ + 20, + 34, + 3, + 25, + 63, + 16 + ], + [ + 40, + 26, + 37, + 22, + 15, + 54 + ], + [ + 4, + 16, + 25, + 28, + 45, + 58 + ], + [ + 33, + 10, + 32, + 24, + 3, + 4 + ], + [ + 0, + 9, + 58, + 3, + 34, + 15 + ], + [ + 16, + 63, + 43, + 59, + 42, + 25 + ], + [ + 16, + 48, + 36, + 33, + 25, + 58 + ], + [ + 20, + 50, + 16, + 23, + 42, + 29 + ], + [ + 60, + 51, + 16, + 22, + 38, + 48 + ], + [ + 31, + 19, + 38, + 3, + 2, + 43 + ], + [ + 56, + 9, + 21, + 40, + 18, + 44 + ], + [ + 11, + 13, + 10, + 45, + 27, + 57 + ], + [ + 56, + 14, + 35, + 44, + 45, + 13 + ], + [ + 58, + 43, + 26, + 33, + 31, + 50 + ], + [ + 13, + 38, + 51, + 46, + 28, + 37 + ], + [ + 58, + 63, + 42, + 10, + 16, + 0 + ], + [ + 0, + 1, + 53, + 52, + 24, + 59 + ], + [ + 17, + 29, + 0, + 40, + 60, + 24 + ], + [ + 17, + 24, + 2, + 26, + 35, + 23 + ], + [ + 44, + 20, + 22, + 4, + 50, + 40 + ], + [ + 9, + 61, + 17, + 16, + 27, + 37 + ], + [ + 63, + 13, + 11, + 16, + 2, + 24 + ], + [ + 44, + 48, + 23, + 49, + 56, + 45 + ], + [ + 6, + 5, + 49, + 33, + 31, + 63 + ], + [ + 49, + 47, + 1, + 57, + 4, + 53 + ], + [ + 2, + 56, + 19, + 63, + 39, + 50 + ] + ], + [ + [ + 16, + 22, + 23, + 35, + 3, + 13 + ], + [ + 16, + 23, + 50, + 9, + 13, + 59 + ], + [ + 11, + 35, + 21, + 7, + 59, + 24 + ], + [ + 53, + 32, + 7, + 60, + 1, + 13 + ], + [ + 24, + 33, + 57, + 35, + 0, + 39 + ], + [ + 15, + 38, + 0, + 45, + 32, + 58 + ], + [ + 59, + 16, + 25, + 49, + 35, + 5 + ], + [ + 45, + 40, + 16, + 14, + 12, + 30 + ], + [ + 20, + 16, + 42, + 6, + 17, + 14 + ], + [ + 60, + 10, + 22, + 50, + 48, + 38 + ], + [ + 31, + 19, + 35, + 27, + 38, + 53 + ], + [ + 33, + 59, + 48, + 16, + 60, + 39 + ], + [ + 9, + 57, + 10, + 11, + 7, + 44 + ], + [ + 13, + 0, + 59, + 61, + 19, + 33 + ], + [ + 18, + 39, + 58, + 21, + 50, + 36 + ], + [ + 57, + 60, + 49, + 19, + 53, + 23 + ], + [ + 6, + 36, + 17, + 38, + 41, + 10 + ], + [ + 63, + 1, + 57, + 20, + 36, + 31 + ], + [ + 40, + 41, + 7, + 0, + 61, + 32 + ], + [ + 39, + 16, + 23, + 14, + 17, + 59 + ], + [ + 51, + 30, + 50, + 7, + 61, + 3 + ], + [ + 9, + 14, + 42, + 15, + 1, + 55 + ], + [ + 33, + 19, + 56, + 58, + 11, + 15 + ], + [ + 9, + 26, + 4, + 41, + 51, + 10 + ], + [ + 4, + 60, + 35, + 16, + 48, + 47 + ], + [ + 3, + 52, + 31, + 16, + 26, + 39 + ], + [ + 4, + 1, + 3, + 61, + 0, + 54 + ] + ], + [ + [ + 38, + 45, + 52, + 43, + 32, + 39 + ], + [ + 29, + 48, + 22, + 60, + 55, + 57 + ], + [ + 5, + 49, + 8, + 20, + 14, + 55 + ], + [ + 35, + 46, + 49, + 32, + 7, + 48 + ], + [ + 24, + 29, + 49, + 16, + 41, + 0 + ], + [ + 56, + 3, + 35, + 15, + 4, + 40 + ], + [ + 49, + 59, + 4, + 16, + 33, + 11 + ], + [ + 12, + 51, + 30, + 56, + 40, + 27 + ], + [ + 23, + 14, + 46, + 55, + 42, + 17 + ], + [ + 60, + 22, + 38, + 6, + 21, + 50 + ], + [ + 11, + 19, + 31, + 35, + 38, + 22 + ], + [ + 59, + 48, + 19, + 33, + 16, + 9 + ], + [ + 9, + 28, + 45, + 57, + 55, + 16 + ], + [ + 59, + 1, + 25, + 47, + 6, + 3 + ], + [ + 9, + 39, + 50, + 18, + 58, + 21 + ], + [ + 60, + 57, + 38, + 53, + 50, + 35 + ], + [ + 41, + 53, + 6, + 38, + 10, + 25 + ], + [ + 1, + 63, + 15, + 24, + 36, + 29 + ], + [ + 41, + 40, + 7, + 42, + 32, + 62 + ], + [ + 4, + 10, + 14, + 37, + 39, + 17 + ], + [ + 61, + 30, + 7, + 50, + 57, + 3 + ], + [ + 9, + 14, + 58, + 30, + 42, + 8 + ], + [ + 33, + 56, + 19, + 60, + 24, + 59 + ], + [ + 9, + 26, + 4, + 51, + 25, + 41 + ], + [ + 4, + 35, + 47, + 41, + 48, + 60 + ], + [ + 39, + 52, + 3, + 26, + 8, + 16 + ], + [ + 4, + 1, + 10, + 61, + 0, + 12 + ] + ], + [ + [ + 51, + 53, + 33, + 13, + 28, + 48 + ], + [ + 63, + 31, + 41, + 39, + 40, + 49 + ], + [ + 42, + 14, + 3, + 24, + 50, + 44 + ], + [ + 11, + 39, + 52, + 10, + 17, + 42 + ], + [ + 7, + 60, + 58, + 15, + 12, + 27 + ], + [ + 34, + 36, + 44, + 28, + 29, + 40 + ], + [ + 52, + 17, + 47, + 42, + 37, + 41 + ], + [ + 62, + 33, + 38, + 39, + 41, + 36 + ], + [ + 50, + 58, + 37, + 10, + 16, + 62 + ], + [ + 41, + 59, + 12, + 62, + 49, + 32 + ], + [ + 3, + 35, + 10, + 41, + 6, + 25 + ], + [ + 39, + 53, + 58, + 21, + 19, + 54 + ], + [ + 42, + 35, + 7, + 39, + 63, + 21 + ], + [ + 15, + 9, + 3, + 54, + 51, + 32 + ], + [ + 3, + 35, + 32, + 36, + 18, + 13 + ], + [ + 20, + 3, + 35, + 45, + 32, + 4 + ], + [ + 46, + 24, + 32, + 33, + 14, + 17 + ], + [ + 16, + 57, + 30, + 19, + 61, + 36 + ], + [ + 41, + 25, + 2, + 26, + 24, + 4 + ], + [ + 42, + 37, + 47, + 48, + 33, + 50 + ], + [ + 54, + 13, + 56, + 53, + 63, + 0 + ], + [ + 34, + 14, + 7, + 51, + 42, + 6 + ], + [ + 6, + 43, + 25, + 58, + 11, + 39 + ], + [ + 0, + 26, + 44, + 4, + 51, + 9 + ], + [ + 37, + 63, + 4, + 13, + 15, + 61 + ], + [ + 9, + 59, + 41, + 1, + 52, + 25 + ], + [ + 56, + 58, + 47, + 1, + 62, + 51 + ] + ], + [ + [ + 52, + 47, + 27, + 36, + 38, + 29 + ], + [ + 43, + 56, + 4, + 25, + 52, + 21 + ], + [ + 25, + 54, + 35, + 18, + 11, + 63 + ], + [ + 17, + 4, + 1, + 18, + 50, + 39 + ], + [ + 36, + 7, + 32, + 4, + 30, + 60 + ], + [ + 34, + 3, + 61, + 48, + 24, + 40 + ], + [ + 17, + 42, + 3, + 12, + 29, + 25 + ], + [ + 33, + 38, + 39, + 55, + 17, + 19 + ], + [ + 17, + 50, + 41, + 13, + 16, + 51 + ], + [ + 41, + 12, + 60, + 16, + 62, + 58 + ], + [ + 3, + 36, + 28, + 39, + 35, + 2 + ], + [ + 48, + 27, + 53, + 54, + 19, + 43 + ], + [ + 63, + 42, + 7, + 35, + 62, + 3 + ], + [ + 56, + 15, + 9, + 51, + 42, + 47 + ], + [ + 35, + 39, + 18, + 36, + 43, + 38 + ], + [ + 60, + 20, + 0, + 3, + 35, + 31 + ], + [ + 24, + 14, + 42, + 17, + 32, + 62 + ], + [ + 24, + 57, + 63, + 0, + 42, + 1 + ], + [ + 41, + 50, + 28, + 32, + 26, + 40 + ], + [ + 2, + 37, + 46, + 17, + 42, + 57 + ], + [ + 49, + 7, + 11, + 54, + 13, + 53 + ], + [ + 14, + 9, + 42, + 6, + 19, + 58 + ], + [ + 63, + 33, + 43, + 19, + 58, + 25 + ], + [ + 9, + 26, + 41, + 4, + 48, + 44 + ], + [ + 4, + 41, + 5, + 15, + 9, + 53 + ], + [ + 14, + 38, + 9, + 3, + 52, + 8 + ], + [ + 23, + 1, + 61, + 4, + 47, + 14 + ] + ], + [ + [ + 37, + 14, + 3, + 5, + 33, + 38 + ], + [ + 34, + 0, + 56, + 58, + 37, + 13 + ], + [ + 61, + 14, + 22, + 29, + 15, + 46 + ], + [ + 50, + 58, + 18, + 44, + 47, + 17 + ], + [ + 4, + 36, + 53, + 7, + 32, + 2 + ], + [ + 22, + 1, + 34, + 61, + 10, + 33 + ], + [ + 28, + 25, + 42, + 30, + 29, + 3 + ], + [ + 33, + 51, + 38, + 39, + 62, + 60 + ], + [ + 17, + 51, + 26, + 16, + 46, + 50 + ], + [ + 12, + 16, + 59, + 6, + 38, + 3 + ], + [ + 39, + 31, + 35, + 3, + 36, + 0 + ], + [ + 48, + 19, + 53, + 56, + 27, + 39 + ], + [ + 7, + 62, + 9, + 63, + 15, + 42 + ], + [ + 56, + 19, + 9, + 47, + 15, + 59 + ], + [ + 39, + 36, + 18, + 35, + 42, + 38 + ], + [ + 60, + 20, + 0, + 57, + 47, + 53 + ], + [ + 6, + 32, + 29, + 62, + 43, + 5 + ], + [ + 24, + 63, + 57, + 1, + 42, + 2 + ], + [ + 41, + 40, + 7, + 62, + 32, + 50 + ], + [ + 37, + 14, + 10, + 2, + 57, + 17 + ], + [ + 7, + 54, + 30, + 53, + 50, + 3 + ], + [ + 9, + 14, + 8, + 6, + 42, + 58 + ], + [ + 33, + 56, + 60, + 19, + 18, + 15 + ], + [ + 9, + 26, + 4, + 41, + 59, + 23 + ], + [ + 4, + 41, + 47, + 9, + 2, + 16 + ], + [ + 3, + 14, + 16, + 30, + 8, + 52 + ], + [ + 1, + 4, + 14, + 61, + 0, + 12 + ] + ], + [ + [ + 27, + 21, + 61, + 30, + 22, + 40 + ], + [ + 33, + 12, + 44, + 0, + 47, + 20 + ], + [ + 14, + 39, + 58, + 37, + 16, + 61 + ], + [ + 47, + 11, + 4, + 18, + 32, + 35 + ], + [ + 2, + 19, + 39, + 20, + 10, + 61 + ], + [ + 39, + 61, + 49, + 15, + 30, + 47 + ], + [ + 12, + 47, + 17, + 49, + 29, + 46 + ], + [ + 39, + 41, + 62, + 30, + 52, + 37 + ], + [ + 21, + 15, + 28, + 48, + 26, + 27 + ], + [ + 12, + 32, + 16, + 24, + 55, + 41 + ], + [ + 43, + 35, + 28, + 31, + 60, + 47 + ], + [ + 3, + 17, + 19, + 49, + 42, + 10 + ], + [ + 7, + 27, + 54, + 59, + 4, + 23 + ], + [ + 35, + 33, + 7, + 6, + 14, + 51 + ], + [ + 51, + 28, + 36, + 42, + 41, + 32 + ], + [ + 45, + 52, + 21, + 60, + 53, + 12 + ], + [ + 53, + 1, + 23, + 5, + 6, + 46 + ], + [ + 35, + 9, + 63, + 16, + 26, + 22 + ], + [ + 41, + 60, + 27, + 40, + 42, + 15 + ], + [ + 42, + 48, + 46, + 9, + 37, + 17 + ], + [ + 31, + 11, + 23, + 58, + 39, + 44 + ], + [ + 34, + 2, + 4, + 14, + 13, + 9 + ], + [ + 60, + 59, + 47, + 40, + 17, + 27 + ], + [ + 60, + 63, + 7, + 42, + 49, + 40 + ], + [ + 59, + 15, + 38, + 62, + 44, + 25 + ], + [ + 23, + 57, + 60, + 31, + 41, + 3 + ], + [ + 38, + 59, + 31, + 51, + 36, + 7 + ] + ], + [ + [ + 12, + 10, + 50, + 23, + 53, + 33 + ], + [ + 48, + 56, + 44, + 11, + 31, + 17 + ], + [ + 33, + 11, + 17, + 54, + 15, + 62 + ], + [ + 31, + 13, + 17, + 40, + 8, + 3 + ], + [ + 42, + 4, + 27, + 15, + 12, + 5 + ], + [ + 34, + 50, + 33, + 29, + 55, + 6 + ], + [ + 16, + 49, + 52, + 29, + 41, + 30 + ], + [ + 0, + 30, + 62, + 57, + 26, + 22 + ], + [ + 26, + 62, + 5, + 58, + 51, + 49 + ], + [ + 5, + 62, + 16, + 46, + 39, + 37 + ], + [ + 31, + 10, + 24, + 45, + 18, + 35 + ], + [ + 50, + 19, + 48, + 54, + 16, + 35 + ], + [ + 59, + 39, + 62, + 7, + 35, + 28 + ], + [ + 20, + 9, + 19, + 6, + 22, + 15 + ], + [ + 35, + 18, + 50, + 32, + 36, + 39 + ], + [ + 3, + 4, + 13, + 38, + 60, + 26 + ], + [ + 46, + 32, + 28, + 3, + 37, + 33 + ], + [ + 30, + 63, + 16, + 19, + 24, + 42 + ], + [ + 41, + 17, + 4, + 2, + 32, + 34 + ], + [ + 42, + 37, + 48, + 3, + 33, + 24 + ], + [ + 56, + 7, + 25, + 11, + 39, + 44 + ], + [ + 42, + 14, + 26, + 13, + 12, + 22 + ], + [ + 25, + 11, + 6, + 42, + 13, + 38 + ], + [ + 0, + 9, + 26, + 41, + 4, + 57 + ], + [ + 37, + 4, + 63, + 41, + 2, + 44 + ], + [ + 9, + 52, + 3, + 41, + 16, + 59 + ], + [ + 58, + 47, + 56, + 1, + 26, + 62 + ] + ], + [ + [ + 54, + 30, + 22, + 26, + 3, + 55 + ], + [ + 44, + 7, + 49, + 50, + 25, + 5 + ], + [ + 54, + 4, + 48, + 58, + 26, + 32 + ], + [ + 20, + 25, + 3, + 9, + 55, + 28 + ], + [ + 7, + 27, + 42, + 12, + 58, + 32 + ], + [ + 50, + 34, + 6, + 42, + 29, + 55 + ], + [ + 16, + 49, + 40, + 3, + 27, + 11 + ], + [ + 51, + 30, + 26, + 62, + 57, + 53 + ], + [ + 3, + 5, + 49, + 28, + 26, + 50 + ], + [ + 16, + 22, + 46, + 6, + 49, + 45 + ], + [ + 31, + 45, + 11, + 10, + 56, + 3 + ], + [ + 21, + 51, + 50, + 19, + 9, + 61 + ], + [ + 28, + 41, + 59, + 13, + 34, + 53 + ], + [ + 23, + 20, + 16, + 9, + 38, + 19 + ], + [ + 50, + 34, + 58, + 20, + 27, + 35 + ], + [ + 8, + 59, + 61, + 29, + 35, + 53 + ], + [ + 3, + 46, + 51, + 10, + 25, + 37 + ], + [ + 62, + 21, + 25, + 9, + 18, + 56 + ], + [ + 34, + 27, + 10, + 29, + 53, + 59 + ], + [ + 56, + 33, + 44, + 24, + 9, + 18 + ], + [ + 44, + 12, + 19, + 8, + 30, + 49 + ], + [ + 47, + 62, + 12, + 51, + 16, + 17 + ], + [ + 11, + 25, + 37, + 58, + 35, + 13 + ], + [ + 4, + 9, + 10, + 39, + 37, + 40 + ], + [ + 22, + 21, + 11, + 48, + 45, + 47 + ], + [ + 10, + 28, + 55, + 12, + 24, + 23 + ], + [ + 30, + 16, + 27, + 32, + 57, + 15 + ] + ], + [ + [ + 16, + 11, + 31, + 46, + 35, + 49 + ], + [ + 13, + 49, + 54, + 5, + 6, + 10 + ], + [ + 36, + 13, + 27, + 46, + 3, + 18 + ], + [ + 24, + 61, + 15, + 0, + 13, + 63 + ], + [ + 17, + 2, + 58, + 50, + 35, + 19 + ], + [ + 8, + 7, + 49, + 52, + 47, + 23 + ], + [ + 61, + 16, + 58, + 20, + 38, + 23 + ], + [ + 20, + 42, + 9, + 51, + 35, + 16 + ], + [ + 25, + 47, + 4, + 1, + 50, + 63 + ], + [ + 54, + 38, + 57, + 27, + 33, + 28 + ], + [ + 1, + 59, + 30, + 60, + 14, + 31 + ], + [ + 31, + 51, + 25, + 14, + 41, + 55 + ], + [ + 34, + 2, + 14, + 11, + 19, + 28 + ], + [ + 40, + 57, + 13, + 61, + 59, + 18 + ], + [ + 44, + 58, + 50, + 45, + 37, + 26 + ], + [ + 5, + 8, + 62, + 24, + 57, + 38 + ], + [ + 13, + 47, + 15, + 3, + 5, + 39 + ], + [ + 25, + 23, + 9, + 18, + 63, + 33 + ], + [ + 1, + 10, + 20, + 8, + 53, + 4 + ], + [ + 38, + 15, + 37, + 34, + 60, + 0 + ], + [ + 41, + 27, + 30, + 57, + 19, + 8 + ], + [ + 11, + 62, + 41, + 14, + 46, + 44 + ], + [ + 11, + 33, + 46, + 31, + 0, + 45 + ], + [ + 4, + 51, + 47, + 16, + 9, + 12 + ], + [ + 11, + 21, + 45, + 47, + 18, + 61 + ], + [ + 10, + 50, + 51, + 12, + 18, + 7 + ], + [ + 17, + 31, + 36, + 5, + 19, + 1 + ] + ], + [ + [ + 22, + 6, + 57, + 39, + 29, + 47 + ], + [ + 27, + 6, + 14, + 17, + 51, + 32 + ], + [ + 1, + 29, + 11, + 26, + 47, + 51 + ], + [ + 14, + 38, + 22, + 31, + 29, + 53 + ], + [ + 14, + 61, + 59, + 29, + 1, + 49 + ], + [ + 30, + 8, + 21, + 47, + 52, + 0 + ], + [ + 4, + 58, + 61, + 23, + 20, + 29 + ], + [ + 20, + 9, + 42, + 35, + 7, + 24 + ], + [ + 47, + 25, + 4, + 1, + 29, + 0 + ], + [ + 54, + 19, + 38, + 29, + 33, + 44 + ], + [ + 14, + 1, + 59, + 40, + 60, + 20 + ], + [ + 51, + 0, + 14, + 62, + 16, + 52 + ], + [ + 2, + 36, + 20, + 29, + 19, + 52 + ], + [ + 57, + 13, + 40, + 22, + 60, + 6 + ], + [ + 37, + 44, + 58, + 5, + 8, + 50 + ], + [ + 24, + 5, + 43, + 62, + 23, + 59 + ], + [ + 13, + 19, + 47, + 39, + 61, + 15 + ], + [ + 58, + 33, + 9, + 7, + 4, + 28 + ], + [ + 1, + 35, + 10, + 19, + 31, + 20 + ], + [ + 15, + 55, + 63, + 18, + 34, + 38 + ], + [ + 27, + 15, + 38, + 30, + 57, + 42 + ], + [ + 62, + 41, + 16, + 29, + 6, + 46 + ], + [ + 46, + 33, + 45, + 10, + 34, + 23 + ], + [ + 4, + 47, + 51, + 1, + 16, + 41 + ], + [ + 11, + 21, + 18, + 7, + 48, + 28 + ], + [ + 18, + 51, + 7, + 50, + 6, + 32 + ], + [ + 9, + 11, + 36, + 55, + 43, + 48 + ] + ], + [ + [ + 47, + 8, + 36, + 61, + 21, + 45 + ], + [ + 46, + 2, + 15, + 32, + 0, + 51 + ], + [ + 24, + 15, + 33, + 61, + 2, + 43 + ], + [ + 60, + 22, + 31, + 27, + 14, + 11 + ], + [ + 59, + 58, + 39, + 57, + 46, + 3 + ], + [ + 57, + 43, + 2, + 31, + 7, + 62 + ], + [ + 9, + 42, + 54, + 19, + 4, + 55 + ], + [ + 46, + 14, + 7, + 24, + 43, + 35 + ], + [ + 47, + 4, + 0, + 37, + 12, + 13 + ], + [ + 54, + 38, + 4, + 47, + 25, + 6 + ], + [ + 47, + 14, + 15, + 24, + 1, + 61 + ], + [ + 16, + 54, + 5, + 0, + 7, + 63 + ], + [ + 49, + 3, + 33, + 13, + 11, + 10 + ], + [ + 18, + 13, + 10, + 39, + 58, + 60 + ], + [ + 58, + 62, + 10, + 33, + 5, + 26 + ], + [ + 48, + 5, + 63, + 53, + 2, + 43 + ], + [ + 24, + 47, + 51, + 15, + 0, + 53 + ], + [ + 44, + 0, + 6, + 3, + 7, + 34 + ], + [ + 58, + 38, + 53, + 61, + 0, + 54 + ], + [ + 55, + 9, + 15, + 27, + 42, + 3 + ], + [ + 43, + 1, + 51, + 41, + 4, + 5 + ], + [ + 49, + 20, + 15, + 6, + 37, + 46 + ], + [ + 11, + 25, + 52, + 5, + 4, + 39 + ], + [ + 38, + 40, + 44, + 51, + 10, + 14 + ], + [ + 13, + 8, + 52, + 63, + 2, + 23 + ], + [ + 23, + 38, + 59, + 57, + 55, + 41 + ], + [ + 23, + 6, + 62, + 0, + 7, + 28 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 61 + ], + [ + 51, + 41, + 5, + 15, + 10, + 61 + ], + [ + 43, + 1, + 29, + 55, + 21, + 60 + ], + [ + 24, + 53, + 25, + 13, + 51, + 32 + ], + [ + 41, + 31, + 57, + 49, + 34, + 11 + ], + [ + 17, + 4, + 35, + 30, + 10, + 38 + ], + [ + 34, + 7, + 56, + 42, + 21, + 19 + ], + [ + 14, + 46, + 7, + 27, + 25, + 52 + ], + [ + 0, + 4, + 6, + 12, + 47, + 60 + ], + [ + 54, + 25, + 4, + 38, + 47, + 6 + ], + [ + 24, + 61, + 15, + 46, + 7, + 22 + ], + [ + 5, + 16, + 57, + 0, + 22, + 55 + ], + [ + 49, + 3, + 26, + 17, + 57, + 52 + ], + [ + 13, + 10, + 61, + 60, + 0, + 58 + ], + [ + 58, + 6, + 49, + 10, + 5, + 2 + ], + [ + 25, + 19, + 2, + 11, + 54, + 53 + ], + [ + 47, + 30, + 27, + 5, + 18, + 10 + ], + [ + 44, + 55, + 63, + 13, + 22, + 31 + ], + [ + 58, + 7, + 38, + 35, + 32, + 40 + ], + [ + 13, + 34, + 16, + 45, + 49, + 55 + ], + [ + 51, + 35, + 30, + 58, + 55, + 4 + ], + [ + 46, + 57, + 15, + 36, + 30, + 27 + ], + [ + 52, + 33, + 23, + 10, + 51, + 15 + ], + [ + 40, + 41, + 4, + 51, + 10, + 31 + ], + [ + 8, + 4, + 59, + 48, + 34, + 9 + ], + [ + 43, + 3, + 27, + 26, + 31, + 19 + ], + [ + 46, + 18, + 8, + 4, + 50, + 29 + ] + ], + [ + [ + 22, + 36, + 35, + 63, + 43, + 23 + ], + [ + 54, + 30, + 4, + 36, + 35, + 55 + ], + [ + 28, + 19, + 23, + 49, + 50, + 59 + ], + [ + 62, + 5, + 50, + 53, + 48, + 42 + ], + [ + 0, + 3, + 61, + 57, + 41, + 49 + ], + [ + 60, + 4, + 29, + 16, + 53, + 30 + ], + [ + 34, + 32, + 33, + 9, + 56, + 35 + ], + [ + 12, + 53, + 14, + 36, + 25, + 61 + ], + [ + 23, + 48, + 35, + 29, + 4, + 16 + ], + [ + 22, + 25, + 4, + 54, + 62, + 5 + ], + [ + 24, + 19, + 27, + 55, + 47, + 25 + ], + [ + 18, + 5, + 22, + 34, + 63, + 30 + ], + [ + 17, + 49, + 30, + 28, + 11, + 42 + ], + [ + 29, + 13, + 54, + 25, + 45, + 47 + ], + [ + 27, + 2, + 10, + 5, + 54, + 33 + ], + [ + 55, + 2, + 57, + 54, + 56, + 22 + ], + [ + 53, + 3, + 60, + 27, + 5, + 50 + ], + [ + 17, + 8, + 47, + 61, + 50, + 44 + ], + [ + 27, + 38, + 32, + 14, + 61, + 39 + ], + [ + 22, + 43, + 32, + 57, + 39, + 34 + ], + [ + 16, + 44, + 37, + 23, + 61, + 27 + ], + [ + 45, + 40, + 55, + 32, + 31, + 3 + ], + [ + 32, + 28, + 41, + 15, + 1, + 52 + ], + [ + 21, + 22, + 31, + 10, + 4, + 40 + ], + [ + 29, + 35, + 62, + 60, + 41, + 1 + ], + [ + 39, + 58, + 1, + 63, + 3, + 35 + ], + [ + 10, + 62, + 31, + 45, + 27, + 7 + ] + ], + [ + [ + 17, + 5, + 10, + 57, + 14, + 27 + ], + [ + 43, + 9, + 33, + 56, + 1, + 20 + ], + [ + 63, + 1, + 35, + 43, + 27, + 10 + ], + [ + 47, + 18, + 6, + 3, + 38, + 15 + ], + [ + 11, + 51, + 61, + 34, + 44, + 55 + ], + [ + 10, + 63, + 53, + 60, + 37, + 58 + ], + [ + 51, + 45, + 63, + 34, + 18, + 60 + ], + [ + 35, + 51, + 52, + 53, + 38, + 45 + ], + [ + 44, + 0, + 35, + 55, + 38, + 9 + ], + [ + 42, + 5, + 43, + 25, + 21, + 6 + ], + [ + 42, + 13, + 27, + 25, + 38, + 32 + ], + [ + 43, + 5, + 1, + 38, + 2, + 22 + ], + [ + 17, + 49, + 0, + 37, + 28, + 30 + ], + [ + 2, + 29, + 54, + 5, + 13, + 60 + ], + [ + 2, + 27, + 10, + 58, + 40, + 28 + ], + [ + 2, + 49, + 54, + 62, + 53, + 57 + ], + [ + 60, + 53, + 12, + 27, + 28, + 0 + ], + [ + 17, + 50, + 51, + 33, + 3, + 11 + ], + [ + 38, + 5, + 19, + 13, + 27, + 32 + ], + [ + 43, + 22, + 0, + 57, + 40, + 63 + ], + [ + 27, + 16, + 35, + 52, + 38, + 39 + ], + [ + 29, + 31, + 55, + 40, + 62, + 27 + ], + [ + 41, + 48, + 32, + 46, + 40, + 57 + ], + [ + 20, + 54, + 4, + 22, + 0, + 55 + ], + [ + 62, + 23, + 25, + 33, + 28, + 20 + ], + [ + 7, + 18, + 60, + 22, + 58, + 3 + ], + [ + 36, + 9, + 11, + 0, + 48, + 31 + ] + ], + [ + [ + 11, + 59, + 52, + 28, + 6, + 45 + ], + [ + 7, + 23, + 15, + 17, + 55, + 61 + ], + [ + 28, + 0, + 29, + 46, + 58, + 14 + ], + [ + 62, + 49, + 39, + 18, + 6, + 13 + ], + [ + 36, + 61, + 7, + 40, + 35, + 33 + ], + [ + 8, + 16, + 29, + 43, + 57, + 37 + ], + [ + 30, + 19, + 21, + 57, + 42, + 14 + ], + [ + 35, + 11, + 53, + 51, + 36, + 33 + ], + [ + 48, + 35, + 4, + 50, + 11, + 40 + ], + [ + 5, + 3, + 21, + 47, + 43, + 38 + ], + [ + 21, + 18, + 2, + 6, + 3, + 10 + ], + [ + 18, + 43, + 5, + 45, + 22, + 4 + ], + [ + 33, + 28, + 37, + 41, + 49, + 10 + ], + [ + 56, + 55, + 22, + 54, + 62, + 15 + ], + [ + 2, + 27, + 57, + 5, + 63, + 19 + ], + [ + 55, + 8, + 54, + 2, + 48, + 59 + ], + [ + 0, + 53, + 51, + 3, + 50, + 52 + ], + [ + 21, + 12, + 62, + 60, + 18, + 43 + ], + [ + 38, + 13, + 53, + 27, + 14, + 28 + ], + [ + 22, + 56, + 44, + 43, + 51, + 53 + ], + [ + 29, + 43, + 12, + 16, + 41, + 52 + ], + [ + 51, + 1, + 35, + 44, + 34, + 48 + ], + [ + 32, + 25, + 4, + 41, + 53, + 54 + ], + [ + 48, + 17, + 25, + 1, + 60, + 62 + ], + [ + 22, + 29, + 5, + 18, + 53, + 20 + ], + [ + 28, + 55, + 15, + 1, + 8, + 49 + ], + [ + 30, + 57, + 6, + 7, + 31, + 50 + ] + ], + [ + [ + 11, + 16, + 31, + 46, + 35, + 49 + ], + [ + 13, + 49, + 31, + 16, + 34, + 10 + ], + [ + 36, + 13, + 27, + 34, + 42, + 18 + ], + [ + 24, + 7, + 61, + 63, + 15, + 34 + ], + [ + 17, + 35, + 2, + 48, + 44, + 62 + ], + [ + 8, + 7, + 23, + 47, + 51, + 26 + ], + [ + 61, + 58, + 29, + 38, + 33, + 45 + ], + [ + 20, + 9, + 42, + 27, + 54, + 29 + ], + [ + 25, + 47, + 34, + 1, + 4, + 38 + ], + [ + 54, + 27, + 57, + 4, + 28, + 25 + ], + [ + 1, + 30, + 59, + 60, + 17, + 22 + ], + [ + 31, + 25, + 51, + 5, + 58, + 49 + ], + [ + 34, + 2, + 19, + 14, + 29, + 36 + ], + [ + 57, + 40, + 6, + 9, + 32, + 14 + ], + [ + 8, + 44, + 34, + 16, + 45, + 47 + ], + [ + 8, + 25, + 40, + 37, + 14, + 59 + ], + [ + 13, + 15, + 19, + 51, + 25, + 37 + ], + [ + 23, + 4, + 44, + 52, + 19, + 15 + ], + [ + 8, + 10, + 1, + 45, + 31, + 53 + ], + [ + 38, + 59, + 15, + 2, + 34, + 0 + ], + [ + 41, + 25, + 57, + 27, + 55, + 24 + ], + [ + 11, + 44, + 41, + 39, + 53, + 62 + ], + [ + 31, + 30, + 42, + 37, + 34, + 59 + ], + [ + 33, + 14, + 12, + 10, + 54, + 51 + ], + [ + 11, + 18, + 45, + 20, + 33, + 48 + ], + [ + 50, + 51, + 54, + 23, + 10, + 2 + ], + [ + 17, + 31, + 37, + 5, + 19, + 36 + ] + ], + [ + [ + 31, + 35, + 51, + 18, + 53, + 61 + ], + [ + 7, + 40, + 39, + 41, + 31, + 37 + ], + [ + 16, + 29, + 26, + 50, + 33, + 10 + ], + [ + 14, + 22, + 37, + 17, + 6, + 25 + ], + [ + 60, + 20, + 46, + 4, + 3, + 57 + ], + [ + 11, + 2, + 49, + 21, + 9, + 27 + ], + [ + 42, + 30, + 54, + 58, + 19, + 56 + ], + [ + 46, + 33, + 38, + 41, + 35, + 49 + ], + [ + 50, + 48, + 38, + 37, + 57, + 10 + ], + [ + 59, + 63, + 47, + 28, + 10, + 35 + ], + [ + 28, + 3, + 2, + 52, + 33, + 43 + ], + [ + 58, + 19, + 17, + 16, + 57, + 63 + ], + [ + 42, + 4, + 3, + 23, + 45, + 57 + ], + [ + 18, + 42, + 51, + 3, + 20, + 15 + ], + [ + 32, + 43, + 12, + 3, + 0, + 35 + ], + [ + 15, + 45, + 4, + 7, + 53, + 21 + ], + [ + 28, + 24, + 14, + 15, + 60, + 31 + ], + [ + 26, + 30, + 0, + 54, + 5, + 36 + ], + [ + 63, + 37, + 11, + 41, + 51, + 4 + ], + [ + 48, + 9, + 33, + 0, + 42, + 54 + ], + [ + 22, + 60, + 11, + 39, + 1, + 49 + ], + [ + 17, + 4, + 34, + 2, + 27, + 53 + ], + [ + 17, + 6, + 40, + 58, + 42, + 39 + ], + [ + 60, + 54, + 35, + 39, + 0, + 32 + ], + [ + 44, + 15, + 37, + 13, + 8, + 25 + ], + [ + 41, + 59, + 25, + 45, + 13, + 6 + ], + [ + 23, + 58, + 13, + 19, + 29, + 62 + ] + ], + [ + [ + 52, + 47, + 27, + 36, + 33, + 38 + ], + [ + 43, + 56, + 4, + 21, + 25, + 6 + ], + [ + 25, + 54, + 35, + 18, + 11, + 57 + ], + [ + 17, + 16, + 1, + 6, + 33, + 45 + ], + [ + 32, + 36, + 0, + 60, + 46, + 57 + ], + [ + 11, + 1, + 3, + 21, + 2, + 34 + ], + [ + 30, + 42, + 58, + 54, + 19, + 13 + ], + [ + 33, + 56, + 35, + 42, + 38, + 45 + ], + [ + 50, + 37, + 36, + 14, + 20, + 48 + ], + [ + 10, + 28, + 4, + 37, + 32, + 6 + ], + [ + 3, + 18, + 28, + 61, + 2, + 44 + ], + [ + 16, + 19, + 56, + 27, + 43, + 46 + ], + [ + 42, + 33, + 57, + 3, + 58, + 26 + ], + [ + 56, + 15, + 51, + 55, + 50, + 13 + ], + [ + 12, + 43, + 40, + 13, + 16, + 29 + ], + [ + 15, + 7, + 28, + 53, + 5, + 20 + ], + [ + 14, + 58, + 24, + 60, + 31, + 51 + ], + [ + 0, + 60, + 3, + 24, + 19, + 44 + ], + [ + 36, + 28, + 11, + 63, + 53, + 15 + ], + [ + 2, + 26, + 9, + 34, + 0, + 3 + ], + [ + 49, + 28, + 43, + 41, + 30, + 11 + ], + [ + 25, + 51, + 12, + 6, + 61, + 16 + ], + [ + 39, + 17, + 41, + 50, + 40, + 21 + ], + [ + 25, + 58, + 48, + 12, + 60, + 33 + ], + [ + 31, + 49, + 5, + 52, + 63, + 3 + ], + [ + 15, + 55, + 38, + 47, + 1, + 49 + ], + [ + 23, + 6, + 32, + 19, + 62, + 7 + ] + ], + [ + [ + 44, + 24, + 56, + 33, + 15, + 7 + ], + [ + 38, + 26, + 24, + 29, + 53, + 19 + ], + [ + 12, + 15, + 29, + 9, + 1, + 63 + ], + [ + 38, + 61, + 58, + 50, + 45, + 6 + ], + [ + 24, + 34, + 4, + 36, + 57, + 31 + ], + [ + 1, + 22, + 43, + 21, + 10, + 7 + ], + [ + 20, + 19, + 54, + 58, + 18, + 42 + ], + [ + 56, + 33, + 14, + 21, + 51, + 18 + ], + [ + 60, + 50, + 14, + 36, + 43, + 4 + ], + [ + 6, + 10, + 9, + 63, + 4, + 38 + ], + [ + 54, + 39, + 46, + 18, + 3, + 2 + ], + [ + 30, + 16, + 37, + 53, + 56, + 43 + ], + [ + 56, + 10, + 42, + 58, + 57, + 26 + ], + [ + 30, + 56, + 13, + 50, + 51, + 0 + ], + [ + 55, + 40, + 62, + 12, + 13, + 30 + ], + [ + 53, + 28, + 11, + 61, + 7, + 19 + ], + [ + 55, + 14, + 17, + 47, + 30, + 5 + ], + [ + 18, + 31, + 20, + 60, + 57, + 32 + ], + [ + 43, + 12, + 53, + 26, + 32, + 61 + ], + [ + 25, + 16, + 26, + 61, + 53, + 3 + ], + [ + 49, + 28, + 51, + 59, + 55, + 11 + ], + [ + 6, + 56, + 32, + 14, + 21, + 10 + ], + [ + 2, + 15, + 58, + 17, + 13, + 62 + ], + [ + 53, + 51, + 4, + 29, + 50, + 25 + ], + [ + 61, + 31, + 19, + 15, + 60, + 49 + ], + [ + 46, + 19, + 44, + 36, + 8, + 40 + ], + [ + 8, + 29, + 46, + 7, + 53, + 20 + ] + ], + [ + [ + 48, + 42, + 38, + 63, + 50, + 7 + ], + [ + 3, + 40, + 2, + 33, + 14, + 60 + ], + [ + 39, + 7, + 45, + 40, + 6, + 44 + ], + [ + 41, + 5, + 20, + 56, + 13, + 0 + ], + [ + 6, + 30, + 37, + 1, + 38, + 52 + ], + [ + 59, + 46, + 4, + 22, + 5, + 6 + ], + [ + 20, + 1, + 44, + 35, + 13, + 3 + ], + [ + 12, + 56, + 8, + 50, + 31, + 2 + ], + [ + 33, + 60, + 41, + 43, + 37, + 36 + ], + [ + 9, + 10, + 0, + 55, + 37, + 40 + ], + [ + 54, + 39, + 9, + 8, + 61, + 46 + ], + [ + 30, + 56, + 53, + 24, + 16, + 11 + ], + [ + 7, + 58, + 57, + 42, + 10, + 52 + ], + [ + 27, + 30, + 42, + 25, + 59, + 13 + ], + [ + 9, + 11, + 55, + 49, + 46, + 0 + ], + [ + 34, + 53, + 54, + 29, + 57, + 37 + ], + [ + 55, + 17, + 5, + 60, + 18, + 31 + ], + [ + 57, + 48, + 43, + 39, + 32, + 36 + ], + [ + 12, + 43, + 42, + 49, + 7, + 61 + ], + [ + 23, + 36, + 1, + 0, + 16, + 61 + ], + [ + 21, + 35, + 11, + 31, + 55, + 7 + ], + [ + 60, + 8, + 42, + 24, + 14, + 6 + ], + [ + 15, + 51, + 58, + 2, + 33, + 30 + ], + [ + 52, + 51, + 4, + 28, + 21, + 19 + ], + [ + 19, + 60, + 15, + 34, + 54, + 46 + ], + [ + 19, + 61, + 58, + 12, + 40, + 3 + ], + [ + 35, + 49, + 54, + 53, + 1, + 25 + ] + ], + [ + [ + 47, + 37, + 59, + 38, + 33, + 10 + ], + [ + 36, + 7, + 21, + 51, + 8, + 47 + ], + [ + 20, + 32, + 44, + 47, + 4, + 54 + ], + [ + 43, + 20, + 42, + 52, + 8, + 19 + ], + [ + 27, + 9, + 39, + 57, + 12, + 54 + ], + [ + 28, + 16, + 29, + 11, + 61, + 58 + ], + [ + 42, + 2, + 30, + 43, + 28, + 25 + ], + [ + 18, + 36, + 32, + 21, + 53, + 15 + ], + [ + 50, + 12, + 37, + 48, + 14, + 52 + ], + [ + 9, + 10, + 13, + 3, + 58, + 26 + ], + [ + 18, + 3, + 9, + 55, + 6, + 61 + ], + [ + 30, + 18, + 45, + 60, + 16, + 24 + ], + [ + 35, + 5, + 42, + 26, + 37, + 58 + ], + [ + 55, + 38, + 15, + 13, + 14, + 54 + ], + [ + 9, + 62, + 2, + 17, + 13, + 26 + ], + [ + 9, + 40, + 37, + 54, + 17, + 47 + ], + [ + 22, + 60, + 51, + 14, + 40, + 39 + ], + [ + 17, + 3, + 21, + 30, + 36, + 25 + ], + [ + 56, + 41, + 24, + 9, + 43, + 14 + ], + [ + 53, + 34, + 26, + 22, + 12, + 35 + ], + [ + 36, + 42, + 24, + 37, + 8, + 4 + ], + [ + 34, + 16, + 8, + 46, + 56, + 6 + ], + [ + 31, + 10, + 16, + 41, + 56, + 32 + ], + [ + 48, + 0, + 42, + 56, + 31, + 30 + ], + [ + 6, + 54, + 26, + 19, + 8, + 7 + ], + [ + 49, + 6, + 22, + 13, + 24, + 59 + ], + [ + 45, + 62, + 27, + 47, + 50, + 7 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 3 + ], + [ + 8, + 60, + 59, + 43, + 10, + 48 + ], + [ + 51, + 45, + 28, + 34, + 59, + 63 + ], + [ + 43, + 2, + 38, + 12, + 20, + 4 + ], + [ + 50, + 57, + 39, + 31, + 63, + 0 + ], + [ + 58, + 53, + 18, + 9, + 30, + 21 + ], + [ + 63, + 51, + 34, + 7, + 20, + 27 + ], + [ + 32, + 21, + 46, + 47, + 25, + 18 + ], + [ + 6, + 12, + 50, + 14, + 33, + 37 + ], + [ + 30, + 10, + 9, + 6, + 13, + 29 + ], + [ + 61, + 18, + 23, + 20, + 44, + 6 + ], + [ + 16, + 12, + 30, + 60, + 0, + 62 + ], + [ + 5, + 26, + 57, + 35, + 37, + 61 + ], + [ + 46, + 55, + 13, + 1, + 17, + 31 + ], + [ + 6, + 10, + 22, + 2, + 16, + 15 + ], + [ + 6, + 54, + 19, + 25, + 57, + 46 + ], + [ + 2, + 30, + 60, + 61, + 18, + 22 + ], + [ + 17, + 4, + 2, + 27, + 3, + 44 + ], + [ + 56, + 46, + 8, + 32, + 6, + 14 + ], + [ + 5, + 13, + 0, + 34, + 14, + 30 + ], + [ + 15, + 23, + 59, + 57, + 24, + 27 + ], + [ + 36, + 61, + 8, + 43, + 57, + 37 + ], + [ + 10, + 15, + 52, + 31, + 29, + 23 + ], + [ + 10, + 38, + 1, + 57, + 31, + 4 + ], + [ + 8, + 32, + 54, + 33, + 3, + 18 + ], + [ + 6, + 33, + 19, + 50, + 2, + 3 + ], + [ + 55, + 43, + 4, + 5, + 25, + 8 + ] + ], + [ + [ + 25, + 14, + 18, + 49, + 51, + 63 + ], + [ + 42, + 21, + 30, + 43, + 24, + 7 + ], + [ + 54, + 39, + 9, + 59, + 28, + 49 + ], + [ + 23, + 1, + 55, + 45, + 43, + 33 + ], + [ + 37, + 41, + 30, + 59, + 21, + 44 + ], + [ + 4, + 41, + 31, + 35, + 19, + 14 + ], + [ + 28, + 55, + 44, + 63, + 9, + 51 + ], + [ + 63, + 12, + 32, + 13, + 47, + 28 + ], + [ + 33, + 12, + 59, + 35, + 6, + 39 + ], + [ + 30, + 40, + 10, + 29, + 52, + 13 + ], + [ + 63, + 19, + 23, + 61, + 8, + 4 + ], + [ + 45, + 62, + 22, + 12, + 38, + 42 + ], + [ + 58, + 26, + 57, + 20, + 45, + 30 + ], + [ + 8, + 59, + 47, + 17, + 25, + 22 + ], + [ + 24, + 54, + 51, + 55, + 10, + 15 + ], + [ + 46, + 22, + 54, + 44, + 57, + 40 + ], + [ + 34, + 60, + 2, + 17, + 18, + 27 + ], + [ + 51, + 6, + 61, + 2, + 39, + 32 + ], + [ + 5, + 49, + 38, + 8, + 32, + 46 + ], + [ + 57, + 4, + 23, + 14, + 5, + 59 + ], + [ + 7, + 36, + 45, + 21, + 53, + 4 + ], + [ + 60, + 3, + 39, + 8, + 14, + 35 + ], + [ + 33, + 57, + 60, + 47, + 15, + 52 + ], + [ + 20, + 11, + 22, + 58, + 19, + 4 + ], + [ + 36, + 34, + 47, + 41, + 60, + 40 + ], + [ + 33, + 3, + 49, + 29, + 14, + 59 + ], + [ + 52, + 24, + 60, + 25, + 35, + 34 + ] + ], + [ + [ + 23, + 54, + 53, + 11, + 58, + 8 + ], + [ + 11, + 30, + 15, + 63, + 59, + 16 + ], + [ + 20, + 29, + 58, + 17, + 52, + 30 + ], + [ + 18, + 1, + 43, + 15, + 3, + 8 + ], + [ + 59, + 55, + 13, + 20, + 44, + 30 + ], + [ + 33, + 45, + 27, + 53, + 63, + 52 + ], + [ + 28, + 57, + 51, + 34, + 53, + 31 + ], + [ + 13, + 63, + 3, + 32, + 44, + 45 + ], + [ + 59, + 33, + 12, + 57, + 6, + 35 + ], + [ + 23, + 30, + 42, + 10, + 29, + 13 + ], + [ + 63, + 23, + 18, + 42, + 38, + 19 + ], + [ + 62, + 2, + 22, + 19, + 45, + 6 + ], + [ + 6, + 26, + 20, + 30, + 5, + 25 + ], + [ + 3, + 6, + 8, + 31, + 17, + 37 + ], + [ + 59, + 2, + 10, + 54, + 55, + 19 + ], + [ + 45, + 7, + 57, + 42, + 54, + 46 + ], + [ + 60, + 55, + 53, + 34, + 41, + 18 + ], + [ + 36, + 2, + 8, + 31, + 32, + 46 + ], + [ + 36, + 45, + 38, + 32, + 61, + 25 + ], + [ + 57, + 39, + 6, + 18, + 19, + 30 + ], + [ + 37, + 39, + 59, + 48, + 53, + 14 + ], + [ + 52, + 28, + 46, + 55, + 47, + 61 + ], + [ + 1, + 28, + 52, + 15, + 31, + 18 + ], + [ + 22, + 21, + 8, + 10, + 46, + 39 + ], + [ + 45, + 41, + 35, + 9, + 54, + 33 + ], + [ + 42, + 39, + 58, + 61, + 24, + 3 + ], + [ + 41, + 46, + 10, + 3, + 15, + 33 + ] + ], + [ + [ + 48, + 63, + 38, + 42, + 7, + 37 + ], + [ + 3, + 10, + 26, + 6, + 2, + 62 + ], + [ + 39, + 7, + 44, + 6, + 45, + 40 + ], + [ + 21, + 39, + 1, + 18, + 15, + 41 + ], + [ + 6, + 33, + 9, + 13, + 3, + 27 + ], + [ + 59, + 33, + 53, + 62, + 21, + 45 + ], + [ + 28, + 10, + 33, + 7, + 50, + 57 + ], + [ + 50, + 13, + 12, + 49, + 3, + 55 + ], + [ + 33, + 59, + 26, + 35, + 48, + 38 + ], + [ + 23, + 30, + 20, + 43, + 10, + 58 + ], + [ + 63, + 23, + 18, + 17, + 38, + 9 + ], + [ + 22, + 62, + 33, + 36, + 6, + 2 + ], + [ + 12, + 17, + 59, + 7, + 26, + 49 + ], + [ + 47, + 27, + 3, + 6, + 24, + 22 + ], + [ + 59, + 57, + 46, + 54, + 2, + 10 + ], + [ + 27, + 45, + 57, + 54, + 34, + 61 + ], + [ + 34, + 60, + 55, + 31, + 58, + 43 + ], + [ + 48, + 42, + 32, + 39, + 2, + 38 + ], + [ + 42, + 49, + 32, + 44, + 12, + 61 + ], + [ + 57, + 36, + 39, + 1, + 30, + 59 + ], + [ + 48, + 7, + 21, + 53, + 17, + 29 + ], + [ + 8, + 60, + 58, + 35, + 14, + 46 + ], + [ + 51, + 15, + 28, + 47, + 33, + 30 + ], + [ + 11, + 19, + 4, + 8, + 58, + 52 + ], + [ + 24, + 5, + 41, + 60, + 40, + 54 + ], + [ + 61, + 3, + 21, + 58, + 19, + 8 + ], + [ + 60, + 35, + 54, + 49, + 1, + 0 + ] + ], + [ + [ + 6, + 24, + 63, + 25, + 26, + 45 + ], + [ + 47, + 13, + 49, + 44, + 20, + 19 + ], + [ + 23, + 32, + 49, + 20, + 24, + 2 + ], + [ + 43, + 21, + 8, + 40, + 39, + 45 + ], + [ + 39, + 29, + 3, + 5, + 41, + 12 + ], + [ + 61, + 33, + 48, + 40, + 29, + 62 + ], + [ + 29, + 28, + 25, + 33, + 44, + 31 + ], + [ + 50, + 33, + 13, + 11, + 30, + 54 + ], + [ + 48, + 26, + 17, + 35, + 55, + 6 + ], + [ + 43, + 23, + 12, + 9, + 25, + 62 + ], + [ + 63, + 35, + 18, + 53, + 38, + 17 + ], + [ + 19, + 48, + 5, + 36, + 59, + 17 + ], + [ + 7, + 26, + 59, + 17, + 12, + 30 + ], + [ + 47, + 22, + 24, + 62, + 25, + 42 + ], + [ + 59, + 46, + 35, + 39, + 57, + 54 + ], + [ + 27, + 9, + 20, + 0, + 57, + 52 + ], + [ + 38, + 58, + 60, + 34, + 43, + 29 + ], + [ + 42, + 32, + 39, + 54, + 38, + 57 + ], + [ + 59, + 32, + 42, + 6, + 21, + 37 + ], + [ + 24, + 36, + 57, + 4, + 30, + 60 + ], + [ + 9, + 48, + 7, + 53, + 21, + 58 + ], + [ + 49, + 31, + 14, + 8, + 22, + 19 + ], + [ + 15, + 33, + 44, + 8, + 14, + 3 + ], + [ + 11, + 4, + 21, + 28, + 41, + 23 + ], + [ + 37, + 27, + 24, + 12, + 9, + 20 + ], + [ + 58, + 9, + 19, + 3, + 12, + 48 + ], + [ + 24, + 14, + 60, + 47, + 25, + 13 + ] + ], + [ + [ + 26, + 62, + 58, + 18, + 38, + 5 + ], + [ + 19, + 12, + 40, + 39, + 31, + 57 + ], + [ + 27, + 38, + 9, + 22, + 23, + 61 + ], + [ + 42, + 20, + 63, + 39, + 45, + 43 + ], + [ + 7, + 36, + 60, + 29, + 57, + 24 + ], + [ + 61, + 34, + 3, + 1, + 44, + 51 + ], + [ + 29, + 42, + 25, + 47, + 30, + 17 + ], + [ + 33, + 38, + 39, + 50, + 62, + 0 + ], + [ + 17, + 26, + 13, + 50, + 41, + 11 + ], + [ + 12, + 43, + 35, + 16, + 55, + 60 + ], + [ + 3, + 18, + 39, + 63, + 35, + 62 + ], + [ + 19, + 48, + 39, + 60, + 54, + 2 + ], + [ + 7, + 42, + 56, + 59, + 37, + 24 + ], + [ + 56, + 47, + 54, + 59, + 3, + 6 + ], + [ + 35, + 39, + 43, + 59, + 2, + 13 + ], + [ + 20, + 27, + 9, + 56, + 0, + 54 + ], + [ + 17, + 60, + 29, + 18, + 58, + 5 + ], + [ + 57, + 42, + 45, + 0, + 24, + 29 + ], + [ + 49, + 28, + 52, + 32, + 50, + 3 + ], + [ + 24, + 2, + 36, + 30, + 59, + 46 + ], + [ + 11, + 7, + 48, + 54, + 53, + 21 + ], + [ + 33, + 14, + 42, + 6, + 8, + 5 + ], + [ + 15, + 36, + 33, + 51, + 17, + 44 + ], + [ + 41, + 11, + 2, + 4, + 8, + 29 + ], + [ + 30, + 15, + 60, + 46, + 5, + 9 + ], + [ + 3, + 21, + 14, + 19, + 61, + 58 + ], + [ + 24, + 60, + 1, + 14, + 35, + 53 + ] + ], + [ + [ + 8, + 56, + 54, + 4, + 37, + 7 + ], + [ + 36, + 9, + 24, + 8, + 1, + 2 + ], + [ + 16, + 57, + 29, + 32, + 58, + 5 + ], + [ + 32, + 63, + 42, + 52, + 4, + 20 + ], + [ + 27, + 24, + 44, + 39, + 20, + 63 + ], + [ + 24, + 11, + 28, + 6, + 15, + 31 + ], + [ + 30, + 41, + 42, + 25, + 52, + 2 + ], + [ + 26, + 36, + 62, + 18, + 50, + 21 + ], + [ + 48, + 50, + 32, + 51, + 27, + 26 + ], + [ + 13, + 43, + 3, + 26, + 12, + 41 + ], + [ + 3, + 32, + 18, + 53, + 39, + 17 + ], + [ + 21, + 19, + 9, + 48, + 36, + 1 + ], + [ + 35, + 59, + 42, + 54, + 63, + 17 + ], + [ + 15, + 12, + 38, + 9, + 51, + 54 + ], + [ + 3, + 2, + 48, + 59, + 57, + 13 + ], + [ + 27, + 9, + 4, + 20, + 22, + 3 + ], + [ + 60, + 40, + 22, + 27, + 46, + 32 + ], + [ + 30, + 21, + 25, + 17, + 0, + 42 + ], + [ + 33, + 27, + 34, + 28, + 38, + 44 + ], + [ + 53, + 32, + 33, + 35, + 31, + 56 + ], + [ + 40, + 11, + 20, + 47, + 48, + 24 + ], + [ + 45, + 62, + 33, + 53, + 17, + 34 + ], + [ + 41, + 9, + 39, + 25, + 17, + 32 + ], + [ + 44, + 30, + 2, + 31, + 0, + 47 + ], + [ + 6, + 62, + 37, + 52, + 55, + 33 + ], + [ + 10, + 24, + 57, + 9, + 49, + 13 + ], + [ + 45, + 24, + 27, + 47, + 19, + 26 + ] + ], + [ + [ + 4, + 59, + 16, + 13, + 3, + 22 + ], + [ + 16, + 23, + 9, + 59, + 13, + 50 + ], + [ + 11, + 35, + 21, + 7, + 59, + 9 + ], + [ + 44, + 1, + 25, + 26, + 15, + 20 + ], + [ + 35, + 57, + 52, + 31, + 24, + 5 + ], + [ + 35, + 15, + 38, + 53, + 12, + 0 + ], + [ + 36, + 20, + 63, + 28, + 60, + 33 + ], + [ + 49, + 50, + 12, + 13, + 8, + 16 + ], + [ + 42, + 48, + 6, + 16, + 35, + 41 + ], + [ + 23, + 9, + 54, + 34, + 30, + 13 + ], + [ + 23, + 63, + 51, + 53, + 55, + 3 + ], + [ + 33, + 36, + 62, + 19, + 59, + 57 + ], + [ + 59, + 38, + 12, + 32, + 17, + 53 + ], + [ + 48, + 0, + 19, + 24, + 61, + 22 + ], + [ + 55, + 6, + 39, + 7, + 60, + 18 + ], + [ + 27, + 57, + 19, + 1, + 60, + 47 + ], + [ + 30, + 40, + 10, + 17, + 36, + 60 + ], + [ + 20, + 13, + 1, + 31, + 17, + 54 + ], + [ + 50, + 32, + 23, + 7, + 33, + 30 + ], + [ + 14, + 19, + 23, + 13, + 10, + 7 + ], + [ + 53, + 23, + 19, + 13, + 5, + 7 + ], + [ + 30, + 18, + 46, + 14, + 1, + 15 + ], + [ + 15, + 18, + 52, + 0, + 51, + 27 + ], + [ + 4, + 21, + 53, + 41, + 10, + 14 + ], + [ + 4, + 9, + 39, + 8, + 16, + 54 + ], + [ + 3, + 19, + 16, + 47, + 30, + 27 + ], + [ + 33, + 8, + 46, + 40, + 29, + 14 + ] + ], + [ + [ + 6, + 26, + 3, + 24, + 11, + 38 + ], + [ + 49, + 19, + 16, + 57, + 0, + 18 + ], + [ + 16, + 60, + 57, + 0, + 22, + 30 + ], + [ + 35, + 46, + 49, + 44, + 26, + 17 + ], + [ + 25, + 54, + 9, + 43, + 45, + 5 + ], + [ + 55, + 33, + 30, + 6, + 28, + 57 + ], + [ + 55, + 31, + 49, + 52, + 25, + 15 + ], + [ + 11, + 36, + 15, + 29, + 30, + 62 + ], + [ + 55, + 42, + 48, + 54, + 6, + 46 + ], + [ + 10, + 3, + 43, + 21, + 62, + 54 + ], + [ + 32, + 56, + 6, + 3, + 18, + 55 + ], + [ + 50, + 21, + 19, + 36, + 4, + 42 + ], + [ + 38, + 35, + 53, + 39, + 41, + 32 + ], + [ + 28, + 24, + 38, + 41, + 15, + 12 + ], + [ + 2, + 40, + 18, + 60, + 55, + 13 + ], + [ + 27, + 57, + 22, + 8, + 54, + 37 + ], + [ + 40, + 27, + 60, + 46, + 44, + 50 + ], + [ + 17, + 21, + 30, + 12, + 29, + 26 + ], + [ + 33, + 38, + 23, + 2, + 13, + 27 + ], + [ + 32, + 56, + 9, + 44, + 31, + 52 + ], + [ + 53, + 12, + 22, + 40, + 5, + 41 + ], + [ + 44, + 45, + 49, + 17, + 14, + 56 + ], + [ + 25, + 39, + 41, + 4, + 9, + 53 + ], + [ + 44, + 1, + 17, + 0, + 13, + 58 + ], + [ + 62, + 37, + 52, + 54, + 42, + 36 + ], + [ + 10, + 9, + 28, + 55, + 2, + 34 + ], + [ + 57, + 30, + 27, + 45, + 47, + 16 + ] + ], + [ + [ + 11, + 16, + 31, + 46, + 0, + 5 + ], + [ + 13, + 49, + 34, + 31, + 16, + 8 + ], + [ + 36, + 13, + 27, + 18, + 34, + 58 + ], + [ + 24, + 61, + 15, + 46, + 63, + 25 + ], + [ + 17, + 2, + 50, + 35, + 58, + 44 + ], + [ + 8, + 23, + 7, + 49, + 26, + 47 + ], + [ + 61, + 38, + 29, + 58, + 36, + 0 + ], + [ + 42, + 20, + 9, + 4, + 11, + 27 + ], + [ + 1, + 34, + 30, + 48, + 26, + 7 + ], + [ + 27, + 10, + 28, + 62, + 13, + 54 + ], + [ + 60, + 1, + 30, + 59, + 6, + 3 + ], + [ + 31, + 62, + 25, + 42, + 21, + 36 + ], + [ + 34, + 32, + 19, + 2, + 14, + 62 + ], + [ + 6, + 40, + 24, + 31, + 57, + 13 + ], + [ + 44, + 2, + 40, + 34, + 8, + 45 + ], + [ + 8, + 27, + 7, + 57, + 50, + 25 + ], + [ + 13, + 60, + 45, + 22, + 52, + 25 + ], + [ + 4, + 23, + 33, + 46, + 58, + 19 + ], + [ + 8, + 33, + 56, + 51, + 45, + 4 + ], + [ + 38, + 53, + 39, + 35, + 31, + 0 + ], + [ + 41, + 11, + 53, + 46, + 8, + 59 + ], + [ + 11, + 44, + 33, + 52, + 55, + 53 + ], + [ + 31, + 1, + 16, + 9, + 15, + 41 + ], + [ + 2, + 31, + 14, + 10, + 44, + 4 + ], + [ + 45, + 18, + 6, + 24, + 62, + 12 + ], + [ + 10, + 50, + 42, + 54, + 24, + 3 + ], + [ + 17, + 31, + 37, + 5, + 10, + 41 + ] + ], + [ + [ + 45, + 37, + 48, + 29, + 30, + 59 + ], + [ + 8, + 59, + 10, + 60, + 43, + 55 + ], + [ + 45, + 51, + 28, + 59, + 34, + 31 + ], + [ + 14, + 31, + 22, + 2, + 19, + 44 + ], + [ + 59, + 50, + 4, + 31, + 9, + 44 + ], + [ + 58, + 23, + 47, + 9, + 53, + 30 + ], + [ + 54, + 13, + 51, + 22, + 29, + 61 + ], + [ + 47, + 42, + 32, + 20, + 24, + 37 + ], + [ + 33, + 12, + 47, + 10, + 30, + 44 + ], + [ + 30, + 27, + 10, + 28, + 33, + 13 + ], + [ + 61, + 23, + 14, + 20, + 1, + 29 + ], + [ + 12, + 14, + 32, + 62, + 31, + 52 + ], + [ + 5, + 36, + 19, + 20, + 2, + 32 + ], + [ + 46, + 24, + 1, + 40, + 17, + 33 + ], + [ + 44, + 8, + 16, + 30, + 2, + 6 + ], + [ + 6, + 46, + 15, + 54, + 21, + 57 + ], + [ + 2, + 30, + 60, + 61, + 10, + 18 + ], + [ + 6, + 17, + 33, + 27, + 58, + 4 + ], + [ + 56, + 46, + 8, + 60, + 16, + 22 + ], + [ + 5, + 13, + 14, + 36, + 35, + 52 + ], + [ + 15, + 53, + 23, + 59, + 6, + 46 + ], + [ + 36, + 30, + 53, + 33, + 60, + 38 + ], + [ + 10, + 15, + 16, + 52, + 38, + 4 + ], + [ + 31, + 1, + 10, + 4, + 41, + 59 + ], + [ + 32, + 9, + 8, + 59, + 18, + 33 + ], + [ + 33, + 19, + 6, + 3, + 45, + 50 + ], + [ + 55, + 4, + 43, + 5, + 25, + 34 + ] + ], + [ + [ + 25, + 14, + 18, + 49, + 51, + 63 + ], + [ + 42, + 21, + 30, + 43, + 24, + 7 + ], + [ + 54, + 39, + 9, + 59, + 28, + 45 + ], + [ + 1, + 23, + 55, + 8, + 16, + 7 + ], + [ + 59, + 37, + 30, + 41, + 16, + 8 + ], + [ + 4, + 31, + 41, + 56, + 45, + 18 + ], + [ + 28, + 55, + 44, + 54, + 9, + 51 + ], + [ + 24, + 47, + 12, + 63, + 32, + 61 + ], + [ + 33, + 47, + 39, + 44, + 38, + 56 + ], + [ + 40, + 30, + 59, + 33, + 58, + 10 + ], + [ + 19, + 8, + 23, + 4, + 25, + 34 + ], + [ + 45, + 14, + 12, + 42, + 62, + 52 + ], + [ + 58, + 45, + 59, + 29, + 25, + 12 + ], + [ + 8, + 59, + 1, + 47, + 33, + 32 + ], + [ + 24, + 30, + 56, + 38, + 4, + 7 + ], + [ + 46, + 22, + 44, + 57, + 12, + 40 + ], + [ + 34, + 60, + 2, + 17, + 18, + 29 + ], + [ + 6, + 61, + 51, + 53, + 39, + 32 + ], + [ + 49, + 5, + 47, + 60, + 21, + 59 + ], + [ + 57, + 4, + 23, + 39, + 45, + 60 + ], + [ + 7, + 36, + 45, + 53, + 21, + 56 + ], + [ + 60, + 3, + 39, + 18, + 53, + 8 + ], + [ + 57, + 33, + 15, + 47, + 38, + 9 + ], + [ + 20, + 11, + 19, + 2, + 4, + 8 + ], + [ + 36, + 34, + 60, + 47, + 1, + 40 + ], + [ + 33, + 14, + 3, + 21, + 19, + 35 + ], + [ + 60, + 24, + 22, + 52, + 35, + 53 + ] + ], + [ + [ + 23, + 54, + 53, + 58, + 11, + 8 + ], + [ + 11, + 30, + 15, + 59, + 55, + 63 + ], + [ + 20, + 58, + 29, + 17, + 42, + 30 + ], + [ + 18, + 1, + 43, + 15, + 8, + 3 + ], + [ + 59, + 55, + 13, + 28, + 26, + 63 + ], + [ + 33, + 45, + 27, + 53, + 63, + 19 + ], + [ + 28, + 57, + 51, + 54, + 34, + 53 + ], + [ + 24, + 13, + 3, + 47, + 45, + 50 + ], + [ + 47, + 59, + 33, + 57, + 37, + 35 + ], + [ + 23, + 42, + 33, + 41, + 48, + 30 + ], + [ + 42, + 63, + 23, + 25, + 17, + 34 + ], + [ + 14, + 62, + 2, + 19, + 45, + 43 + ], + [ + 12, + 6, + 20, + 30, + 29, + 17 + ], + [ + 6, + 3, + 17, + 8, + 27, + 31 + ], + [ + 19, + 59, + 2, + 10, + 8, + 55 + ], + [ + 45, + 61, + 19, + 42, + 57, + 17 + ], + [ + 55, + 60, + 41, + 34, + 35, + 53 + ], + [ + 36, + 2, + 8, + 31, + 14, + 5 + ], + [ + 36, + 45, + 16, + 38, + 51, + 21 + ], + [ + 57, + 39, + 48, + 6, + 19, + 60 + ], + [ + 39, + 37, + 59, + 48, + 42, + 14 + ], + [ + 52, + 46, + 33, + 28, + 18, + 55 + ], + [ + 1, + 15, + 28, + 13, + 52, + 18 + ], + [ + 10, + 21, + 4, + 23, + 2, + 31 + ], + [ + 45, + 9, + 41, + 18, + 54, + 3 + ], + [ + 42, + 61, + 36, + 3, + 19, + 27 + ], + [ + 41, + 46, + 33, + 3, + 10, + 15 + ] + ], + [ + [ + 49, + 52, + 60, + 63, + 21, + 0 + ], + [ + 14, + 7, + 25, + 52, + 58, + 36 + ], + [ + 46, + 57, + 28, + 24, + 49, + 12 + ], + [ + 10, + 21, + 39, + 1, + 11, + 17 + ], + [ + 53, + 3, + 2, + 16, + 46, + 44 + ], + [ + 25, + 40, + 39, + 15, + 34, + 30 + ], + [ + 62, + 24, + 47, + 52, + 17, + 49 + ], + [ + 34, + 41, + 39, + 11, + 33, + 15 + ], + [ + 36, + 26, + 48, + 39, + 15, + 56 + ], + [ + 23, + 43, + 36, + 20, + 21, + 10 + ], + [ + 48, + 63, + 51, + 10, + 62, + 18 + ], + [ + 10, + 42, + 36, + 39, + 49, + 19 + ], + [ + 22, + 38, + 51, + 56, + 50, + 7 + ], + [ + 4, + 31, + 19, + 22, + 17, + 7 + ], + [ + 41, + 7, + 39, + 2, + 57, + 26 + ], + [ + 33, + 27, + 58, + 28, + 20, + 45 + ], + [ + 58, + 8, + 37, + 18, + 34, + 62 + ], + [ + 24, + 34, + 43, + 39, + 17, + 19 + ], + [ + 23, + 50, + 12, + 32, + 43, + 31 + ], + [ + 9, + 7, + 51, + 16, + 30, + 60 + ], + [ + 25, + 62, + 14, + 53, + 58, + 0 + ], + [ + 53, + 4, + 18, + 22, + 14, + 29 + ], + [ + 6, + 14, + 15, + 48, + 58, + 0 + ], + [ + 35, + 4, + 45, + 3, + 59, + 39 + ], + [ + 17, + 57, + 23, + 16, + 40, + 37 + ], + [ + 9, + 19, + 3, + 36, + 11, + 47 + ], + [ + 39, + 14, + 22, + 33, + 62, + 42 + ] + ], + [ + [ + 14, + 17, + 2, + 39, + 47, + 63 + ], + [ + 23, + 58, + 2, + 25, + 5, + 18 + ], + [ + 27, + 33, + 0, + 56, + 6, + 7 + ], + [ + 23, + 5, + 30, + 7, + 21, + 39 + ], + [ + 39, + 53, + 27, + 38, + 54, + 41 + ], + [ + 37, + 25, + 50, + 40, + 33, + 36 + ], + [ + 49, + 37, + 24, + 55, + 29, + 28 + ], + [ + 34, + 15, + 36, + 16, + 30, + 11 + ], + [ + 5, + 36, + 2, + 26, + 16, + 48 + ], + [ + 56, + 46, + 10, + 23, + 39, + 45 + ], + [ + 10, + 56, + 62, + 45, + 49, + 3 + ], + [ + 35, + 6, + 36, + 50, + 11, + 1 + ], + [ + 50, + 46, + 39, + 51, + 38, + 41 + ], + [ + 41, + 19, + 31, + 6, + 48, + 24 + ], + [ + 2, + 7, + 57, + 18, + 47, + 63 + ], + [ + 27, + 58, + 22, + 8, + 57, + 45 + ], + [ + 37, + 59, + 51, + 44, + 55, + 18 + ], + [ + 52, + 21, + 25, + 12, + 34, + 39 + ], + [ + 23, + 34, + 60, + 13, + 27, + 32 + ], + [ + 32, + 56, + 26, + 24, + 44, + 30 + ], + [ + 41, + 12, + 53, + 2, + 5, + 56 + ], + [ + 44, + 18, + 39, + 33, + 38, + 46 + ], + [ + 25, + 0, + 59, + 18, + 11, + 53 + ], + [ + 1, + 7, + 17, + 30, + 4, + 37 + ], + [ + 62, + 48, + 19, + 54, + 37, + 22 + ], + [ + 10, + 28, + 3, + 16, + 11, + 42 + ], + [ + 26, + 30, + 57, + 42, + 41, + 33 + ] + ], + [ + [ + 36, + 31, + 37, + 16, + 43, + 63 + ], + [ + 26, + 51, + 0, + 48, + 42, + 21 + ], + [ + 18, + 41, + 37, + 34, + 24, + 30 + ], + [ + 58, + 4, + 22, + 44, + 18, + 51 + ], + [ + 18, + 4, + 26, + 61, + 14, + 31 + ], + [ + 9, + 23, + 34, + 29, + 3, + 31 + ], + [ + 12, + 58, + 36, + 46, + 16, + 25 + ], + [ + 4, + 56, + 9, + 33, + 60, + 32 + ], + [ + 14, + 26, + 43, + 30, + 46, + 35 + ], + [ + 2, + 0, + 10, + 43, + 28, + 27 + ], + [ + 61, + 18, + 19, + 17, + 2, + 33 + ], + [ + 60, + 19, + 23, + 62, + 37, + 43 + ], + [ + 56, + 32, + 8, + 61, + 58, + 38 + ], + [ + 25, + 28, + 22, + 0, + 26, + 29 + ], + [ + 22, + 49, + 7, + 60, + 23, + 55 + ], + [ + 46, + 57, + 19, + 37, + 12, + 27 + ], + [ + 22, + 60, + 11, + 30, + 19, + 20 + ], + [ + 36, + 20, + 17, + 2, + 37, + 55 + ], + [ + 13, + 16, + 32, + 58, + 41, + 10 + ], + [ + 50, + 55, + 13, + 25, + 14, + 41 + ], + [ + 14, + 3, + 44, + 48, + 60, + 6 + ], + [ + 36, + 18, + 57, + 22, + 1, + 43 + ], + [ + 10, + 22, + 15, + 52, + 13, + 18 + ], + [ + 4, + 56, + 10, + 31, + 14, + 52 + ], + [ + 9, + 4, + 16, + 54, + 8, + 59 + ], + [ + 3, + 43, + 27, + 45, + 6, + 30 + ], + [ + 33, + 4, + 43, + 40, + 46, + 18 + ] + ], + [ + [ + 48, + 42, + 38, + 63, + 47, + 7 + ], + [ + 3, + 62, + 10, + 6, + 26, + 2 + ], + [ + 39, + 44, + 6, + 7, + 45, + 40 + ], + [ + 56, + 33, + 2, + 41, + 48, + 62 + ], + [ + 6, + 18, + 37, + 41, + 30, + 27 + ], + [ + 59, + 45, + 23, + 31, + 53, + 9 + ], + [ + 0, + 13, + 12, + 44, + 33, + 16 + ], + [ + 12, + 56, + 50, + 60, + 28, + 63 + ], + [ + 14, + 33, + 26, + 24, + 32, + 16 + ], + [ + 43, + 2, + 0, + 50, + 28, + 58 + ], + [ + 61, + 9, + 63, + 8, + 17, + 39 + ], + [ + 60, + 19, + 59, + 23, + 24, + 30 + ], + [ + 56, + 59, + 53, + 32, + 7, + 38 + ], + [ + 28, + 25, + 22, + 24, + 59, + 42 + ], + [ + 22, + 49, + 46, + 23, + 60, + 7 + ], + [ + 46, + 34, + 27, + 29, + 12, + 0 + ], + [ + 17, + 22, + 5, + 31, + 20, + 27 + ], + [ + 39, + 48, + 57, + 37, + 36, + 17 + ], + [ + 42, + 49, + 32, + 7, + 16, + 12 + ], + [ + 1, + 7, + 23, + 14, + 36, + 29 + ], + [ + 21, + 7, + 48, + 60, + 35, + 14 + ], + [ + 8, + 18, + 24, + 60, + 1, + 0 + ], + [ + 51, + 15, + 33, + 36, + 5, + 30 + ], + [ + 52, + 4, + 29, + 8, + 9, + 26 + ], + [ + 4, + 9, + 16, + 40, + 58, + 60 + ], + [ + 3, + 17, + 21, + 61, + 19, + 8 + ], + [ + 35, + 54, + 1, + 53, + 60, + 0 + ] + ], + [ + [ + 19, + 1, + 31, + 52, + 49, + 63 + ], + [ + 7, + 47, + 5, + 60, + 22, + 46 + ], + [ + 59, + 30, + 3, + 11, + 0, + 19 + ], + [ + 43, + 42, + 19, + 62, + 8, + 56 + ], + [ + 61, + 15, + 25, + 18, + 39, + 27 + ], + [ + 61, + 50, + 36, + 45, + 33, + 44 + ], + [ + 8, + 37, + 52, + 1, + 2, + 41 + ], + [ + 36, + 30, + 53, + 11, + 16, + 29 + ], + [ + 14, + 58, + 46, + 49, + 3, + 26 + ], + [ + 62, + 43, + 0, + 45, + 22, + 46 + ], + [ + 6, + 56, + 45, + 18, + 10, + 41 + ], + [ + 60, + 21, + 50, + 47, + 30, + 35 + ], + [ + 11, + 53, + 28, + 56, + 41, + 39 + ], + [ + 23, + 9, + 33, + 28, + 22, + 26 + ], + [ + 23, + 56, + 34, + 27, + 2, + 63 + ], + [ + 22, + 9, + 44, + 41, + 37, + 47 + ], + [ + 1, + 11, + 46, + 27, + 3, + 52 + ], + [ + 51, + 37, + 17, + 21, + 61, + 30 + ], + [ + 13, + 19, + 32, + 5, + 2, + 9 + ], + [ + 41, + 32, + 6, + 47, + 29, + 56 + ], + [ + 53, + 33, + 9, + 35, + 38, + 12 + ], + [ + 40, + 19, + 51, + 7, + 26, + 22 + ], + [ + 37, + 5, + 25, + 46, + 34, + 53 + ], + [ + 16, + 55, + 20, + 24, + 44, + 52 + ], + [ + 21, + 46, + 11, + 0, + 36, + 7 + ], + [ + 18, + 60, + 32, + 3, + 34, + 28 + ], + [ + 52, + 9, + 36, + 48, + 11, + 41 + ] + ], + [ + [ + 53, + 15, + 34, + 0, + 36, + 56 + ], + [ + 8, + 12, + 41, + 11, + 19, + 55 + ], + [ + 56, + 13, + 31, + 36, + 23, + 47 + ], + [ + 36, + 51, + 30, + 7, + 26, + 54 + ], + [ + 58, + 13, + 50, + 2, + 53, + 34 + ], + [ + 49, + 52, + 23, + 32, + 7, + 26 + ], + [ + 61, + 38, + 23, + 0, + 4, + 28 + ], + [ + 42, + 27, + 17, + 9, + 18, + 20 + ], + [ + 1, + 34, + 45, + 4, + 12, + 7 + ], + [ + 54, + 27, + 57, + 38, + 44, + 6 + ], + [ + 59, + 40, + 1, + 60, + 48, + 30 + ], + [ + 25, + 31, + 32, + 51, + 62, + 8 + ], + [ + 62, + 19, + 14, + 2, + 37, + 15 + ], + [ + 40, + 57, + 37, + 35, + 22, + 61 + ], + [ + 45, + 16, + 34, + 42, + 37, + 48 + ], + [ + 50, + 16, + 62, + 33, + 25, + 37 + ], + [ + 42, + 13, + 39, + 47, + 3, + 63 + ], + [ + 46, + 23, + 28, + 27, + 4, + 15 + ], + [ + 56, + 62, + 31, + 35, + 59, + 45 + ], + [ + 15, + 38, + 13, + 4, + 63, + 48 + ], + [ + 34, + 15, + 57, + 38, + 13, + 24 + ], + [ + 62, + 36, + 41, + 54, + 46, + 29 + ], + [ + 22, + 53, + 46, + 34, + 30, + 27 + ], + [ + 24, + 10, + 4, + 47, + 18, + 36 + ], + [ + 50, + 57, + 51, + 11, + 49, + 3 + ], + [ + 51, + 7, + 0, + 18, + 11, + 44 + ], + [ + 39, + 37, + 9, + 42, + 40, + 44 + ] + ], + [ + [ + 36, + 25, + 57, + 55, + 47, + 63 + ], + [ + 0, + 2, + 46, + 3, + 51, + 34 + ], + [ + 24, + 2, + 46, + 15, + 33, + 43 + ], + [ + 22, + 31, + 17, + 19, + 10, + 55 + ], + [ + 58, + 59, + 3, + 9, + 40, + 57 + ], + [ + 23, + 31, + 43, + 57, + 2, + 38 + ], + [ + 9, + 62, + 13, + 42, + 47, + 52 + ], + [ + 24, + 7, + 14, + 10, + 46, + 59 + ], + [ + 47, + 18, + 4, + 37, + 0, + 13 + ], + [ + 54, + 4, + 25, + 47, + 36, + 38 + ], + [ + 15, + 47, + 1, + 24, + 58, + 14 + ], + [ + 54, + 5, + 16, + 63, + 14, + 7 + ], + [ + 49, + 3, + 33, + 13, + 46, + 10 + ], + [ + 18, + 10, + 11, + 13, + 63, + 39 + ], + [ + 19, + 62, + 32, + 58, + 10, + 43 + ], + [ + 43, + 48, + 63, + 5, + 55, + 53 + ], + [ + 24, + 51, + 47, + 15, + 59, + 32 + ], + [ + 44, + 0, + 34, + 43, + 3, + 6 + ], + [ + 58, + 38, + 54, + 47, + 11, + 59 + ], + [ + 55, + 27, + 15, + 9, + 42, + 31 + ], + [ + 43, + 41, + 1, + 51, + 5, + 29 + ], + [ + 49, + 27, + 20, + 6, + 4, + 13 + ], + [ + 11, + 25, + 2, + 27, + 54, + 50 + ], + [ + 38, + 44, + 40, + 54, + 33, + 14 + ], + [ + 13, + 63, + 52, + 2, + 8, + 29 + ], + [ + 23, + 41, + 59, + 57, + 38, + 15 + ], + [ + 23, + 6, + 62, + 50, + 51, + 34 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 23 + ], + [ + 51, + 41, + 5, + 15, + 40, + 21 + ], + [ + 43, + 1, + 29, + 55, + 21, + 35 + ], + [ + 24, + 53, + 25, + 51, + 32, + 29 + ], + [ + 41, + 31, + 49, + 57, + 60, + 34 + ], + [ + 17, + 4, + 35, + 30, + 10, + 38 + ], + [ + 34, + 7, + 21, + 9, + 48, + 31 + ], + [ + 14, + 24, + 7, + 46, + 25, + 27 + ], + [ + 47, + 0, + 12, + 6, + 37, + 60 + ], + [ + 4, + 30, + 25, + 47, + 36, + 54 + ], + [ + 24, + 61, + 15, + 47, + 46, + 1 + ], + [ + 5, + 14, + 24, + 16, + 57, + 63 + ], + [ + 49, + 3, + 17, + 26, + 36, + 44 + ], + [ + 13, + 10, + 61, + 0, + 11, + 22 + ], + [ + 10, + 6, + 2, + 49, + 58, + 46 + ], + [ + 19, + 11, + 2, + 25, + 54, + 18 + ], + [ + 10, + 5, + 52, + 24, + 18, + 17 + ], + [ + 44, + 13, + 55, + 31, + 63, + 38 + ], + [ + 58, + 7, + 25, + 32, + 38, + 14 + ], + [ + 49, + 55, + 34, + 13, + 16, + 40 + ], + [ + 51, + 23, + 59, + 35, + 5, + 4 + ], + [ + 57, + 15, + 46, + 27, + 42, + 32 + ], + [ + 33, + 15, + 23, + 52, + 24, + 27 + ], + [ + 4, + 41, + 33, + 10, + 26, + 40 + ], + [ + 8, + 13, + 59, + 4, + 9, + 54 + ], + [ + 19, + 3, + 31, + 27, + 43, + 2 + ], + [ + 46, + 61, + 25, + 8, + 29, + 50 + ] + ], + [ + [ + 48, + 38, + 62, + 63, + 42, + 50 + ], + [ + 3, + 2, + 26, + 17, + 10, + 6 + ], + [ + 39, + 44, + 6, + 45, + 7, + 40 + ], + [ + 5, + 62, + 53, + 50, + 41, + 3 + ], + [ + 6, + 41, + 49, + 37, + 30, + 23 + ], + [ + 59, + 60, + 46, + 4, + 53, + 29 + ], + [ + 44, + 34, + 7, + 15, + 13, + 43 + ], + [ + 12, + 14, + 24, + 25, + 58, + 7 + ], + [ + 33, + 35, + 4, + 37, + 8, + 36 + ], + [ + 40, + 30, + 25, + 36, + 20, + 54 + ], + [ + 9, + 8, + 24, + 25, + 5, + 63 + ], + [ + 59, + 34, + 5, + 24, + 6, + 57 + ], + [ + 17, + 49, + 44, + 26, + 55, + 7 + ], + [ + 47, + 13, + 59, + 27, + 22, + 26 + ], + [ + 49, + 2, + 44, + 10, + 46, + 54 + ], + [ + 34, + 2, + 54, + 55, + 57, + 53 + ], + [ + 5, + 17, + 29, + 31, + 43, + 52 + ], + [ + 48, + 57, + 38, + 63, + 39, + 43 + ], + [ + 42, + 38, + 49, + 32, + 7, + 40 + ], + [ + 59, + 1, + 16, + 23, + 60, + 10 + ], + [ + 21, + 7, + 35, + 53, + 48, + 31 + ], + [ + 8, + 60, + 24, + 42, + 14, + 35 + ], + [ + 33, + 51, + 15, + 28, + 23, + 5 + ], + [ + 4, + 41, + 9, + 11, + 8, + 51 + ], + [ + 60, + 24, + 19, + 48, + 9, + 4 + ], + [ + 19, + 3, + 26, + 58, + 12, + 61 + ], + [ + 35, + 54, + 1, + 60, + 53, + 49 + ] + ], + [ + [ + 21, + 7, + 53, + 56, + 63, + 33 + ], + [ + 3, + 34, + 57, + 16, + 20, + 51 + ], + [ + 55, + 11, + 16, + 60, + 0, + 13 + ], + [ + 62, + 43, + 5, + 50, + 8, + 53 + ], + [ + 35, + 52, + 9, + 43, + 27, + 0 + ], + [ + 16, + 60, + 29, + 61, + 28, + 58 + ], + [ + 31, + 34, + 43, + 30, + 2, + 18 + ], + [ + 36, + 25, + 53, + 24, + 35, + 48 + ], + [ + 48, + 4, + 35, + 32, + 8, + 60 + ], + [ + 25, + 54, + 30, + 10, + 4, + 41 + ], + [ + 9, + 3, + 19, + 20, + 61, + 24 + ], + [ + 30, + 5, + 17, + 45, + 18, + 49 + ], + [ + 49, + 21, + 17, + 26, + 37, + 42 + ], + [ + 38, + 25, + 10, + 51, + 54, + 13 + ], + [ + 2, + 27, + 26, + 10, + 58, + 30 + ], + [ + 9, + 2, + 54, + 21, + 25, + 13 + ], + [ + 22, + 23, + 33, + 27, + 51, + 52 + ], + [ + 30, + 21, + 35, + 55, + 5, + 17 + ], + [ + 38, + 27, + 24, + 56, + 21, + 35 + ], + [ + 53, + 45, + 22, + 28, + 0, + 32 + ], + [ + 40, + 42, + 48, + 37, + 52, + 8 + ], + [ + 46, + 45, + 57, + 36, + 32, + 51 + ], + [ + 62, + 10, + 38, + 42, + 41, + 54 + ], + [ + 60, + 1, + 10, + 31, + 44, + 36 + ], + [ + 29, + 6, + 9, + 8, + 56, + 7 + ], + [ + 9, + 59, + 22, + 0, + 35, + 57 + ], + [ + 45, + 27, + 62, + 47, + 3, + 28 + ] + ], + [ + [ + 27, + 13, + 18, + 8, + 63, + 55 + ], + [ + 36, + 21, + 57, + 8, + 46, + 55 + ], + [ + 43, + 61, + 10, + 13, + 41, + 37 + ], + [ + 43, + 16, + 24, + 6, + 26, + 61 + ], + [ + 60, + 29, + 35, + 31, + 23, + 16 + ], + [ + 9, + 58, + 0, + 60, + 17, + 38 + ], + [ + 63, + 16, + 7, + 13, + 31, + 18 + ], + [ + 16, + 25, + 24, + 2, + 47, + 58 + ], + [ + 8, + 35, + 6, + 14, + 48, + 4 + ], + [ + 30, + 25, + 54, + 4, + 10, + 34 + ], + [ + 20, + 23, + 19, + 7, + 38, + 61 + ], + [ + 5, + 3, + 24, + 32, + 12, + 42 + ], + [ + 49, + 17, + 10, + 26, + 32, + 60 + ], + [ + 13, + 10, + 29, + 22, + 58, + 54 + ], + [ + 44, + 8, + 2, + 58, + 6, + 5 + ], + [ + 25, + 6, + 2, + 54, + 19, + 53 + ], + [ + 61, + 5, + 30, + 17, + 1, + 27 + ], + [ + 22, + 55, + 63, + 57, + 19, + 33 + ], + [ + 35, + 46, + 6, + 32, + 14, + 7 + ], + [ + 45, + 13, + 51, + 14, + 7, + 5 + ], + [ + 15, + 23, + 50, + 51, + 13, + 59 + ], + [ + 57, + 36, + 53, + 61, + 6, + 14 + ], + [ + 10, + 23, + 15, + 33, + 27, + 38 + ], + [ + 4, + 10, + 33, + 41, + 26, + 36 + ], + [ + 8, + 4, + 33, + 9, + 47, + 48 + ], + [ + 3, + 43, + 6, + 19, + 2, + 30 + ], + [ + 55, + 4, + 46, + 40, + 18, + 20 + ] + ], + [ + [ + 48, + 38, + 63, + 37, + 7, + 42 + ], + [ + 3, + 26, + 10, + 35, + 6, + 2 + ], + [ + 39, + 44, + 7, + 45, + 6, + 40 + ], + [ + 56, + 23, + 61, + 16, + 33, + 41 + ], + [ + 6, + 37, + 44, + 16, + 30, + 26 + ], + [ + 59, + 17, + 60, + 19, + 5, + 52 + ], + [ + 44, + 55, + 33, + 63, + 13, + 31 + ], + [ + 12, + 25, + 63, + 2, + 24, + 54 + ], + [ + 33, + 8, + 42, + 4, + 35, + 41 + ], + [ + 30, + 40, + 25, + 21, + 48, + 51 + ], + [ + 8, + 23, + 34, + 5, + 19, + 22 + ], + [ + 34, + 59, + 42, + 11, + 5, + 3 + ], + [ + 10, + 26, + 50, + 55, + 53, + 59 + ], + [ + 59, + 54, + 13, + 10, + 47, + 25 + ], + [ + 24, + 2, + 49, + 46, + 38, + 8 + ], + [ + 34, + 2, + 53, + 54, + 40, + 57 + ], + [ + 17, + 29, + 5, + 27, + 18, + 43 + ], + [ + 48, + 57, + 63, + 39, + 38, + 32 + ], + [ + 42, + 49, + 7, + 32, + 38, + 61 + ], + [ + 1, + 59, + 23, + 14, + 16, + 10 + ], + [ + 21, + 7, + 35, + 53, + 13, + 11 + ], + [ + 8, + 60, + 24, + 25, + 42, + 14 + ], + [ + 33, + 15, + 5, + 51, + 28, + 23 + ], + [ + 4, + 9, + 51, + 8, + 41, + 39 + ], + [ + 60, + 4, + 19, + 48, + 9, + 24 + ], + [ + 19, + 3, + 26, + 12, + 21, + 53 + ], + [ + 53, + 35, + 1, + 54, + 60, + 20 + ] + ], + [ + [ + 17, + 37, + 31, + 32, + 63, + 50 + ], + [ + 12, + 2, + 9, + 32, + 47, + 17 + ], + [ + 3, + 57, + 56, + 50, + 33, + 38 + ], + [ + 43, + 42, + 19, + 52, + 8, + 17 + ], + [ + 61, + 39, + 27, + 12, + 15, + 57 + ], + [ + 6, + 33, + 36, + 44, + 29, + 61 + ], + [ + 2, + 41, + 42, + 15, + 52, + 5 + ], + [ + 36, + 37, + 15, + 53, + 18, + 62 + ], + [ + 50, + 58, + 3, + 5, + 16, + 4 + ], + [ + 16, + 22, + 55, + 32, + 41, + 26 + ], + [ + 3, + 41, + 56, + 45, + 38, + 10 + ], + [ + 21, + 53, + 50, + 34, + 38, + 35 + ], + [ + 35, + 11, + 37, + 16, + 53, + 42 + ], + [ + 9, + 15, + 54, + 38, + 12, + 52 + ], + [ + 56, + 2, + 23, + 24, + 9, + 5 + ], + [ + 2, + 44, + 9, + 53, + 35, + 54 + ], + [ + 27, + 46, + 1, + 5, + 60, + 50 + ], + [ + 51, + 61, + 50, + 54, + 33, + 44 + ], + [ + 2, + 38, + 19, + 9, + 5, + 32 + ], + [ + 32, + 28, + 6, + 15, + 0, + 33 + ], + [ + 26, + 53, + 27, + 13, + 5, + 4 + ], + [ + 34, + 40, + 45, + 55, + 62, + 53 + ], + [ + 41, + 5, + 46, + 4, + 45, + 3 + ], + [ + 20, + 4, + 22, + 47, + 59, + 58 + ], + [ + 62, + 36, + 25, + 28, + 53, + 33 + ], + [ + 22, + 33, + 7, + 3, + 54, + 0 + ], + [ + 58, + 27, + 52, + 48, + 45, + 17 + ] + ], + [ + [ + 3, + 43, + 17, + 42, + 4, + 35 + ], + [ + 0, + 42, + 36, + 34, + 32, + 24 + ], + [ + 43, + 12, + 19, + 20, + 2, + 29 + ], + [ + 28, + 4, + 54, + 22, + 58, + 23 + ], + [ + 12, + 60, + 52, + 57, + 46, + 7 + ], + [ + 43, + 2, + 1, + 63, + 20, + 11 + ], + [ + 42, + 19, + 57, + 31, + 15, + 21 + ], + [ + 35, + 6, + 46, + 25, + 36, + 33 + ], + [ + 11, + 4, + 35, + 50, + 32, + 52 + ], + [ + 21, + 25, + 38, + 47, + 31, + 58 + ], + [ + 3, + 2, + 22, + 45, + 42, + 15 + ], + [ + 2, + 21, + 5, + 7, + 43, + 38 + ], + [ + 33, + 10, + 41, + 42, + 60, + 16 + ], + [ + 44, + 56, + 30, + 54, + 9, + 43 + ], + [ + 9, + 5, + 33, + 28, + 58, + 49 + ], + [ + 8, + 53, + 59, + 54, + 31, + 2 + ], + [ + 47, + 0, + 48, + 27, + 24, + 51 + ], + [ + 0, + 60, + 21, + 12, + 56, + 28 + ], + [ + 38, + 13, + 53, + 51, + 9, + 7 + ], + [ + 19, + 28, + 56, + 17, + 21, + 26 + ], + [ + 12, + 43, + 26, + 4, + 0, + 5 + ], + [ + 43, + 1, + 12, + 55, + 4, + 40 + ], + [ + 41, + 5, + 21, + 45, + 25, + 12 + ], + [ + 4, + 20, + 1, + 22, + 30, + 37 + ], + [ + 5, + 62, + 25, + 54, + 48, + 47 + ], + [ + 10, + 28, + 55, + 23, + 0, + 53 + ], + [ + 30, + 57, + 8, + 5, + 17, + 29 + ] + ], + [ + [ + 11, + 16, + 31, + 0, + 35, + 46 + ], + [ + 13, + 49, + 31, + 50, + 16, + 19 + ], + [ + 36, + 13, + 53, + 27, + 4, + 18 + ], + [ + 24, + 7, + 29, + 26, + 12, + 32 + ], + [ + 17, + 35, + 2, + 44, + 10, + 48 + ], + [ + 8, + 7, + 23, + 5, + 51, + 26 + ], + [ + 58, + 15, + 61, + 29, + 38, + 62 + ], + [ + 20, + 9, + 42, + 35, + 3, + 6 + ], + [ + 4, + 47, + 25, + 11, + 1, + 52 + ], + [ + 54, + 25, + 55, + 38, + 21, + 27 + ], + [ + 1, + 60, + 14, + 59, + 22, + 30 + ], + [ + 51, + 31, + 5, + 25, + 14, + 52 + ], + [ + 34, + 2, + 10, + 26, + 52, + 47 + ], + [ + 40, + 57, + 13, + 54, + 9, + 6 + ], + [ + 8, + 44, + 58, + 5, + 16, + 1 + ], + [ + 8, + 53, + 59, + 25, + 24, + 52 + ], + [ + 13, + 47, + 45, + 0, + 42, + 8 + ], + [ + 23, + 44, + 55, + 33, + 38, + 7 + ], + [ + 53, + 38, + 11, + 1, + 8, + 24 + ], + [ + 15, + 38, + 14, + 28, + 0, + 19 + ], + [ + 41, + 27, + 8, + 42, + 57, + 40 + ], + [ + 11, + 62, + 55, + 16, + 10, + 41 + ], + [ + 31, + 20, + 46, + 37, + 34, + 41 + ], + [ + 47, + 4, + 55, + 33, + 49, + 22 + ], + [ + 11, + 45, + 47, + 48, + 54, + 36 + ], + [ + 50, + 10, + 23, + 51, + 18, + 2 + ], + [ + 17, + 31, + 4, + 5, + 36, + 20 + ] + ], + [ + [ + 22, + 6, + 39, + 57, + 29, + 28 + ], + [ + 27, + 6, + 14, + 17, + 51, + 55 + ], + [ + 1, + 11, + 29, + 26, + 47, + 4 + ], + [ + 14, + 38, + 31, + 22, + 29, + 6 + ], + [ + 14, + 59, + 61, + 16, + 1, + 19 + ], + [ + 30, + 8, + 23, + 21, + 47, + 1 + ], + [ + 58, + 4, + 15, + 61, + 27, + 31 + ], + [ + 20, + 42, + 3, + 9, + 35, + 6 + ], + [ + 47, + 4, + 25, + 8, + 36, + 0 + ], + [ + 54, + 55, + 21, + 19, + 33, + 25 + ], + [ + 14, + 4, + 60, + 20, + 24, + 40 + ], + [ + 51, + 0, + 5, + 32, + 52, + 3 + ], + [ + 2, + 36, + 10, + 52, + 26, + 32 + ], + [ + 40, + 13, + 54, + 36, + 57, + 46 + ], + [ + 44, + 8, + 5, + 37, + 58, + 2 + ], + [ + 6, + 53, + 24, + 2, + 54, + 37 + ], + [ + 13, + 47, + 61, + 5, + 19, + 17 + ], + [ + 58, + 55, + 44, + 38, + 63, + 6 + ], + [ + 35, + 46, + 31, + 1, + 19, + 32 + ], + [ + 15, + 13, + 63, + 45, + 9, + 55 + ], + [ + 27, + 15, + 23, + 6, + 35, + 63 + ], + [ + 36, + 62, + 57, + 41, + 10, + 16 + ], + [ + 10, + 33, + 41, + 20, + 5, + 46 + ], + [ + 47, + 4, + 26, + 55, + 10, + 49 + ], + [ + 4, + 11, + 8, + 48, + 36, + 33 + ], + [ + 18, + 51, + 43, + 33, + 50, + 3 + ], + [ + 4, + 55, + 9, + 36, + 43, + 5 + ] + ], + [ + [ + 48, + 38, + 47, + 63, + 7, + 42 + ], + [ + 3, + 26, + 10, + 6, + 2, + 12 + ], + [ + 39, + 44, + 6, + 7, + 45, + 8 + ], + [ + 60, + 31, + 22, + 0, + 54, + 45 + ], + [ + 6, + 59, + 14, + 16, + 37, + 19 + ], + [ + 59, + 30, + 8, + 47, + 60, + 17 + ], + [ + 44, + 56, + 4, + 13, + 15, + 9 + ], + [ + 12, + 24, + 20, + 58, + 61, + 28 + ], + [ + 47, + 33, + 8, + 4, + 36, + 42 + ], + [ + 40, + 54, + 55, + 33, + 21, + 51 + ], + [ + 8, + 14, + 22, + 39, + 31, + 7 + ], + [ + 34, + 59, + 14, + 5, + 52, + 0 + ], + [ + 36, + 10, + 52, + 26, + 44, + 59 + ], + [ + 59, + 44, + 13, + 1, + 36, + 22 + ], + [ + 24, + 5, + 46, + 44, + 2, + 38 + ], + [ + 34, + 53, + 6, + 40, + 30, + 2 + ], + [ + 17, + 29, + 50, + 5, + 47, + 27 + ], + [ + 48, + 38, + 57, + 63, + 32, + 60 + ], + [ + 42, + 7, + 49, + 46, + 32, + 1 + ], + [ + 59, + 1, + 23, + 14, + 10, + 16 + ], + [ + 21, + 35, + 7, + 53, + 6, + 17 + ], + [ + 8, + 24, + 60, + 13, + 55, + 14 + ], + [ + 33, + 5, + 51, + 15, + 3, + 23 + ], + [ + 4, + 9, + 51, + 26, + 41, + 28 + ], + [ + 60, + 4, + 19, + 47, + 40, + 48 + ], + [ + 19, + 3, + 26, + 21, + 53, + 8 + ], + [ + 53, + 1, + 35, + 54, + 60, + 20 + ] + ], + [ + [ + 37, + 46, + 39, + 54, + 27, + 55 + ], + [ + 34, + 5, + 16, + 47, + 6, + 42 + ], + [ + 32, + 38, + 16, + 42, + 3, + 20 + ], + [ + 43, + 19, + 60, + 42, + 52, + 11 + ], + [ + 7, + 61, + 39, + 57, + 12, + 46 + ], + [ + 50, + 36, + 44, + 24, + 28, + 31 + ], + [ + 15, + 42, + 41, + 2, + 40, + 32 + ], + [ + 36, + 35, + 38, + 53, + 58, + 51 + ], + [ + 50, + 4, + 16, + 3, + 36, + 58 + ], + [ + 59, + 16, + 32, + 38, + 26, + 56 + ], + [ + 3, + 45, + 37, + 41, + 21, + 33 + ], + [ + 21, + 53, + 34, + 35, + 5, + 39 + ], + [ + 35, + 42, + 63, + 16, + 11, + 46 + ], + [ + 23, + 9, + 51, + 54, + 15, + 38 + ], + [ + 2, + 5, + 20, + 43, + 24, + 29 + ], + [ + 53, + 44, + 9, + 2, + 54, + 20 + ], + [ + 27, + 46, + 1, + 47, + 50, + 5 + ], + [ + 61, + 54, + 57, + 44, + 51, + 43 + ], + [ + 19, + 9, + 33, + 61, + 38, + 37 + ], + [ + 0, + 33, + 15, + 32, + 6, + 9 + ], + [ + 26, + 27, + 53, + 5, + 47, + 54 + ], + [ + 55, + 34, + 12, + 62, + 3, + 4 + ], + [ + 41, + 5, + 46, + 40, + 4, + 32 + ], + [ + 55, + 20, + 44, + 26, + 4, + 40 + ], + [ + 62, + 12, + 28, + 34, + 23, + 33 + ], + [ + 18, + 7, + 22, + 3, + 54, + 14 + ], + [ + 36, + 9, + 27, + 52, + 48, + 11 + ] + ], + [ + [ + 46, + 37, + 61, + 18, + 36, + 63 + ], + [ + 22, + 34, + 28, + 59, + 24, + 56 + ], + [ + 32, + 15, + 17, + 60, + 38, + 20 + ], + [ + 28, + 4, + 58, + 16, + 30, + 35 + ], + [ + 7, + 36, + 9, + 57, + 33, + 23 + ], + [ + 43, + 63, + 2, + 30, + 11, + 19 + ], + [ + 19, + 42, + 57, + 15, + 3, + 22 + ], + [ + 35, + 46, + 47, + 6, + 58, + 8 + ], + [ + 32, + 4, + 37, + 36, + 35, + 57 + ], + [ + 21, + 47, + 55, + 58, + 38, + 54 + ], + [ + 3, + 42, + 2, + 38, + 5, + 33 + ], + [ + 43, + 18, + 2, + 21, + 5, + 49 + ], + [ + 10, + 49, + 24, + 32, + 25, + 36 + ], + [ + 30, + 21, + 63, + 49, + 16, + 51 + ], + [ + 62, + 2, + 13, + 5, + 29, + 56 + ], + [ + 53, + 2, + 9, + 63, + 46, + 54 + ], + [ + 27, + 0, + 63, + 47, + 5, + 14 + ], + [ + 60, + 57, + 43, + 44, + 0, + 12 + ], + [ + 38, + 9, + 61, + 33, + 60, + 13 + ], + [ + 25, + 0, + 15, + 34, + 35, + 11 + ], + [ + 26, + 54, + 28, + 47, + 53, + 37 + ], + [ + 55, + 1, + 34, + 4, + 21, + 50 + ], + [ + 41, + 5, + 40, + 21, + 17, + 23 + ], + [ + 25, + 43, + 52, + 26, + 4, + 55 + ], + [ + 52, + 29, + 35, + 17, + 45, + 60 + ], + [ + 38, + 10, + 15, + 7, + 50, + 3 + ], + [ + 23, + 6, + 19, + 56, + 41, + 15 + ] + ], + [ + [ + 44, + 14, + 20, + 47, + 19, + 56 + ], + [ + 28, + 34, + 2, + 56, + 0, + 11 + ], + [ + 46, + 15, + 61, + 14, + 22, + 60 + ], + [ + 15, + 50, + 36, + 47, + 25, + 21 + ], + [ + 25, + 12, + 13, + 36, + 23, + 57 + ], + [ + 22, + 43, + 1, + 37, + 36, + 30 + ], + [ + 28, + 42, + 19, + 31, + 14, + 21 + ], + [ + 35, + 51, + 47, + 33, + 7, + 46 + ], + [ + 32, + 4, + 16, + 11, + 19, + 35 + ], + [ + 21, + 38, + 16, + 47, + 62, + 15 + ], + [ + 3, + 45, + 39, + 50, + 21, + 2 + ], + [ + 21, + 5, + 56, + 43, + 53, + 52 + ], + [ + 10, + 41, + 33, + 11, + 63, + 37 + ], + [ + 56, + 16, + 9, + 63, + 33, + 45 + ], + [ + 58, + 52, + 2, + 5, + 30, + 56 + ], + [ + 53, + 8, + 59, + 55, + 2, + 50 + ], + [ + 0, + 47, + 51, + 45, + 37, + 14 + ], + [ + 0, + 43, + 21, + 12, + 60, + 62 + ], + [ + 38, + 53, + 60, + 36, + 34, + 9 + ], + [ + 27, + 9, + 56, + 0, + 53, + 11 + ], + [ + 28, + 12, + 43, + 54, + 62, + 5 + ], + [ + 50, + 55, + 34, + 16, + 4, + 21 + ], + [ + 4, + 21, + 5, + 40, + 32, + 54 + ], + [ + 25, + 57, + 49, + 1, + 44, + 43 + ], + [ + 5, + 35, + 25, + 42, + 2, + 22 + ], + [ + 28, + 15, + 38, + 55, + 35, + 37 + ], + [ + 19, + 30, + 6, + 57, + 39, + 33 + ] + ], + [ + [ + 11, + 46, + 0, + 49, + 31, + 16 + ], + [ + 13, + 49, + 50, + 16, + 31, + 19 + ], + [ + 36, + 13, + 27, + 34, + 4, + 52 + ], + [ + 24, + 32, + 7, + 59, + 13, + 15 + ], + [ + 17, + 35, + 2, + 44, + 10, + 63 + ], + [ + 8, + 7, + 23, + 26, + 56, + 42 + ], + [ + 58, + 61, + 29, + 38, + 62, + 50 + ], + [ + 20, + 42, + 3, + 35, + 61, + 47 + ], + [ + 4, + 10, + 7, + 47, + 25, + 34 + ], + [ + 54, + 55, + 18, + 38, + 27, + 28 + ], + [ + 60, + 1, + 14, + 59, + 3, + 30 + ], + [ + 51, + 31, + 5, + 21, + 25, + 52 + ], + [ + 34, + 10, + 2, + 26, + 11, + 47 + ], + [ + 16, + 40, + 23, + 34, + 9, + 48 + ], + [ + 34, + 8, + 1, + 31, + 5, + 32 + ], + [ + 25, + 59, + 8, + 52, + 53, + 27 + ], + [ + 13, + 47, + 45, + 48, + 42, + 0 + ], + [ + 4, + 23, + 53, + 10, + 3, + 25 + ], + [ + 53, + 8, + 38, + 11, + 47, + 24 + ], + [ + 38, + 15, + 0, + 21, + 11, + 8 + ], + [ + 27, + 23, + 41, + 62, + 8, + 42 + ], + [ + 55, + 11, + 10, + 41, + 16, + 21 + ], + [ + 31, + 21, + 41, + 54, + 34, + 9 + ], + [ + 55, + 25, + 22, + 33, + 44, + 47 + ], + [ + 47, + 45, + 20, + 35, + 12, + 42 + ], + [ + 50, + 10, + 47, + 35, + 53, + 57 + ], + [ + 17, + 31, + 5, + 37, + 36, + 20 + ] + ], + [ + [ + 22, + 6, + 29, + 39, + 44, + 57 + ], + [ + 27, + 6, + 17, + 14, + 55, + 57 + ], + [ + 11, + 1, + 29, + 26, + 47, + 4 + ], + [ + 14, + 38, + 31, + 22, + 36, + 29 + ], + [ + 14, + 61, + 59, + 44, + 16, + 63 + ], + [ + 30, + 8, + 47, + 23, + 21, + 0 + ], + [ + 58, + 4, + 54, + 61, + 15, + 62 + ], + [ + 20, + 42, + 58, + 3, + 35, + 0 + ], + [ + 10, + 47, + 4, + 8, + 41, + 19 + ], + [ + 54, + 55, + 33, + 11, + 38, + 21 + ], + [ + 14, + 1, + 4, + 60, + 20, + 40 + ], + [ + 51, + 5, + 14, + 0, + 32, + 62 + ], + [ + 2, + 10, + 36, + 49, + 32, + 52 + ], + [ + 13, + 40, + 54, + 36, + 22, + 11 + ], + [ + 2, + 44, + 37, + 58, + 5, + 8 + ], + [ + 6, + 24, + 2, + 53, + 19, + 52 + ], + [ + 13, + 47, + 61, + 50, + 5, + 30 + ], + [ + 55, + 58, + 4, + 63, + 22, + 38 + ], + [ + 35, + 32, + 46, + 14, + 7, + 58 + ], + [ + 15, + 13, + 45, + 0, + 51, + 60 + ], + [ + 15, + 23, + 27, + 50, + 6, + 17 + ], + [ + 57, + 36, + 10, + 55, + 14, + 16 + ], + [ + 10, + 23, + 41, + 33, + 30, + 54 + ], + [ + 33, + 26, + 4, + 10, + 49, + 55 + ], + [ + 4, + 8, + 33, + 9, + 20, + 12 + ], + [ + 43, + 6, + 3, + 50, + 18, + 38 + ], + [ + 4, + 55, + 43, + 36, + 46, + 5 + ] + ], + [ + [ + 48, + 38, + 47, + 63, + 7, + 42 + ], + [ + 3, + 26, + 10, + 2, + 6, + 35 + ], + [ + 39, + 44, + 6, + 7, + 45, + 8 + ], + [ + 60, + 31, + 22, + 0, + 54, + 27 + ], + [ + 6, + 59, + 16, + 14, + 37, + 44 + ], + [ + 59, + 8, + 30, + 47, + 48, + 3 + ], + [ + 44, + 54, + 4, + 56, + 15, + 13 + ], + [ + 12, + 24, + 20, + 58, + 31, + 61 + ], + [ + 47, + 33, + 10, + 8, + 4, + 36 + ], + [ + 54, + 40, + 55, + 33, + 11, + 51 + ], + [ + 8, + 14, + 23, + 29, + 22, + 31 + ], + [ + 34, + 59, + 14, + 5, + 57, + 50 + ], + [ + 44, + 36, + 10, + 59, + 52, + 9 + ], + [ + 59, + 44, + 13, + 9, + 1, + 54 + ], + [ + 24, + 2, + 46, + 44, + 38, + 47 + ], + [ + 34, + 53, + 6, + 2, + 57, + 40 + ], + [ + 17, + 29, + 47, + 5, + 50, + 34 + ], + [ + 48, + 63, + 57, + 38, + 60, + 32 + ], + [ + 42, + 7, + 49, + 32, + 46, + 1 + ], + [ + 1, + 59, + 23, + 14, + 16, + 10 + ], + [ + 21, + 35, + 7, + 53, + 6, + 17 + ], + [ + 8, + 24, + 60, + 14, + 13, + 55 + ], + [ + 33, + 5, + 51, + 15, + 23, + 3 + ], + [ + 9, + 4, + 51, + 26, + 41, + 8 + ], + [ + 4, + 60, + 19, + 47, + 40, + 9 + ], + [ + 19, + 3, + 26, + 21, + 17, + 8 + ], + [ + 1, + 35, + 53, + 54, + 60, + 20 + ] + ], + [ + [ + 12, + 41, + 14, + 62, + 24, + 10 + ], + [ + 10, + 53, + 39, + 35, + 41, + 58 + ], + [ + 33, + 32, + 50, + 31, + 3, + 34 + ], + [ + 43, + 10, + 42, + 11, + 17, + 47 + ], + [ + 42, + 12, + 11, + 19, + 58, + 54 + ], + [ + 36, + 50, + 55, + 61, + 25, + 56 + ], + [ + 41, + 22, + 16, + 52, + 2, + 15 + ], + [ + 26, + 36, + 62, + 53, + 15, + 51 + ], + [ + 58, + 16, + 5, + 53, + 3, + 49 + ], + [ + 32, + 46, + 26, + 45, + 16, + 62 + ], + [ + 41, + 45, + 56, + 49, + 11, + 3 + ], + [ + 52, + 34, + 35, + 50, + 21, + 53 + ], + [ + 59, + 53, + 46, + 30, + 39, + 37 + ], + [ + 20, + 9, + 52, + 2, + 7, + 33 + ], + [ + 20, + 50, + 24, + 29, + 23, + 2 + ], + [ + 53, + 2, + 44, + 41, + 9, + 13 + ], + [ + 47, + 27, + 1, + 5, + 45, + 46 + ], + [ + 61, + 7, + 51, + 30, + 35, + 9 + ], + [ + 19, + 34, + 32, + 17, + 2, + 14 + ], + [ + 15, + 32, + 6, + 45, + 9, + 11 + ], + [ + 35, + 27, + 6, + 53, + 13, + 60 + ], + [ + 26, + 62, + 16, + 28, + 41, + 3 + ], + [ + 5, + 20, + 46, + 37, + 11, + 55 + ], + [ + 55, + 47, + 4, + 16, + 14, + 27 + ], + [ + 36, + 11, + 27, + 62, + 33, + 7 + ], + [ + 33, + 7, + 54, + 3, + 32, + 12 + ], + [ + 58, + 52, + 27, + 26, + 48, + 38 + ] + ], + [ + [ + 6, + 52, + 19, + 63, + 38, + 46 + ], + [ + 8, + 42, + 4, + 47, + 57, + 56 + ], + [ + 31, + 46, + 32, + 4, + 14, + 10 + ], + [ + 28, + 27, + 4, + 37, + 58, + 20 + ], + [ + 57, + 59, + 60, + 62, + 22, + 14 + ], + [ + 30, + 9, + 2, + 57, + 11, + 13 + ], + [ + 20, + 19, + 57, + 42, + 51, + 27 + ], + [ + 46, + 35, + 47, + 32, + 7, + 0 + ], + [ + 12, + 4, + 35, + 10, + 47, + 50 + ], + [ + 15, + 47, + 54, + 25, + 38, + 51 + ], + [ + 18, + 42, + 5, + 15, + 38, + 61 + ], + [ + 18, + 22, + 8, + 16, + 5, + 7 + ], + [ + 5, + 3, + 24, + 35, + 4, + 30 + ], + [ + 17, + 13, + 55, + 20, + 36, + 22 + ], + [ + 22, + 10, + 21, + 54, + 6, + 47 + ], + [ + 30, + 53, + 6, + 19, + 2, + 54 + ], + [ + 22, + 26, + 5, + 7, + 47, + 21 + ], + [ + 4, + 41, + 13, + 46, + 55, + 43 + ], + [ + 17, + 56, + 32, + 45, + 14, + 6 + ], + [ + 5, + 49, + 53, + 28, + 34, + 60 + ], + [ + 42, + 55, + 57, + 17, + 28, + 22 + ], + [ + 40, + 23, + 28, + 57, + 21, + 16 + ], + [ + 51, + 21, + 35, + 24, + 44, + 10 + ], + [ + 38, + 25, + 4, + 14, + 62, + 31 + ], + [ + 31, + 50, + 13, + 56, + 39, + 53 + ], + [ + 6, + 51, + 55, + 8, + 0, + 21 + ], + [ + 29, + 46, + 18, + 55, + 37, + 50 + ] + ], + [ + [ + 46, + 31, + 35, + 49, + 14, + 21 + ], + [ + 13, + 16, + 50, + 31, + 33, + 8 + ], + [ + 13, + 36, + 27, + 52, + 3, + 19 + ], + [ + 9, + 24, + 29, + 12, + 55, + 61 + ], + [ + 35, + 17, + 2, + 10, + 41, + 48 + ], + [ + 51, + 23, + 30, + 57, + 52, + 17 + ], + [ + 34, + 48, + 0, + 38, + 27, + 51 + ], + [ + 47, + 32, + 2, + 35, + 58, + 55 + ], + [ + 15, + 12, + 35, + 45, + 52, + 4 + ], + [ + 36, + 15, + 4, + 59, + 11, + 54 + ], + [ + 60, + 5, + 59, + 29, + 50, + 18 + ], + [ + 18, + 8, + 31, + 23, + 24, + 3 + ], + [ + 34, + 4, + 5, + 58, + 14, + 27 + ], + [ + 17, + 57, + 36, + 55, + 51, + 6 + ], + [ + 10, + 45, + 8, + 54, + 22, + 47 + ], + [ + 30, + 50, + 14, + 6, + 16, + 37 + ], + [ + 26, + 0, + 22, + 5, + 13, + 4 + ], + [ + 4, + 46, + 23, + 43, + 60, + 41 + ], + [ + 45, + 8, + 56, + 62, + 17, + 51 + ], + [ + 4, + 38, + 53, + 15, + 58, + 5 + ], + [ + 57, + 24, + 34, + 15, + 55, + 42 + ], + [ + 23, + 37, + 40, + 41, + 11, + 54 + ], + [ + 48, + 29, + 51, + 31, + 34, + 9 + ], + [ + 62, + 14, + 18, + 31, + 4, + 59 + ], + [ + 50, + 31, + 49, + 45, + 56, + 57 + ], + [ + 51, + 50, + 56, + 8, + 12, + 52 + ], + [ + 17, + 37, + 5, + 39, + 44, + 20 + ] + ], + [ + [ + 45, + 13, + 63, + 37, + 38, + 56 + ], + [ + 63, + 6, + 12, + 18, + 27, + 51 + ], + [ + 3, + 21, + 4, + 48, + 17, + 27 + ], + [ + 14, + 55, + 9, + 37, + 29, + 26 + ], + [ + 35, + 57, + 2, + 13, + 41, + 10 + ], + [ + 39, + 30, + 57, + 13, + 53, + 23 + ], + [ + 34, + 48, + 38, + 27, + 56, + 51 + ], + [ + 47, + 35, + 46, + 2, + 32, + 7 + ], + [ + 12, + 56, + 4, + 35, + 52, + 50 + ], + [ + 15, + 36, + 14, + 30, + 47, + 48 + ], + [ + 5, + 60, + 59, + 22, + 15, + 46 + ], + [ + 8, + 22, + 23, + 47, + 5, + 26 + ], + [ + 4, + 5, + 30, + 58, + 26, + 8 + ], + [ + 17, + 29, + 13, + 50, + 14, + 6 + ], + [ + 54, + 10, + 41, + 14, + 21, + 5 + ], + [ + 50, + 6, + 54, + 30, + 16, + 56 + ], + [ + 0, + 47, + 7, + 4, + 5, + 34 + ], + [ + 4, + 46, + 23, + 41, + 59, + 55 + ], + [ + 62, + 45, + 1, + 47, + 57, + 32 + ], + [ + 4, + 53, + 15, + 50, + 60, + 1 + ], + [ + 34, + 57, + 24, + 31, + 15, + 53 + ], + [ + 23, + 41, + 45, + 12, + 57, + 14 + ], + [ + 51, + 29, + 49, + 48, + 18, + 33 + ], + [ + 14, + 4, + 62, + 26, + 18, + 28 + ], + [ + 50, + 49, + 7, + 24, + 9, + 48 + ], + [ + 56, + 20, + 51, + 3, + 14, + 26 + ], + [ + 55, + 37, + 50, + 14, + 42, + 20 + ] + ], + [ + [ + 51, + 43, + 27, + 30, + 5, + 12 + ], + [ + 24, + 16, + 48, + 15, + 7, + 30 + ], + [ + 26, + 21, + 50, + 52, + 4, + 56 + ], + [ + 19, + 17, + 2, + 14, + 57, + 22 + ], + [ + 3, + 35, + 37, + 45, + 1, + 6 + ], + [ + 13, + 30, + 23, + 39, + 57, + 60 + ], + [ + 9, + 34, + 48, + 17, + 27, + 26 + ], + [ + 47, + 32, + 59, + 2, + 28, + 57 + ], + [ + 18, + 12, + 31, + 20, + 52, + 4 + ], + [ + 14, + 15, + 36, + 30, + 58, + 31 + ], + [ + 5, + 4, + 26, + 19, + 22, + 59 + ], + [ + 22, + 8, + 34, + 26, + 42, + 52 + ], + [ + 20, + 5, + 52, + 4, + 30, + 33 + ], + [ + 34, + 17, + 2, + 5, + 39, + 20 + ], + [ + 54, + 61, + 30, + 14, + 20, + 25 + ], + [ + 56, + 41, + 2, + 5, + 54, + 37 + ], + [ + 12, + 34, + 47, + 7, + 54, + 59 + ], + [ + 9, + 11, + 2, + 43, + 33, + 50 + ], + [ + 8, + 10, + 19, + 20, + 32, + 18 + ], + [ + 17, + 38, + 6, + 29, + 49, + 41 + ], + [ + 38, + 52, + 45, + 57, + 63, + 27 + ], + [ + 26, + 28, + 3, + 40, + 5, + 47 + ], + [ + 60, + 49, + 35, + 34, + 38, + 3 + ], + [ + 61, + 24, + 4, + 55, + 45, + 16 + ], + [ + 46, + 11, + 27, + 0, + 56, + 48 + ], + [ + 60, + 45, + 44, + 25, + 32, + 3 + ], + [ + 11, + 38, + 52, + 48, + 9, + 21 + ] + ], + [ + [ + 22, + 19, + 46, + 31, + 3, + 23 + ], + [ + 32, + 62, + 15, + 54, + 10, + 55 + ], + [ + 47, + 30, + 38, + 5, + 7, + 60 + ], + [ + 15, + 13, + 1, + 8, + 25, + 43 + ], + [ + 13, + 59, + 5, + 6, + 62, + 52 + ], + [ + 27, + 63, + 62, + 45, + 12, + 56 + ], + [ + 9, + 50, + 8, + 51, + 48, + 18 + ], + [ + 59, + 57, + 2, + 28, + 61, + 6 + ], + [ + 18, + 59, + 6, + 52, + 39, + 57 + ], + [ + 14, + 23, + 11, + 36, + 15, + 32 + ], + [ + 26, + 5, + 42, + 25, + 22, + 23 + ], + [ + 33, + 22, + 55, + 28, + 16, + 24 + ], + [ + 6, + 20, + 33, + 14, + 15, + 52 + ], + [ + 17, + 6, + 5, + 39, + 2, + 34 + ], + [ + 54, + 62, + 25, + 61, + 21, + 14 + ], + [ + 41, + 45, + 14, + 5, + 2, + 54 + ], + [ + 47, + 34, + 10, + 31, + 5, + 41 + ], + [ + 9, + 2, + 36, + 6, + 43, + 38 + ], + [ + 57, + 20, + 41, + 10, + 32, + 18 + ], + [ + 47, + 6, + 49, + 15, + 34, + 7 + ], + [ + 3, + 57, + 44, + 38, + 50, + 53 + ], + [ + 47, + 28, + 57, + 26, + 19, + 22 + ], + [ + 11, + 1, + 18, + 5, + 46, + 22 + ], + [ + 55, + 9, + 4, + 3, + 25, + 10 + ], + [ + 45, + 11, + 0, + 63, + 48, + 57 + ], + [ + 42, + 10, + 3, + 43, + 49, + 0 + ], + [ + 38, + 15, + 17, + 41, + 10, + 3 + ] + ], + [ + [ + 19, + 18, + 51, + 25, + 60, + 55 + ], + [ + 56, + 27, + 61, + 42, + 55, + 23 + ], + [ + 32, + 39, + 37, + 46, + 20, + 52 + ], + [ + 41, + 21, + 37, + 13, + 57, + 2 + ], + [ + 10, + 9, + 3, + 46, + 58, + 32 + ], + [ + 62, + 57, + 27, + 43, + 2, + 53 + ], + [ + 50, + 21, + 19, + 48, + 15, + 6 + ], + [ + 14, + 21, + 54, + 8, + 57, + 28 + ], + [ + 6, + 18, + 4, + 52, + 39, + 24 + ], + [ + 32, + 38, + 23, + 53, + 25, + 17 + ], + [ + 26, + 22, + 25, + 5, + 42, + 33 + ], + [ + 5, + 22, + 16, + 9, + 61, + 55 + ], + [ + 49, + 33, + 30, + 25, + 20, + 22 + ], + [ + 18, + 17, + 25, + 63, + 39, + 11 + ], + [ + 62, + 54, + 10, + 5, + 58, + 37 + ], + [ + 42, + 41, + 5, + 24, + 37, + 54 + ], + [ + 47, + 34, + 27, + 10, + 22, + 55 + ], + [ + 60, + 43, + 44, + 7, + 52, + 37 + ], + [ + 12, + 32, + 39, + 38, + 1, + 20 + ], + [ + 34, + 52, + 49, + 15, + 28, + 40 + ], + [ + 28, + 39, + 3, + 26, + 30, + 22 + ], + [ + 29, + 1, + 40, + 22, + 19, + 63 + ], + [ + 41, + 5, + 53, + 33, + 26, + 39 + ], + [ + 25, + 38, + 4, + 34, + 49, + 51 + ], + [ + 29, + 52, + 48, + 47, + 20, + 33 + ], + [ + 50, + 3, + 16, + 38, + 53, + 15 + ], + [ + 19, + 62, + 6, + 23, + 10, + 36 + ] + ], + [ + [ + 5, + 14, + 17, + 57, + 10, + 27 + ], + [ + 43, + 9, + 56, + 1, + 14, + 33 + ], + [ + 63, + 35, + 43, + 1, + 10, + 27 + ], + [ + 51, + 50, + 57, + 41, + 20, + 54 + ], + [ + 11, + 43, + 40, + 33, + 30, + 54 + ], + [ + 27, + 53, + 63, + 30, + 15, + 58 + ], + [ + 51, + 48, + 21, + 57, + 9, + 50 + ], + [ + 21, + 14, + 59, + 52, + 28, + 8 + ], + [ + 6, + 18, + 4, + 0, + 12, + 59 + ], + [ + 14, + 53, + 30, + 4, + 42, + 36 + ], + [ + 26, + 42, + 5, + 58, + 22, + 19 + ], + [ + 22, + 5, + 34, + 38, + 52, + 29 + ], + [ + 49, + 33, + 0, + 20, + 26, + 8 + ], + [ + 17, + 2, + 39, + 13, + 44, + 63 + ], + [ + 10, + 54, + 20, + 37, + 5, + 59 + ], + [ + 41, + 43, + 24, + 5, + 53, + 37 + ], + [ + 12, + 47, + 34, + 57, + 27, + 5 + ], + [ + 9, + 33, + 7, + 38, + 43, + 31 + ], + [ + 19, + 10, + 20, + 32, + 18, + 1 + ], + [ + 36, + 63, + 15, + 49, + 40, + 50 + ], + [ + 39, + 3, + 38, + 27, + 36, + 35 + ], + [ + 29, + 19, + 40, + 47, + 62, + 14 + ], + [ + 5, + 46, + 33, + 53, + 49, + 21 + ], + [ + 55, + 4, + 27, + 61, + 51, + 34 + ], + [ + 11, + 0, + 59, + 48, + 14, + 27 + ], + [ + 7, + 32, + 3, + 51, + 18, + 14 + ], + [ + 11, + 9, + 36, + 48, + 0, + 46 + ] + ], + [ + [ + 63, + 62, + 60, + 19, + 23, + 56 + ], + [ + 48, + 32, + 1, + 35, + 5, + 21 + ], + [ + 22, + 24, + 46, + 58, + 59, + 60 + ], + [ + 27, + 37, + 50, + 28, + 61, + 6 + ], + [ + 10, + 12, + 15, + 58, + 35, + 23 + ], + [ + 2, + 43, + 57, + 36, + 30, + 20 + ], + [ + 21, + 14, + 19, + 63, + 41, + 42 + ], + [ + 46, + 7, + 35, + 43, + 21, + 36 + ], + [ + 6, + 18, + 0, + 35, + 40, + 4 + ], + [ + 47, + 53, + 32, + 38, + 1, + 58 + ], + [ + 26, + 18, + 22, + 15, + 5, + 46 + ], + [ + 7, + 22, + 24, + 63, + 53, + 5 + ], + [ + 11, + 3, + 10, + 37, + 18, + 24 + ], + [ + 63, + 18, + 13, + 39, + 0, + 17 + ], + [ + 62, + 54, + 5, + 18, + 6, + 14 + ], + [ + 19, + 43, + 31, + 30, + 57, + 42 + ], + [ + 24, + 5, + 51, + 34, + 47, + 55 + ], + [ + 59, + 13, + 33, + 44, + 11, + 41 + ], + [ + 58, + 32, + 1, + 7, + 39, + 38 + ], + [ + 49, + 13, + 28, + 9, + 34, + 7 + ], + [ + 32, + 51, + 3, + 53, + 48, + 13 + ], + [ + 40, + 15, + 37, + 57, + 1, + 23 + ], + [ + 5, + 53, + 35, + 51, + 22, + 38 + ], + [ + 4, + 44, + 14, + 41, + 11, + 38 + ], + [ + 47, + 48, + 8, + 61, + 13, + 9 + ], + [ + 23, + 3, + 27, + 43, + 0, + 61 + ], + [ + 46, + 23, + 59, + 62, + 18, + 50 + ] + ], + [ + [ + 62, + 0, + 9, + 32, + 26, + 41 + ], + [ + 45, + 29, + 35, + 7, + 30, + 62 + ], + [ + 56, + 31, + 23, + 53, + 28, + 2 + ], + [ + 36, + 2, + 5, + 4, + 48, + 41 + ], + [ + 18, + 0, + 15, + 23, + 16, + 11 + ], + [ + 4, + 36, + 57, + 56, + 14, + 5 + ], + [ + 59, + 56, + 21, + 8, + 7, + 33 + ], + [ + 12, + 23, + 7, + 2, + 28, + 34 + ], + [ + 17, + 6, + 23, + 19, + 62, + 27 + ], + [ + 53, + 32, + 51, + 38, + 41, + 58 + ], + [ + 18, + 63, + 19, + 22, + 26, + 5 + ], + [ + 22, + 7, + 60, + 11, + 12, + 19 + ], + [ + 24, + 7, + 11, + 17, + 30, + 37 + ], + [ + 47, + 13, + 63, + 50, + 39, + 45 + ], + [ + 62, + 54, + 18, + 14, + 29, + 56 + ], + [ + 56, + 30, + 0, + 55, + 51, + 10 + ], + [ + 12, + 5, + 21, + 34, + 63, + 29 + ], + [ + 42, + 8, + 38, + 57, + 41, + 44 + ], + [ + 32, + 54, + 1, + 37, + 49, + 40 + ], + [ + 5, + 57, + 49, + 28, + 34, + 10 + ], + [ + 46, + 32, + 13, + 7, + 61, + 48 + ], + [ + 40, + 3, + 30, + 60, + 39, + 1 + ], + [ + 5, + 44, + 33, + 36, + 28, + 31 + ], + [ + 4, + 11, + 28, + 41, + 51, + 5 + ], + [ + 48, + 47, + 61, + 28, + 60, + 27 + ], + [ + 14, + 3, + 26, + 12, + 53, + 61 + ], + [ + 54, + 60, + 49, + 35, + 10, + 62 + ] + ], + [ + [ + 27, + 62, + 63, + 23, + 47, + 56 + ], + [ + 7, + 4, + 2, + 35, + 10, + 36 + ], + [ + 3, + 0, + 27, + 62, + 50, + 60 + ], + [ + 36, + 3, + 42, + 18, + 2, + 48 + ], + [ + 61, + 12, + 27, + 10, + 15, + 14 + ], + [ + 36, + 33, + 50, + 6, + 29, + 16 + ], + [ + 2, + 41, + 8, + 43, + 40, + 59 + ], + [ + 36, + 53, + 15, + 7, + 37, + 2 + ], + [ + 3, + 58, + 4, + 19, + 5, + 27 + ], + [ + 22, + 26, + 53, + 45, + 25, + 38 + ], + [ + 18, + 41, + 22, + 3, + 34, + 27 + ], + [ + 21, + 53, + 5, + 22, + 9, + 28 + ], + [ + 11, + 24, + 28, + 16, + 35, + 8 + ], + [ + 38, + 9, + 21, + 52, + 41, + 39 + ], + [ + 27, + 54, + 14, + 5, + 8, + 63 + ], + [ + 56, + 55, + 43, + 30, + 35, + 51 + ], + [ + 51, + 3, + 14, + 5, + 60, + 26 + ], + [ + 62, + 30, + 8, + 21, + 44, + 35 + ], + [ + 38, + 34, + 27, + 32, + 22, + 8 + ], + [ + 22, + 53, + 28, + 60, + 49, + 12 + ], + [ + 49, + 32, + 41, + 53, + 12, + 33 + ], + [ + 40, + 45, + 22, + 44, + 1, + 34 + ], + [ + 5, + 4, + 53, + 25, + 41, + 63 + ], + [ + 4, + 11, + 25, + 43, + 31, + 14 + ], + [ + 31, + 52, + 5, + 48, + 58, + 43 + ], + [ + 23, + 1, + 28, + 3, + 0, + 55 + ], + [ + 27, + 45, + 32, + 4, + 30, + 6 + ] + ], + [ + [ + 41, + 2, + 42, + 16, + 50, + 32 + ], + [ + 51, + 5, + 41, + 40, + 21, + 44 + ], + [ + 43, + 1, + 29, + 55, + 21, + 35 + ], + [ + 36, + 58, + 25, + 3, + 18, + 54 + ], + [ + 31, + 60, + 24, + 12, + 61, + 41 + ], + [ + 4, + 10, + 35, + 36, + 0, + 43 + ], + [ + 45, + 43, + 63, + 35, + 36, + 48 + ], + [ + 3, + 9, + 8, + 7, + 43, + 27 + ], + [ + 0, + 43, + 4, + 40, + 18, + 44 + ], + [ + 34, + 15, + 50, + 38, + 53, + 25 + ], + [ + 24, + 46, + 26, + 13, + 2, + 18 + ], + [ + 57, + 37, + 5, + 7, + 22, + 53 + ], + [ + 3, + 9, + 38, + 26, + 37, + 23 + ], + [ + 60, + 26, + 0, + 39, + 13, + 55 + ], + [ + 54, + 21, + 58, + 6, + 55, + 14 + ], + [ + 11, + 19, + 43, + 57, + 56, + 37 + ], + [ + 10, + 24, + 60, + 21, + 5, + 58 + ], + [ + 31, + 20, + 63, + 44, + 55, + 33 + ], + [ + 7, + 25, + 40, + 32, + 54, + 58 + ], + [ + 49, + 30, + 50, + 13, + 55, + 28 + ], + [ + 30, + 3, + 51, + 61, + 0, + 10 + ], + [ + 57, + 46, + 43, + 40, + 30, + 22 + ], + [ + 52, + 33, + 51, + 19, + 27, + 29 + ], + [ + 14, + 4, + 41, + 40, + 0, + 3 + ], + [ + 4, + 13, + 8, + 61, + 47, + 9 + ], + [ + 31, + 27, + 43, + 1, + 3, + 0 + ], + [ + 8, + 4, + 46, + 9, + 3, + 50 + ] + ] ] } } \ No newline at end of file From e0d3dd52d09ad1d781de4a855aa031ae4c535b09 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 10 Feb 2026 13:54:00 -0800 Subject: [PATCH 43/56] add routing indices to metrics --- .../model_config.yaml | 1 + .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml index d33a854e49e..5ab6d37f17e 100644 --- a/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq/model_config.yaml @@ -89,3 +89,4 @@ MODEL_ARGS: METRICS: - "generated_tokens" - "logprobs" + - "routing_indices" diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index b9d78097bbd..a4fd86c4219 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -39,7 +39,7 @@ spec: ARGUMENTS=( "CHECKPOINT_LOAD_PATH=/mnt/artifacts" "CHECKPOINT_SAVE_PATH=/tmp/checkpoints" - "DATA_PATH=null" + "DATA_PATH=/mnt/artifacts/" "DATA_CACHE_PATH=/workspace/data/cache" "TRAINING_SCRIPT_PATH=examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py" "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" From dda0b6a29dac10364474ebeccfdffa0b2560abae Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 11 Feb 2026 04:45:35 -0800 Subject: [PATCH 44/56] move test to github --- .../recipes/h100/moe-dynamic-inference-with-coordinator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml index a4fd86c4219..4ce11808bdc 100644 --- a/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml +++ b/tests/test_utils/recipes/h100/moe-dynamic-inference-with-coordinator.yaml @@ -63,6 +63,6 @@ products: - test_case: [gpt_dynamic_inference_tp4_etp1_pp1_ep8_16B_logitsmatch_cudagraph_zmq] products: - environment: [dev] - scope: [mr] + scope: [mr-github] platforms: [dgx_h100] From b632151e9f75dc8dd635c6f5ba95f21de4628651 Mon Sep 17 00:00:00 2001 From: Siddharth Singh <136645615+sidsingh-nvidia@users.noreply.github.com> Date: Wed, 11 Feb 2026 05:27:23 -0800 Subject: [PATCH 45/56] Update test_inference_regular_pipeline.py --- .../python_test_utils/test_inference_regular_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py index 00a4890af45..93bfa6e1299 100644 --- a/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_inference_regular_pipeline.py @@ -172,7 +172,7 @@ def test_inference_pipeline( f"\nCurrent (truncated to {min_len} chars): {generated_text_current[:min_len]}" ) - if "routing_indices" in groundtruth_results: + if "routing_indices" in groundtruth_results and "routing_indices" in metrics: at_least_one_test_loop = True routing_indices_groundtruth = groundtruth_results["routing_indices"] routing_indices_current = current_results["routing_indices"] From 2594c3921b6f1addec936ca54a7b8fa1093da416 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Wed, 11 Feb 2026 15:42:08 -0600 Subject: [PATCH 46/56] Fix new GRPO functional test --- .../test_cases/gpt/gpt_grpo_basic_function/model_config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml index 76fccecb827..0143a39f017 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_basic_function/model_config.yaml @@ -43,8 +43,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam From 14be3e15ce4ac4c62bd27595ca5aba25d1a5efd8 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Thu, 12 Feb 2026 11:08:01 -0600 Subject: [PATCH 47/56] Update uv.lock --- pyproject.toml | 1 + uv.lock | 127 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 70653a19af8..9b75fcf3596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ dev = [ "emerging_optimizers", "flask[async]", "hypercorn", + "openai", ] lts = [ diff --git a/uv.lock b/uv.lock index d3d779ddf9a..13a344ac365 100644 --- a/uv.lock +++ b/uv.lock @@ -1227,6 +1227,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -1946,6 +1955,103 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/5e/4ec91646aee381d01cdb9974e30882c9cd3b8c5d1079d6b5ff4af522439a/jiter-0.13.0.tar.gz", hash = "sha256:f2839f9c2c7e2dffc1bc5929a510e14ce0a946be9365fd1219e7ef342dae14f4", size = 164847, upload-time = "2026-02-02T12:37:56.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/5a/41da76c5ea07bec1b0472b6b2fdb1b651074d504b19374d7e130e0cdfb25/jiter-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e", size = 311164, upload-time = "2026-02-02T12:35:17.688Z" }, + { url = "https://files.pythonhosted.org/packages/40/cb/4a1bf994a3e869f0d39d10e11efb471b76d0ad70ecbfb591427a46c880c2/jiter-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a", size = 320296, upload-time = "2026-02-02T12:35:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/09/82/acd71ca9b50ecebadc3979c541cd717cce2fe2bc86236f4fa597565d8f1a/jiter-0.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19928b5d1ce0ff8c1ee1b9bdef3b5bfc19e8304f1b904e436caf30bc15dc6cf5", size = 352742, upload-time = "2026-02-02T12:35:21.258Z" }, + { url = "https://files.pythonhosted.org/packages/71/03/d1fc996f3aecfd42eb70922edecfb6dd26421c874503e241153ad41df94f/jiter-0.13.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:309549b778b949d731a2f0e1594a3f805716be704a73bf3ad9a807eed5eb5721", size = 363145, upload-time = "2026-02-02T12:35:24.653Z" }, + { url = "https://files.pythonhosted.org/packages/f1/61/a30492366378cc7a93088858f8991acd7d959759fe6138c12a4644e58e81/jiter-0.13.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bcdabaea26cb04e25df3103ce47f97466627999260290349a88c8136ecae0060", size = 487683, upload-time = "2026-02-02T12:35:26.162Z" }, + { url = "https://files.pythonhosted.org/packages/20/4e/4223cffa9dbbbc96ed821c5aeb6bca510848c72c02086d1ed3f1da3d58a7/jiter-0.13.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a377af27b236abbf665a69b2bdd680e3b5a0bd2af825cd3b81245279a7606c", size = 373579, upload-time = "2026-02-02T12:35:27.582Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c9/b0489a01329ab07a83812d9ebcffe7820a38163c6d9e7da644f926ff877c/jiter-0.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe49d3ff6db74321f144dff9addd4a5874d3105ac5ba7c5b77fac099cfae31ae", size = 362904, upload-time = "2026-02-02T12:35:28.925Z" }, + { url = "https://files.pythonhosted.org/packages/05/af/53e561352a44afcba9a9bc67ee1d320b05a370aed8df54eafe714c4e454d/jiter-0.13.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2113c17c9a67071b0f820733c0893ed1d467b5fcf4414068169e5c2cabddb1e2", size = 392380, upload-time = "2026-02-02T12:35:30.385Z" }, + { url = "https://files.pythonhosted.org/packages/76/2a/dd805c3afb8ed5b326c5ae49e725d1b1255b9754b1b77dbecdc621b20773/jiter-0.13.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab1185ca5c8b9491b55ebf6c1e8866b8f68258612899693e24a92c5fdb9455d5", size = 517939, upload-time = "2026-02-02T12:35:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/20/2a/7b67d76f55b8fe14c937e7640389612f05f9a4145fc28ae128aaa5e62257/jiter-0.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9621ca242547edc16400981ca3231e0c91c0c4c1ab8573a596cd9bb3575d5c2b", size = 551696, upload-time = "2026-02-02T12:35:33.306Z" }, + { url = "https://files.pythonhosted.org/packages/85/9c/57cdd64dac8f4c6ab8f994fe0eb04dc9fd1db102856a4458fcf8a99dfa62/jiter-0.13.0-cp310-cp310-win32.whl", hash = "sha256:a7637d92b1c9d7a771e8c56f445c7f84396d48f2e756e5978840ecba2fac0894", size = 204592, upload-time = "2026-02-02T12:35:34.58Z" }, + { url = "https://files.pythonhosted.org/packages/a7/38/f4f3ea5788b8a5bae7510a678cdc747eda0c45ffe534f9878ff37e7cf3b3/jiter-0.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c1b609e5cbd2f52bb74fb721515745b407df26d7b800458bd97cb3b972c29e7d", size = 206016, upload-time = "2026-02-02T12:35:36.435Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/499f8c9eaa8a16751b1c0e45e6f5f1761d180da873d417996cc7bddc8eef/jiter-0.13.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ea026e70a9a28ebbdddcbcf0f1323128a8db66898a06eaad3a4e62d2f554d096", size = 311157, upload-time = "2026-02-02T12:35:37.758Z" }, + { url = "https://files.pythonhosted.org/packages/50/f6/566364c777d2ab450b92100bea11333c64c38d32caf8dc378b48e5b20c46/jiter-0.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66aa3e663840152d18cc8ff1e4faad3dd181373491b9cfdc6004b92198d67911", size = 319729, upload-time = "2026-02-02T12:35:39.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/560f13ec5e4f116d8ad2658781646cca91b617ae3b8758d4a5076b278f70/jiter-0.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3524798e70655ff19aec58c7d05adb1f074fecff62da857ea9be2b908b6d701", size = 354766, upload-time = "2026-02-02T12:35:40.662Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0d/061faffcfe94608cbc28a0d42a77a74222bdf5055ccdbe5fd2292b94f510/jiter-0.13.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec7e287d7fbd02cb6e22f9a00dd9c9cd504c40a61f2c61e7e1f9690a82726b4c", size = 362587, upload-time = "2026-02-02T12:35:42.025Z" }, + { url = "https://files.pythonhosted.org/packages/92/c9/c66a7864982fd38a9773ec6e932e0398d1262677b8c60faecd02ffb67bf3/jiter-0.13.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:47455245307e4debf2ce6c6e65a717550a0244231240dcf3b8f7d64e4c2f22f4", size = 487537, upload-time = "2026-02-02T12:35:43.459Z" }, + { url = "https://files.pythonhosted.org/packages/6c/86/84eb4352cd3668f16d1a88929b5888a3fe0418ea8c1dfc2ad4e7bf6e069a/jiter-0.13.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ee9da221dca6e0429c2704c1b3655fe7b025204a71d4d9b73390c759d776d165", size = 373717, upload-time = "2026-02-02T12:35:44.928Z" }, + { url = "https://files.pythonhosted.org/packages/6e/09/9fe4c159358176f82d4390407a03f506a8659ed13ca3ac93a843402acecf/jiter-0.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24ab43126d5e05f3d53a36a8e11eb2f23304c6c1117844aaaf9a0aa5e40b5018", size = 362683, upload-time = "2026-02-02T12:35:46.636Z" }, + { url = "https://files.pythonhosted.org/packages/c9/5e/85f3ab9caca0c1d0897937d378b4a515cae9e119730563572361ea0c48ae/jiter-0.13.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9da38b4fedde4fb528c740c2564628fbab737166a0e73d6d46cb4bb5463ff411", size = 392345, upload-time = "2026-02-02T12:35:48.088Z" }, + { url = "https://files.pythonhosted.org/packages/12/4c/05b8629ad546191939e6f0c2f17e29f542a398f4a52fb987bc70b6d1eb8b/jiter-0.13.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0b34c519e17658ed88d5047999a93547f8889f3c1824120c26ad6be5f27b6cf5", size = 517775, upload-time = "2026-02-02T12:35:49.482Z" }, + { url = "https://files.pythonhosted.org/packages/4d/88/367ea2eb6bc582c7052e4baf5ddf57ebe5ab924a88e0e09830dfb585c02d/jiter-0.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2a6394e6af690d462310a86b53c47ad75ac8c21dc79f120714ea449979cb1d3", size = 551325, upload-time = "2026-02-02T12:35:51.104Z" }, + { url = "https://files.pythonhosted.org/packages/f3/12/fa377ffb94a2f28c41afaed093e0d70cfe512035d5ecb0cad0ae4792d35e/jiter-0.13.0-cp311-cp311-win32.whl", hash = "sha256:0f0c065695f616a27c920a56ad0d4fc46415ef8b806bf8fc1cacf25002bd24e1", size = 204709, upload-time = "2026-02-02T12:35:52.467Z" }, + { url = "https://files.pythonhosted.org/packages/cb/16/8e8203ce92f844dfcd3d9d6a5a7322c77077248dbb12da52d23193a839cd/jiter-0.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:0733312953b909688ae3c2d58d043aa040f9f1a6a75693defed7bc2cc4bf2654", size = 204560, upload-time = "2026-02-02T12:35:53.925Z" }, + { url = "https://files.pythonhosted.org/packages/44/26/97cc40663deb17b9e13c3a5cf29251788c271b18ee4d262c8f94798b8336/jiter-0.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:5d9b34ad56761b3bf0fbe8f7e55468704107608512350962d3317ffd7a4382d5", size = 189608, upload-time = "2026-02-02T12:35:55.304Z" }, + { url = "https://files.pythonhosted.org/packages/2e/30/7687e4f87086829955013ca12a9233523349767f69653ebc27036313def9/jiter-0.13.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0a2bd69fc1d902e89925fc34d1da51b2128019423d7b339a45d9e99c894e0663", size = 307958, upload-time = "2026-02-02T12:35:57.165Z" }, + { url = "https://files.pythonhosted.org/packages/c3/27/e57f9a783246ed95481e6749cc5002a8a767a73177a83c63ea71f0528b90/jiter-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f917a04240ef31898182f76a332f508f2cc4b57d2b4d7ad2dbfebbfe167eb505", size = 318597, upload-time = "2026-02-02T12:35:58.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/e5719a60ac5d4d7c5995461a94ad5ef962a37c8bf5b088390e6fad59b2ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e2b199f446d3e82246b4fd9236d7cb502dc2222b18698ba0d986d2fecc6152", size = 348821, upload-time = "2026-02-02T12:36:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/61/db/c1efc32b8ba4c740ab3fc2d037d8753f67685f475e26b9d6536a4322bcdd/jiter-0.13.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04670992b576fa65bd056dbac0c39fe8bd67681c380cb2b48efa885711d9d726", size = 364163, upload-time = "2026-02-02T12:36:01.937Z" }, + { url = "https://files.pythonhosted.org/packages/55/8a/fb75556236047c8806995671a18e4a0ad646ed255276f51a20f32dceaeec/jiter-0.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1aff1fbdb803a376d4d22a8f63f8e7ccbce0b4890c26cc7af9e501ab339ef0", size = 483709, upload-time = "2026-02-02T12:36:03.41Z" }, + { url = "https://files.pythonhosted.org/packages/7e/16/43512e6ee863875693a8e6f6d532e19d650779d6ba9a81593ae40a9088ff/jiter-0.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b3fb8c2053acaef8580809ac1d1f7481a0a0bdc012fd7f5d8b18fb696a5a089", size = 370480, upload-time = "2026-02-02T12:36:04.791Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4c/09b93e30e984a187bc8aaa3510e1ec8dcbdcd71ca05d2f56aac0492453aa/jiter-0.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdaba7d87e66f26a2c45d8cbadcbfc4bf7884182317907baf39cfe9775bb4d93", size = 360735, upload-time = "2026-02-02T12:36:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1b/46c5e349019874ec5dfa508c14c37e29864ea108d376ae26d90bee238cd7/jiter-0.13.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b88d649135aca526da172e48083da915ec086b54e8e73a425ba50999468cc08", size = 391814, upload-time = "2026-02-02T12:36:08.368Z" }, + { url = "https://files.pythonhosted.org/packages/15/9e/26184760e85baee7162ad37b7912797d2077718476bf91517641c92b3639/jiter-0.13.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e404ea551d35438013c64b4f357b0474c7abf9f781c06d44fcaf7a14c69ff9e2", size = 513990, upload-time = "2026-02-02T12:36:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/e9/34/2c9355247d6debad57a0a15e76ab1566ab799388042743656e566b3b7de1/jiter-0.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f4748aad1b4a93c8bdd70f604d0f748cdc0e8744c5547798acfa52f10e79228", size = 548021, upload-time = "2026-02-02T12:36:11.376Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4a/9f2c23255d04a834398b9c2e0e665382116911dc4d06b795710503cdad25/jiter-0.13.0-cp312-cp312-win32.whl", hash = "sha256:0bf670e3b1445fc4d31612199f1744f67f889ee1bbae703c4b54dc097e5dd394", size = 203024, upload-time = "2026-02-02T12:36:12.682Z" }, + { url = "https://files.pythonhosted.org/packages/09/ee/f0ae675a957ae5a8f160be3e87acea6b11dc7b89f6b7ab057e77b2d2b13a/jiter-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:15db60e121e11fe186c0b15236bd5d18381b9ddacdcf4e659feb96fc6c969c92", size = 205424, upload-time = "2026-02-02T12:36:13.93Z" }, + { url = "https://files.pythonhosted.org/packages/1b/02/ae611edf913d3cbf02c97cdb90374af2082c48d7190d74c1111dde08bcdd/jiter-0.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:41f92313d17989102f3cb5dd533a02787cdb99454d494344b0361355da52fcb9", size = 186818, upload-time = "2026-02-02T12:36:15.308Z" }, + { url = "https://files.pythonhosted.org/packages/91/9c/7ee5a6ff4b9991e1a45263bfc46731634c4a2bde27dfda6c8251df2d958c/jiter-0.13.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1f8a55b848cbabf97d861495cd65f1e5c590246fabca8b48e1747c4dfc8f85bf", size = 306897, upload-time = "2026-02-02T12:36:16.748Z" }, + { url = "https://files.pythonhosted.org/packages/7c/02/be5b870d1d2be5dd6a91bdfb90f248fbb7dcbd21338f092c6b89817c3dbf/jiter-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f556aa591c00f2c45eb1b89f68f52441a016034d18b65da60e2d2875bbbf344a", size = 317507, upload-time = "2026-02-02T12:36:18.351Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/b25d2ec333615f5f284f3a4024f7ce68cfa0604c322c6808b2344c7f5d2b/jiter-0.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7e1d61da332ec412350463891923f960c3073cf1aae93b538f0bb4c8cd46efb", size = 350560, upload-time = "2026-02-02T12:36:19.746Z" }, + { url = "https://files.pythonhosted.org/packages/be/ec/74dcb99fef0aca9fbe56b303bf79f6bd839010cb18ad41000bf6cc71eec0/jiter-0.13.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3097d665a27bc96fd9bbf7f86178037db139f319f785e4757ce7ccbf390db6c2", size = 363232, upload-time = "2026-02-02T12:36:21.243Z" }, + { url = "https://files.pythonhosted.org/packages/1b/37/f17375e0bb2f6a812d4dd92d7616e41917f740f3e71343627da9db2824ce/jiter-0.13.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d01ecc3a8cbdb6f25a37bd500510550b64ddf9f7d64a107d92f3ccb25035d0f", size = 483727, upload-time = "2026-02-02T12:36:22.688Z" }, + { url = "https://files.pythonhosted.org/packages/77/d2/a71160a5ae1a1e66c1395b37ef77da67513b0adba73b993a27fbe47eb048/jiter-0.13.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed9bbc30f5d60a3bdf63ae76beb3f9db280d7f195dfcfa61af792d6ce912d159", size = 370799, upload-time = "2026-02-02T12:36:24.106Z" }, + { url = "https://files.pythonhosted.org/packages/01/99/ed5e478ff0eb4e8aa5fd998f9d69603c9fd3f32de3bd16c2b1194f68361c/jiter-0.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98fbafb6e88256f4454de33c1f40203d09fc33ed19162a68b3b257b29ca7f663", size = 359120, upload-time = "2026-02-02T12:36:25.519Z" }, + { url = "https://files.pythonhosted.org/packages/16/be/7ffd08203277a813f732ba897352797fa9493faf8dc7995b31f3d9cb9488/jiter-0.13.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5467696f6b827f1116556cb0db620440380434591e93ecee7fd14d1a491b6daa", size = 390664, upload-time = "2026-02-02T12:36:26.866Z" }, + { url = "https://files.pythonhosted.org/packages/d1/84/e0787856196d6d346264d6dcccb01f741e5f0bd014c1d9a2ebe149caf4f3/jiter-0.13.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:2d08c9475d48b92892583df9da592a0e2ac49bcd41fae1fec4f39ba6cf107820", size = 513543, upload-time = "2026-02-02T12:36:28.217Z" }, + { url = "https://files.pythonhosted.org/packages/65/50/ecbd258181c4313cf79bca6c88fb63207d04d5bf5e4f65174114d072aa55/jiter-0.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:aed40e099404721d7fcaf5b89bd3b4568a4666358bcac7b6b15c09fb6252ab68", size = 547262, upload-time = "2026-02-02T12:36:29.678Z" }, + { url = "https://files.pythonhosted.org/packages/27/da/68f38d12e7111d2016cd198161b36e1f042bd115c169255bcb7ec823a3bf/jiter-0.13.0-cp313-cp313-win32.whl", hash = "sha256:36ebfbcffafb146d0e6ffb3e74d51e03d9c35ce7c625c8066cdbfc7b953bdc72", size = 200630, upload-time = "2026-02-02T12:36:31.808Z" }, + { url = "https://files.pythonhosted.org/packages/25/65/3bd1a972c9a08ecd22eb3b08a95d1941ebe6938aea620c246cf426ae09c2/jiter-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:8d76029f077379374cf0dbc78dbe45b38dec4a2eb78b08b5194ce836b2517afc", size = 202602, upload-time = "2026-02-02T12:36:33.679Z" }, + { url = "https://files.pythonhosted.org/packages/15/fe/13bd3678a311aa67686bb303654792c48206a112068f8b0b21426eb6851e/jiter-0.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:bb7613e1a427cfcb6ea4544f9ac566b93d5bf67e0d48c787eca673ff9c9dff2b", size = 185939, upload-time = "2026-02-02T12:36:35.065Z" }, + { url = "https://files.pythonhosted.org/packages/49/19/a929ec002ad3228bc97ca01dbb14f7632fffdc84a95ec92ceaf4145688ae/jiter-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fa476ab5dd49f3bf3a168e05f89358c75a17608dbabb080ef65f96b27c19ab10", size = 316616, upload-time = "2026-02-02T12:36:36.579Z" }, + { url = "https://files.pythonhosted.org/packages/52/56/d19a9a194afa37c1728831e5fb81b7722c3de18a3109e8f282bfc23e587a/jiter-0.13.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ade8cb6ff5632a62b7dbd4757d8c5573f7a2e9ae285d6b5b841707d8363205ef", size = 346850, upload-time = "2026-02-02T12:36:38.058Z" }, + { url = "https://files.pythonhosted.org/packages/36/4a/94e831c6bf287754a8a019cb966ed39ff8be6ab78cadecf08df3bb02d505/jiter-0.13.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9950290340acc1adaded363edd94baebcee7dabdfa8bee4790794cd5cfad2af6", size = 358551, upload-time = "2026-02-02T12:36:39.417Z" }, + { url = "https://files.pythonhosted.org/packages/a2/ec/a4c72c822695fa80e55d2b4142b73f0012035d9fcf90eccc56bc060db37c/jiter-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2b4972c6df33731aac0742b64fd0d18e0a69bc7d6e03108ce7d40c85fd9e3e6d", size = 201950, upload-time = "2026-02-02T12:36:40.791Z" }, + { url = "https://files.pythonhosted.org/packages/b6/00/393553ec27b824fbc29047e9c7cd4a3951d7fbe4a76743f17e44034fa4e4/jiter-0.13.0-cp313-cp313t-win_arm64.whl", hash = "sha256:701a1e77d1e593c1b435315ff625fd071f0998c5f02792038a5ca98899261b7d", size = 185852, upload-time = "2026-02-02T12:36:42.077Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f5/f1997e987211f6f9bd71b8083047b316208b4aca0b529bb5f8c96c89ef3e/jiter-0.13.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:cc5223ab19fe25e2f0bf2643204ad7318896fe3729bf12fde41b77bfc4fafff0", size = 308804, upload-time = "2026-02-02T12:36:43.496Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8f/5482a7677731fd44881f0204981ce2d7175db271f82cba2085dd2212e095/jiter-0.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9776ebe51713acf438fd9b4405fcd86893ae5d03487546dae7f34993217f8a91", size = 318787, upload-time = "2026-02-02T12:36:45.071Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b9/7257ac59778f1cd025b26a23c5520a36a424f7f1b068f2442a5b499b7464/jiter-0.13.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:879e768938e7b49b5e90b7e3fecc0dbec01b8cb89595861fb39a8967c5220d09", size = 353880, upload-time = "2026-02-02T12:36:47.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/87/719eec4a3f0841dad99e3d3604ee4cba36af4419a76f3cb0b8e2e691ad67/jiter-0.13.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:682161a67adea11e3aae9038c06c8b4a9a71023228767477d683f69903ebc607", size = 366702, upload-time = "2026-02-02T12:36:48.871Z" }, + { url = "https://files.pythonhosted.org/packages/d2/65/415f0a75cf6921e43365a1bc227c565cb949caca8b7532776e430cbaa530/jiter-0.13.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a13b68cd1cd8cc9de8f244ebae18ccb3e4067ad205220ef324c39181e23bbf66", size = 486319, upload-time = "2026-02-02T12:36:53.006Z" }, + { url = "https://files.pythonhosted.org/packages/54/a2/9e12b48e82c6bbc6081fd81abf915e1443add1b13d8fc586e1d90bb02bb8/jiter-0.13.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87ce0f14c6c08892b610686ae8be350bf368467b6acd5085a5b65441e2bf36d2", size = 372289, upload-time = "2026-02-02T12:36:54.593Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c1/e4693f107a1789a239c759a432e9afc592366f04e901470c2af89cfd28e1/jiter-0.13.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c365005b05505a90d1c47856420980d0237adf82f70c4aff7aebd3c1cc143ad", size = 360165, upload-time = "2026-02-02T12:36:56.112Z" }, + { url = "https://files.pythonhosted.org/packages/17/08/91b9ea976c1c758240614bd88442681a87672eebc3d9a6dde476874e706b/jiter-0.13.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1317fdffd16f5873e46ce27d0e0f7f4f90f0cdf1d86bf6abeaea9f63ca2c401d", size = 389634, upload-time = "2026-02-02T12:36:57.495Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/58325ef99390d6d40427ed6005bf1ad54f2577866594bcf13ce55675f87d/jiter-0.13.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c05b450d37ba0c9e21c77fef1f205f56bcee2330bddca68d344baebfc55ae0df", size = 514933, upload-time = "2026-02-02T12:36:58.909Z" }, + { url = "https://files.pythonhosted.org/packages/5b/25/69f1120c7c395fd276c3996bb8adefa9c6b84c12bb7111e5c6ccdcd8526d/jiter-0.13.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:775e10de3849d0631a97c603f996f518159272db00fdda0a780f81752255ee9d", size = 548842, upload-time = "2026-02-02T12:37:00.433Z" }, + { url = "https://files.pythonhosted.org/packages/18/05/981c9669d86850c5fbb0d9e62bba144787f9fba84546ba43d624ee27ef29/jiter-0.13.0-cp314-cp314-win32.whl", hash = "sha256:632bf7c1d28421c00dd8bbb8a3bac5663e1f57d5cd5ed962bce3c73bf62608e6", size = 202108, upload-time = "2026-02-02T12:37:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/cdcf54dd0b0341db7d25413229888a346c7130bd20820530905fdb65727b/jiter-0.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:f22ef501c3f87ede88f23f9b11e608581c14f04db59b6a801f354397ae13739f", size = 204027, upload-time = "2026-02-02T12:37:03.075Z" }, + { url = "https://files.pythonhosted.org/packages/fb/f9/724bcaaab7a3cd727031fe4f6995cb86c4bd344909177c186699c8dec51a/jiter-0.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:07b75fe09a4ee8e0c606200622e571e44943f47254f95e2436c8bdcaceb36d7d", size = 187199, upload-time = "2026-02-02T12:37:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/62/92/1661d8b9fd6a3d7a2d89831db26fe3c1509a287d83ad7838831c7b7a5c7e/jiter-0.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:964538479359059a35fb400e769295d4b315ae61e4105396d355a12f7fef09f0", size = 318423, upload-time = "2026-02-02T12:37:05.806Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/f77d342a54d4ebcd128e520fc58ec2f5b30a423b0fd26acdfc0c6fef8e26/jiter-0.13.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e104da1db1c0991b3eaed391ccd650ae8d947eab1480c733e5a3fb28d4313e40", size = 351438, upload-time = "2026-02-02T12:37:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/ba9a69f0e4209bd3331470c723c2f5509e6f0482e416b612431a5061ed71/jiter-0.13.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e3a5f0cde8ff433b8e88e41aa40131455420fb3649a3c7abdda6145f8cb7202", size = 364774, upload-time = "2026-02-02T12:37:08.579Z" }, + { url = "https://files.pythonhosted.org/packages/b3/16/6cdb31fa342932602458dbb631bfbd47f601e03d2e4950740e0b2100b570/jiter-0.13.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57aab48f40be1db920a582b30b116fe2435d184f77f0e4226f546794cedd9cf0", size = 487238, upload-time = "2026-02-02T12:37:10.066Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b1/956cc7abaca8d95c13aa8d6c9b3f3797241c246cd6e792934cc4c8b250d2/jiter-0.13.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7772115877c53f62beeb8fd853cab692dbc04374ef623b30f997959a4c0e7e95", size = 372892, upload-time = "2026-02-02T12:37:11.656Z" }, + { url = "https://files.pythonhosted.org/packages/26/c4/97ecde8b1e74f67b8598c57c6fccf6df86ea7861ed29da84629cdbba76c4/jiter-0.13.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1211427574b17b633cfceba5040de8081e5abf114f7a7602f73d2e16f9fdaa59", size = 360309, upload-time = "2026-02-02T12:37:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d7/eabe3cf46715854ccc80be2cd78dd4c36aedeb30751dbf85a1d08c14373c/jiter-0.13.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7beae3a3d3b5212d3a55d2961db3c292e02e302feb43fce6a3f7a31b90ea6dfe", size = 389607, upload-time = "2026-02-02T12:37:14.881Z" }, + { url = "https://files.pythonhosted.org/packages/df/2d/03963fc0804e6109b82decfb9974eb92df3797fe7222428cae12f8ccaa0c/jiter-0.13.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:e5562a0f0e90a6223b704163ea28e831bd3a9faa3512a711f031611e6b06c939", size = 514986, upload-time = "2026-02-02T12:37:16.326Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6c/8c83b45eb3eb1c1e18d841fe30b4b5bc5619d781267ca9bc03e005d8fd0a/jiter-0.13.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:6c26a424569a59140fb51160a56df13f438a2b0967365e987889186d5fc2f6f9", size = 548756, upload-time = "2026-02-02T12:37:17.736Z" }, + { url = "https://files.pythonhosted.org/packages/47/66/eea81dfff765ed66c68fd2ed8c96245109e13c896c2a5015c7839c92367e/jiter-0.13.0-cp314-cp314t-win32.whl", hash = "sha256:24dc96eca9f84da4131cdf87a95e6ce36765c3b156fc9ae33280873b1c32d5f6", size = 201196, upload-time = "2026-02-02T12:37:19.101Z" }, + { url = "https://files.pythonhosted.org/packages/ff/32/4ac9c7a76402f8f00d00842a7f6b83b284d0cf7c1e9d4227bc95aa6d17fa/jiter-0.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0a8d76c7524087272c8ae913f5d9d608bd839154b62c4322ef65723d2e5bb0b8", size = 204215, upload-time = "2026-02-02T12:37:20.495Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, + { url = "https://files.pythonhosted.org/packages/79/b3/3c29819a27178d0e461a8571fb63c6ae38be6dc36b78b3ec2876bbd6a910/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b1cbfa133241d0e6bdab48dcdc2604e8ba81512f6bbd68ec3e8e1357dd3c316c", size = 307016, upload-time = "2026-02-02T12:37:42.755Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ae/60993e4b07b1ac5ebe46da7aa99fdbb802eb986c38d26e3883ac0125c4e0/jiter-0.13.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:db367d8be9fad6e8ebbac4a7578b7af562e506211036cba2c06c3b998603c3d2", size = 305024, upload-time = "2026-02-02T12:37:44.774Z" }, + { url = "https://files.pythonhosted.org/packages/77/fa/2227e590e9cf98803db2811f172b2d6460a21539ab73006f251c66f44b14/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45f6f8efb2f3b0603092401dc2df79fa89ccbc027aaba4174d2d4133ed661434", size = 339337, upload-time = "2026-02-02T12:37:46.668Z" }, + { url = "https://files.pythonhosted.org/packages/2d/92/015173281f7eb96c0ef580c997da8ef50870d4f7f4c9e03c845a1d62ae04/jiter-0.13.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:597245258e6ad085d064780abfb23a284d418d3e61c57362d9449c6c7317ee2d", size = 346395, upload-time = "2026-02-02T12:37:48.09Z" }, + { url = "https://files.pythonhosted.org/packages/80/60/e50fa45dd7e2eae049f0ce964663849e897300433921198aef94b6ffa23a/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:3d744a6061afba08dd7ae375dcde870cffb14429b7477e10f67e9e6d68772a0a", size = 305169, upload-time = "2026-02-02T12:37:50.376Z" }, + { url = "https://files.pythonhosted.org/packages/d2/73/a009f41c5eed71c49bec53036c4b33555afcdee70682a18c6f66e396c039/jiter-0.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:ff732bd0a0e778f43d5009840f20b935e79087b4dc65bd36f1cd0f9b04b8ff7f", size = 303808, upload-time = "2026-02-02T12:37:52.092Z" }, + { url = "https://files.pythonhosted.org/packages/c4/10/528b439290763bff3d939268085d03382471b442f212dca4ff5f12802d43/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab44b178f7981fcaea7e0a5df20e773c663d06ffda0198f1a524e91b2fde7e59", size = 337384, upload-time = "2026-02-02T12:37:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, +] + [[package]] name = "jmespath" version = "1.0.1" @@ -2301,6 +2407,7 @@ dev = [ { name = "nvidia-resiliency-ext" }, { name = "nvtx" }, { name = "onnxscript" }, + { name = "openai" }, { name = "opentelemetry-api" }, { name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tensorstore", version = "0.1.80", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, @@ -2425,6 +2532,7 @@ requires-dist = [ { name = "nvtx", marker = "extra == 'lts'", specifier = "~=0.2" }, { name = "onnxscript", marker = "extra == 'dev'" }, { name = "onnxscript", marker = "extra == 'lts'" }, + { name = "openai", marker = "extra == 'dev'" }, { name = "opentelemetry-api", marker = "extra == 'dev'", specifier = "~=1.33.1" }, { name = "opentelemetry-api", marker = "extra == 'lts'", specifier = "~=1.33.1" }, { name = "packaging", specifier = ">=24.2" }, @@ -3398,6 +3506,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/ec/1656ea93be1e50baf429c20603dce249fa3571f3a180407cee79b1afa013/onnxscript-0.5.7-py3-none-any.whl", hash = "sha256:f94a66059c56d13b44908e9b7fd9dae4b4faa6681c784f3fd4c29cfa863e454e", size = 693353, upload-time = "2025-12-16T20:47:17.897Z" }, ] +[[package]] +name = "openai" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/5a/f495777c02625bfa18212b6e3b73f1893094f2bf660976eb4bc6f43a1ca2/openai-2.20.0.tar.gz", hash = "sha256:2654a689208cd0bf1098bb9462e8d722af5cbe961e6bba54e6f19fb843d88db1", size = 642355, upload-time = "2026-02-10T19:02:54.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/a0/cf4297aa51bbc21e83ef0ac018947fa06aea8f2364aad7c96cbf148590e6/openai-2.20.0-py3-none-any.whl", hash = "sha256:38d989c4b1075cd1f76abc68364059d822327cf1a932531d429795f4fc18be99", size = 1098479, upload-time = "2026-02-10T19:02:52.157Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.33.1" From f2f0b6c2112334c1b2924abab21f97f2c01fe1e2 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Thu, 12 Feb 2026 18:47:52 -0600 Subject: [PATCH 48/56] Fix test --- .../model_config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml index 46b0474056f..f3c0c4ecc5b 100644 --- a/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt_grpo_tp4_pp1_dp2_8b_cudagraphs_throughput/model_config.yaml @@ -40,8 +40,6 @@ MODEL_ARGS: --attention-softmax-in-fp32: true --tokenizer-type: HuggingFaceTokenizer --tokenizer-model: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer - --langrl-inference-server-type: inplace_megatron_chat - --langrl-inference-server-conversation-template: ${CHECKPOINT_LOAD_PATH}/model/qwen3-8b-dist/tokenizer --vocab-size: 151936 --make-vocab-size-divisible-by: 128 --optimizer: adam From be42aa6a431474dbffca41ca7c85d5037e13c870 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Thu, 12 Feb 2026 18:56:22 -0600 Subject: [PATCH 49/56] Fix BOS issue --- .../dynamic_text_gen_server/endpoints/chat_completions.py | 5 ++++- megatron/rl/inference/megatron.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 349e41f7787..a71e08ce694 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -61,6 +61,8 @@ async def chat_completions(): # Check for 'logprobs' (bool) and 'top_logprobs' (int) return_log_probs = bool(req.get("logprobs", False)) top_n_logprobs = int(req.get("top_logprobs", 0)) if return_log_probs else 0 + skip_prompt_log_probs = bool(req.get("skip_prompt_log_probs", False)) + add_BOS = bool(req.get("add_BOS", False)) sampling_params = SamplingParams( temperature=temperature, @@ -73,7 +75,8 @@ async def chat_completions(): if ((max_tokens := req.get("max_tokens", None)) is not None) else None ), - skip_prompt_log_probs=True, + skip_prompt_log_probs=skip_prompt_log_probs, + add_BOS=add_BOS, ) except ValueError as e: return f"Invalid sampling parameter: {e}", 400 diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 2fa64ed6988..5e26879eecd 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -37,6 +37,8 @@ class MegatronLocal(InferenceServer, ReturnsTokens, ReturnsRaw): async def base_generate(self, request: InferenceRequest) -> InferenceResponse: assert self._server_task is not None, "Inference server is not initialized" + tokenizer = get_tokenizer() + args = get_args() from openai import AsyncOpenAI client = AsyncOpenAI(base_url=f"http://{self.host}:{self.port}", api_key="NONE") @@ -51,6 +53,8 @@ async def base_generate(self, request: InferenceRequest) -> InferenceResponse: top_p=request.generation_args.top_p or 0.0, n=1, logprobs=True, + skip_prompt_logprobs=True, + add_BOS=(not args.rl_skip_bos_token and tokenizer.bos is not None), ) choice = response.choices[0] From 0aa4aa66bdf2208f1d77802f7cf30a926db52251 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Thu, 12 Feb 2026 21:00:03 -0600 Subject: [PATCH 50/56] Fix typo --- megatron/rl/inference/megatron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index 5e26879eecd..bd7ca4e0d67 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -53,7 +53,7 @@ async def base_generate(self, request: InferenceRequest) -> InferenceResponse: top_p=request.generation_args.top_p or 0.0, n=1, logprobs=True, - skip_prompt_logprobs=True, + skip_prompt_log_probs=True, add_BOS=(not args.rl_skip_bos_token and tokenizer.bos is not None), ) From 53d8f74eb4b909f3aa0849374d8065ef471af4f6 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 10:21:34 -0600 Subject: [PATCH 51/56] Fix extra body --- megatron/rl/inference/megatron.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/megatron/rl/inference/megatron.py b/megatron/rl/inference/megatron.py index bd7ca4e0d67..61eb4602d02 100644 --- a/megatron/rl/inference/megatron.py +++ b/megatron/rl/inference/megatron.py @@ -53,8 +53,10 @@ async def base_generate(self, request: InferenceRequest) -> InferenceResponse: top_p=request.generation_args.top_p or 0.0, n=1, logprobs=True, - skip_prompt_log_probs=True, - add_BOS=(not args.rl_skip_bos_token and tokenizer.bos is not None), + extra_body={ + "skip_prompt_log_probs": True, + "add_BOS": (not args.rl_skip_bos_token and tokenizer.bos is not None), + }, ) choice = response.choices[0] From cf2d71b90920d7b8e48f9215617be71f8502af5c Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 11:31:33 -0600 Subject: [PATCH 52/56] Account for engine modifying prompts --- .../endpoints/chat_completions.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index a71e08ce694..629a2e6d04a 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -102,13 +102,16 @@ async def chat_completions(): # --- 4. Format OpenAI Response --- choices = [] total_completion_tokens = 0 - prompt_token_count = len(prompt_tokens) # Calculated once + prompt_tokens_counts = [] request_idx = 0 for record in batch_results: assert len(record.requests) == 1, "Each record should contain one request result." result = record.merge() + prompt_tokens = result.prompt_tokens # The engine can modify prompt_tokens. text_output = result.generated_text + prompt_tokens_count = len(prompt_tokens) if prompt_tokens is not None else 0 + prompt_tokens_counts.append(prompt_tokens_count) logprobs_content = None if sampling_params.return_log_probs: @@ -176,10 +179,9 @@ async def chat_completions(): logging.info(result) if result.routing_indices is not None: choice_data["moe_topk_indices"] = result.routing_indices.tolist() - prompt_length = len(result.prompt_tokens) if result.prompt_tokens is not None else 0 - if prompt_length: + if prompt_tokens_count: choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ - :prompt_length + :prompt_tokens_count ].tolist() choices.append(choice_data) total_completion_tokens += len(result.generated_tokens) @@ -188,7 +190,7 @@ async def chat_completions(): response = { "choices": choices, "usage": { - "prompt_tokens": prompt_token_count, + "prompt_tokens": max(prompt_tokens_counts), "completion_tokens": total_completion_tokens, "total_tokens": prompt_token_count + total_completion_tokens, }, From d6df874fe9aa736ca1feeae6cc804810033c93cd Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 12:29:33 -0600 Subject: [PATCH 53/56] Fix README --- examples/rl/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/rl/README.md b/examples/rl/README.md index 9c2de3ec088..b66b2abbc32 100644 --- a/examples/rl/README.md +++ b/examples/rl/README.md @@ -172,7 +172,6 @@ torchrun \ --save $CHECKPOINT_DIR \ --load $CHECKPOINT_DIR \ --tensorboard-dir $TB_DIR \ - --langrl-inference-server-type inplace_megatron \ --seed $SEED \ --sequence-parallel \ --finetune \ From c0dba9740ea0db51a3acbd46fda64d49bc295abd Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 12:31:23 -0600 Subject: [PATCH 54/56] Fix typo --- .../dynamic_text_gen_server/endpoints/chat_completions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 629a2e6d04a..fede926de7b 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -187,10 +187,11 @@ async def chat_completions(): total_completion_tokens += len(result.generated_tokens) request_idx += 0 + prompt_token_count = max(prompt_tokens_counts) response = { "choices": choices, "usage": { - "prompt_tokens": max(prompt_tokens_counts), + "prompt_tokens": prompt_token_count, "completion_tokens": total_completion_tokens, "total_tokens": prompt_token_count + total_completion_tokens, }, From ca5110410aa19e8e10e5586687ea26c4a89bb799 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 13:44:14 -0600 Subject: [PATCH 55/56] Serialize correctly --- .../endpoints/chat_completions.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index fede926de7b..7a33f08ff7f 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -107,19 +107,24 @@ async def chat_completions(): request_idx = 0 for record in batch_results: assert len(record.requests) == 1, "Each record should contain one request result." - result = record.merge() - prompt_tokens = result.prompt_tokens # The engine can modify prompt_tokens. - text_output = result.generated_text + result = record.merge().serialize() + # Unwrap ("tensor", [...]) tuples from serialize() into plain lists. + result = { + k: v[1] if isinstance(v, (list, tuple)) and len(v) == 2 and v[0] == "tensor" else v + for k, v in result.items() + } + prompt_tokens = result["prompt_tokens"] # The engine can modify prompt_tokens. + text_output = result["generated_text"] prompt_tokens_count = len(prompt_tokens) if prompt_tokens is not None else 0 prompt_tokens_counts.append(prompt_tokens_count) logprobs_content = None if sampling_params.return_log_probs: - token_logprobs = getattr(result, 'log_probs', []) - tokens = [tokenizer.detokenize([tok]) for tok in result.generated_tokens] + token_logprobs = result.get('log_probs', []) + tokens = [tokenizer.detokenize([tok]) for tok in result["generated_tokens"]] # Get top_n_logprobs if available - generated_top_n_logprobs = getattr(result, 'generated_top_n_logprobs', None) + generated_top_n_logprobs = result.get('generated_top_n_logprobs') logprobs_content = [] for i, (tok, lp) in enumerate(zip(tokens, token_logprobs)): @@ -166,10 +171,10 @@ async def chat_completions(): choice_data = { "index": request_idx, "message": message, - "prompt_token_ids": prompt_tokens, - "generation_token_ids": result.generated_tokens, - "generation_log_probs": result.generated_log_probs, - "raw_text": result.prompt + result.generated_text, + "prompt_token_ids": result["prompt_tokens"], + "generation_token_ids": result["generated_tokens"], + "generation_log_probs": result["generated_log_probs"], + "raw_text": result["prompt"] + result["generated_text"], # 'logprobs' in chat API is an object containing 'content' "logprobs": {"content": logprobs_content} if logprobs_content else None, "finish_reason": ( @@ -177,15 +182,15 @@ async def chat_completions(): ), # Original code hardcoded this. } logging.info(result) - if result.routing_indices is not None: - choice_data["moe_topk_indices"] = result.routing_indices.tolist() + if result["routing_indices"] is not None: + choice_data["moe_topk_indices"] = result["routing_indices"] if prompt_tokens_count: - choices[-1]["prompt_moe_topk_indices"] = result.routing_indices[ + choices[-1]["prompt_moe_topk_indices"] = result["routing_indices"][ :prompt_tokens_count - ].tolist() + ] choices.append(choice_data) - total_completion_tokens += len(result.generated_tokens) - request_idx += 0 + total_completion_tokens += len(result["generated_tokens"]) + request_idx += 1 prompt_token_count = max(prompt_tokens_counts) response = { From 58fb09985a5fff683a5e60fed77f9c451eed1140 Mon Sep 17 00:00:00 2001 From: Teodor-Dumitru Ene Date: Fri, 13 Feb 2026 15:35:36 -0600 Subject: [PATCH 56/56] Fix add_BOS --- .../endpoints/chat_completions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 7a33f08ff7f..8d3ecba235f 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -64,6 +64,15 @@ async def chat_completions(): skip_prompt_log_probs = bool(req.get("skip_prompt_log_probs", False)) add_BOS = bool(req.get("add_BOS", False)) + # The engine only handles add_BOS for string prompts, not pre-tokenized + # input. Since we pre-tokenize via apply_chat_template, we must handle + # BOS ourselves, matching the logic in tokenize_prompt(). + if hasattr(tokenizer, 'bos') and tokenizer.bos is not None: + while prompt_tokens and prompt_tokens[0] == tokenizer.bos: + prompt_tokens.pop(0) + if add_BOS: + prompt_tokens = [tokenizer.bos] + prompt_tokens + sampling_params = SamplingParams( temperature=temperature, top_k=top_k,