【Hackathon 9th No.60、62】add eagle_get_hidden_states

tests/operators/test_eagle_get_hidden_states.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import paddle
+
+from fastdeploy.model_executor.ops.gpu import eagle_get_hidden_states
+
+
+def ComputeOrderKernel(
+    seq_lens_this_time,
+    seq_lens_encoder,
+    base_model_seq_lens_this_time,
+    base_model_seq_lens_encoder,
+    accept_nums,
+    position_map,
+    output_token_num,
+    bsz,
+    actual_draft_token_num,
+    input_token_num,
+):
+    in_offset = 0
+    out_offset = 0
+    for i in range(bsz):
+        cur_base_model_seq_lens_this_time = base_model_seq_lens_this_time[i]
+        # cur_base_model_seq_lens_encoder = base_model_seq_lens_encoder[i]
+        cur_seq_lens_this_time = seq_lens_this_time[i]
+        accept_num = accept_nums[i]
+        cur_seq_lens_encoder = seq_lens_encoder[i]
+        # 1. eagle encoder. Base step=1
+        if cur_seq_lens_encoder > 0:
+            for j in range(cur_seq_lens_encoder):
+                position_map[in_offset] = out_offset
+                in_offset += 1
+                out_offset += 1
+        # 2. Base model stop at last verify-step.
+        elif cur_base_model_seq_lens_this_time != 0 and cur_seq_lens_this_time == 0:
+            in_offset += cur_base_model_seq_lens_this_time
+        # 4. stopped
+        elif cur_base_model_seq_lens_this_time == 0 and cur_seq_lens_this_time == 0:  # end
+            pass
+        else:
+            for i in range(accept_num):
+                position_map[in_offset] = out_offset
+                in_offset += 1
+                out_offset += 1
+            in_offset += cur_base_model_seq_lens_this_time - accept_num
+        output_token_num[0] = out_offset
+
+
+def rebuildHiddenStatesKernel(input, position_map, out, dim_embed, elem_cnt):
+    for elem_idx in range(elem_cnt):
+        ori_token_idx = int(elem_idx / dim_embed)
+        token_idx = position_map[ori_token_idx]
+        if token_idx >= 0:
+            offset = elem_idx % dim_embed
+            out[token_idx][offset] = input[ori_token_idx][offset]
+
+
+def eagle_get_hidden_states_ref(
+    input,
+    seq_lens_this_time,
+    seq_lens_encoder,
+    seq_lens_decoder,
+    stop_flags,
+    accept_nums,
+    base_model_seq_lens_this_time,
+    base_model_seq_lens_encoder,
+    actual_draft_token_num,
+):
+    input_token_num = input.shape[0]
+    dim_embed = input.shape[1]
+    bsz = seq_lens_this_time.shape[0]
+    position_map = paddle.full([input_token_num], 0xFFFFFFFF, seq_lens_this_time.dtype)
+    output_token_num = paddle.empty([1], seq_lens_this_time.dtype)
+    ComputeOrderKernel(
+        seq_lens_this_time,
+        seq_lens_encoder,
+        base_model_seq_lens_this_time,
+        base_model_seq_lens_encoder,
+        accept_nums,
+        position_map,
+        output_token_num,
+        bsz,
+        actual_draft_token_num,
+        input_token_num,
+    )
+
+    output_token_num_cpu = output_token_num[0]
+    out = paddle.empty([output_token_num_cpu, dim_embed], input.dtype)
+    elem_cnt = input_token_num * dim_embed
+    rebuildHiddenStatesKernel(input, position_map, out, dim_embed, elem_cnt)
+    return out
+
+
+class TestEagleGetHiddenStates(unittest.TestCase):
+    def test_eagle_get_hidden_states(self):
+        np.random.seed(2023)
+        paddle.seed(2023)
+        bs = 2
+        input_token_num = 10
+        dim_embed = 512
+        actual_draft_token_num = np.random.randint(2, 6, dtype=np.int32)
+
+        seq_lens_this_time = np.random.randint(0, 2, bs, dtype=np.int32)
+        seq_lens_encoder = np.random.randint(0, input_token_num // bs + 1, bs, dtype=np.int32)
+        accept_nums = np.random.randint(0, actual_draft_token_num + 1, bs, dtype=np.int32)
+        base_model_seq_lens_this_time = np.random.randint(0, input_token_num // bs + 1, bs, dtype=np.int32)
+        base_model_seq_lens_encoder = np.random.randint(0, 2, bs, dtype=np.int32)
+
+        seq_lens_decoder = np.random.randint(0, input_token_num // bs + 1, bs, dtype=np.int32)
+        stop_flags = np.random.randint(0, 2, bs, dtype=np.int32)
+
+        seq_lens_this_time_tensor = paddle.to_tensor(seq_lens_this_time, dtype=paddle.int32)
+        seq_lens_encoder_tensor = paddle.to_tensor(seq_lens_encoder, dtype=paddle.int32)
+        accept_nums_tensor = paddle.to_tensor(accept_nums, dtype=paddle.int32)
+        base_model_seq_lens_this_time_tensor = paddle.to_tensor(base_model_seq_lens_this_time, dtype=paddle.int32)
+        base_model_seq_lens_encoder_tensor = paddle.to_tensor(base_model_seq_lens_encoder, dtype=paddle.int32)
+
+        seq_lens_decoder_tensor = paddle.to_tensor(seq_lens_decoder, dtype=paddle.int32)
+        stop_flags_tensor = paddle.to_tensor(stop_flags, dtype=paddle.int32)
+
+        input = np.random.randint(0, 10, (input_token_num, dim_embed), dtype=np.int32)
+        input_tensor = paddle.to_tensor(input, dtype=paddle.float16)
+        out = eagle_get_hidden_states(
+            input_tensor,
+            seq_lens_this_time_tensor,
+            seq_lens_encoder_tensor,
+            seq_lens_decoder_tensor,
+            stop_flags_tensor,
+            accept_nums_tensor,
+            base_model_seq_lens_this_time_tensor,
+            base_model_seq_lens_encoder_tensor,
+            actual_draft_token_num,
+        )
+        out_ref = eagle_get_hidden_states_ref(
+            input_tensor,
+            seq_lens_this_time_tensor,
+            seq_lens_encoder_tensor,
+            seq_lens_decoder_tensor,
+            stop_flags_tensor,
+            accept_nums_tensor,
+            base_model_seq_lens_this_time_tensor,
+            base_model_seq_lens_encoder_tensor,
+            actual_draft_token_num,
+        )
+        np.testing.assert_allclose(out.numpy(), out_ref.numpy())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/operators/test_eagle_get_self_hidden_states.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import paddle
+
+from fastdeploy.model_executor.ops.gpu import eagle_get_self_hidden_states
+
+
+def computeOrderKernel(last_seq_lens_this_time, seq_lens_this_time, step_idx, src_map, output_token_num, bsz):
+    in_offset = 0
+    out_offset = 0
+    for i in range(bsz):
+        cur_seq_lens_this_time = seq_lens_this_time[i]
+        cur_last_seq_lens_this_time = last_seq_lens_this_time[i]
+        # 1. encoder
+        if step_idx[i] == 1 and cur_seq_lens_this_time > 0:
+            in_offset += 1
+            src_map[out_offset] = in_offset - 1
+            out_offset += 1
+        # 2. decoder
+        elif cur_seq_lens_this_time > 0:  # =1
+            in_offset += cur_last_seq_lens_this_time
+            src_map[out_offset] = in_offset - 1
+            out_offset += 1
+        # 3. stop
+        else:
+            # first token end
+            if step_idx[i] == 1:
+                in_offset += 1 if cur_last_seq_lens_this_time > 0 else 0
+            # normal end
+            else:
+                in_offset += cur_last_seq_lens_this_time
+    output_token_num[0] = out_offset
+
+
+def rebuildSelfHiddenStatesKernel(input, src_map, output, dim_embed, elem_cnt):
+    for elem_id in range(elem_cnt):
+        output_token_idx = int(elem_id / dim_embed)
+        input_token_idx = src_map[output_token_idx]
+        offset = elem_id % dim_embed
+        output[output_token_idx][offset] = input[input_token_idx][offset]
+
+
+def eagle_get_self_hidden_states_ref(input, last_seq_lens_this_time, seq_lens_this_time, step_idx):
+    input_token_num = input.shape[0]
+    dim_embed = input.shape[1]
+    bsz = seq_lens_this_time.shape[0]
+    src_map = paddle.full([input_token_num], -1, seq_lens_this_time.dtype)
+    output_token_num = paddle.full([1], 0, seq_lens_this_time.dtype)
+
+    computeOrderKernel(last_seq_lens_this_time, seq_lens_this_time, step_idx, src_map, output_token_num, bsz)
+
+    output_token_num_cpu = output_token_num[0]
+    out = paddle.full([output_token_num_cpu, dim_embed], -1, input.dtype)
+
+    elem_cnt = output_token_num_cpu * dim_embed
+    rebuildSelfHiddenStatesKernel(input, src_map, out, dim_embed, elem_cnt)
+
+    return out
+
+
+class TestEagleGetSelfHiddenStates(unittest.TestCase):
+    def test_eagle_get_self_hidden_states(self):
+        paddle.seed(2023)
+        np.random.seed(2023)
+        bs = 2
+        input_token_num = 10
+        dim_embed = 512
+
+        last_seq_lens_this_time = np.random.randint(0, input_token_num // bs, bs, dtype=np.int32)
+        seq_lens_this_time = np.random.randint(0, input_token_num // bs, bs, dtype=np.int32)
+        step_idx = np.arange(0, bs, dtype=np.int32)
+
+        last_seq_lens_this_time_tensor = paddle.to_tensor(last_seq_lens_this_time, dtype=paddle.int32)
+        seq_lens_this_time_tensor = paddle.to_tensor(seq_lens_this_time, dtype=paddle.int32)
+        step_idx_tensor = paddle.to_tensor(step_idx, dtype=paddle.int64)
+
+        input = np.random.randint(0, 10, (input_token_num, dim_embed), dtype=np.int32)
+        input_tensor = paddle.to_tensor(input, dtype=paddle.float16)
+        out = eagle_get_self_hidden_states(
+            input_tensor,
+            last_seq_lens_this_time_tensor,
+            seq_lens_this_time_tensor,
+            step_idx_tensor,
+        )
+        out_ref = eagle_get_self_hidden_states_ref(
+            input_tensor,
+            last_seq_lens_this_time_tensor,
+            seq_lens_this_time_tensor,
+            step_idx_tensor,
+        )
+        np.testing.assert_allclose(out.numpy(), out_ref.numpy())
+
+
+if __name__ == "__main__":
+    unittest.main()