Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[XPU] update XHPC date and refine FA ut #60598

Merged
merged 3 commits into from
Jan 10, 2024
Merged

[XPU] update XHPC date and refine FA ut #60598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dc20917
[XPU] update XHPC date
houj04 Jan 6, 2024
d1e9bda
update comments for ut
houj04 Jan 9, 2024
690a863
Merge branch 'develop' into 20240106-refine-fa-ut
houj04 Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmake/external/xpu.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ if(NOT DEFINED XPU_BASE_DATE)
set(XPU_BASE_DATE "20231218")
endif()
if(NOT DEFINED XPU_XHPC_BASE_DATE)
set(XPU_XHPC_BASE_DATE "20231229")
set(XPU_XHPC_BASE_DATE "20240105")
endif()
set(XPU_XCCL_BASE_VERSION "1.1.8.1")
if(NOT DEFINED XPU_XFT_BASE_VERSION)
Expand Down
29 changes: 13 additions & 16 deletions test/xpu/test_flash_attention_op_xpu.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,6 @@ def run_case(self, dtype, tolerance, tolerance_dv):
float_out = paddle.cast(out, "float32")
float_out_ = paddle.cast(out_, "float32")

np.testing.assert_allclose(
float_out, float_out_, rtol=tolerance, atol=tolerance
)
# TODO(houj04) remove debug codes after correctness check
max_diff_forward = np.max(
np.abs(float_out.numpy() - float_out_.numpy())
Expand All @@ -139,6 +136,10 @@ def run_case(self, dtype, tolerance, tolerance_dv):
print("max_diff_forward:", max_diff_forward)
print("mean_diff_forward:", mean_diff_forward)

np.testing.assert_allclose(
float_out, float_out_, rtol=tolerance, atol=tolerance
)

# backward shape
self.assertEqual(q.grad.shape, q.shape)
self.assertEqual(q_.grad.shape, q.shape)
Expand Down Expand Up @@ -194,17 +195,17 @@ def run_case(self, dtype, tolerance, tolerance_dv):
)


# TODO(houj04) un-comment following DEBUG cases after correctness check
# class TestFlashAttentionAPITest1(TestFlashAttentionAPI):
# def setUp(self):
# self.place = paddle.XPUPlace(0)
# self.shape = (2, 128, 1, 32)
# self.dropout = 0.0
# self.causal = True
# self.return_softmax = False
class TestFlashAttentionAPITest1(TestFlashAttentionAPI):
def setUp(self):
self.place = paddle.XPUPlace(0)
self.shape = (2, 128, 1, 32)
self.dropout = 0.0
self.causal = True
self.return_softmax = False


# The following three REAL unit tests are disabled because they take a VERY LONG time to run, although they all pass under XHPC v20240105.

# TODO(houj04) un-comment following REAL cases after correctness check
# class TestFlashAttentionAPITestEB(TestFlashAttentionAPI):
# def setUp(self):
# self.place = paddle.XPUPlace(0)
Expand All @@ -213,8 +214,6 @@ def run_case(self, dtype, tolerance, tolerance_dv):
# self.causal = True
# self.return_softmax = False


# TODO(houj04) un-comment following REAL cases after correctness check
# class TestFlashAttentionAPITestLlama7B(TestFlashAttentionAPI):
# def setUp(self):
# self.place = paddle.XPUPlace(0)
Expand All @@ -223,8 +222,6 @@ def run_case(self, dtype, tolerance, tolerance_dv):
# self.causal = True
# self.return_softmax = False


# TODO(houj04) un-comment following REAL cases after correctness check
# class TestFlashAttentionAPITestLlama65B(TestFlashAttentionAPI):
# def setUp(self):
# self.place = paddle.XPUPlace(0)
Expand Down