Skip to content

Update spec test to 2460ad02b51fb5ed5824f44de287a8638b19a5f8 #889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 15, 2021
Merged

Update spec test to 2460ad02b51fb5ed5824f44de287a8638b19a5f8 #889

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/iwasm/interpreter/wasm_loader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3342,6 +3342,7 @@ create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list,
if (last_section_index != (uint8)-1
&& (section_index <= last_section_index)) {
set_error_buf(error_buf, error_buf_size,
"unexpected content after last section or "
"junk after last section");
return false;
}
Expand Down
158 changes: 62 additions & 96 deletions tests/wamr-test-suites/spec-test-script/all.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,70 +204,59 @@ def test_suite(
xip_flag=False,
clean_up_flag=True,
verbose_flag=True,
parl_flag=False,
):
suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
if not suite_path.exists():
print(f"can not find spec test cases at {suite_path}")
return False

case_list = sorted(suite_path.glob("**/*.wast"))
case_list = sorted(suite_path.glob("*.wast"))
if simd_flag:
simd_case_list = sorted(suite_path.glob("simd/*.wast"))
case_list.extend(simd_case_list)

case_count = len(case_list)
failed_case = 0
successful_case = 0
for case_path in case_list:
try:
test_case(
str(case_path),
target,
aot_flag,
sgx_flag,
multi_module_flag,
multi_thread_flag,
simd_flag,
xip_flag,
clean_up_flag,
verbose_flag,
)
successful_case += 1
except Exception:
failed_case += 1
break

print(
f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP"
)

return 0 == failed_case

if parl_flag:
print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----")
with mp.Pool() as pool:
results = {}
for case_path in case_list:
results[case_path.stem] = pool.apply_async(
test_case,
[
str(case_path),
target,
aot_flag,
sgx_flag,
multi_module_flag,
multi_thread_flag,
simd_flag,
xip_flag,
clean_up_flag,
verbose_flag,
],
)

def test_suite_parallelly(
target,
aot_flag=False,
sgx_flag=False,
multi_module_flag=False,
multi_thread_flag=False,
simd_flag=False,
xip_flag=False,
clean_up_flag=False,
verbose_flag=False,
):

suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
if not suite_path.exists():
print(f"can not find spec test cases at {suite_path}")
return False

case_list = sorted(suite_path.glob("**/*.wast"))
case_count = len(case_list)
failed_case = 0
successful_case = 0
print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----")
with mp.Pool() as pool:
results = {}
for case_name, result in results.items():
try:
# 5 min / case
result.wait(300)
if not result.successful():
failed_case += 1
else:
successful_case += 1
except mp.TimeoutError:
print(f"{case_name} meets TimeoutError")
failed_case += 1
else:
print(f"----- Run the whole spec test suite -----")
for case_path in case_list:
results[case_path.stem] = pool.apply_async(
test_case,
[
try:
test_case(
str(case_path),
target,
aot_flag,
Expand All @@ -278,20 +267,11 @@ def test_suite_parallelly(
xip_flag,
clean_up_flag,
verbose_flag,
],
)

for case_name, result in results.items():
try:
# 5 min / case
result.wait(300)
if not result.successful():
failed_case += 1
else:
successful_case += 1
except mp.TimeoutError:
print(f"{case_name} meets TimeoutError")
)
successful_case += 1
except Exception:
failed_case += 1
break

print(
f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP"
Expand Down Expand Up @@ -396,37 +376,23 @@ def main():
options.clean_up_flag = False
options.verbose_flag = False

start = time.time_ns()
ret = test_suite_parallelly(
options.target,
options.aot_flag,
options.sgx_flag,
options.multi_module_flag,
options.multi_thread_flag,
options.simd_flag,
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
)
end = time.time_ns()
print(
f"It takes {((end - start) / 1000000):,} ms to run test_suite_parallelly"
)
else:
start = time.time_ns()
ret = test_suite(
options.target,
options.aot_flag,
options.sgx_flag,
options.multi_module_flag,
options.multi_thread_flag,
options.simd_flag,
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
)
end = time.time_ns()
print(f"It takes {((end - start) / 1000000):,} ms to run test_suite")
start = time.time_ns()
ret = test_suite(
options.target,
options.aot_flag,
options.sgx_flag,
options.multi_module_flag,
options.multi_thread_flag,
options.simd_flag,
options.xip_flag,
options.clean_up_flag,
options.verbose_flag,
options.parl_flag,
)
end = time.time_ns()
print(
f"It takes {((end - start) / 1000000):,} ms to run test_suite {'parallelly' if options.parl_flag else ''}"
)
else:
try:
for case in options.cases:
Expand Down
20 changes: 5 additions & 15 deletions tests/wamr-test-suites/test_wamr.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,12 @@ function spec_test()
# restore from XX_ignore_cases.patch
# resotre branch
git checkout -B master
git reset --hard 397399a70565609bf142d211891724e21bffd01f
# [spec] Fix instruction table (#1402) Thu Dec 2 17:21:54 2021 +0100
git reset --hard 2460ad02b51fb5ed5824f44de287a8638b19a5f8
git apply ../../spec-test-script/ignore_cases.patch
if [[ ${ENABLE_SIMD} == 1 ]]; then
git apply ../../spec-test-script/simd_ignore_cases.patch
fi

# udpate thread cases
if [ ${ENABLE_MULTI_THREAD} == 1 ]; then
Expand All @@ -298,20 +302,6 @@ function spec_test()
git apply ../../spec-test-script/thread_proposal_ignore_cases.patch
fi

# udpate SIMD cases
if [[ ${ENABLE_SIMD} == 1 ]]; then
echo "checkout spec for SIMD proposal"
# check spec test cases for simd
if [[ -z $(git remote | grep "\<simd\>") ]]; then
git remote add simd https://github.com/WebAssembly/simd.git
fi

git fetch simd
git checkout simd/main -- test/core/simd

git apply ../../spec-test-script/simd_ignore_cases.patch
fi

popd
echo $(pwd)

Expand Down