test: Migrate host function recursion limit tests

This reworks tests which have host function recursion and moves them
to the execute_call_depth suite.

test/unittests/execute_call_depth_test.cpp

@@ -237,6 +237,59 @@ TEST(execute_call_depth, call_host_function_calling_wasm_function)
     EXPECT_EQ(recorded_depth, -1000);
 }
 
+TEST(execute_call_depth, call_host_function_calling_another_wasm_module)
+{
+    // Test for "wasm-host-wasm" sandwich.
+    // The host function is obligated to bump depth and pass it along.
+
+    /* wat2wasm
+    (func $host_f (import "host" "f") (result i32))
+    (func (result i32) (call $host_f))
+    */
+    const auto wasm =
+        from_hex("0061736d010000000105016000017f020a0104686f737401660000030201000a0601040010000b");
+
+    /* wat2wasm
+    (func (result i32) (i32.const 1))
+    */
+    const auto another_wasm = from_hex("0061736d010000000105016000017f030201000a0601040041010b");
+
+    static int recorded_depth;
+    constexpr auto host_f = [](std::any& host_context, Instance&, const Value*,
+                                int depth) noexcept {
+        recorded_depth = depth;
+        auto instance = *std::any_cast<Instance*>(&host_context);
+        return fizzy::execute(*instance, 0, {}, depth + 1);
+    };
+
+    auto another_instance = instantiate(parse(another_wasm));
+
+    auto host_context = std::make_any<Instance*>(another_instance.get());
+
+    const auto module = parse(wasm);
+    auto instance = instantiate(*module, {{{host_f, host_context}, module->typesec[0]}});
+
+    recorded_depth = -1000;
+    EXPECT_THAT(execute(*instance, 1, {}), Result(1_u32));
+    EXPECT_EQ(recorded_depth, 1);
+
+    recorded_depth = -1000;
+    EXPECT_THAT(execute(*instance, 1, {}, DepthLimit - 3), Result(1_u32));
+    EXPECT_EQ(recorded_depth, DepthLimit - 2);
+
+    recorded_depth = -1000;
+    EXPECT_THAT(execute(*instance, 1, {}, DepthLimit - 2), Traps());
+    EXPECT_EQ(recorded_depth, DepthLimit - 1);
+
+    recorded_depth = -1000;
+    EXPECT_THAT(execute(*instance, 1, {}, DepthLimit - 1), Traps());
+    EXPECT_EQ(recorded_depth, -1000);
+
+    recorded_depth = -1000;
+    EXPECT_THAT(execute(*instance, 1, {}, DepthLimit), Traps());
+    EXPECT_EQ(recorded_depth, -1000);
+}
+
 
 /// Infinite recursion
 
@@ -325,3 +378,144 @@ TEST(execute_call_depth, execute_imported_wasm_function_infinite_recursion)
     EXPECT_THAT(execute(*executor, 1, {}, DepthLimit), Traps());
     EXPECT_EQ(counter.i64, 0);
 }
+
+TEST(execute_call_depth, execute_host_function_within_wasm_recursion_limit)
+{
+    // In this test the host_f host function uses wasm call depth limit
+    // to protect itself against infinite recursion.
+
+    /* wat2wasm
+    (func $host_f (import "host" "f") (result i32))
+    */
+    const auto wasm = from_hex("0061736d010000000105016000017f020a0104686f737401660000");
+
+    static int max_recorded_wasm_recursion_depth;
+
+    constexpr auto host_f = [](std::any&, Instance& instance, const Value*, int depth) noexcept {
+        max_recorded_wasm_recursion_depth = std::max(max_recorded_wasm_recursion_depth, depth);
+        return fizzy::execute(instance, 0, {}, depth + 1);
+    };
+
+    const auto module = parse(wasm);
+    auto instance = instantiate(*module, {{{host_f}, module->typesec[0]}});
+
+    EXPECT_EQ(max_recorded_wasm_recursion_depth, 0);
+    EXPECT_THAT(execute(*instance, 0, {}), Traps());
+    EXPECT_EQ(max_recorded_wasm_recursion_depth, DepthLimit - 1);
+}
+
+TEST(execute_call_depth, execute_host_function_with_custom_recursion_limit)
+{
+    // In this test the host_f host function implements independent recursion depth limit.
+
+    /* wat2wasm
+    (func $host_f (import "host" "f") (result i32))
+    */
+    const auto wasm = from_hex("0061736d010000000105016000017f020a0104686f737401660000");
+
+    static constexpr int host_recursion_limit = 10;
+    static int host_recursion_depth;
+    static int max_recorded_wasm_recursion_depth;
+    static int max_recorded_host_recursion_depth;
+
+    constexpr auto host_f = [](std::any&, Instance& instance, const Value*, int depth) noexcept {
+        ++host_recursion_depth;
+
+        assert(depth == 0);
+        max_recorded_wasm_recursion_depth = std::max(max_recorded_wasm_recursion_depth, depth);
+        max_recorded_host_recursion_depth =
+            std::max(max_recorded_host_recursion_depth, host_recursion_depth);
+
+        const auto result = (host_recursion_depth < host_recursion_limit) ?
+                                fizzy::execute(instance, 0, {}, depth) :
+                                ExecutionResult{Value{1}};
+        --host_recursion_depth;
+        return result;
+    };
+
+    const auto module = parse(wasm);
+    auto instance = instantiate(*module, {{{host_f}, module->typesec[0]}});
+
+    EXPECT_EQ(host_recursion_depth, 0);
+    EXPECT_EQ(max_recorded_host_recursion_depth, 0);
+    EXPECT_EQ(max_recorded_wasm_recursion_depth, 0);
+    EXPECT_THAT(execute(*instance, 0, {}), Result(1_u32));
+    EXPECT_EQ(max_recorded_wasm_recursion_depth, 0);
+    EXPECT_EQ(max_recorded_host_recursion_depth, host_recursion_limit);
+    EXPECT_EQ(host_recursion_depth, 0);
+}
+
+TEST(execute_call, call_host_function_calling_wasm_interleaved_infinite_recursion_inclusive)
+{
+    // In this test the host function host_f bumps the wasm call depth
+    // including itself in the call depth limit.
+
+    /* wat2wasm
+    (func $host_f (import "host" "f") (result i32))
+    (func (result i32) (call $host_f))
+    */
+    const auto wasm =
+        from_hex("0061736d010000000105016000017f020a0104686f737401660000030201000a0601040010000b");
+
+    static int counter = 0;
+    constexpr auto host_f = [](std::any&, Instance& instance, const Value*, int depth) noexcept {
+        EXPECT_LT(depth, DepthLimit);
+        ++counter;
+        return fizzy::execute(instance, 1, {}, depth + 1);
+    };
+
+    const auto module = parse(wasm);
+    auto instance = instantiate(*module, {{{host_f}, module->typesec[0]}});
+
+    // Start with the imported host function.
+    counter = 0;
+    EXPECT_THAT(execute(*instance, 0, {}), Traps());
+    EXPECT_EQ(counter, DepthLimit / 2);
+
+    // Start with the internal wasm function.
+    counter = 0;
+    EXPECT_THAT(execute(*instance, 1, {}), Traps());
+    EXPECT_EQ(counter, DepthLimit / 2);
+}
+
+TEST(execute_call, call_host_function_calling_wasm_interleaved_infinite_recursion_exclusive)
+{
+    // In this test the host function host_f only passes the wasm call depth along
+    // excluding itself in the call depth limit.
+
+    /* wat2wasm
+    (func $host_f (import "host" "f") (result i32))
+    (func (result i32) (call $host_f))
+    */
+    const auto wasm =
+        from_hex("0061736d010000000105016000017f020a0104686f737401660000030201000a0601040010000b");
+
+    static int counter = 0;
+    constexpr auto host_f = [](std::any&, Instance& instance, const Value*, int depth) noexcept {
+        EXPECT_LT(depth, DepthLimit);
+        ++counter;
+        return fizzy::execute(instance, 1, {}, depth);
+    };
+
+    const auto module = parse(wasm);
+    auto instance = instantiate(*module, {{{host_f}, module->typesec[0]}});
+
+    // Warning! Going up to the wasm call depth limit with host functions not counted in
+    // causes OS stack overflow when build with GCC's ASan. Therefore the test starts at the depth
+    // being the 1/3 of the limit.
+    constexpr auto start_depth = DepthLimit / 3;
+
+    // Start with the imported host function.
+    // Wasm and host functions are executed the same number of times.
+    // host, wasm, ... , host, wasm, host, wasm:TRAP.
+    counter = 0;
+    EXPECT_THAT(execute(*instance, 0, {}, start_depth), Traps());
+    EXPECT_EQ(counter, DepthLimit - start_depth);
+
+    // Start with the internal wasm function.
+    // Host function is execute one time less than the wasm function.
+    // wasm, host, ... , wasm, host, wasm:TRAP.
+    counter = 0;
+    EXPECT_THAT(execute(*instance, 1, {}, start_depth), Traps());
+    EXPECT_EQ(counter, DepthLimit - start_depth - 1);
+}