Skip to content

Harden the known wall-clock test flakes #2657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lahma merged 1 commit into from
Jul 13, 2026
Merged

Harden the known wall-clock test flakes #2657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions Jint.Tests.PublicInterface/TimeSystemTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,30 @@ public void CanProduceValidDatesUsingNodaTimeIntegration(string input, long expe
[Fact]
public void CanUseTimeProvider()
{
// Bracket the evaluation instead of assuming it completes within a fixed window: the
// engine reads the same system clock, so the script's "now" must land between wall-clock
// readings taken around it (small slack for coarse timer ticks / clock adjustments). The
// previous form captured only "before" and allowed 100 ms — evaluation on a loaded CI
// runner regularly exceeded that and flaked the test.
var defaultEngine = new Engine();
var defaultNow = DateTimeOffset.Now.ToUnixTimeMilliseconds();
var beforeDefault = DateTimeOffset.Now.ToUnixTimeMilliseconds();
var defaultScriptNow = defaultEngine.Evaluate("new Date() * 1").AsNumber();
Assert.InRange(defaultScriptNow, defaultNow, defaultNow + 100);

var afterDefault = DateTimeOffset.Now.ToUnixTimeMilliseconds();
Assert.InRange(defaultScriptNow, beforeDefault - 100, afterDefault + 100);

var timeProvider = new FakeTimeProvider();
timeProvider.SetUtcNow(new DateTimeOffset(2023, 11, 6, 0, 0, 0, 0, TimeSpan.Zero));

var timeProviderEngine = new Engine(options =>
{
options.TimeSystem = new TimeProviderTimeSystem(timeProvider);
});

// the fake provider is frozen, so the script must read exactly its instant
var timeProviderNow = timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
var timeProviderScriptNow = timeProviderEngine.Evaluate("new Date() * 1").AsNumber();
Assert.InRange(timeProviderNow, timeProviderScriptNow, timeProviderScriptNow + 100);
Assert.Equal(timeProviderNow, timeProviderScriptNow);

Assert.NotInRange(timeProviderScriptNow, defaultScriptNow - 10000, defaultScriptNow + 10000);
}
}
Expand Down
19 changes: 18 additions & 1 deletion Jint.Tests.Test262/Test262Test.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,23 @@ namespace Jint.Tests.Test262;

public abstract partial class Test262Test
{
// Tests that legitimately do seconds of work (large code-point loops): they pass in a few
// seconds when run alone but blow the default 30-second wall-clock TimeoutInterval when the
// parallel suite starves the CPU. The constraint exists to catch runaway scripts, so these
// known-heavy files get a raised budget instead of a global increase — a genuine hang is
// still caught, everything else keeps the tight limit.
private static readonly HashSet<string> _slowTestFiles = new(StringComparer.Ordinal)
{
"annexB/built-ins/RegExp/RegExp-leading-escape-BMP.js",
"annexB/built-ins/RegExp/RegExp-trailing-escape-BMP.js",
// 4-level byte-range loops, ~1.3M decodeURI calls per script (~1.5s alone)
"built-ins/decodeURI/S15.1.3.1_A2.5_T1.js",
"built-ins/decodeURIComponent/S15.1.3.2_A2.5_T1.js",
};

private static TimeSpan GetTimeoutInterval(Test262File file)
=> _slowTestFiles.Contains(file.FileName) ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(30);

// Thread-local storage for agent manager (tests run in parallel)
[ThreadStatic]
private static Test262AgentManager? _currentAgentManager;
Expand All @@ -28,7 +45,7 @@ private static Engine BuildTestExecutor(Test262File file, Test262AgentManager? a
var relativePath = Path.GetDirectoryName(file.FileName) ?? "";
cfg.EnableModules(new Test262ModuleLoader(State.Test262Stream.Options.FileSystem, relativePath));
cfg.ExperimentalFeatures = ExperimentalFeature.All;
cfg.TimeoutInterval(TimeSpan.FromSeconds(30));
cfg.TimeoutInterval(GetTimeoutInterval(file));
// Configure agent blocking based on test flags
if (file.Flags.Contains("CanBlockIsFalse"))
{
Expand Down
72 changes: 44 additions & 28 deletions Jint.Tests/Runtime/AsyncTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ namespace Jint.Tests.Runtime;

public class AsyncTests
{
// Wall-clock promise budgets are a CI flake trap: on a loaded runner, thread-pool starvation
// can delay a Task.Delay continuation by many seconds (a 100 ms delay has been observed to
// blow the default 10-second PromiseTimeout on a two-core Windows runner). Tests that assert
// BEHAVIOR of the task-interop / async-wake path — not latency — use this budget so they
// cannot lose that race; tests that assert a timeout FIRES keep tight budgets (starvation
// only helps the timeout fire).
private static readonly TimeSpan GenerousPromiseTimeout = TimeSpan.FromMinutes(2);

[Fact]
public void AwaitPropagationAgainstPrimitiveValue()
{
Expand Down Expand Up @@ -725,12 +733,14 @@ public void ShouldReturnedTaskConvertedToPromiseInJS()
Assert.Equal(AsyncTestClass.TestString, result);
}

[Fact(Skip = "Flaky test")]
[Fact]
public void ShouldRespectCustomProvidedTimeoutWhenUnwrapping()
{
Engine engine = new(options => options.ExperimentalFeatures = ExperimentalFeature.TaskInterop);
engine.SetValue("asyncTestClass", new AsyncTestClass());
var result = engine.Evaluate("asyncTestClass.ReturnDelayedTaskAsync().then(x=>x)");
// A promise that never settles makes the timeout deterministic. The previous version raced
// a 1 ms timeout against a 100 ms delayed task — under load the task could win, no
// exception was thrown, and the test flaked (it was skipped for that reason).
Engine engine = new();
var result = engine.Evaluate("new Promise(function () {})");
var timeout = TimeSpan.FromMilliseconds(1);
var exception = Assert.Throws<PromiseRejectedException>(() => result.UnwrapIfPromise(timeout));
Assert.Equal($"Promise was rejected with value Timeout of {timeout} reached", exception.Message);
Expand All @@ -739,14 +749,16 @@ public void ShouldRespectCustomProvidedTimeoutWhenUnwrapping()
[Fact]
public void ShouldAwaitUnwrapPromiseWithCustomTimeout()
{
Engine engine = new(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = TimeSpan.FromMilliseconds(500); });
// the custom budget must be generous: this asserts the SUCCESS path, and a tight budget
// races the delayed task's continuation scheduling under CI load
Engine engine = new(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = GenerousPromiseTimeout; });
engine.SetValue("asyncTestClass", new AsyncTestClass());
engine.Execute("""
engine.Execute("""
async function test() {
return await asyncTestClass.ReturnDelayedTaskAsync();
}
""");
var result = engine.Invoke("test").UnwrapIfPromise();
var result = engine.Invoke("test").UnwrapIfPromise(GenerousPromiseTimeout);
Assert.Equal(AsyncTestClass.TestString, result);
}

Expand Down Expand Up @@ -1074,14 +1086,15 @@ public void ShouldReturnedValueTaskOfTCatchWhenThrowError()
[Fact]
public void ShouldAwaitUnwrapValueTaskOfTPromiseWithCustomTimeout()
{
Engine engine = new(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = TimeSpan.FromMilliseconds(500); });
// see ShouldAwaitUnwrapPromiseWithCustomTimeout — success path must not race CI load
Engine engine = new(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = GenerousPromiseTimeout; });
engine.SetValue("asyncTestClass", new AsyncTestClass());
engine.Execute("""
async function test() {
return await asyncTestClass.ReturnDelayedValueTaskAsync();
}
""");
var result = engine.Invoke("test").UnwrapIfPromise();
var result = engine.Invoke("test").UnwrapIfPromise(GenerousPromiseTimeout);
Assert.Equal(AsyncTestClass.TestString, result);
}

Expand Down Expand Up @@ -1951,7 +1964,7 @@ async function main() {
[Fact]
public async Task EvaluateAsyncWithTaskInteropShouldWork()
{
var engine = new Engine(options => options.ExperimentalFeatures = ExperimentalFeature.TaskInterop);
var engine = new Engine(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = GenerousPromiseTimeout; });
engine.SetValue("asyncTestClass", new AsyncTestClass());

var result = await engine.EvaluateAsync("""
Expand All @@ -1966,7 +1979,7 @@ async function main() {
[Fact]
public async Task InvokeAsyncWithMultipleAwaitsShouldWork()
{
var engine = new Engine(options => options.ExperimentalFeatures = ExperimentalFeature.TaskInterop);
var engine = new Engine(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = GenerousPromiseTimeout; });
engine.SetValue("asyncTestClass", new AsyncTestClass());
engine.Execute("""
async function test() {
Expand All @@ -1988,7 +2001,7 @@ public async Task EvaluateAsyncShouldNotBlockDuringClrTaskDelay()
{
// Verifies the async wake path: EvaluateAsync releases the thread during
// a .NET Task.Delay (simulating IO like gRPC), and resumes correctly.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("simulateIO", new Func<Task<int>>(async () =>
{
await Task.Delay(100);
Expand All @@ -2009,7 +2022,7 @@ async function main() {
public async Task EvaluateAsyncShouldHandleMultipleSequentialClrTasks()
{
// Multiple sequential .NET async calls, each releasing and re-acquiring the thread.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
var callOrder = new List<string>();

engine.SetValue("step1", new Func<Task<string>>(async () =>
Expand Down Expand Up @@ -2050,7 +2063,7 @@ public async Task EvaluateAsyncShouldHandleConcurrentClrTasksViaPromiseAll()
{
// Promise.all with multiple .NET Tasks running concurrently.
// All tasks should start before any completes.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
var startTimes = new ConcurrentDictionary<string, DateTime>();

engine.SetValue("fetch", new Func<string, Task<string>>(async (id) =>
Expand Down Expand Up @@ -2105,7 +2118,7 @@ async function main() {
public async Task EvaluateAsyncShouldHandleClrTaskRejection()
{
// .NET Task that throws should propagate as a rejected promise.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("failingIO", new Func<Task<string>>(async () =>
{
await Task.Delay(50);
Expand All @@ -2131,7 +2144,7 @@ async function main() {
public async Task EvaluateAsyncShouldHandleNestedJsToClrToJsAsync()
{
// Nested async: JS → .NET async → back into JS engine (via callback) → .NET async
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);

engine.SetValue("fetchData", new Func<Task<string>>(async () =>
{
Expand Down Expand Up @@ -2186,7 +2199,7 @@ public async Task EvaluateAsyncShouldNotBlockCallerThread()
{
// Proves the caller thread is released: start EvaluateAsync, then verify
// we can do other work before it completes.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
var ioStarted = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);

engine.SetValue("simulateIO", new Func<Task<int>>(async () =>
Expand Down Expand Up @@ -2228,7 +2241,8 @@ public async Task EvaluateAsyncMultipleEnginesConcurrently()
var idx = i;
tasks[i] = Task.Run(async () =>
{
var engine = new Engine();
// 20 engines contending for the pool makes starvation likely by construction
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("compute", new Func<int, Task<int>>(async (n) =>
{
await Task.Delay(50);
Expand All @@ -2252,7 +2266,7 @@ public async Task EvaluateAsyncMultipleEnginesConcurrently()
public async Task InvokeAsyncWithClrTaskInterop()
{
// InvokeAsync with .NET Task interop, verifying the complete path.
var engine = new Engine(options => options.ExperimentalFeatures = ExperimentalFeature.TaskInterop);
var engine = new Engine(options => { options.ExperimentalFeatures = ExperimentalFeature.TaskInterop; options.Constraints.PromiseTimeout = GenerousPromiseTimeout; });

engine.SetValue("asyncTestClass", new AsyncTestClass());
engine.Execute("""
Expand All @@ -2271,7 +2285,7 @@ public async Task EvaluateAsyncWithSetTimeoutPattern()
{
// setTimeout pattern using .NET Task.Delay, exercising the async wake path
// with event loop scheduling (not direct Task interop).
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("setTimeout", (Action action, int ms) =>
{
Task.Delay(ms).ContinueWith(_ => action());
Expand Down Expand Up @@ -2303,7 +2317,7 @@ public async Task SequentialEvaluateAsyncOnSameEngineShouldWork()
// After the first EvaluateAsync completes, the event loop's _eventAvailable
// TCS has been consumed. The second call must detect the stale/null TCS and
// install a fresh one without hanging or busy-looping.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("io", new Func<int, Task<int>>(async (n) =>
{
await Task.Delay(50);
Expand All @@ -2326,7 +2340,7 @@ public async Task MixedSyncThenAsyncOnSameEngineShouldWork()
// Sync Evaluate+UnwrapIfPromise sets _waitingThreadId during the spin-wait.
// A subsequent EvaluateAsync must not be blocked by a leftover _waitingThreadId
// from the sync path. This tests the handoff between the two execution models.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("io", new Func<Task<string>>(async () =>
{
await Task.Delay(50);
Expand Down Expand Up @@ -2374,8 +2388,8 @@ await Assert.ThrowsAsync<PromiseRejectedException>(async () =>
await engine.EvaluateAsync("(async () => await slowIO())()");
});

// Increase timeout for recovery
engine.Options.Constraints.PromiseTimeout = TimeSpan.FromSeconds(5);
// Increase timeout for recovery (generous — the recovery half asserts success)
engine.Options.Constraints.PromiseTimeout = GenerousPromiseTimeout;

// Engine should still work
var result = await engine.EvaluateAsync("(async () => await fastIO())()");
Expand All @@ -2387,7 +2401,7 @@ public async Task EngineReusableAfterEvaluateAsyncCancellation()
{
// CancellationToken fires during a pending EvaluateAsync. The TCS in
// WaitForEventAsync gets cancelled. Verify the engine is still usable.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("slowIO", new Func<Task<int>>(async () =>
{
await Task.Delay(10_000);
Expand Down Expand Up @@ -2421,7 +2435,9 @@ public async Task EvaluateAsyncShouldPropagateUncaughtClrTaskFailure()
{
// A .NET Task that throws, with NO try/catch in JS, should surface
// as PromiseRejectedException to the .NET caller via EvaluateAsync.
var engine = new Engine();
// (A tight budget could throw PromiseRejectedException for the WRONG reason —
// timeout instead of the propagated failure — and fail the message assert.)
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("failingIO", new Func<Task<string>>(async () =>
{
await Task.Delay(50);
Expand Down Expand Up @@ -2461,7 +2477,7 @@ public async Task EvaluateAsyncWithVoidTaskShouldResolveUndefined()
// path in ConvertTaskToPromise — there's no Task<T>.Result property.
// The promise should resolve with undefined, not throw or return a
// wrapped VoidTaskResult.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
var sideEffect = false;

engine.SetValue("doWork", new Func<Task>(async () =>
Expand All @@ -2480,7 +2496,7 @@ public async Task EvaluateAsyncWithPreparedScriptShouldWork()
{
// Tests the EvaluateAsync(Prepared<Script>) overload which has its own
// code path and was previously untested.
var engine = new Engine();
var engine = new Engine(options => options.Constraints.PromiseTimeout = GenerousPromiseTimeout);
engine.SetValue("io", new Func<Task<int>>(async () =>
{
await Task.Delay(50);
Expand Down
17 changes: 14 additions & 3 deletions Jint/Native/Atomics/AtomicsInstance.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -660,14 +660,25 @@ private JsValue WaitAsync(JsValue thisObject, JsValue typedArray, JsValue index,
// Handle timeout
if (!double.IsPositiveInfinity(q))
{
var timeoutMs = (int) System.Math.Min(q, int.MaxValue);
var timeoutMs = (int) System.Math.Min(System.Math.Ceiling(q), int.MaxValue);
var capturedWaiterList = waiterList;
var capturedAsyncWaiter = asyncWaiter;

// Use a timer to resolve with "timed-out" after the timeout
// Use a timer to resolve with "timed-out" after the timeout. Task.Delay can
// complete slightly ahead of the requested interval (timer granularity /
// coalescing), and the wait must not report timed-out before the timeout has
// actually elapsed — script can observe the lapse (test262 asserts
// lapse >= timeout) — so re-delay until a monotonic clock agrees.
_ = Task.Run(async () =>
{
await Task.Delay(timeoutMs).ConfigureAwait(false);
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var remainingMs = timeoutMs;
do
{
await Task.Delay(remainingMs).ConfigureAwait(false);
remainingMs = timeoutMs - (int) stopwatch.ElapsedMilliseconds;
} while (remainingMs > 0);

if (!capturedAsyncWaiter.Resolved)
{
capturedWaiterList.RemoveAsync(capturedAsyncWaiter);
Expand Down
Loading