Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 77 additions & 65 deletions osprey_async_worker/src/osprey/async_worker/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ async def _execute_async_udf(
caught_exception = e
finally:
_record_udf_metric(metric_tags, execution_result, caught_exception)
return execution_result
return execution_result


async def _execute_async_batch(
Expand Down Expand Up @@ -433,71 +433,83 @@ async def execute(

ready_sync, ready_async = _get_ready_sync_and_async(allow_async, context)

while ready_sync or ready_async or in_progress_singlets or in_progress_batches:
# Check for already-finished tasks (non-blocking)
finished_singlets = [t for t in in_progress_singlets if t.done()]
finished_batches = [t for t in in_progress_batches if t.done()]

if not ready_sync and not ready_async and not finished_singlets and not finished_batches:
# Block until at least one async task finishes
all_pending: set[asyncio.Task[Any]] = set(in_progress_singlets.keys()) | set(in_progress_batches.keys())
if all_pending:
with tracer.start_span('osprey.rules.async_wait_nodes', child_of=parent_tracer_span):
done, _ = await asyncio.wait(all_pending, return_when=asyncio.FIRST_COMPLETED)
finished_singlets = [t for t in done if t in in_progress_singlets]
finished_batches = [t for t in done if t in in_progress_batches]

# Process finished singlets
for task in finished_singlets:
chain = in_progress_singlets.pop(task)
context.set_resolved_value(chain, task.result())

# Process finished batches
# Distinct loop variable from the singlet `task` above: the two dicts hold
# tasks with different result types (Task[NodeResult] vs Task[Sequence[NodeResult]]),
# and reusing one name would unify them to the singlet type.
for batch_task in finished_batches:
chains = in_progress_batches.pop(batch_task)
results = batch_task.result()
for i, chain in enumerate(chains):
context.set_resolved_value(chain, results[i])

# Enqueue async tasks
if allow_async and ready_async:
with tracer.start_span('osprey.rules.try_enqueue_batches', child_of=parent_tracer_span):
remaining_ready_async, new_batch_tasks = await _enqueue_batches(
loop, semaphore, context, error_infos, ready_async
)
in_progress_batches.update(new_batch_tasks)

for async_chain, pre_resolved_arguments in remaining_ready_async:
# Native async UDF → await on event loop
if isinstance(async_chain.executor, CallExecutor) and isinstance(
async_chain.executor._udf, (AsyncUDFBase, AsyncBatchableUDFBase)
):
task = asyncio.create_task(
_execute_async_udf(semaphore, async_chain, context, error_infos, pre_resolved_arguments)
)
else:
# Legacy sync UDF with execute_async=True → thread pool fallback
task = asyncio.create_task(
_execute_legacy_in_executor(loop, semaphore, async_chain, context, error_infos)
try:
while ready_sync or ready_async or in_progress_singlets or in_progress_batches:
# Check for already-finished tasks (non-blocking)
finished_singlets = [t for t in in_progress_singlets if t.done()]
finished_batches = [t for t in in_progress_batches if t.done()]

if not ready_sync and not ready_async and not finished_singlets and not finished_batches:
# Block until at least one async task finishes
all_pending: set[asyncio.Task[Any]] = set(in_progress_singlets.keys()) | set(in_progress_batches.keys())
if all_pending:
with tracer.start_span('osprey.rules.async_wait_nodes', child_of=parent_tracer_span):
done, _ = await asyncio.wait(all_pending, return_when=asyncio.FIRST_COMPLETED)
finished_singlets = [t for t in done if t in in_progress_singlets]
finished_batches = [t for t in done if t in in_progress_batches]

# Process finished singlets
for task in finished_singlets:
chain = in_progress_singlets.pop(task)
context.set_resolved_value(chain, task.result())

# Process finished batches
# Distinct loop variable from the singlet `task` above: the two dicts hold
# tasks with different result types (Task[NodeResult] vs Task[Sequence[NodeResult]]),
# and reusing one name would unify them to the singlet type.
for batch_task in finished_batches:
chains = in_progress_batches.pop(batch_task)
results = batch_task.result()
for i, chain in enumerate(chains):
context.set_resolved_value(chain, results[i])

# Enqueue async tasks
if allow_async and ready_async:
with tracer.start_span('osprey.rules.try_enqueue_batches', child_of=parent_tracer_span):
remaining_ready_async, new_batch_tasks = await _enqueue_batches(
loop, semaphore, context, error_infos, ready_async
)
in_progress_singlets[task] = async_chain

# Execute sync chains inline (pure computation, fast, no I/O).
# Only yield deep into a long sync round when async tasks are in flight.
# Each sleep(0) triggers a full event loop cycle including gRPC C-core polling,
# so unnecessary yields cause significant context-switch overhead.
# Short rounds (<100 chains) skip yielding entirely — the asyncio.wait() at the
# top of the loop provides natural yield points between rounds.
for i, sync_chain in enumerate(ready_sync):
if (in_progress_singlets or in_progress_batches) and i > 0 and i % 100 == 0:
await asyncio.sleep(0)
result = _execute_sync(sync_chain, context, error_infos)
context.set_resolved_value(sync_chain, result)

ready_sync, ready_async = _get_ready_sync_and_async(allow_async, context)
in_progress_batches.update(new_batch_tasks)

for async_chain, pre_resolved_arguments in remaining_ready_async:
# Native async UDF → await on event loop
if isinstance(async_chain.executor, CallExecutor) and isinstance(
async_chain.executor._udf, (AsyncUDFBase, AsyncBatchableUDFBase)
):
task = asyncio.create_task(
_execute_async_udf(semaphore, async_chain, context, error_infos, pre_resolved_arguments)
)
else:
# Legacy sync UDF with execute_async=True → thread pool fallback
task = asyncio.create_task(
_execute_legacy_in_executor(loop, semaphore, async_chain, context, error_infos)
)
in_progress_singlets[task] = async_chain

# Execute sync chains inline (pure computation, fast, no I/O).
# Only yield deep into a long sync round when async tasks are in flight.
# Each sleep(0) triggers a full event loop cycle including gRPC C-core polling,
# so unnecessary yields cause significant context-switch overhead.
# Short rounds (<100 chains) skip yielding entirely — the asyncio.wait() at the
# top of the loop provides natural yield points between rounds.
for i, sync_chain in enumerate(ready_sync):
if (in_progress_singlets or in_progress_batches) and i > 0 and i % 100 == 0:
await asyncio.sleep(0)
result = _execute_sync(sync_chain, context, error_infos)
context.set_resolved_value(sync_chain, result)

ready_sync, ready_async = _get_ready_sync_and_async(allow_async, context)
except BaseException:
owned_tasks = [*in_progress_singlets, *in_progress_batches]
for owned_task in owned_tasks:
owned_task.cancel()
if owned_tasks:
cleanup = asyncio.gather(*owned_tasks, return_exceptions=True)
try:
await asyncio.shield(cleanup)
except asyncio.CancelledError:
await cleanup
Comment thread
github-code-quality[bot] marked this conversation as resolved.
Fixed
raise

# --- Build result ---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from osprey.engine.stdlib import get_config_registry
from osprey.engine.udf.arguments import ArgumentsBase
from osprey.engine.udf.registry import UDFRegistry
from result import Ok
from result import Ok, Result


class CountingBatchableArguments(ArgumentsBase):
Expand Down Expand Up @@ -83,6 +83,15 @@ class GatedArguments(ArgumentsBase):
value: str


class CancellationArguments(ArgumentsBase):
value: str


class BatchCancellationArguments(ArgumentsBase):
id: str
routing_key: str


class GatedAsyncUdf(AsyncUDFBase[GatedArguments, str]):
entered = 0
both_entered: asyncio.Event
Expand Down Expand Up @@ -473,3 +482,155 @@ async def test_batch_of_one_resolve_failure_surfaces_once(
assert isinstance(result.error_infos[0].error, RuntimeError)
assert str(result.error_infos[0].error) == 'resolve boom'
assert counting_batchable_udf.resolve_call_count == 1


@pytest.mark.asyncio
async def test_cancelled_udf_task_cancels_execution(async_execute_with_result):
started = asyncio.Event()
cancelled = asyncio.Event()

class SlowUDF(AsyncUDFBase[CancellationArguments, str]):
@classmethod
def _get_udf_base_args(cls):
return (CancellationArguments, str)

async def async_execute(self, execution_context: ExecutionContext, arguments: CancellationArguments) -> str:
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise

execution = asyncio.create_task(
async_execute_with_result(
'Result = SlowUDF(value="test")',
udf_registry=UDFRegistry.with_udfs(SlowUDF),
)
)
await asyncio.wait_for(started.wait(), timeout=1)

udf_tasks = [
task
for task in asyncio.all_tasks()
if task is not asyncio.current_task() and getattr(task.get_coro(), '__name__', '') == '_execute_async_udf'
]
assert len(udf_tasks) == 1
udf_tasks[0].cancel()

with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(execution, timeout=1)
assert cancelled.is_set()


@pytest.mark.asyncio
async def test_cancelled_execution_cancels_owned_udf_tasks(async_execute_with_result):
singlet_started = asyncio.Event()
singlet_cancelled = asyncio.Event()
singlet_release = asyncio.Event()
batch_started = asyncio.Event()
batch_cancelled = asyncio.Event()
batch_release = asyncio.Event()

class SlowSingletUDF(AsyncUDFBase[CancellationArguments, str]):
@classmethod
def _get_udf_base_args(cls):
return (CancellationArguments, str)

async def async_execute(self, execution_context: ExecutionContext, arguments: CancellationArguments) -> str:
singlet_started.set()
try:
await singlet_release.wait()
except asyncio.CancelledError:
singlet_cancelled.set()
raise
return arguments.value

class SlowBatchUDF(AsyncBatchableUDFBase[BatchCancellationArguments, str, BatchCancellationArguments]):
@classmethod
def _get_udf_base_args(cls):
return (BatchCancellationArguments, str, BatchCancellationArguments)

def get_batchable_arguments(self, arguments: BatchCancellationArguments) -> BatchCancellationArguments:
return arguments

async def async_execute(
self, execution_context: ExecutionContext, arguments: BatchCancellationArguments
) -> str:
return arguments.id

async def async_execute_batch(
self,
execution_context: ExecutionContext,
udfs: Sequence['SlowBatchUDF'],
arguments: Sequence[BatchCancellationArguments],
) -> Sequence[Result[str, Exception]]:
batch_started.set()
try:
await batch_release.wait()
except asyncio.CancelledError:
batch_cancelled.set()
raise
return [Ok(argument.id) for argument in arguments]

execution = asyncio.create_task(
async_execute_with_result(
"""
Single = SlowSingletUDF(value="single")
Batch1 = SlowBatchUDF(id="one", routing_key="shared")
Batch2 = SlowBatchUDF(id="two", routing_key="shared")
""",
udf_registry=UDFRegistry.with_udfs(SlowSingletUDF, SlowBatchUDF),
)
)

await asyncio.wait_for(asyncio.gather(singlet_started.wait(), batch_started.wait()), timeout=1)
execution.cancel()
try:
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(execution, timeout=1)
assert singlet_cancelled.is_set()
assert batch_cancelled.is_set()
finally:
singlet_release.set()
batch_release.set()


@pytest.mark.asyncio
async def test_repeated_cancellation_waits_for_owned_task_cleanup(async_execute_with_result):
started = asyncio.Event()
cleanup_started = asyncio.Event()
cleanup_release = asyncio.Event()

class SlowUDF(AsyncUDFBase[CancellationArguments, str]):
@classmethod
def _get_udf_base_args(cls):
return (CancellationArguments, str)

async def async_execute(self, execution_context: ExecutionContext, arguments: CancellationArguments) -> str:
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cleanup_started.set()
await cleanup_release.wait()
raise

execution = asyncio.create_task(
async_execute_with_result(
'Result = SlowUDF(value="test")',
udf_registry=UDFRegistry.with_udfs(SlowUDF),
)
)
await asyncio.wait_for(started.wait(), timeout=1)

execution.cancel()
await asyncio.wait_for(cleanup_started.wait(), timeout=1)
execution.cancel()
for _ in range(10):
await asyncio.sleep(0)

assert not execution.done()
cleanup_release.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(execution, timeout=1)
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,20 @@ async def test_batch_get_deduplicates_duplicate_keys():
assert service.batch_call_count == 1


@pytest.mark.asyncio
async def test_batch_get_resolves_its_owned_future_after_cache_replacement():
service = GatedBatchService()
accessor = ExternalServiceAccessor(service)
batch_get = asyncio.create_task(accessor.batch_get(['a']))
await asyncio.wait_for(service.started.wait(), timeout=1)

assert await accessor.get_without_cache('a') == 'value_a'
service.release.set()

assert await asyncio.wait_for(batch_get, timeout=1) == [Ok('batch_a')]
assert await accessor.get('a') == 'value_a'


@pytest.mark.asyncio
async def test_cancelled_batch_loader_evicts_its_cache_entries():
service = GatedBatchService()
Expand Down
Loading