From b52d7552f60908bb081626c685a4a2f59044a88b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 19 May 2025 17:00:11 -0700 Subject: [PATCH 01/39] Set asyncio_default_test_loop_scope to avoid warning about the value being unset Signed-off-by: David Gardner --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d8f3edaa5..d46ea27b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -203,8 +203,10 @@ filterwarnings = [ ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" pytest_plugins = ["aiqtoolkit-test"] + ## Pylint configuration begins here [tool.pylint.main] From 7ac47f382e9a9a56a4421cc92e21a412c2456c90 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 19 May 2025 17:09:46 -0700 Subject: [PATCH 02/39] Define the default event loop scope of asynchronous tests avoids warning about this not being set. Remove definition of pytest_plugins this is not a supported pytest config and triggers a warning Signed-off-by: David Gardner --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d46ea27b8..32ed9e224 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -203,8 +203,7 @@ filterwarnings = [ ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" -asyncio_default_fixture_loop_scope = "function" -pytest_plugins = ["aiqtoolkit-test"] +asyncio_default_fixture_loop_scope = "session" ## Pylint configuration begins here From f7da9972ac026f27992b78d172061f4c8e8820b0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 09:25:36 -0700 Subject: [PATCH 03/39] Silence known deprecation warnings being triggered by weave Signed-off-by: David Gardner --- src/aiq/observability/async_otel_listener.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/aiq/observability/async_otel_listener.py b/src/aiq/observability/async_otel_listener.py index ab8616315..06b42052c 100644 --- a/src/aiq/observability/async_otel_listener.py +++ b/src/aiq/observability/async_otel_listener.py @@ -15,6 +15,7 @@ import logging import re +import warnings from contextlib import asynccontextmanager from contextlib import contextmanager from typing import Any @@ -30,10 +31,20 @@ from aiq.utils.optional_imports import try_import_opentelemetry try: - from weave.trace.context import weave_client_context - from weave.trace.context.call_context import get_current_call - from weave.trace.context.call_context import set_call_stack - from weave.trace.weave_client import Call + with warnings.catch_warnings(): + # Ignore deprecation warnings being triggered by weave. https://github.com/wandb/weave/issues/3666 + # and https://github.com/wandb/weave/issues/4533 + warnings.filterwarnings("ignore", category=DeprecationWarning, message=r"^`sentry_sdk\.Hub` is deprecated") + warnings.filterwarnings("ignore", + category=DeprecationWarning, + message=r"^Using extra keyword arguments on `Field` is deprecated") + warnings.filterwarnings("ignore", + category=DeprecationWarning, + message=r"^`include` is deprecated and does nothing") + from weave.trace.context import weave_client_context + from weave.trace.context.call_context import get_current_call + from weave.trace.context.call_context import set_call_stack + from weave.trace.weave_client import Call WEAVE_AVAILABLE = True except ImportError: WEAVE_AVAILABLE = False From fb6d99c89a30de9bcf3c5e6e62f78c50abaec9ff Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 09:57:49 -0700 Subject: [PATCH 04/39] Rename config classes such that pytest doesn't believe they are test classes to be collected Signed-off-by: David Gardner --- tests/aiq/builder/test_builder.py | 94 +++++++++++++++---------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/aiq/builder/test_builder.py b/tests/aiq/builder/test_builder.py index 0266d06f0..9b59ca886 100644 --- a/tests/aiq/builder/test_builder.py +++ b/tests/aiq/builder/test_builder.py @@ -59,19 +59,19 @@ class FunctionReturningDerivedConfig(FunctionBaseConfig, name="fn_return_derived pass -class TestLLMProviderConfig(LLMBaseConfig, name="test_llm"): +class TLLMProviderConfig(LLMBaseConfig, name="test_llm"): raise_error: bool = False -class TestEmbedderProviderConfig(EmbedderBaseConfig, name="test_embedder_provider"): +class TEmbedderProviderConfig(EmbedderBaseConfig, name="test_embedder_provider"): raise_error: bool = False -class TestMemoryConfig(MemoryBaseConfig, name="test_memory"): +class TMemoryConfig(MemoryBaseConfig, name="test_memory"): raise_error: bool = False -class TestRetrieverProviderConfig(RetrieverBaseConfig, name="test_retriever"): +class TRetrieverProviderConfig(RetrieverBaseConfig, name="test_retriever"): raise_error: bool = False @@ -116,24 +116,24 @@ async def _astream(self, value: str): yield DerivedFunction(config) - @register_llm_provider(config_type=TestLLMProviderConfig) - async def register4(config: TestLLMProviderConfig, b: Builder): + @register_llm_provider(config_type=TLLMProviderConfig) + async def register4(config: TLLMProviderConfig, b: Builder): if (config.raise_error): raise ValueError("Error") yield LLMProviderInfo(config=config, description="A test client.") - @register_embedder_provider(config_type=TestEmbedderProviderConfig) - async def registe5(config: TestEmbedderProviderConfig, b: Builder): + @register_embedder_provider(config_type=TEmbedderProviderConfig) + async def registe5(config: TEmbedderProviderConfig, b: Builder): if (config.raise_error): raise ValueError("Error") yield EmbedderProviderInfo(config=config, description="A test client.") - @register_memory(config_type=TestMemoryConfig) - async def register6(config: TestMemoryConfig, b: Builder): + @register_memory(config_type=TMemoryConfig) + async def register6(config: TMemoryConfig, b: Builder): if (config.raise_error): raise ValueError("Error") @@ -152,8 +152,8 @@ async def remove_items(self, **kwargs) -> None: yield TestMemoryEditor() # Register mock provider - @register_retriever_provider(config_type=TestRetrieverProviderConfig) - async def register7(config: TestRetrieverProviderConfig, builder: Builder): + @register_retriever_provider(config_type=TRetrieverProviderConfig) + async def register7(config: TRetrieverProviderConfig, builder: Builder): if (config.raise_error): raise ValueError("Error") @@ -318,33 +318,33 @@ async def test_add_llm(): async with WorkflowBuilder() as builder: - await builder.add_llm("llm_name", TestLLMProviderConfig()) + await builder.add_llm("llm_name", TLLMProviderConfig()) with pytest.raises(ValueError): - await builder.add_llm("llm_name2", TestLLMProviderConfig(raise_error=True)) + await builder.add_llm("llm_name2", TLLMProviderConfig(raise_error=True)) # Try and add a llm with the same name with pytest.raises(ValueError): - await builder.add_llm("llm_name", TestLLMProviderConfig()) + await builder.add_llm("llm_name", TLLMProviderConfig()) async def test_get_llm(): - @register_llm_client(config_type=TestLLMProviderConfig, wrapper_type="test_framework") - async def register(config: TestLLMProviderConfig, b: Builder): + @register_llm_client(config_type=TLLMProviderConfig, wrapper_type="test_framework") + async def register(config: TLLMProviderConfig, b: Builder): class TestFrameworkLLM(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) - config: TestLLMProviderConfig + config: TLLMProviderConfig builder: Builder yield TestFrameworkLLM(config=config, builder=b) async with WorkflowBuilder() as builder: - config = TestLLMProviderConfig() + config = TLLMProviderConfig() await builder.add_llm("llm_name", config) @@ -360,7 +360,7 @@ async def test_get_llm_config(): async with WorkflowBuilder() as builder: - config = TestLLMProviderConfig() + config = TLLMProviderConfig() await builder.add_llm("llm_name", config) @@ -374,33 +374,33 @@ async def test_add_embedder(): async with WorkflowBuilder() as builder: - await builder.add_embedder("embedder_name", TestEmbedderProviderConfig()) + await builder.add_embedder("embedder_name", TEmbedderProviderConfig()) with pytest.raises(ValueError): - await builder.add_embedder("embedder_name2", TestEmbedderProviderConfig(raise_error=True)) + await builder.add_embedder("embedder_name2", TEmbedderProviderConfig(raise_error=True)) # Try and add the same name with pytest.raises(ValueError): - await builder.add_embedder("embedder_name", TestEmbedderProviderConfig()) + await builder.add_embedder("embedder_name", TEmbedderProviderConfig()) async def test_get_embedder(): - @register_embedder_client(config_type=TestEmbedderProviderConfig, wrapper_type="test_framework") - async def register(config: TestEmbedderProviderConfig, b: Builder): + @register_embedder_client(config_type=TEmbedderProviderConfig, wrapper_type="test_framework") + async def register(config: TEmbedderProviderConfig, b: Builder): class TestFrameworkEmbedder(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) - config: TestEmbedderProviderConfig + config: TEmbedderProviderConfig builder: Builder yield TestFrameworkEmbedder(config=config, builder=b) async with WorkflowBuilder() as builder: - config = TestEmbedderProviderConfig() + config = TEmbedderProviderConfig() await builder.add_embedder("embedder_name", config) @@ -416,7 +416,7 @@ async def test_get_embedder_config(): async with WorkflowBuilder() as builder: - config = TestEmbedderProviderConfig() + config = TEmbedderProviderConfig() await builder.add_embedder("embedder_name", config) @@ -430,21 +430,21 @@ async def test_add_memory(): async with WorkflowBuilder() as builder: - await builder.add_memory_client("memory_name", TestMemoryConfig()) + await builder.add_memory_client("memory_name", TMemoryConfig()) with pytest.raises(ValueError): - await builder.add_memory_client("memory_name2", TestMemoryConfig(raise_error=True)) + await builder.add_memory_client("memory_name2", TMemoryConfig(raise_error=True)) # Try and add the same name with pytest.raises(ValueError): - await builder.add_memory_client("memory_name", TestMemoryConfig()) + await builder.add_memory_client("memory_name", TMemoryConfig()) async def test_get_memory(): async with WorkflowBuilder() as builder: - config = TestMemoryConfig() + config = TMemoryConfig() memory = await builder.add_memory_client("memory_name", config) @@ -458,7 +458,7 @@ async def test_get_memory_config(): async with WorkflowBuilder() as builder: - config = TestMemoryConfig() + config = TMemoryConfig() await builder.add_memory_client("memory_name", config) @@ -471,31 +471,31 @@ async def test_get_memory_config(): async def test_add_retriever(): async with WorkflowBuilder() as builder: - await builder.add_retriever("retriever_name", TestRetrieverProviderConfig()) + await builder.add_retriever("retriever_name", TRetrieverProviderConfig()) with pytest.raises(ValueError): - await builder.add_retriever("retriever_name2", TestRetrieverProviderConfig(raise_error=True)) + await builder.add_retriever("retriever_name2", TRetrieverProviderConfig(raise_error=True)) with pytest.raises(ValueError): - await builder.add_retriever("retriever_name", TestRetrieverProviderConfig()) + await builder.add_retriever("retriever_name", TRetrieverProviderConfig()) async def get_retriever(): - @register_retriever_client(config_type=TestRetrieverProviderConfig, wrapper_type="test_framework") - async def register(config: TestRetrieverProviderConfig, b: Builder): + @register_retriever_client(config_type=TRetrieverProviderConfig, wrapper_type="test_framework") + async def register(config: TRetrieverProviderConfig, b: Builder): class TestFrameworkRetriever(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) - config: TestRetrieverProviderConfig + config: TRetrieverProviderConfig builder: Builder yield TestFrameworkRetriever(config=config, builder=b) - @register_retriever_client(config_type=TestRetrieverProviderConfig, wrapper_type=None) - async def register_no_framework(config: TestRetrieverProviderConfig, builder: Builder): + @register_retriever_client(config_type=TRetrieverProviderConfig, wrapper_type=None) + async def register_no_framework(config: TRetrieverProviderConfig, builder: Builder): class TestRetriever(AIQRetriever): @@ -515,7 +515,7 @@ async def remove_items(self, **kwargs): async with WorkflowBuilder() as builder: - config = TestRetrieverProviderConfig() + config = TRetrieverProviderConfig() await builder.add_retriever("retriever_name", config) @@ -535,7 +535,7 @@ async def get_retriever_config(): async with WorkflowBuilder() as builder: - config = TestRetrieverProviderConfig() + config = TRetrieverProviderConfig() await builder.add_retriever("retriever_name", config) @@ -550,10 +550,10 @@ async def test_built_config(): general_config = GeneralConfig(cache_dir="Something else") function_config = FunctionReturningFunctionConfig() workflow_config = FunctionReturningFunctionConfig() - llm_config = TestLLMProviderConfig() - embedder_config = TestEmbedderProviderConfig() - memory_config = TestMemoryConfig() - retriever_config = TestRetrieverProviderConfig() + llm_config = TLLMProviderConfig() + embedder_config = TEmbedderProviderConfig() + memory_config = TMemoryConfig() + retriever_config = TRetrieverProviderConfig() async with WorkflowBuilder(general_config=general_config) as builder: From 57030b10bc2076ae77292f2e6b39b2d35c5137f1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 10:08:31 -0700 Subject: [PATCH 05/39] Rename config classes such that pytest doesn't believe they are test classes to be collected Signed-off-by: David Gardner --- .../aiq/front_ends/fastapi/test_fastapi_front_end_plugin.py | 4 ++-- tests/aiq/retriever/test_retrievers.py | 4 ++-- tests/aiq/utils/test_yaml_tools.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/aiq/front_ends/fastapi/test_fastapi_front_end_plugin.py b/tests/aiq/front_ends/fastapi/test_fastapi_front_end_plugin.py index ecbdbcddf..9eeb46d7a 100644 --- a/tests/aiq/front_ends/fastapi/test_fastapi_front_end_plugin.py +++ b/tests/aiq/front_ends/fastapi/test_fastapi_front_end_plugin.py @@ -36,7 +36,7 @@ from aiq.utils.type_utils import override -class TestCustomWorker(FastApiFrontEndPluginWorker): +class CustomWorker(FastApiFrontEndPluginWorker): @override async def add_routes(self, app: FastAPI, builder: WorkflowBuilder): @@ -169,7 +169,7 @@ async def test_custom_endpoint(): workflow=EchoFunctionConfig(), ) - async with _build_client(config, worker_class=TestCustomWorker) as client: + async with _build_client(config, worker_class=CustomWorker) as client: response = await client.get("/custom") assert response.status_code == 200 diff --git a/tests/aiq/retriever/test_retrievers.py b/tests/aiq/retriever/test_retrievers.py index 62dee6e39..f33e9fa90 100644 --- a/tests/aiq/retriever/test_retrievers.py +++ b/tests/aiq/retriever/test_retrievers.py @@ -25,7 +25,7 @@ from aiq.retriever.nemo_retriever.retriever import NemoRetriever -class TestMilvusClient: +class CustomMilvusClient: def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -167,7 +167,7 @@ def embed_documents(self, texts): @pytest.fixture(name="milvus_retriever", scope="module") def _get_milvus_retriever(): - test_client = TestMilvusClient() + test_client = CustomMilvusClient() return MilvusRetriever( client=test_client, diff --git a/tests/aiq/utils/test_yaml_tools.py b/tests/aiq/utils/test_yaml_tools.py index 6419f2447..ffbc63c6a 100644 --- a/tests/aiq/utils/test_yaml_tools.py +++ b/tests/aiq/utils/test_yaml_tools.py @@ -66,7 +66,7 @@ def fixture_env_vars(): del os.environ[var] -class TestConfig(FunctionBaseConfig, name="my_test_fn"): +class CustomConfig(FunctionBaseConfig, name="my_test_fn"): string_input: str int_input: int float_input: float @@ -80,8 +80,8 @@ class TestConfig(FunctionBaseConfig, name="my_test_fn"): @pytest.fixture(scope="module", autouse=True) async def fixture_register_test_fn(): - @register_function(config_type=TestConfig) - async def register(config: TestConfig, b: Builder): + @register_function(config_type=CustomConfig) + async def register(config: CustomConfig, b: Builder): async def _inner(some_input: str) -> str: return some_input From abd0c4460b6923a20cbf1b8d8a370a81255055c4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 10:16:55 -0700 Subject: [PATCH 06/39] Replace usage of update_forward_refs() with model_rebuild() per https://docs.pydantic.dev/2.10/migration/#changes-to-pydanticbasemodel Signed-off-by: David Gardner --- src/aiq/profiler/inference_optimization/data_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiq/profiler/inference_optimization/data_models.py b/src/aiq/profiler/inference_optimization/data_models.py index 64c88c920..2c09e57f4 100644 --- a/src/aiq/profiler/inference_optimization/data_models.py +++ b/src/aiq/profiler/inference_optimization/data_models.py @@ -220,7 +220,7 @@ def _repr(self, level: int) -> str: return "\n".join([info] + child_strs) -CallNode.update_forward_refs() +CallNode.model_rebuild() class NodeMetrics(BaseModel): @@ -296,7 +296,7 @@ class ConcurrencyCallNode(CallNode): llm_text_output: str | None = None -ConcurrencyCallNode.update_forward_refs() +ConcurrencyCallNode.model_rebuild() class ConcurrencySpikeInfo(BaseModel): From 70169d42e010fae13d184e270e17556225942c8e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 10:33:04 -0700 Subject: [PATCH 07/39] Silence expected builder warnings with explicitly testing that they are being emitted Signed-off-by: David Gardner --- tests/aiq/builder/test_builder.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/aiq/builder/test_builder.py b/tests/aiq/builder/test_builder.py index 9b59ca886..1867cd41e 100644 --- a/tests/aiq/builder/test_builder.py +++ b/tests/aiq/builder/test_builder.py @@ -247,17 +247,22 @@ async def register2(config: FunctionReturningBadConfig, b: Builder): fn = await builder.set_workflow(FunctionReturningFunctionConfig()) assert isinstance(fn, Function) - fn = await builder.set_workflow(FunctionReturningInfoConfig()) + with pytest.warns(UserWarning, match=r"^Overwriting existing workflow$"): + fn = await builder.set_workflow(FunctionReturningInfoConfig()) + assert isinstance(fn, Function) - fn = await builder.set_workflow(FunctionReturningDerivedConfig()) + with pytest.warns(UserWarning, match=r"^Overwriting existing workflow$"): + fn = await builder.set_workflow(FunctionReturningDerivedConfig()) + assert isinstance(fn, Function) with pytest.raises(ValueError): - await builder.set_workflow(FunctionReturningBadConfig()) + with pytest.warns(UserWarning, match=r"^Overwriting existing workflow$"): + await builder.set_workflow(FunctionReturningBadConfig()) # Try and add a function with the same name - with pytest.warns(): + with pytest.warns(UserWarning, match=r"^Overwriting existing workflow$"): await builder.set_workflow(FunctionReturningFunctionConfig()) From 3feefc4da2f452fc65f4d6e3834702d7516e256f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 10:47:22 -0700 Subject: [PATCH 08/39] Silence expected warning at test completion Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index 24c6644ce..8d20bd249 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -199,6 +199,7 @@ async def test_run_workflow_local_success(evaluation_run, session_manager, gener assert not evaluation_run.workflow_interrupted +@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_errors(evaluation_run, session_manager): """Test workflow with no 'single output' fails gracefully.""" From 4ea2590ad204943551a9511fa6dab55f25d14892 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 11:20:55 -0700 Subject: [PATCH 09/39] Replace deprecated dict method with model_dump Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index 8d20bd249..5fb9260f9 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -383,7 +383,7 @@ def test_write_output(evaluation_run, default_eval_config, eval_input, eval_outp eval_input_item.output_obj = generated_answer mock_dataset_handler = MagicMock() - workflow_output = json.dumps([item.dict() for item in eval_input.eval_input_items]) + workflow_output = json.dumps([item.model_dump() for item in eval_input.eval_input_items]) mock_dataset_handler.publish_eval_input.return_value = workflow_output # Mock evaluation results From 99627f7a67991f863ab152aa96b27ff9da020e22 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 11:53:52 -0700 Subject: [PATCH 10/39] Ignore expected warning about the mock method not being called Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index 5fb9260f9..545a081fa 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -246,6 +246,7 @@ async def test_run_workflow_local_skip_completed(evaluation_run, session_manager assert pending_item.output_obj == generated_answer, "Pending item output should have been processed" +@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_workflow_interrupted(evaluation_run, eval_input, session_manager): """Test that workflow_interrupted is set to True when an exception occurs during workflow execution.""" From c280f82dff8756e1254fc183b730de2d9489975d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 12:01:59 -0700 Subject: [PATCH 11/39] Silence expected warning Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index 545a081fa..b39924b52 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -210,6 +210,7 @@ async def test_run_workflow_local_errors(evaluation_run, session_manager): await evaluation_run.run_workflow_local(session_manager) +@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_skip_completed(evaluation_run, session_manager, generated_answer): """Test that 'skip_completed_entries=True' skips completed items and processes only unfinished ones.""" From 8268d6a0a0a28d99e06f06f3560d6574eeebd76f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 12:03:08 -0700 Subject: [PATCH 12/39] Replace deprecated MultiCommand with Group Signed-off-by: David Gardner --- src/aiq/cli/commands/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiq/cli/commands/start.py b/src/aiq/cli/commands/start.py index 44c6d1316..202c920ad 100644 --- a/src/aiq/cli/commands/start.py +++ b/src/aiq/cli/commands/start.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) -class StartCommandGroup(click.MultiCommand): +class StartCommandGroup(click.Group): # pylint: disable=too-many-positional-arguments def __init__( From a602c1a00ea4810ea32cbeadfdfbca090875dc50 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 12:39:53 -0700 Subject: [PATCH 13/39] Avoid deprecation warning about calling apply on dataframe group https://pandas.pydata.org/docs/reference/api/pandas.core.groupby.DataFrameGroupBy.apply.html Signed-off-by: David Gardner --- src/aiq/profiler/inference_optimization/llm_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiq/profiler/inference_optimization/llm_metrics.py b/src/aiq/profiler/inference_optimization/llm_metrics.py index 5d7b46f77..7fd058a3c 100644 --- a/src/aiq/profiler/inference_optimization/llm_metrics.py +++ b/src/aiq/profiler/inference_optimization/llm_metrics.py @@ -176,8 +176,8 @@ def _rowwise_calc(row): return subdf # Apply the group metrics - df = (df.groupby(['example_number', 'function_name'], - group_keys=False).apply(_compute_group_metrics).sort_index()) + df_group = df.groupby(['example_number', 'function_name'], group_keys=False) + df = df_group[df.columns].apply(_compute_group_metrics).sort_index() # --------------------------------------------------------------------- # 5. NOVA-Predicted-OSL From 02c63a1adb89800fa886abcdc96026ea2720bbb4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 12:51:34 -0700 Subject: [PATCH 14/39] Always await the search method, fix type-o in collection name Signed-off-by: David Gardner --- tests/aiq/retriever/test_retrievers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/aiq/retriever/test_retrievers.py b/tests/aiq/retriever/test_retrievers.py index f33e9fa90..88541f54c 100644 --- a/tests/aiq/retriever/test_retrievers.py +++ b/tests/aiq/retriever/test_retrievers.py @@ -232,15 +232,15 @@ async def test_milvus_retriever_binding(milvus_retriever): _ = await milvus_retriever.search(query="Test query", collection_name="collection_not_exist", top_k=4) milvus_retriever.bind(top_k=2) - _ = milvus_retriever.search(query="Test query", collection_name="collection2") + _ = await milvus_retriever.search(query="Test query", collection_name="collection2") # Test not supplying enough parameters with pytest.raises(TypeError): _ = await milvus_retriever.search(query="Test query no collection name") # Test that binding those parameters makes the same call work - milvus_retriever.bind(top_k=2, collection_name="collecion1") - _ = milvus_retriever.search(query="Test query") + milvus_retriever.bind(top_k=2, collection_name="collection1") + _ = await milvus_retriever.search(query="Test query") async def test_milvus_validation(milvus_retriever): From d8b5a6727cc3f4e883243bde9e51376fb2a0a08e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 12:55:49 -0700 Subject: [PATCH 15/39] Remove edgecolor argument which was being overriden by the color argument Signed-off-by: David Gardner --- .../profiler_agent/src/aiq_profiler_agent/tool/flow_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/profiler_agent/src/aiq_profiler_agent/tool/flow_chart.py b/examples/profiler_agent/src/aiq_profiler_agent/tool/flow_chart.py index d2e2c1372..c9ca41ec3 100644 --- a/examples/profiler_agent/src/aiq_profiler_agent/tool/flow_chart.py +++ b/examples/profiler_agent/src/aiq_profiler_agent/tool/flow_chart.py @@ -155,7 +155,7 @@ def create_trace_flow_diagram(df: pd.DataFrame, temp_dir: str) -> TraceFlowInfo: # Draw span box color = colors.get(span_kind, "lightgray") - rect = plt.Rectangle((x_start, 0.5 * i - 0.2), x_end - x_start, 0.4, color=color, alpha=0.8, edgecolor="black") + rect = plt.Rectangle((x_start, 0.5 * i - 0.2), x_end - x_start, 0.4, color=color, alpha=0.8) ax.add_patch(rect) From e5c945cb7eac0196da0a68137525f6acb3af6036 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 13:43:58 -0700 Subject: [PATCH 16/39] Replace deprecated import of DeterministicFakeEmbedding, add missing dependency on langchain-community (this is currently a transitory dep of ragas) Signed-off-by: David Gardner --- packages/aiqtoolkit_test/pyproject.toml | 1 + packages/aiqtoolkit_test/src/aiq/test/embedder.py | 2 +- uv.lock | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/aiqtoolkit_test/pyproject.toml b/packages/aiqtoolkit_test/pyproject.toml index 1189b88e3..ea5f86e7b 100644 --- a/packages/aiqtoolkit_test/pyproject.toml +++ b/packages/aiqtoolkit_test/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ # version when adding a new package. If unsure, default to using `~=` instead of `==`. Does not apply to aiq packages. # Keep sorted!!! "aiqtoolkit~=1.1", + "langchain-community~=0.3", "pytest~=8.3", ] requires-python = ">=3.11,<3.13" diff --git a/packages/aiqtoolkit_test/src/aiq/test/embedder.py b/packages/aiqtoolkit_test/src/aiq/test/embedder.py index d49f9eb95..cf253bee3 100644 --- a/packages/aiqtoolkit_test/src/aiq/test/embedder.py +++ b/packages/aiqtoolkit_test/src/aiq/test/embedder.py @@ -39,6 +39,6 @@ async def embedder_test_provider(config: EmbedderTestConfig, builder: Builder): @register_embedder_client(config_type=EmbedderTestConfig, wrapper_type=LLMFrameworkEnum.LANGCHAIN) async def embedder_langchain_test_client(config: EmbedderTestConfig, builder: Builder): - from langchain.embeddings import DeterministicFakeEmbedding + from langchain_community.embeddings import DeterministicFakeEmbedding yield DeterministicFakeEmbedding(size=config.embedding_size) diff --git a/uv.lock b/uv.lock index 73fe316b1..0f64d7e26 100644 --- a/uv.lock +++ b/uv.lock @@ -706,12 +706,14 @@ name = "aiqtoolkit-test" source = { editable = "packages/aiqtoolkit_test" } dependencies = [ { name = "aiqtoolkit" }, + { name = "langchain-community" }, { name = "pytest" }, ] [package.metadata] requires-dist = [ { name = "aiqtoolkit", editable = "." }, + { name = "langchain-community", specifier = "~=0.3" }, { name = "pytest", specifier = "~=8.3" }, ] From dc367a946b6b97d96a9c97ec96e4bd3b3de4be10 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 20 May 2025 14:24:28 -0700 Subject: [PATCH 17/39] Attempt to await all coroutines, filter warnings for the rest Signed-off-by: David Gardner --- .../tests/test_tool_wrapper.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py index 7bdffc12a..d99fc327b 100644 --- a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py +++ b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio from unittest.mock import AsyncMock from unittest.mock import MagicMock from unittest.mock import patch @@ -28,6 +29,7 @@ from aiq.plugins.agno.tool_wrapper import process_result +@pytest.mark.filterwarnings("ignore:coroutine '.*' was never awaited.*:RuntimeWarning") class TestToolWrapper: """Tests for the agno_tool_wrapper function.""" @@ -37,6 +39,14 @@ def mock_event_loop(self): loop = MagicMock() return loop + def _waiter(self, func, *args, **kwargs): + """Helper function to await a coroutine.""" + + async def awaiter(): + _ = await func(*args, **kwargs) + + asyncio.run(awaiter()) + @pytest.fixture def mock_function(self): """Create a mock Function object.""" @@ -49,7 +59,9 @@ async def mock_acall_invoke(*args, **kwargs): return "test_result" mock_fn.acall_invoke = mock_acall_invoke - return mock_fn + yield mock_fn + + self._waiter(mock_fn.acall_invoke) @pytest.fixture def mock_model_schema_function(self): @@ -75,7 +87,9 @@ async def mock_acall_invoke(*args, **kwargs): return "test_result" mock_fn.acall_invoke = mock_acall_invoke - return mock_fn + yield mock_fn + + self._waiter(mock_fn.acall_invoke) @pytest.fixture def mock_builder(self): @@ -249,6 +263,8 @@ def test_execute_agno_tool_search_api_empty_query(self, mock_run_coroutine_threa # Verify that run_coroutine_threadsafe was not called since we blocked the empty query mock_run_coroutine_threadsafe.assert_not_called() + self._waiter(mock_coroutine_fn) + @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"test_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {"test_tool": False}) @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") From 27e4becb53ca1042ea993565390844c8dd56acf0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 07:23:58 -0700 Subject: [PATCH 18/39] Revert async hacks Signed-off-by: David Gardner --- .../tests/test_tool_wrapper.py | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py index d99fc327b..7bdffc12a 100644 --- a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py +++ b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio from unittest.mock import AsyncMock from unittest.mock import MagicMock from unittest.mock import patch @@ -29,7 +28,6 @@ from aiq.plugins.agno.tool_wrapper import process_result -@pytest.mark.filterwarnings("ignore:coroutine '.*' was never awaited.*:RuntimeWarning") class TestToolWrapper: """Tests for the agno_tool_wrapper function.""" @@ -39,14 +37,6 @@ def mock_event_loop(self): loop = MagicMock() return loop - def _waiter(self, func, *args, **kwargs): - """Helper function to await a coroutine.""" - - async def awaiter(): - _ = await func(*args, **kwargs) - - asyncio.run(awaiter()) - @pytest.fixture def mock_function(self): """Create a mock Function object.""" @@ -59,9 +49,7 @@ async def mock_acall_invoke(*args, **kwargs): return "test_result" mock_fn.acall_invoke = mock_acall_invoke - yield mock_fn - - self._waiter(mock_fn.acall_invoke) + return mock_fn @pytest.fixture def mock_model_schema_function(self): @@ -87,9 +75,7 @@ async def mock_acall_invoke(*args, **kwargs): return "test_result" mock_fn.acall_invoke = mock_acall_invoke - yield mock_fn - - self._waiter(mock_fn.acall_invoke) + return mock_fn @pytest.fixture def mock_builder(self): @@ -263,8 +249,6 @@ def test_execute_agno_tool_search_api_empty_query(self, mock_run_coroutine_threa # Verify that run_coroutine_threadsafe was not called since we blocked the empty query mock_run_coroutine_threadsafe.assert_not_called() - self._waiter(mock_coroutine_fn) - @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"test_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {"test_tool": False}) @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") From 7568c9563d6bc6f327c5fbbba429646c27eb20a9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 07:26:20 -0700 Subject: [PATCH 19/39] Revert async hacks Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index b39924b52..bb763fc54 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -199,7 +199,6 @@ async def test_run_workflow_local_success(evaluation_run, session_manager, gener assert not evaluation_run.workflow_interrupted -@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_errors(evaluation_run, session_manager): """Test workflow with no 'single output' fails gracefully.""" @@ -210,7 +209,6 @@ async def test_run_workflow_local_errors(evaluation_run, session_manager): await evaluation_run.run_workflow_local(session_manager) -@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_skip_completed(evaluation_run, session_manager, generated_answer): """Test that 'skip_completed_entries=True' skips completed items and processes only unfinished ones.""" @@ -247,7 +245,6 @@ async def test_run_workflow_local_skip_completed(evaluation_run, session_manager assert pending_item.output_obj == generated_answer, "Pending item output should have been processed" -@pytest.mark.filterwarnings("ignore:coroutine 'AsyncMockMixin._execute_mock_call' was never awaited.*:RuntimeWarning") async def test_run_workflow_local_workflow_interrupted(evaluation_run, eval_input, session_manager): """Test that workflow_interrupted is set to True when an exception occurs during workflow execution.""" From f60f859cb6d052b41a4fe660aceddf1689925db6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 09:12:51 -0700 Subject: [PATCH 20/39] Create a fixture that yields a loop that runs in another thread Signed-off-by: David Gardner --- .../tests/test_tool_wrapper.py | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py index 7bdffc12a..a61530ec9 100644 --- a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py +++ b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import threading from unittest.mock import AsyncMock from unittest.mock import MagicMock from unittest.mock import patch @@ -28,6 +30,40 @@ from aiq.plugins.agno.tool_wrapper import process_result +class RunLoopThread(threading.Thread): + + def __init__(self, loop: asyncio.AbstractEventLoop, release_event: threading.Event): + super().__init__() + self._loop = loop + self._release_event = release_event + + def run(self): + asyncio.set_event_loop(self._loop) + self._release_event.set() + self._loop.run_forever() + + +@pytest.fixture(name="run_loop_thread") +def fixture_run_loop_thread(): + """ + Fixture to create an asyncio event loop running in another thread. + Useful for creating a loop that can be used with the asyncio.run_coroutine_threadsafe function. + """ + loop = asyncio.new_event_loop() + release_event = threading.Event() + thread = RunLoopThread(loop=loop, release_event=release_event) + thread.start() + + # Wait for the thread to set the event + release_event.wait() + + yield loop + + # Stop the loop and join the thread + loop.call_soon_threadsafe(loop.stop) + thread.join() + + class TestToolWrapper: """Tests for the agno_tool_wrapper function.""" @@ -205,19 +241,15 @@ async def mock_acall_invoke(*args, **kwargs): @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {}) - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_execute_agno_tool_initialization(self, mock_run_coroutine_threadsafe, mock_event_loop): + def test_execute_agno_tool_initialization(self, run_loop_thread: asyncio.AbstractEventLoop): """Test that execute_agno_tool correctly handles tool initialization.""" - # Set up the mock future - mock_future = MagicMock() - mock_future.result.return_value = "initialization_result" - mock_run_coroutine_threadsafe.return_value = mock_future # Create a mock coroutine function mock_coroutine_fn = AsyncMock() + mock_coroutine_fn.return_value = "initialization_result" # Call the function under test for a tool with an empty kwargs dict (initialization) - result = execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], mock_event_loop) + result = execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], run_loop_thread) # Verify that the counters and initialization flags were set correctly from aiq.plugins.agno.tool_wrapper import _tool_call_counters @@ -226,7 +258,7 @@ def test_execute_agno_tool_initialization(self, mock_run_coroutine_threadsafe, m assert "test_tool" in _tool_initialization_done # Verify that run_coroutine_threadsafe was called with the coroutine function - mock_run_coroutine_threadsafe.assert_called() + mock_coroutine_fn.assert_called_once_with() # Verify the result assert result == "initialization_result" From 0f7b857d9f03041b1c086c69f931e2b947b59668 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 09:40:33 -0700 Subject: [PATCH 21/39] Make the RunLoopThread class private to the fixture, update tests to use new fixture Signed-off-by: David Gardner --- .../tests/test_tool_wrapper.py | 85 +++++++------------ 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py index a61530ec9..0d522d489 100644 --- a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py +++ b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py @@ -30,25 +30,25 @@ from aiq.plugins.agno.tool_wrapper import process_result -class RunLoopThread(threading.Thread): - - def __init__(self, loop: asyncio.AbstractEventLoop, release_event: threading.Event): - super().__init__() - self._loop = loop - self._release_event = release_event - - def run(self): - asyncio.set_event_loop(self._loop) - self._release_event.set() - self._loop.run_forever() - - @pytest.fixture(name="run_loop_thread") def fixture_run_loop_thread(): """ Fixture to create an asyncio event loop running in another thread. Useful for creating a loop that can be used with the asyncio.run_coroutine_threadsafe function. """ + + class RunLoopThread(threading.Thread): + + def __init__(self, loop: asyncio.AbstractEventLoop, release_event: threading.Event): + super().__init__() + self._loop = loop + self._release_event = release_event + + def run(self): + asyncio.set_event_loop(self._loop) + self._release_event.set() + self._loop.run_forever() + loop = asyncio.new_event_loop() release_event = threading.Event() thread = RunLoopThread(loop=loop, release_event=release_event) @@ -257,7 +257,7 @@ def test_execute_agno_tool_initialization(self, run_loop_thread: asyncio.Abstrac assert "test_tool" in _tool_call_counters assert "test_tool" in _tool_initialization_done - # Verify that run_coroutine_threadsafe was called with the coroutine function + # Verify that the coroutine function was called mock_coroutine_fn.assert_called_once_with() # Verify the result @@ -265,107 +265,80 @@ def test_execute_agno_tool_initialization(self, run_loop_thread: asyncio.Abstrac @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"search_api_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {"search_api_tool": True}) - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_execute_agno_tool_search_api_empty_query(self, mock_run_coroutine_threadsafe, mock_event_loop): + def test_execute_agno_tool_search_api_empty_query(self, run_loop_thread): """Test that execute_agno_tool correctly handles search API tools with empty queries.""" # Create a mock coroutine function mock_coroutine_fn = AsyncMock() # Call the function under test for a search tool with an empty query - result = execute_agno_tool("search_api_tool", mock_coroutine_fn, ["query"], mock_event_loop, query="") + result = execute_agno_tool("search_api_tool", mock_coroutine_fn, ["query"], run_loop_thread, query="") # Verify that an error message is returned for empty query after initialization assert "ERROR" in result assert "requires a valid query" in result - # Verify that run_coroutine_threadsafe was not called since we blocked the empty query - mock_run_coroutine_threadsafe.assert_not_called() + # Verify that coroutine was not called since we called execute_agno_tool with an empty query + mock_coroutine_fn.assert_not_called() @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"test_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {"test_tool": False}) - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_execute_agno_tool_filtered_kwargs(self, mock_run_coroutine_threadsafe, mock_event_loop): + def test_execute_agno_tool_filtered_kwargs(self, run_loop_thread: asyncio.AbstractEventLoop): """Test that execute_agno_tool correctly filters reserved keywords.""" - # Set up the mock future - mock_future = MagicMock() - mock_future.result.return_value = "filtered_result" - mock_run_coroutine_threadsafe.return_value = mock_future - - # Create a mock process_result future - process_future = MagicMock() - process_future.result.return_value = "processed_result" - mock_run_coroutine_threadsafe.side_effect = [mock_future, process_future] # Create a mock coroutine function mock_coroutine_fn = AsyncMock() + mock_coroutine_fn.return_value = "processed_result" # Call the function under test with kwargs containing reserved keywords result = execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], - mock_event_loop, + run_loop_thread, query="test query", model_config="should be filtered", _type="should be filtered") # Verify that run_coroutine_threadsafe was called with filtered kwargs - args, kwargs = mock_coroutine_fn.call_args - assert "query" in kwargs - assert "model_config" not in kwargs - assert "_type" not in kwargs + mock_coroutine_fn.assert_called_once_with(query="test query") # Verify the result assert result == "processed_result" @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"test_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._tool_initialization_done", {"test_tool": False}) - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_execute_agno_tool_wrapped_kwargs(self, mock_run_coroutine_threadsafe, mock_event_loop): + def test_execute_agno_tool_wrapped_kwargs(self, run_loop_thread: asyncio.AbstractEventLoop): """Test that execute_agno_tool correctly unwraps nested kwargs.""" - # Set up the mock future - mock_future = MagicMock() - mock_future.result.return_value = "unwrapped_result" - - # Create a mock process_result future - process_future = MagicMock() - process_future.result.return_value = "processed_result" - mock_run_coroutine_threadsafe.side_effect = [mock_future, process_future] - # Create a mock coroutine function mock_coroutine_fn = AsyncMock() + mock_coroutine_fn.return_value = "processed_result" # Call the function under test with wrapped kwargs result = execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], - mock_event_loop, + run_loop_thread, kwargs={ "query": "test query", "other_param": "value" }) # Verify that run_coroutine_threadsafe was called with unwrapped kwargs - args, kwargs = mock_coroutine_fn.call_args - assert "query" in kwargs - assert kwargs["query"] == "test query" - assert "other_param" in kwargs - assert kwargs["other_param"] == "value" + mock_coroutine_fn.assert_called_once_with(query="test query", other_param="value") # Verify the result assert result == "processed_result" @patch("aiq.plugins.agno.tool_wrapper._tool_call_counters", {"test_tool": 0}) @patch("aiq.plugins.agno.tool_wrapper._MAX_EMPTY_CALLS", 2) - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_execute_agno_tool_infinite_loop_detection(self, mock_run_coroutine_threadsafe, mock_event_loop): + def test_execute_agno_tool_infinite_loop_detection(self, run_loop_thread: asyncio.AbstractEventLoop): """Test that execute_agno_tool detects and prevents infinite loops.""" # Create a mock coroutine function mock_coroutine_fn = AsyncMock() # First call with only metadata should increment counter but proceed - execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], mock_event_loop, model_config="metadata only") + execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], run_loop_thread, model_config="metadata only") # Second call with only metadata should detect potential infinite loop result2 = execute_agno_tool("test_tool", mock_coroutine_fn, ["query"], - mock_event_loop, + run_loop_thread, model_config="metadata only") # Verify that the second call returned an error about infinite loops From 72d7fe2f7ca614dcfd5ccbc2f878f8d0bb24daa8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 12:24:40 -0700 Subject: [PATCH 22/39] Avoid mocking the asyncio.run_coroutine_threadsafe method Signed-off-by: David Gardner --- .../aiqtoolkit_agno/tests/test_tool_wrapper.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py index 0d522d489..70004b665 100644 --- a/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py +++ b/packages/aiqtoolkit_agno/tests/test_tool_wrapper.py @@ -297,7 +297,7 @@ def test_execute_agno_tool_filtered_kwargs(self, run_loop_thread: asyncio.Abstra model_config="should be filtered", _type="should be filtered") - # Verify that run_coroutine_threadsafe was called with filtered kwargs + # Verify that mock_coroutine_fn was called with filtered kwargs mock_coroutine_fn.assert_called_once_with(query="test query") # Verify the result @@ -319,7 +319,7 @@ def test_execute_agno_tool_wrapped_kwargs(self, run_loop_thread: asyncio.Abstrac "query": "test query", "other_param": "value" }) - # Verify that run_coroutine_threadsafe was called with unwrapped kwargs + # Verify that mock_coroutine_fn was called with unwrapped kwargs mock_coroutine_fn.assert_called_once_with(query="test query", other_param="value") # Verify the result @@ -417,8 +417,11 @@ def __init__(self, choices): assert result == "OpenAI response content" @patch("aiq.plugins.agno.tool_wrapper.tool") - @patch("aiq.plugins.agno.tool_wrapper.asyncio.run_coroutine_threadsafe") - def test_different_calling_styles(self, mock_run_coroutine_threadsafe, mock_tool, mock_function, mock_builder): + def test_different_calling_styles(self, + mock_tool, + mock_function, + mock_builder, + run_loop_thread: asyncio.AbstractEventLoop): """Test that execute_agno_tool handles different function calling styles.""" # Mock the tool decorator to return a function that returns its input mock_tool.return_value = lambda x: x @@ -433,11 +436,6 @@ def test_different_calling_styles(self, mock_run_coroutine_threadsafe, mock_tool process_future = MagicMock() process_future.result.return_value = "processed_result" - mock_run_coroutine_threadsafe.side_effect = [future1, future2, process_future] - - # Create a mock coroutine function - AsyncMock() - # Call the function under test wrapper_func = agno_tool_wrapper("test_tool", mock_function, mock_builder) From b4b2396149201e28d5c991eefd70dd3cf2a32a0d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 13:16:00 -0700 Subject: [PATCH 23/39] Check session_manager.workflow.has_single_output early before calling pull_intermediate, avoids issue where pull_intermediate is left unawaited when the workflow has multiple outputs Signed-off-by: David Gardner --- src/aiq/eval/evaluate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/aiq/eval/evaluate.py b/src/aiq/eval/evaluate.py index 82a99fa28..fc304bb95 100644 --- a/src/aiq/eval/evaluate.py +++ b/src/aiq/eval/evaluate.py @@ -85,14 +85,14 @@ async def run_one(item: EvalInputItem): async with session_manager.run(item.input_obj) as runner: try: - # Start usage stats and intermediate steps collection in parallel - intermediate_future = pull_intermediate() - if session_manager.workflow.has_single_output: - base_output = await runner.result() - else: + if not session_manager.workflow.has_single_output: # raise an error if the workflow has multiple outputs raise NotImplementedError("Multiple outputs are not supported") + + # Start usage stats and intermediate steps collection in parallel + intermediate_future = pull_intermediate() + base_output = await runner.result() intermediate_steps = await intermediate_future except NotImplementedError as e: # raise original error From 012bac02f4d0a6943bca75dabcdf7cd5104f0840 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 13:34:06 -0700 Subject: [PATCH 24/39] Ensure that any running coroutines are canceled in the exception handler Signed-off-by: David Gardner --- src/aiq/eval/evaluate.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/aiq/eval/evaluate.py b/src/aiq/eval/evaluate.py index fc304bb95..ae198e432 100644 --- a/src/aiq/eval/evaluate.py +++ b/src/aiq/eval/evaluate.py @@ -84,11 +84,14 @@ async def run_one(item: EvalInputItem): return "", [] async with session_manager.run(item.input_obj) as runner: - try: + if not session_manager.workflow.has_single_output: + # raise an error if the workflow has multiple outputs + raise NotImplementedError("Multiple outputs are not supported") + + base_output = None + intermediate_future = None - if not session_manager.workflow.has_single_output: - # raise an error if the workflow has multiple outputs - raise NotImplementedError("Multiple outputs are not supported") + try: # Start usage stats and intermediate steps collection in parallel intermediate_future = pull_intermediate() @@ -101,6 +104,13 @@ async def run_one(item: EvalInputItem): logger.exception("Failed to run the workflow: %s", e, exc_info=True) # stop processing if a workflow error occurs self.workflow_interrupted = True + + # Cancel any coroutines that are still running, avoiding a warning about unawaited coroutines + # (typically one of these two is what raised the exception and the other is still running) + for coro in (base_output, intermediate_future): + if coro is not None: + asyncio.ensure_future(coro).cancel() + stop_event.set() return From 88c66054b0ef6fe69e8001a12ffe24ce4ce7238a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 14:11:42 -0700 Subject: [PATCH 25/39] Don't patch the AIQSessionManager constructor, instead just pass in session_manager, avoids another warning about an unwaited coroutine Signed-off-by: David Gardner --- tests/aiq/eval/test_evaluate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/aiq/eval/test_evaluate.py b/tests/aiq/eval/test_evaluate.py index bb763fc54..7afffd76c 100644 --- a/tests/aiq/eval/test_evaluate.py +++ b/tests/aiq/eval/test_evaluate.py @@ -483,7 +483,6 @@ async def mock_eval_builder(config): # Patch functions and classes. Goal here is simply to ensure calls are made to the right functions. with patch("aiq.runtime.loader.load_config", mock_load_config), \ patch("aiq.builder.eval_builder.WorkflowEvalBuilder.from_config", side_effect=mock_eval_builder), \ - patch("aiq.runtime.session.AIQSessionManager", return_value=session_manager), \ patch("aiq.eval.evaluate.DatasetHandler", return_value=mock_dataset_handler), \ patch("aiq.eval.evaluate.OutputUploader", return_value=mock_uploader), \ patch.object(evaluation_run, "run_workflow_local", @@ -493,7 +492,7 @@ async def mock_eval_builder(config): patch.object(evaluation_run, "write_output", MagicMock()) as mock_write_output: # Run the function - await evaluation_run.run_and_evaluate() + await evaluation_run.run_and_evaluate(session_manager=session_manager) # Ensure config is loaded assert evaluation_run.eval_config == default_eval_config, "Evaluation config should be set correctly" From 5712df33c268a5c40469ce93941edc263d9d4a43 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 15:46:12 -0700 Subject: [PATCH 26/39] Ensure the NVIDIA_API_KEY environment variable is set (to a fake value) to avoid warning from the nim client Signed-off-by: David Gardner --- tests/aiq/builder/test_component_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/aiq/builder/test_component_utils.py b/tests/aiq/builder/test_component_utils.py index 17a588d95..2d7ee866b 100644 --- a/tests/aiq/builder/test_component_utils.py +++ b/tests/aiq/builder/test_component_utils.py @@ -410,6 +410,7 @@ def test_build_dependency_sequence(nested_aiq_config: AIQConfig): assert noref_instance_ids == list(noref_order.keys()) +@pytest.mark.usefixtures("set_test_api_keys") async def test_load_hierarchial_workflow(nested_aiq_config: AIQConfig): # Validate nested workflow instantiation From 0adc43f50eb8f00a8211567bf8fcce8412511fad Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:04:43 -0700 Subject: [PATCH 27/39] Silence syntax warnings being emitted from the qdrant-client (used by mem0), remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved Signed-off-by: David Gardner --- tests/conftest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 354837526..7bb1565cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ import sys import typing import uuid +import warnings from collections.abc import AsyncGenerator from collections.abc import Callable from unittest import mock @@ -359,7 +360,13 @@ def _run(self, @pytest.fixture(scope="function", autouse=True) def patched_async_memory_client(monkeypatch): - from mem0.client.main import MemoryClient + with warnings.catch_warnings(): + # Ignore syntax warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens + # the first time the module is imported and parsed, after that the pyc files in the __pycache__ directory are + # used which don't trigger the warnings. + # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. + warnings.filterwarnings("ignore", category=SyntaxWarning, message=r"^invalid escape sequence.*") + from mem0.client.main import MemoryClient mock_method = mock.MagicMock(return_value=None) monkeypatch.setattr(MemoryClient, "_validate_api_key", mock_method) From acaeaf1f922e04dcbf6d6a6911d4e23088681ecc Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:17:04 -0700 Subject: [PATCH 28/39] Fix return type hint, and avoid unneeded import Signed-off-by: David Gardner --- packages/aiqtoolkit_mem0ai/tests/test_mem0_editor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/aiqtoolkit_mem0ai/tests/test_mem0_editor.py b/packages/aiqtoolkit_mem0ai/tests/test_mem0_editor.py index 661800168..27475f723 100644 --- a/packages/aiqtoolkit_mem0ai/tests/test_mem0_editor.py +++ b/packages/aiqtoolkit_mem0ai/tests/test_mem0_editor.py @@ -16,14 +16,13 @@ from unittest.mock import AsyncMock import pytest -from mem0 import AsyncMemoryClient from aiq.memory.models import MemoryItem from aiq.plugins.mem0ai.mem0_editor import Mem0Editor @pytest.fixture(name="mock_mem0_client") -def mock_mem0_client_fixture() -> AsyncMemoryClient: +def mock_mem0_client_fixture() -> AsyncMock: """Fixture to provide a mocked AsyncMemoryClient.""" return AsyncMock() From 1d25deb9df8c91708a970a88a6f06510b8ed0dc6 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:25:05 -0700 Subject: [PATCH 29/39] Replace the warnings.catch_warnings statement with a global pytest ignore, since this warning was also bein emitted from other parts of the code base Signed-off-by: David Gardner --- pyproject.toml | 6 +++++- tests/conftest.py | 9 +-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0eb057e3e..07d241881 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,7 +200,11 @@ markers = [ "slow: Slow tests", ] filterwarnings = [ - # Add warnings to ignore as a part of pytest here + # Ignore syntax warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens + # the first time the module is imported and parsed, after that the pyc files in the __pycache__ directory are + # used which don't trigger the warnings. + # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. + "ignore:invalid escape sequence '\\&':SyntaxWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" diff --git a/tests/conftest.py b/tests/conftest.py index 7bb1565cc..cbd265c02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -359,14 +359,7 @@ def _run(self, @pytest.fixture(scope="function", autouse=True) def patched_async_memory_client(monkeypatch): - - with warnings.catch_warnings(): - # Ignore syntax warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens - # the first time the module is imported and parsed, after that the pyc files in the __pycache__ directory are - # used which don't trigger the warnings. - # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. - warnings.filterwarnings("ignore", category=SyntaxWarning, message=r"^invalid escape sequence.*") - from mem0.client.main import MemoryClient + from mem0.client.main import MemoryClient mock_method = mock.MagicMock(return_value=None) monkeypatch.setattr(MemoryClient, "_validate_api_key", mock_method) From e13e3498b29126389e50d4c922309753823becb7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:30:25 -0700 Subject: [PATCH 30/39] Fix the warning filter Signed-off-by: David Gardner --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 07d241881..0266faac4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,7 +204,7 @@ filterwarnings = [ # the first time the module is imported and parsed, after that the pyc files in the __pycache__ directory are # used which don't trigger the warnings. # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. - "ignore:invalid escape sequence '\\&':SyntaxWarning" + "ignore:^invalid escape sequence:SyntaxWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" From 2346a466c810e065159ca2bf630b283eb1f2f6ba Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:31:54 -0700 Subject: [PATCH 31/39] Remove unused import Signed-off-by: David Gardner --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index cbd265c02..61cec3fde 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,6 @@ import sys import typing import uuid -import warnings from collections.abc import AsyncGenerator from collections.abc import Callable from unittest import mock From c08f320a6b39f5251da3159753bf0f2850a4e837 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:34:11 -0700 Subject: [PATCH 32/39] Remove unintented change Signed-off-by: David Gardner --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 61cec3fde..354837526 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -358,6 +358,7 @@ def _run(self, @pytest.fixture(scope="function", autouse=True) def patched_async_memory_client(monkeypatch): + from mem0.client.main import MemoryClient mock_method = mock.MagicMock(return_value=None) From ca738c8a1003b7dbb200d1fb9fa86af4d6428972 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:40:59 -0700 Subject: [PATCH 33/39] Warning is different in Python 3.11 Signed-off-by: David Gardner --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0266faac4..6cb44ed68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,11 +200,12 @@ markers = [ "slow: Slow tests", ] filterwarnings = [ - # Ignore syntax warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens - # the first time the module is imported and parsed, after that the pyc files in the __pycache__ directory are - # used which don't trigger the warnings. + # Ignore warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens the first + # time the module is imported and parsed, after that the pyc files in the __pycache__ directory are used which don't + # trigger the warnings. In Python 3.12 this triggers a SyntaxWarning, in Python 3.11 it triggers a DeprecationWarning # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. "ignore:^invalid escape sequence:SyntaxWarning" + "ignore:^invalid escape sequence:DeprecationWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" From c4010dff5a39e4ac28d1686f7efc7f3cf18cac60 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 16:52:59 -0700 Subject: [PATCH 34/39] Fix toml Signed-off-by: David Gardner --- .vale.ini | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vale.ini b/.vale.ini index 2d27c9e73..624a3b1e4 100644 --- a/.vale.ini +++ b/.vale.ini @@ -5,7 +5,7 @@ MinAlertLevel = error Vocab = aiq # Configs for markdown and reStructuredText files -[*{.md,.rst}] +[*{.md,.mdx,.rst}] BasedOnStyles = Vale diff --git a/pyproject.toml b/pyproject.toml index 6cb44ed68..7b2590013 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,7 +204,7 @@ filterwarnings = [ # time the module is imported and parsed, after that the pyc files in the __pycache__ directory are used which don't # trigger the warnings. In Python 3.12 this triggers a SyntaxWarning, in Python 3.11 it triggers a DeprecationWarning # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. - "ignore:^invalid escape sequence:SyntaxWarning" + "ignore:^invalid escape sequence:SyntaxWarning", "ignore:^invalid escape sequence:DeprecationWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] From b4381896596794bfa9c1a5744e388bf0e54a9a1e Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 21 May 2025 17:07:55 -0700 Subject: [PATCH 35/39] Fix ignore for 3.11 Signed-off-by: David Gardner --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b2590013..699787799 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,7 @@ filterwarnings = [ # trigger the warnings. In Python 3.12 this triggers a SyntaxWarning, in Python 3.11 it triggers a DeprecationWarning # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. "ignore:^invalid escape sequence:SyntaxWarning", - "ignore:^invalid escape sequence:DeprecationWarning" + "ignore: invalid escape sequence:DeprecationWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" From 363e4bd327b2429fd90833f5d5e03ef09e6e828f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 May 2025 08:35:21 -0700 Subject: [PATCH 36/39] Remove attempt to filter the 3.11 deprecation warning Signed-off-by: David Gardner --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 699787799..6123eb89f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -203,9 +203,9 @@ filterwarnings = [ # Ignore warnings from qdrant-client (used by mem0) with Python 3.12+ of note is that this only happens the first # time the module is imported and parsed, after that the pyc files in the __pycache__ directory are used which don't # trigger the warnings. In Python 3.12 this triggers a SyntaxWarning, in Python 3.11 it triggers a DeprecationWarning + # which unfortunately pytest is unable to filter. # Remove once https://github.com/qdrant/qdrant-client/issues/983 is resolved. - "ignore:^invalid escape sequence:SyntaxWarning", - "ignore: invalid escape sequence:DeprecationWarning" + "ignore:^invalid escape sequence:SyntaxWarning" ] testpaths = ["tests", "examples/*/tests", "packages/*/tests"] asyncio_mode = "auto" From a38c9412be1abc3ca82e74dba4527d6861f93892 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 22 May 2025 09:02:24 -0700 Subject: [PATCH 37/39] Explcitly cast entry point objects to a list to avoid a warning in Python 3.11 Signed-off-by: David Gardner --- src/aiq/runtime/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiq/runtime/loader.py b/src/aiq/runtime/loader.py index 4edaad5bc..8cf069ca0 100644 --- a/src/aiq/runtime/loader.py +++ b/src/aiq/runtime/loader.py @@ -132,7 +132,7 @@ def discover_entrypoints(plugin_type: PluginTypes): plugin_groups.append("aiq.evaluators") # Get the entry points for the specified groups - aiq_plugins = reduce(lambda x, y: x + y, [entry_points.select(group=y) for y in plugin_groups]) + aiq_plugins = reduce(lambda x, y: list(x) + list(y), [entry_points.select(group=y) for y in plugin_groups]) return aiq_plugins From f1023e3bc87694d64e3934e07bdbdcad0690f0d8 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 May 2025 16:25:49 -0700 Subject: [PATCH 38/39] base_output is not a coroutine Signed-off-by: David Gardner --- src/aiq/eval/evaluate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aiq/eval/evaluate.py b/src/aiq/eval/evaluate.py index ae198e432..094c7b25b 100644 --- a/src/aiq/eval/evaluate.py +++ b/src/aiq/eval/evaluate.py @@ -88,14 +88,15 @@ async def run_one(item: EvalInputItem): # raise an error if the workflow has multiple outputs raise NotImplementedError("Multiple outputs are not supported") - base_output = None + runner_result = None intermediate_future = None try: # Start usage stats and intermediate steps collection in parallel intermediate_future = pull_intermediate() - base_output = await runner.result() + runner_result = runner.result() + base_output = await runner_result intermediate_steps = await intermediate_future except NotImplementedError as e: # raise original error @@ -107,7 +108,7 @@ async def run_one(item: EvalInputItem): # Cancel any coroutines that are still running, avoiding a warning about unawaited coroutines # (typically one of these two is what raised the exception and the other is still running) - for coro in (base_output, intermediate_future): + for coro in (runner_result, intermediate_future): if coro is not None: asyncio.ensure_future(coro).cancel() From 24b3a3793b1e6a8035a1715bbd5b3c2af8c16753 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 27 May 2025 16:28:00 -0700 Subject: [PATCH 39/39] Revert unintended change Signed-off-by: David Gardner --- .vale.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vale.ini b/.vale.ini index 624a3b1e4..2d27c9e73 100644 --- a/.vale.ini +++ b/.vale.ini @@ -5,7 +5,7 @@ MinAlertLevel = error Vocab = aiq # Configs for markdown and reStructuredText files -[*{.md,.mdx,.rst}] +[*{.md,.rst}] BasedOnStyles = Vale