From cda63f498a6f826ba24f3b3ead866b32bb589f3a Mon Sep 17 00:00:00 2001 From: Dean Schmigelski Date: Wed, 3 Sep 2025 10:02:00 -0400 Subject: [PATCH 1/3] fix(tests): adjust test_bedrock_guardrails to account for async behavior --- src/strands/tools/registry.py | 6 +-- tests_integ/test_bedrock_guardrails.py | 61 +++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/strands/tools/registry.py b/src/strands/tools/registry.py index 6bb76f560..471472a64 100644 --- a/src/strands/tools/registry.py +++ b/src/strands/tools/registry.py @@ -192,9 +192,9 @@ def register_tool(self, tool: AgentTool) -> None: # Check duplicate tool name, throw on duplicate tool names except if hot_reloading is enabled if tool.tool_name in self.registry and not tool.supports_hot_reload: - raise ValueError( - f"Tool name '{tool.tool_name}' already exists. Cannot register tools with exact same name." - ) + raise ValueError( + f"Tool name '{tool.tool_name}' already exists. Cannot register tools with exact same name." + ) # Check for normalized name conflicts (- vs _) if self.registry.get(tool.tool_name) is None: diff --git a/tests_integ/test_bedrock_guardrails.py b/tests_integ/test_bedrock_guardrails.py index 4683918cb..450319b39 100644 --- a/tests_integ/test_bedrock_guardrails.py +++ b/tests_integ/test_bedrock_guardrails.py @@ -138,9 +138,25 @@ def test_guardrail_output_intervention(boto_session, bedrock_guardrail, processi response1 = agent("Say the word.") response2 = agent("Hello!") assert response1.stop_reason == "guardrail_intervened" - assert BLOCKED_OUTPUT in str(response1) - assert response2.stop_reason != "guardrail_intervened" - assert BLOCKED_OUTPUT not in str(response2) + + """ + In async streaming: The buffering is non-blocking. + Tokens are streamed while Guardrails processes the buffered content in the background. + This means the response may be returned before Guardrails has finished processing. + As a result, we cannot guarantee that the REDACT_MESSAGE is in the response + """ + if processing_mode == "sync": + assert BLOCKED_OUTPUT in str(response1) + assert response2.stop_reason != "guardrail_intervened" + assert BLOCKED_OUTPUT not in str(response2) + else: + cactus_returned_in_response1_blocked_by_input_guardrail = BLOCKED_INPUT in str(response2) + cactus_blocked_in_response1_allows_next_response = ( + BLOCKED_OUTPUT not in str(response2) and response2.stop_reason != "guardrail_intervened" + ) + assert ( + cactus_returned_in_response1_blocked_by_input_guardrail or cactus_blocked_in_response1_allows_next_response + ) @pytest.mark.parametrize("processing_mode", ["sync", "async"]) @@ -164,10 +180,45 @@ def test_guardrail_output_intervention_redact_output(bedrock_guardrail, processi response1 = agent("Say the word.") response2 = agent("Hello!") + + assert response1.stop_reason == "guardrail_intervened" + + """ + In async streaming: The buffering is non-blocking. + Tokens are streamed while Guardrails processes the buffered content in the background. + This means the response may be returned before Guardrails has finished processing. + As a result, we cannot guarantee that the REDACT_MESSAGE is in the response + """ + if processing_mode == "sync": + assert REDACT_MESSAGE in str(response1) + assert response2.stop_reason != "guardrail_intervened" + assert REDACT_MESSAGE not in str(response2) + else: + cactus_returned_in_response1_blocked_by_input_guardrail = BLOCKED_INPUT in str(response2) + cactus_blocked_in_response1_allows_next_response = ( + REDACT_MESSAGE not in str(response2) and response2.stop_reason != "guardrail_intervened" + ) + assert ( + cactus_returned_in_response1_blocked_by_input_guardrail or cactus_blocked_in_response1_allows_next_response + ) + + +def test_guardrail_input_intervention_properly_redacts(boto_session, bedrock_guardrail): + bedrock_model = BedrockModel( + guardrail_id=bedrock_guardrail, + guardrail_version="DRAFT", + boto_session=boto_session, + ) + + agent = Agent(model=bedrock_model, system_prompt="You are a helpful assistant.", callback_handler=None) + + response1 = agent("CACTUS") + response2 = agent("Hello!") + assert response1.stop_reason == "guardrail_intervened" - assert REDACT_MESSAGE in str(response1) + assert str(response1).strip() == BLOCKED_INPUT assert response2.stop_reason != "guardrail_intervened" - assert REDACT_MESSAGE not in str(response2) + assert str(response2).strip() != BLOCKED_INPUT def test_guardrail_input_intervention_properly_redacts_in_session(boto_session, bedrock_guardrail, temp_dir): From 16fbb9fe58b99c2f67bbaf52a81f482a75ceb478 Mon Sep 17 00:00:00 2001 From: Dean Schmigelski Date: Wed, 3 Sep 2025 10:07:52 -0400 Subject: [PATCH 2/3] fix(tests): revert test_guardrail_input_intervention_properly_redacts --- tests_integ/test_bedrock_guardrails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests_integ/test_bedrock_guardrails.py b/tests_integ/test_bedrock_guardrails.py index 450319b39..fee169f6d 100644 --- a/tests_integ/test_bedrock_guardrails.py +++ b/tests_integ/test_bedrock_guardrails.py @@ -216,9 +216,9 @@ def test_guardrail_input_intervention_properly_redacts(boto_session, bedrock_gua response2 = agent("Hello!") assert response1.stop_reason == "guardrail_intervened" - assert str(response1).strip() == BLOCKED_INPUT + assert REDACT_MESSAGE in str(response1) assert response2.stop_reason != "guardrail_intervened" - assert str(response2).strip() != BLOCKED_INPUT + assert REDACT_MESSAGE not in str(response2) def test_guardrail_input_intervention_properly_redacts_in_session(boto_session, bedrock_guardrail, temp_dir): From d98cfce06a096403a626d4de9de2895b45075225 Mon Sep 17 00:00:00 2001 From: Dean Schmigelski Date: Wed, 3 Sep 2025 10:10:32 -0400 Subject: [PATCH 3/3] fix: remove extra test --- tests_integ/test_bedrock_guardrails.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests_integ/test_bedrock_guardrails.py b/tests_integ/test_bedrock_guardrails.py index fee169f6d..e25bf3cca 100644 --- a/tests_integ/test_bedrock_guardrails.py +++ b/tests_integ/test_bedrock_guardrails.py @@ -203,24 +203,6 @@ def test_guardrail_output_intervention_redact_output(bedrock_guardrail, processi ) -def test_guardrail_input_intervention_properly_redacts(boto_session, bedrock_guardrail): - bedrock_model = BedrockModel( - guardrail_id=bedrock_guardrail, - guardrail_version="DRAFT", - boto_session=boto_session, - ) - - agent = Agent(model=bedrock_model, system_prompt="You are a helpful assistant.", callback_handler=None) - - response1 = agent("CACTUS") - response2 = agent("Hello!") - - assert response1.stop_reason == "guardrail_intervened" - assert REDACT_MESSAGE in str(response1) - assert response2.stop_reason != "guardrail_intervened" - assert REDACT_MESSAGE not in str(response2) - - def test_guardrail_input_intervention_properly_redacts_in_session(boto_session, bedrock_guardrail, temp_dir): bedrock_model = BedrockModel( guardrail_id=bedrock_guardrail,