Skip to content

fix: Add elapsed time logging for strix fallback models #334

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
seonghobae merged 8 commits into from
Jun 2, 2026
Merged

fix: Add elapsed time logging for strix fallback models #334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
85c628f
fix(strix): add elapsed time to fallback success message
seonghobae Jun 1, 2026
dd72336
fix(strix): add elapsed time to fallback success message
seonghobae Jun 1, 2026
a5598e4
Merge remote-tracking branch 'origin/master' into fix-timing-message-…
seonghobae Jun 2, 2026
c731fb7
test: require Strix timing durations
seonghobae Jun 2, 2026
0762752
Merge remote-tracking branch 'origin/master' into fix-timing-message-…
seonghobae Jun 2, 2026
035f3de
Merge remote-tracking branch 'origin/master' into fix-timing-message-…
seonghobae Jun 2, 2026
739cb86
ci: serialize Strix GitHub Models scans
seonghobae Jun 2, 2026
1f0a7fb
ci: retry GitHub Models connection failures
seonghobae Jun 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/strix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ on:
type: string

concurrency:
group: strix-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
group: strix-${{ github.repository }}
# cancel-in-progress deliberately disabled: an attacker could force-push
# a benign commit to cancel an in-progress scan of a malicious commit.
cancel-in-progress: false
Expand Down
2 changes: 2 additions & 0 deletions backend/tests/test_release_governance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ def test_strix_workflow_uses_github_models_default_and_narrow_warning_filter() -
workflow = read_repo_text(".github/workflows/strix.yml")
gate_script = read_repo_text("scripts/ci/strix_quick_gate.sh")

assert 'group: strix-${{ github.repository }}' in workflow
assert "cancel-in-progress: false" in workflow
assert "models: read" in workflow
assert "provider_mode=github_models" in workflow
assert "strix_llm:" in workflow
Expand Down
11 changes: 10 additions & 1 deletion scripts/ci/strix_quick_gate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2121,6 +2121,12 @@ is_llm_api_connection_error() {
return 0
fi

if grep -Eiq 'litellm(\.exceptions)?\.InternalServerError' "$STRIX_LOG" &&
grep -Eiq 'OpenAIException[[:space:]]*-[[:space:]]*Connection error' "$STRIX_LOG" &&
grep -Eiq '(openai|LLM CONNECTION FAILED|Could not establish connection to the language model)' "$STRIX_LOG"; then
return 0
fi

return 1
}

Expand Down Expand Up @@ -2916,13 +2922,16 @@ run_current_target_scan() {
echo "Primary model unavailable; retrying with fallback '$candidate'."
fi
local fallback_scan_rc=0
local fallback_start_epoch
fallback_start_epoch="$(date +%s)"
run_strix_with_transient_retry "$candidate" || fallback_scan_rc=$?
local fallback_elapsed=$(( $(date +%s) - fallback_start_epoch ))
if [ "$fallback_scan_rc" -eq 0 ]; then
if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then
echo "Strix fallback scan had provider infrastructure or failure-signal output; failing closed." >&2
return 1
fi
echo "Strix quick scan succeeded with fallback model '$candidate'."
echo "Strix quick scan succeeded with fallback model '$candidate' in ${fallback_elapsed}s." >&2
return 0
fi
if [ "$fallback_scan_rc" -eq 2 ]; then
Expand Down
86 changes: 62 additions & 24 deletions scripts/ci/test_strix_quick_gate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,16 @@ assert_file_contains() {
fi
}

assert_file_matches() {
local file_path="$1"
local pattern="$2"
local message="$3"

if ! grep -Eq -- "$pattern" "$file_path"; then
record_failure "$message (missing pattern '$pattern')"
fi
}

assert_file_not_contains() {
local file_path="$1"
local needle="$2"
Expand All @@ -63,6 +73,8 @@ assert_strix_workflow_pr_trigger_hardened() {

assert_file_contains "$workflow_file" "branches: [master]" "strix workflow scans the protected default branch"
assert_file_contains "$workflow_file" "pull_request_target:" "strix workflow uses trusted PR trigger"
assert_file_contains "$workflow_file" 'group: strix-${{ github.repository }}' "strix workflow serializes scans per repository for GitHub Models quota"
assert_file_contains "$workflow_file" "cancel-in-progress: false" "strix workflow never cancels in-progress security evidence"
assert_file_contains "$workflow_file" "models: read" "strix workflow grants only the GitHub Models read permission needed for Strix"
assert_file_contains "$workflow_file" "Materialize trusted workspace" "strix workflow materializes trusted workspace"
assert_file_contains "$workflow_file" "TRUSTED_WORKSPACE_SHA" "strix workflow pins trusted workspace SHA"
Expand Down Expand Up @@ -670,16 +682,22 @@ case "${FAKE_STRIX_SCENARIO:?}" in
;;
vertex-primary-api-connection-retry-same-model-success)
case "${STRIX_LLM:-}" in
gemini/retry-api-connection-primary|vertex_ai/retry-api-connection-primary)
gemini/retry-api-connection-primary|vertex_ai/retry-api-connection-primary|openai/openai/retry-api-connection-primary)
attempt="0"
if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then
attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")"
fi
attempt="$((attempt + 1))"
echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}"
if [ "$attempt" -eq 1 ]; then
echo "LLM CONNECTION FAILED"
echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response."
if [ "${STRIX_LLM:-}" = "openai/openai/retry-api-connection-primary" ]; then
echo "LLM CONNECTION FAILED"
echo "Could not establish connection to the language model."
echo "Error: litellm.InternalServerError: InternalServerError: OpenAIException - Connection error."
else
echo "LLM CONNECTION FAILED"
echo "litellm.APIConnectionError: GeminiException - Server disconnected without sending a response."
fi
exit 1
fi
echo "scan ok after same-model api connection retry"
Expand Down Expand Up @@ -2091,7 +2109,14 @@ EOS
assert_equals "$expected_exit" "$rc" "scenario=$scenario exit code"

if [ -n "$expected_message" ]; then
assert_file_contains "$output_log" "$expected_message" "scenario=$scenario output"
case "$expected_message" in
REGEX:*)
assert_file_matches "$output_log" "${expected_message#REGEX:}" "scenario=$scenario output"
;;
*)
assert_file_contains "$output_log" "$expected_message" "scenario=$scenario output"
;;
esac
fi

local call_count
Expand Down Expand Up @@ -4664,7 +4689,7 @@ run_gate_case "vertex-primary-notfound-fallback-success" \
"vertex_ai/missing-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand Down Expand Up @@ -4700,7 +4725,7 @@ run_gate_case "provider-prefix-fallback-normalization" \
"missing-primary" \
"fallback-one fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand Down Expand Up @@ -4728,7 +4753,7 @@ run_gate_case "provider-prefix-resource-path-primary-notfound-fallback-success"
"projects/p1/locations/us-central1/publishers/google/models/missing-primary" \
"projects/p1/locations/us-central1/publishers/google/models/fallback-one projects/p1/locations/us-central1/publishers/google/models/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4749,7 +4774,7 @@ run_gate_case "vertex-notfound-without-status-fallback-success" \
"vertex_ai/missing-primary" \
"vertex_ai/fallback-one" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4758,7 +4783,7 @@ run_gate_case "vertex-notfound-compact-status-fallback-success" \
"vertex_ai/missing-primary" \
"vertex_ai/fallback-one" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4776,7 +4801,7 @@ run_gate_case "primary-duplicate-in-fallback" \
"missing-primary" \
"vertex_ai/missing-primary fallback-one" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4785,7 +4810,7 @@ run_gate_case "multiline-fallback-success" \
"vertex_ai/missing-primary" \
$'vertex_ai/fallback-one\nvertex_ai/fallback-two' \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-two'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-two' in [0-9]+s\\." \
"3" \
"vertex_ai/missing-primary|vertex_ai/fallback-one|vertex_ai/fallback-two" \
"<unset>|<unset>|<unset>"
Expand All @@ -4794,7 +4819,7 @@ run_gate_case_allow_provider_signal "vertex-primary-ratelimit-fallback-success"
"vertex_ai/ratelimit-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/ratelimit-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4803,7 +4828,7 @@ run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-
"vertex_ai/resource-exhausted-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/resource-exhausted-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4812,7 +4837,7 @@ run_gate_case_allow_provider_signal "vertex-primary-429-fallback-success" \
"vertex_ai/http429-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/http429-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand All @@ -4821,7 +4846,7 @@ run_gate_case_allow_provider_signal "vertex-primary-midstream-fallback-success"
"vertex_ai/midstream-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/midstream-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand Down Expand Up @@ -4866,6 +4891,19 @@ run_gate_case_allow_provider_signal "vertex-primary-api-connection-retry-same-mo
"" \
"1"

run_gate_case_allow_provider_signal "github-models-internal-server-connection-retry-same-model-success" \
"openai/openai/retry-api-connection-primary" \
"" \
"0" \
"scan ok after same-model api connection retry" \
"2" \
"openai/openai/retry-api-connection-primary|openai/openai/retry-api-connection-primary" \
"https://models.github.ai/inference|https://models.github.ai/inference" \
"openai" \
"https://models.github.ai/inference" \
"" \
"1"

run_gate_case_allow_provider_signal "gemini-high-demand-retry-same-model-success" \
"gemini/retry-high-demand-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
Expand All @@ -4883,7 +4921,7 @@ run_gate_case_allow_provider_signal "gemini-timeout-direct-fallback-success" \
"gemini/retry-timeout-primary" \
"gemini/fallback-one gemini/fallback-two" \
"0" \
"scan ok after timeout fallback" \
"REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \
"2" \
"gemini/retry-timeout-primary|gemini/fallback-one" \
"https://example.invalid|https://example.invalid" \
Expand All @@ -4896,7 +4934,7 @@ run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \
"gemini/timeout-fallback-primary" \
"gemini/fallback-one gemini/fallback-two" \
"0" \
"scan ok after gemini fallback" \
"REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \
"2" \
"gemini/timeout-fallback-primary|gemini/fallback-one" \
"https://example.invalid|https://example.invalid" \
Expand All @@ -4909,7 +4947,7 @@ run_gate_case_allow_provider_signal "gemini-generic-fallback-success" \
"gemini/timeout-fallback-primary" \
"" \
"0" \
"scan ok after gemini fallback" \
"REGEX:Strix quick scan succeeded with fallback model 'gemini/fallback-one' in [0-9]+s\\." \
"2" \
"gemini/timeout-fallback-primary|gemini/fallback-one" \
"https://example.invalid|https://example.invalid" \
Expand Down Expand Up @@ -5244,7 +5282,7 @@ run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \
"vertex_ai/hallucination-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/hallucination-primary|vertex_ai/fallback-one" \
"<unset>|<unset>"
Expand Down Expand Up @@ -5510,7 +5548,7 @@ run_gate_case "target-path-src-default-source-dirs" \
"vertex_ai/hallucination-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/hallucination-primary|vertex_ai/fallback-one" \
"<unset>|<unset>" \
Expand Down Expand Up @@ -5560,7 +5598,7 @@ run_gate_case "default-fallback-order-fast-first" \
"vertex_ai/missing-primary" \
"" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/gemini-2.5-pro'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/gemini-2[.]5-pro' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|vertex_ai/gemini-2.5-pro" \
"<unset>|<unset>"
Expand All @@ -5581,7 +5619,7 @@ run_gate_case_allow_provider_signal "vertex-primary-timeout-retry-reason-message
"vertex_ai/retry-timeout-primary" \
"vertex_ai/fallback-one vertex_ai/fallback-two" \
"0" \
"Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one'." \
"REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \
"2" \
"vertex_ai/retry-timeout-primary|vertex_ai/fallback-one" \
"<unset>|<unset>" \
Expand Down Expand Up @@ -5609,7 +5647,7 @@ run_gate_case "vertex-primary-success-timing-message" \
"vertex_ai/ready-primary" \
"" \
"0" \
"Strix run succeeded for model 'vertex_ai/ready-primary' in " \
"REGEX:Strix run succeeded for model 'vertex_ai/ready-primary' in [0-9]+s\\." \
"1" \
"vertex_ai/ready-primary" \
"<unset>"
Expand Down Expand Up @@ -6830,7 +6868,7 @@ run_gate_case "github-models-fallback-success" \
"vertex_ai/missing-primary" \
"openai/openai/gpt-5.4" \
"0" \
"Strix quick scan succeeded with fallback model 'openai/openai/gpt-5.4'." \
"REGEX:Strix quick scan succeeded with fallback model 'openai/openai/gpt-5[.]4' in [0-9]+s\\." \
"2" \
"vertex_ai/missing-primary|openai/openai/gpt-5.4" \
"<unset>|https://models.github.ai/inference" \
Expand Down