diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index c8c054e422..9b30488388 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -674,7 +674,7 @@ jobs: echo "LLM_API_BASE_FILE=$llm_api_base_file" >> "$GITHUB_ENV" - name: Prepare GitHub Models fallback credentials - if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' + if: steps.gate.outputs.provider_mode == 'github_models' || steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' env: GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} OPENAI_FALLBACK_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} @@ -706,6 +706,9 @@ jobs: openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" printf '%s' "$openai_trimmed" > "$openai_fallback_key_file" echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + openai_fallback_api_base_file="$RUNNER_TEMP/openai_fallback_api_base.txt" + printf '%s' 'https://api.openai.com/v1' > "$openai_fallback_api_base_file" + echo "STRIX_OPENAI_FALLBACK_API_BASE_FILE=$openai_fallback_api_base_file" >> "$GITHUB_ENV" fi - name: Prepare Vertex AI credentials @@ -838,6 +841,7 @@ jobs: STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} + STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 36ec3e5f8e..fa676a0d5a 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2417,7 +2417,14 @@ resolved_llm_api_base_for_model() { local api_base_file="$LLM_API_BASE_FILE" local api_base_file_name="LLM_API_BASE_FILE" - if is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then + if is_explicit_openai_model "$model" && [ -n "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ]; then + # Cross-provider fallback: openai-direct/* candidates must reach the + # direct OpenAI API even when the primary provider selected a + # different LLM_API_BASE_FILE endpoint (e.g. NVIDIA NIM). Without + # this the fallback hits the primary gateway and 404s. + api_base_file="$STRIX_OPENAI_FALLBACK_API_BASE_FILE" + api_base_file_name="STRIX_OPENAI_FALLBACK_API_BASE_FILE" + elif is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then # Cross-provider fallback: when the active primary provider uses a # different API base (for example OpenRouter), github_models/* fallback # attempts must still route through the GitHub Models inference endpoint. diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 945eb3fb30..571bf4dd9a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -365,6 +365,9 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'github_models' || steps.gate.outputs.provider_mode == 'openai_direct'" "strix workflow provisions direct-OpenAI fallback credentials for GitHub Models scans" + assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" + assert_file_contains "$workflow_file" "https://api.openai.com/v1" "strix workflow uses the OpenAI platform endpoint for direct fallbacks" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" @@ -3421,6 +3424,54 @@ REPORT ;; esac ;; + nvidia-nim-openai-direct-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/primary-model) + if [ "${LLM_API_KEY:-}" != "dummy" ] || [ "${LLM_API_BASE:-}" != "https://integrate.api.nvidia.com/v1" ]; then + echo "unexpected NVIDIA NIM primary credentials or endpoint" >&2 + exit 17 + fi + echo "nvidia_nim.RateLimitError: Error code: 429 - provider quota exhausted" + exit 1 + ;; + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "openai-fallback-key" ] || [ "${LLM_API_BASE:-}" != "https://api.openai.com/v1" ]; then + echo "unexpected direct-OpenAI fallback credentials or endpoint" >&2 + exit 18 + fi + echo "scan ok after direct-OpenAI fallback" + exit 0 + ;; + *) + echo "unexpected NVIDIA-to-OpenAI fallback model ${STRIX_LLM:-}" >&2 + exit 19 + ;; + esac + ;; + github-models-openai-direct-fallback-success) + case "${STRIX_LLM:-}" in + openai/gpt-5) + if [ "${LLM_API_KEY:-}" != "dummy" ] || [ "${LLM_API_BASE:-}" != "https://models.github.ai/inference" ]; then + echo "unexpected GitHub Models primary credentials or endpoint" >&2 + exit 20 + fi + echo "openai.RateLimitError: Error code: 429 - GitHub Models quota exhausted" + exit 1 + ;; + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "openai-fallback-key" ] || [ "${LLM_API_BASE:-}" != "https://api.openai.com/v1" ]; then + echo "unexpected GitHub-to-OpenAI fallback credentials or endpoint" >&2 + exit 21 + fi + echo "scan ok after GitHub-to-OpenAI fallback" + exit 0 + ;; + *) + echo "unexpected GitHub-to-OpenAI fallback model ${STRIX_LLM:-}" >&2 + exit 22 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5648,6 +5699,18 @@ PY env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") env_cmd+=(STRIX_GITHUB_MODELS_KEY_FILE="$tmp_dir/github_models_key.txt") fi + if [ "$scenario" = "nvidia-nim-openai-direct-fallback-success" ]; then + printf '%s' 'openai-fallback-key' >"$tmp_dir/openai_fallback_key.txt" + printf '%s' 'https://api.openai.com/v1' >"$tmp_dir/openai_fallback_api_base.txt" + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + env_cmd+=(STRIX_OPENAI_FALLBACK_API_BASE_FILE="$tmp_dir/openai_fallback_api_base.txt") + fi + if [ "$scenario" = "github-models-openai-direct-fallback-success" ]; then + printf '%s' 'openai-fallback-key' >"$tmp_dir/openai_fallback_key.txt" + printf '%s' 'https://api.openai.com/v1' >"$tmp_dir/openai_fallback_api_base.txt" + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + env_cmd+=(STRIX_OPENAI_FALLBACK_API_BASE_FILE="$tmp_dir/openai_fallback_api_base.txt") + fi if [ "$min_fail_severity" = "__UNSET__" ]; then local next_env_cmd=() local env_pair @@ -5902,6 +5965,68 @@ run_gate_case_allow_provider_signal() { run_gate_case_with_provider_signal_mode "0" "$@" } +run_nvidia_nim_openai_direct_fallback_case() { + run_gate_case_allow_provider_signal "nvidia-nim-openai-direct-fallback-success" \ + "nvidia_nim/primary-model" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.6-luna' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/primary-model|openai/gpt-5.6-luna" \ + "https://integrate.api.nvidia.com/v1|https://api.openai.com/v1" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "openai-direct/gpt-5.6-luna" +} + +run_github_models_openai_direct_fallback_case() { + run_gate_case_allow_provider_signal "github-models-openai-direct-fallback-success" \ + "openai/gpt-5" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'openai-direct/gpt-5.6-luna' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5|openai/gpt-5.6-luna" \ + "https://models.github.ai/inference|https://api.openai.com/v1" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "openai-direct/gpt-5.6-luna" +} + run_github_models_http410_case() { local scenario="$1" local expected_exit="$2" @@ -6124,6 +6249,12 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/openai/o3" ;; + nvidia-nim-openai-direct-fallback-success) + run_nvidia_nim_openai_direct_fallback_case + ;; + github-models-openai-direct-fallback-success) + run_github_models_openai_direct_fallback_case + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -12514,6 +12645,13 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "" \ "github_models/openai/o3" +# NVIDIA NIM primary scans must route an explicit direct-OpenAI fallback to +# the OpenAI endpoint and its own credential, not the primary provider input. +run_nvidia_nim_openai_direct_fallback_case + +# GitHub Models primary scans use the same direct-OpenAI fallback contract. +run_github_models_openai_direct_fallback_case + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \