Skip to content

tests:harness tests with compat and fixes #4379

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akshaydeo merged 1 commit into from
Jun 15, 2026
+26 −1
Merged

tests:harness tests with compat and fixes #4379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ FLOW ?=
VERSION ?= dev-build
LOCAL ?=
DEBUG ?=
COMPAT ?=

# Colors for output
RED=\033[0;31m
Expand Down Expand Up @@ -1771,6 +1772,21 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov
exit 0; \
fi
@if [ -n "$(HELP)" ]; then exit 0; fi; \
if [ "$(COMPAT)" = "both" ]; then \
mkdir -p tmp; \
$(ECHO) "$(CYAN)COMPAT=both: running harness with compat OFF then ON (sub-runs forced CI=1 to skip the interactive viewer)...$(NC)"; \
for mode in off on; do \
$(ECHO) "$(CYAN)=== Harness run: compat $$mode ===$(NC)"; \
$(MAKE) run-provider-harness-test COMPAT=$$mode CI=1; \
RC=$$?; \
mv -f tmp/newman-report.json "tmp/newman-report-compat-$$mode.json" 2>/dev/null || true; \
mv -f tmp/newman-report.html "tmp/newman-report-compat-$$mode.html" 2>/dev/null || true; \
mv -f tmp/harness-failures.md "tmp/harness-failures-compat-$$mode.md" 2>/dev/null || true; \
if [ "$$RC" -ne 0 ]; then $(ECHO) "$(RED)compat $$mode run failed (exit $$RC)$(NC)"; BOTH_RC=$$RC; fi; \
done; \
$(ECHO) "$(GREEN)COMPAT=both complete. Reports: tmp/newman-report-compat-{off,on}.{json,html}, tmp/harness-failures-compat-{off,on}.md$(NC)"; \
exit $${BOTH_RC:-0}; \
fi; \
$(EXPOSE_ENV); \
mkdir -p tmp; \
BASE_URL_VAL="$(or $(BASE_URL),http://localhost:8080)"; \
Expand Down Expand Up @@ -1893,6 +1909,7 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov
( \
newman run "tmp/harness-filtered-$$p.json" \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
Comment thread
TejasGhatte marked this conversation as resolved.
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \
Expand Down Expand Up @@ -1975,6 +1992,7 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov
echo $$! > tmp/harness-monitor.pid; \
newman run "$$COLLECTION_FILE" \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \
Expand All @@ -1998,6 +2016,7 @@ run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost prov
else \
newman run "$$COLLECTION_FILE" \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \
Expand Down
2 changes: 2 additions & 0 deletions framework/modelcatalog/datasheet/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,8 @@ func extractSupportedParams(parsed *modelParametersParseResult) []string {
case "web_search":
addParam("web_search_options") // chat-path param
addParam("web_search") // responses-path server tool
case "stop_sequences":
addParam("stop")
case "promptTools", "image_detail", "stream":
// skip — not top-level request parameters
default:
Expand Down
6 changes: 5 additions & 1 deletion tests/e2e/api/collections/provider-harness.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
"// Used in the criss-cross matrix (§8) to mark provider+modality cells that return NewUnsupportedOperationError by design (e.g., anthropic embeddings, bedrock audio).",
"if (name.indexOf('[SKIP]') !== -1 && (pm.environment.get('include_skip') || pm.variables.get('include_skip') || '') !== '1') {",
" pm.execution.skipRequest();",
"}"
"}",
"// Per-request compat override: when compat is set (via Newman --env-var compat=true), inject the x-bf-compat header on every request.",
"// Absent => harness baseline (compat config off). Lets the suite run once with compat off and once with compat on.",
"var __compat = (pm.environment.get('compat') || pm.variables.get('compat') || '');",
"if (__compat) { pm.request.headers.upsert({ key: 'x-bf-compat', value: String(__compat) }); }"
]
}
},
Expand Down
Loading