-
Notifications
You must be signed in to change notification settings - Fork 3.1k
feat(policy): add opt-in Tavily web-search preset for Deep Agents Code (#5621) #5651
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
0a6e24a
feat(policy): add opt-in Tavily web-search preset for Deep Agents Cod…
abhi-0906 a885daf
test(policy): assert node and curl binary scoping in the Tavily opt-i…
abhi-0906 763e917
test(policy): move Tavily opt-in assertions to a dedicated test file
abhi-0906 f04efa8
Merge branch 'main' into feat/issue-5621-tavily-preset
cv 303d69b
test(policy): drop redundant source-shape assertions from tavily pres…
abhi-0906 9798432
test(policy): prove dcode tavily opt-in
cv 97770f6
Merge remote-tracking branch 'origin/main' into feat/issue-5621-tavil…
cv File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| preset: | ||
| name: tavily | ||
| description: "Tavily web search API access (opt-in)" | ||
|
|
||
| network_policies: | ||
| tavily: | ||
| name: tavily | ||
| endpoints: | ||
| - host: api.tavily.com | ||
| port: 443 | ||
| protocol: rest | ||
| enforcement: enforce | ||
| rules: | ||
| - allow: { method: GET, path: "/**" } | ||
| - allow: { method: POST, path: "/**" } | ||
| binaries: | ||
| - { path: /opt/venv/bin/python3* } | ||
| - { path: /usr/bin/python3* } | ||
| - { path: /usr/local/bin/python3* } | ||
| - { path: /usr/local/bin/node } | ||
| - { path: /usr/bin/node } | ||
| - { path: /usr/bin/curl } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
144 changes: 144 additions & 0 deletions
144
test/e2e/e2e-cloud-experimental/checks/09-deepagents-code-tavily-opt-in.sh
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| #!/bin/bash | ||
| # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
| # Case: Deep Agents Code Tavily opt-in policy (#5739). | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}}" | ||
| PREFIX="09-deepagents-code-tavily-opt-in" | ||
| REPO="${REPO:-$(pwd)}" | ||
| CLI="${NEMOCLAW_E2E_CLI:-${REPO}/bin/nemoclaw.js}" | ||
|
|
||
| ok() { printf '%s\n' "${PREFIX}: OK ($*)"; } | ||
| info() { printf '%s\n' "${PREFIX}: $*"; } | ||
| fail_test() { | ||
| printf '%s\n' "${PREFIX}: FAIL: $1" >&2 | ||
| FAILED=$((FAILED + 1)) | ||
| } | ||
| pass() { | ||
| ok "$1" | ||
| PASSED=$((PASSED + 1)) | ||
| } | ||
|
|
||
| sandbox_exec() { | ||
| openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 | ||
| } | ||
|
|
||
| nemoclaw_cli() { | ||
| if [ -f "$CLI" ]; then | ||
| node "$CLI" "$@" | ||
| else | ||
| nemoclaw "$@" | ||
| fi | ||
| } | ||
|
|
||
| python_probe() { | ||
| local url="$1" | ||
| sandbox_exec "python3 - ${url@Q} <<'PY' | ||
| import sys | ||
| import urllib.error | ||
| import urllib.request | ||
|
|
||
| DENIAL_MARKERS = ( | ||
| 'access denied', | ||
| 'blocked by', | ||
| 'connection forbidden', | ||
| 'egress denied', | ||
| 'network is unreachable', | ||
| 'network policy', | ||
| 'operation not permitted', | ||
| 'permission denied', | ||
| 'policy denied', | ||
| 'tunnel connection failed', | ||
| ) | ||
|
|
||
|
|
||
| def is_policy_denial(text): | ||
| lowered = text.lower() | ||
| return any(marker in lowered for marker in DENIAL_MARKERS) | ||
|
|
||
|
|
||
| url = sys.argv[1] | ||
| try: | ||
| with urllib.request.urlopen(url, timeout=8) as response: | ||
| print(f'REACHED:{response.status}') | ||
| except urllib.error.HTTPError as exc: | ||
| body = '' | ||
| try: | ||
| body = exc.read(512).decode('utf-8', 'replace') | ||
| except Exception: | ||
| body = '' | ||
| details = f'{exc} {body}'.strip() | ||
| if is_policy_denial(details): | ||
| print(f'BLOCKED:HTTPError:{details}') | ||
| else: | ||
| print(f'REACHED:{exc.code}') | ||
| except urllib.error.URLError as exc: | ||
| details = str(exc.reason if getattr(exc, 'reason', None) is not None else exc) | ||
| if is_policy_denial(details): | ||
| print(f'BLOCKED:URLError:{details}') | ||
| else: | ||
| print(f'ERROR:URLError:{details}') | ||
| except OSError as exc: | ||
| details = str(exc) | ||
| if is_policy_denial(details): | ||
| print(f'BLOCKED:{type(exc).__name__}:{details}') | ||
| else: | ||
| print(f'ERROR:{type(exc).__name__}:{details}') | ||
| except Exception as exc: | ||
| print(f'ERROR:{type(exc).__name__}:{exc}') | ||
| PY | ||
| " | ||
| } | ||
|
|
||
| PASSED=0 | ||
| FAILED=0 | ||
|
|
||
| if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then | ||
| info "SKIP: sandbox '${SANDBOX_NAME}' is not a Deep Agents Code sandbox" | ||
| exit 0 | ||
| fi | ||
|
|
||
| info "Running Deep Agents Code Tavily opt-in check in sandbox: $SANDBOX_NAME" | ||
|
|
||
| # shellcheck disable=SC2016 # command substitution must run inside the sandbox. | ||
| PYTHON_REAL="$(sandbox_exec 'readlink -f "$(command -v python3)"' || true)" | ||
| if [[ "$PYTHON_REAL" == /opt/venv/* ]]; then | ||
| pass "sandbox python resolves through the managed Deep Agents Code venv" | ||
| else | ||
| fail_test "sandbox python does not resolve through /opt/venv: $PYTHON_REAL" | ||
| fi | ||
|
|
||
| DRY_RUN_OUTPUT="$(nemoclaw_cli "$SANDBOX_NAME" policy-add tavily --dry-run 2>&1)" || { | ||
| fail_test "policy-add tavily --dry-run failed: $DRY_RUN_OUTPUT" | ||
| printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" | ||
| exit 1 | ||
| } | ||
| if echo "$DRY_RUN_OUTPUT" | grep -q "api.tavily.com"; then | ||
| pass "tavily dry-run shows api.tavily.com" | ||
| else | ||
| fail_test "tavily dry-run did not show api.tavily.com: $DRY_RUN_OUTPUT" | ||
| fi | ||
|
|
||
| APPLY_OUTPUT="$(nemoclaw_cli "$SANDBOX_NAME" policy-add tavily --yes 2>&1)" || { | ||
| fail_test "policy-add tavily failed: $APPLY_OUTPUT" | ||
| printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" | ||
| exit 1 | ||
| } | ||
| pass "tavily policy preset applies" | ||
|
|
||
| sleep "${NEMOCLAW_E2E_POLICY_SETTLE_SECONDS:-5}" | ||
|
|
||
| PROBE_OUTPUT="$(python_probe "https://api.tavily.com/")" | ||
| if echo "$PROBE_OUTPUT" | grep -q "REACHED:"; then | ||
| pass "managed Deep Agents Code python can reach Tavily after policy-add" | ||
| elif echo "$PROBE_OUTPUT" | grep -q "BLOCKED:"; then | ||
| fail_test "managed Deep Agents Code python is still policy-blocked after policy-add: $PROBE_OUTPUT" | ||
| else | ||
| fail_test "Tavily probe lacked reachability evidence after policy-add: $PROBE_OUTPUT" | ||
| fi | ||
|
|
||
| printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" | ||
| [ "$FAILED" -eq 0 ] || exit 1 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| import { describe, expect, it } from "vitest"; | ||
| import YAML from "yaml"; | ||
| import * as policies from "../dist/lib/policy"; | ||
|
|
||
| type TavilyEndpoint = { | ||
| host: string; | ||
| port: number; | ||
| protocol: string; | ||
| enforcement: string; | ||
| rules: Array<{ allow: { method: string; path: string } }>; | ||
| tls?: string; | ||
| }; | ||
|
|
||
| type TavilyPolicy = { | ||
| endpoints?: TavilyEndpoint[]; | ||
| binaries?: Array<{ path: string }>; | ||
| access?: string; | ||
| }; | ||
|
|
||
| describe("tavily opt-in preset", () => { | ||
| it("declares narrow api.tavily.com egress for the interpreter binaries it allows", () => { | ||
| const tavily = policies.loadPreset("tavily"); | ||
| expect(tavily).not.toBeNull(); | ||
| const content = String(tavily); | ||
| const parsed = YAML.parse(content) as { | ||
| network_policies?: { | ||
| tavily?: TavilyPolicy; | ||
| }; | ||
| }; | ||
| const policy = parsed.network_policies?.tavily; | ||
|
|
||
| expect(policy?.endpoints).toEqual([ | ||
| { | ||
| host: "api.tavily.com", | ||
| port: 443, | ||
| protocol: "rest", | ||
| enforcement: "enforce", | ||
| rules: [ | ||
| { allow: { method: "GET", path: "/**" } }, | ||
| { allow: { method: "POST", path: "/**" } }, | ||
| ], | ||
| }, | ||
| ]); | ||
| expect(policy?.binaries).toEqual([ | ||
| { path: "/opt/venv/bin/python3*" }, | ||
| { path: "/usr/bin/python3*" }, | ||
| { path: "/usr/local/bin/python3*" }, | ||
| { path: "/usr/local/bin/node" }, | ||
| { path: "/usr/bin/node" }, | ||
| { path: "/usr/bin/curl" }, | ||
| ]); | ||
| expect(policy).not.toHaveProperty("access", "full"); | ||
| expect(policy?.endpoints?.[0]).not.toHaveProperty("tls", "skip"); | ||
| }); | ||
| }); |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.