diff --git a/docs/ADRs/0024-harness-definitions.md b/docs/ADRs/0024-harness-definitions.md index 232439e159..095bcfcf67 100644 --- a/docs/ADRs/0024-harness-definitions.md +++ b/docs/ADRs/0024-harness-definitions.md @@ -384,6 +384,7 @@ agent_input: # another agent, and where that agent runs, is an open question. validation_loop: script: scripts/.sh # exit 0 = pass, non-zero = retry + schema: schemas/result.schema.json # output schema (resolved like script) max_iterations: 3 # how many times the agent can retry feedback_mode: append # append validation output to agent prompt @@ -542,6 +543,7 @@ post_script: scripts/code-post.sh validation_loop: script: scripts/validate-lint.sh + schema: schemas/code-result.schema.json max_iterations: 3 feedback_mode: append diff --git a/docs/ADRs/0045-forge-portable-harness-schema.md b/docs/ADRs/0045-forge-portable-harness-schema.md index 1b89b937d0..9358306bb8 100644 --- a/docs/ADRs/0045-forge-portable-harness-schema.md +++ b/docs/ADRs/0045-forge-portable-harness-schema.md @@ -205,9 +205,10 @@ skills: - skills/issue-labels - skills/output-schema-validation runner_env: - FULLSEND_OUTPUT_SCHEMA: ${FULLSEND_DIR}/schemas/triage-result.schema.json + FULLSEND_OUTPUT_SCHEMA: ${FULLSEND_DIR}/schemas/triage-result.schema.json # legacy — prefer validation_loop.schema validation_loop: script: scripts/validate-output-schema.sh + schema: schemas/triage-result.schema.json max_iterations: 2 forge: @@ -295,9 +296,10 @@ skills: - skills/issue-labels validation_loop: script: scripts/validate-output-schema.sh + schema: schemas/triage-result.schema.json max_iterations: 2 runner_env: - FULLSEND_OUTPUT_SCHEMA: ${FULLSEND_DIR}/schemas/triage-result.schema.json + FULLSEND_OUTPUT_SCHEMA: ${FULLSEND_DIR}/schemas/triage-result.schema.json # legacy — prefer validation_loop.schema forge: github: diff --git a/docs/guides/user/building-custom-agents.md b/docs/guides/user/building-custom-agents.md index 47088242e5..f9e0c452ab 100644 --- a/docs/guides/user/building-custom-agents.md +++ b/docs/guides/user/building-custom-agents.md @@ -139,6 +139,7 @@ pre_script: customized/scripts/pre-my-agent.sh validation_loop: script: scripts/validate-output-schema.sh + schema: customized/schemas/my-agent-result.schema.json max_iterations: 2 post_script: customized/scripts/post-my-agent.sh @@ -148,7 +149,6 @@ env: MY_VAR: "${MY_VAR}" ISSUE_KEY: "${ISSUE_KEY}" GH_TOKEN: "${GH_TOKEN}" # auto-minted in CI when --mint-url is provided - FULLSEND_OUTPUT_SCHEMA: ${FULLSEND_DIR}/customized/schemas/my-agent-result.schema.json timeout_minutes: 20 diff --git a/docs/guides/user/customizing-agents.md b/docs/guides/user/customizing-agents.md index 9f9c1c583e..cab457eaab 100644 --- a/docs/guides/user/customizing-agents.md +++ b/docs/guides/user/customizing-agents.md @@ -38,6 +38,7 @@ post_script: scripts/post-code.sh validation_loop: script: scripts/validate-output-schema.sh + schema: schemas/result.schema.json max_iterations: 2 env: @@ -53,7 +54,9 @@ env: providers: # Inference providers (loaded from providers/ dir) - vertex # References providers/vertex.yaml -validation_loop: +validation_loop: # script is required; these sub-fields are optional + script: scripts/validate-output-schema.sh + schema: schemas/result.schema.json # JSON Schema file for output validation (optional) feedback_mode: stderr # "stderr", "stdout", or "exit_code" (optional) allowed_remote_resources: # URL prefixes allowed for remote skills/agents/policies @@ -357,6 +360,7 @@ post_script: scripts/post-code.sh validation_loop: script: scripts/custom-validate.sh # Changed script + schema: schemas/custom-result.schema.json max_iterations: 5 # Changed from: 2 env: diff --git a/internal/cli/lock.go b/internal/cli/lock.go index 37a90a10fb..86736932d4 100644 --- a/internal/cli/lock.go +++ b/internal/cli/lock.go @@ -675,6 +675,12 @@ func resolveFromLock(h *harness.Harness, entry *lock.HarnessLock, workspaceRoot // Same as forge pre_script above. case strings.HasPrefix(m.field, "forge.") && strings.HasSuffix(m.field, ".validation_loop.script"): // Same as forge pre_script above. + case m.field == "validation_loop.schema": + if h.ValidationLoop != nil { + h.ValidationLoop.Schema = m.localPath + } + case strings.HasPrefix(m.field, "forge.") && strings.HasSuffix(m.field, ".validation_loop.schema"): + // Same as forge pre_script above. default: var idx int if _, err := fmt.Sscanf(m.field, "skills[%d]", &idx); err == nil && idx >= 0 && idx < len(h.Skills) { diff --git a/internal/cli/lock_test.go b/internal/cli/lock_test.go index 27c6a06af0..925612b866 100644 --- a/internal/cli/lock_test.go +++ b/internal/cli/lock_test.go @@ -922,6 +922,41 @@ func TestResolveFromLock_BaseFieldNoOp(t *testing.T) { assert.True(t, baseDep.CacheHit) } +func TestResolveFromLock_ValidationLoopSchema(t *testing.T) { + agentContent := []byte("You are a coding agent.") + agentHash := fetch.ComputeSHA256(agentContent) + schemaContent := []byte(`{"type":"object"}`) + schemaHash := fetch.ComputeSHA256(schemaContent) + + root := t.TempDir() + require.NoError(t, fetch.CachePut(root, "https://example.com/agents/code.md", agentContent)) + require.NoError(t, fetch.CachePut(root, "https://example.com/schemas/result.json", schemaContent)) + + entry := &lock.HarnessLock{ + Dependencies: []lock.DependencyEntry{ + {Field: "agent", URL: "https://example.com/agents/code.md", SHA256: agentHash}, + {Field: "validation_loop.schema", URL: "https://example.com/schemas/result.json", SHA256: schemaHash}, + }, + } + + h := &harness.Harness{ + Agent: "https://example.com/agents/code.md#sha256=" + agentHash, + ValidationLoop: &harness.ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "https://example.com/schemas/result.json#sha256=" + schemaHash, + }, + } + + printer := ui.New(os.Stdout) + deps, err := resolveFromLock(h, entry, root, printer) + require.NoError(t, err) + require.Len(t, deps, 2) + + assert.Equal(t, "validation_loop.schema", deps[1].Field) + assert.True(t, deps[1].CacheHit) + assert.True(t, strings.HasSuffix(h.ValidationLoop.Schema, "/content")) +} + func TestRunLock_URLBaseOnlyDeps(t *testing.T) { // A child harness with a URL base and no other URL references. // The baseDeps conversion loop runs and the base-only-deps path is taken diff --git a/internal/cli/run.go b/internal/cli/run.go index 8316761095..b89d4e8570 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -415,6 +415,12 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } } + // Expand ${VAR} references in validation_loop.schema so the path + // resolves before ValidateFilesExist stat-checks it. + if h.ValidationLoop != nil && strings.Contains(h.ValidationLoop.Schema, "${") { + h.ValidationLoop.Schema = os.Expand(h.ValidationLoop.Schema, expander) + } + if err := h.ValidateFilesExist(); err != nil { printer.StepFail("File validation failed") return fmt.Errorf("validating files: %w", err) @@ -1406,8 +1412,16 @@ func bootstrapEnv(sandboxName, remoteRepositoryDir string, h *harness.Harness, r // Expose output schema and expected filename inside the sandbox so // agents can self-check output with fullsend-check-output. See #1107. + // Prefer validation_loop.schema (already resolved by compose); fall + // back to the legacy RunnerEnv path for backward compatibility. remoteSchemaPath := sandbox.SandboxWorkspace + "/.fullsend/output-schema.json" - if schemaHost, ok := h.RunnerEnv["FULLSEND_OUTPUT_SCHEMA"]; ok && schemaHost != "" { + var schemaHost string + if h.ValidationLoop != nil && h.ValidationLoop.Schema != "" { + schemaHost = h.ValidationLoop.Schema + } else if v, ok := h.RunnerEnv["FULLSEND_OUTPUT_SCHEMA"]; ok && v != "" { + schemaHost = v + } + if schemaHost != "" { if _, statErr := os.Stat(schemaHost); statErr != nil { fmt.Fprintf(os.Stderr, "WARNING: schema file not found on host: %s\n", schemaHost) } else { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 7fa030eae9..bdb09d95c4 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1628,6 +1628,72 @@ func TestBootstrapEnv_SkipsFetchVarsWhenEmpty(t *testing.T) { assert.Contains(t, err.Error(), "copying .env file to sandbox") } +func TestBootstrapEnv_ValidationLoopSchemaPrecedence(t *testing.T) { + dir := t.TempDir() + schemaFile := filepath.Join(dir, "schema.json") + require.NoError(t, os.WriteFile(schemaFile, []byte(`{"type":"object"}`), 0o644)) + + h := &harness.Harness{ + Agent: "agents/test.md", + ValidationLoop: &harness.ValidationLoop{ + Script: "scripts/validate.sh", + Schema: schemaFile, + }, + RunnerEnv: map[string]string{ + "FULLSEND_OUTPUT_SCHEMA": "/should/not/be/used", + }, + } + + err := bootstrapEnv("nonexistent-sandbox", "/workspace/repo", h, nil) + + // Expected to fail at sandbox operations — the schema code path is + // exercised before the failure. + require.Error(t, err) +} + +func TestBootstrapEnv_ValidationLoopSchemaFallback(t *testing.T) { + h := &harness.Harness{ + Agent: "agents/test.md", + RunnerEnv: map[string]string{ + "FULLSEND_OUTPUT_SCHEMA": "/nonexistent/schema.json", + }, + } + + err := bootstrapEnv("nonexistent-sandbox", "/workspace/repo", h, nil) + + require.Error(t, err) + assert.Contains(t, err.Error(), "copying .env file to sandbox") +} + +func TestExpandValidationLoopSchema(t *testing.T) { + dir := t.TempDir() + schemaDir := filepath.Join(dir, "schemas") + require.NoError(t, os.MkdirAll(schemaDir, 0o755)) + schemaPath := filepath.Join(schemaDir, "result.json") + require.NoError(t, os.WriteFile(schemaPath, []byte(`{"type":"object"}`), 0o644)) + + expander := func(key string) string { + if key == "FULLSEND_DIR" { + return dir + } + return "" + } + + h := &harness.Harness{ + ValidationLoop: &harness.ValidationLoop{ + Schema: "${FULLSEND_DIR}/schemas/result.json", + }, + } + + if h.ValidationLoop != nil && strings.Contains(h.ValidationLoop.Schema, "${") { + h.ValidationLoop.Schema = os.Expand(h.ValidationLoop.Schema, expander) + } + + assert.Equal(t, schemaPath, h.ValidationLoop.Schema) + _, err := os.Stat(h.ValidationLoop.Schema) + require.NoError(t, err, "expanded schema path should exist") +} + func TestBuildSandboxEnvLines_FromEnvSandbox(t *testing.T) { h := &harness.Harness{ Agent: "agents/test.md", diff --git a/internal/harness/compose.go b/internal/harness/compose.go index 4eacea9e1e..026e851ae9 100644 --- a/internal/harness/compose.go +++ b/internal/harness/compose.go @@ -600,6 +600,17 @@ func resolveBaseScripts(ctx context.Context, base *Harness, baseURL string, allo base.ValidationLoop.Script = cachePath deps = append(deps, dep) } + if base.ValidationLoop != nil && base.ValidationLoop.Schema != "" { + if err := validateBaseRelPath("validation_loop.schema", base.ValidationLoop.Schema); err != nil { + return nil, err + } + dep, cachePath, err := fetchBaseFile(ctx, "validation_loop.schema", baseURLDir, base.ValidationLoop.Schema, allowlist, opts, "resource", false) + if err != nil { + return nil, err + } + base.ValidationLoop.Schema = cachePath + deps = append(deps, dep) + } for platform, fc := range base.Forge { if fc == nil { @@ -638,6 +649,18 @@ func resolveBaseScripts(ctx context.Context, base *Harness, baseURL string, allo fc.ValidationLoop.Script = cachePath deps = append(deps, dep) } + if fc.ValidationLoop != nil && fc.ValidationLoop.Schema != "" { + fieldName := fmt.Sprintf("forge.%s.validation_loop.schema", platform) + if err := validateBaseRelPath(fieldName, fc.ValidationLoop.Schema); err != nil { + return nil, err + } + dep, cachePath, err := fetchBaseFile(ctx, fieldName, baseURLDir, fc.ValidationLoop.Schema, allowlist, opts, "resource", false) + if err != nil { + return nil, err + } + fc.ValidationLoop.Schema = cachePath + deps = append(deps, dep) + } } // agent_input is a directory at runtime (uploaded recursively) and cannot diff --git a/internal/harness/compose_test.go b/internal/harness/compose_test.go index e47933d22b..620c4f7f09 100644 --- a/internal/harness/compose_test.go +++ b/internal/harness/compose_test.go @@ -1360,6 +1360,97 @@ base: `+baseURL+` assert.Equal(t, "resource", deps[2].Type) } +func TestLoadWithBase_URLBase_ValidationLoopSchemaFetched(t *testing.T) { + validateScript := []byte("#!/bin/bash\necho validate") + schemaContent := []byte(`{"type":"object","properties":{"action":{"type":"string"}}}`) + + baseContent := []byte(` +agent: agents/triage.md +role: test +validation_loop: + script: scripts/validate.sh + schema: schemas/result.schema.json + max_iterations: 2 +`) + + server, policy := setupScriptTestServer(t, baseContent, map[string][]byte{ + "/scripts/validate.sh": validateScript, + "/schemas/result.schema.json": schemaContent, + }) + + hash := computeHash(baseContent) + dir := t.TempDir() + cacheDir := filepath.Join(dir, "cache") + baseURL := server.URL + "/harness/triage.yaml#sha256=" + hash + + path := writeTestHarness(t, dir, "child.yaml", ` +agent: agents/child.md +role: test +base: `+baseURL+` +`) + + h, deps, err := LoadWithBase(context.Background(), path, ComposeOpts{ + WorkspaceRoot: cacheDir, + FetchPolicy: policy, + OrgAllowlist: []string{server.URL + "/"}, + }) + require.NoError(t, err) + + require.NotNil(t, h.ValidationLoop) + assert.True(t, filepath.IsAbs(h.ValidationLoop.Schema)) + assert.Equal(t, 2, h.ValidationLoop.MaxIterations) + + content, err := os.ReadFile(h.ValidationLoop.Schema) + require.NoError(t, err) + assert.Equal(t, schemaContent, content) + + // 1 base + 1 validation script + 1 schema + 1 agent resource + require.Len(t, deps, 4) + assert.Equal(t, "validation_loop.script", deps[1].Field) + assert.Equal(t, "script", deps[1].Type) + assert.Equal(t, "validation_loop.schema", deps[2].Field) + assert.Equal(t, "resource", deps[2].Type) + assert.Equal(t, "agent", deps[3].Field) + assert.Equal(t, "resource", deps[3].Type) +} + +func TestLoadWithBase_URLBase_ValidationLoopSchemaFetchError(t *testing.T) { + validateScript := []byte("#!/bin/bash\necho validate") + + baseContent := []byte(` +agent: agents/triage.md +role: test +validation_loop: + script: scripts/validate.sh + schema: schemas/missing.schema.json + max_iterations: 2 +`) + + // The server does NOT serve /schemas/missing.schema.json + server, policy := setupScriptTestServer(t, baseContent, map[string][]byte{ + "/scripts/validate.sh": validateScript, + }) + + hash := computeHash(baseContent) + dir := t.TempDir() + cacheDir := filepath.Join(dir, "cache") + baseURL := server.URL + "/harness/triage.yaml#sha256=" + hash + + path := writeTestHarness(t, dir, "child.yaml", ` +agent: agents/child.md +role: test +base: `+baseURL+` +`) + + _, _, err := LoadWithBase(context.Background(), path, ComposeOpts{ + WorkspaceRoot: cacheDir, + FetchPolicy: policy, + OrgAllowlist: []string{server.URL + "/"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation_loop.schema") +} + func TestLoadWithBase_URLBase_ForgeScriptsFetched(t *testing.T) { forgePre := []byte("#!/bin/bash\necho forge-pre") forgePost := []byte("#!/bin/bash\necho forge-post") @@ -1770,6 +1861,98 @@ base: `+baseURL+` assert.Equal(t, "forge.github.validation_loop.script", deps[1].Field) } +func TestLoadWithBase_URLBase_ForgeValidationLoopSchemaFetched(t *testing.T) { + forgeValidate := []byte("#!/bin/bash\necho forge-validate") + schemaContent := []byte(`{"type":"object"}`) + + baseContent := []byte(` +agent: agents/triage.md +role: test +forge: + github: + validation_loop: + script: scripts/gh-validate.sh + schema: schemas/result.schema.json + max_iterations: 2 +`) + + server, policy := setupScriptTestServer(t, baseContent, map[string][]byte{ + "/scripts/gh-validate.sh": forgeValidate, + "/schemas/result.schema.json": schemaContent, + }) + + hash := computeHash(baseContent) + dir := t.TempDir() + cacheDir := filepath.Join(dir, "cache") + baseURL := server.URL + "/harness/triage.yaml#sha256=" + hash + + path := writeTestHarness(t, dir, "child.yaml", ` +agent: agents/child.md +role: test +base: `+baseURL+` +`) + + h, deps, err := LoadWithBase(context.Background(), path, ComposeOpts{ + WorkspaceRoot: cacheDir, + FetchPolicy: policy, + OrgAllowlist: []string{server.URL + "/"}, + ForgePlatform: "github", + }) + require.NoError(t, err) + + require.NotNil(t, h.ValidationLoop) + assert.True(t, filepath.IsAbs(h.ValidationLoop.Schema)) + + content, err := os.ReadFile(h.ValidationLoop.Schema) + require.NoError(t, err) + assert.Equal(t, schemaContent, content) + + // 1 base + 1 forge validation_loop script + 1 forge schema + 1 agent resource + require.Len(t, deps, 4) + assert.Equal(t, "forge.github.validation_loop.script", deps[1].Field) + assert.Equal(t, "forge.github.validation_loop.schema", deps[2].Field) + assert.Equal(t, "resource", deps[2].Type) +} + +func TestLoadWithBase_URLBase_ForgeValidationLoopSchemaFetchError(t *testing.T) { + forgeValidate := []byte("#!/bin/bash\necho forge-validate") + + baseContent := []byte(` +agent: agents/triage.md +role: test +forge: + github: + validation_loop: + script: scripts/gh-validate.sh + schema: schemas/missing.schema.json + max_iterations: 2 +`) + + server, policy := setupScriptTestServer(t, baseContent, map[string][]byte{ + "/scripts/gh-validate.sh": forgeValidate, + }) + + hash := computeHash(baseContent) + dir := t.TempDir() + cacheDir := filepath.Join(dir, "cache") + baseURL := server.URL + "/harness/triage.yaml#sha256=" + hash + + path := writeTestHarness(t, dir, "child.yaml", ` +agent: agents/child.md +role: test +base: `+baseURL+` +`) + + _, _, err := LoadWithBase(context.Background(), path, ComposeOpts{ + WorkspaceRoot: cacheDir, + FetchPolicy: policy, + OrgAllowlist: []string{server.URL + "/"}, + ForgePlatform: "github", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation_loop.schema") +} + func TestLoadWithBase_URLBase_AgentInputNotFetched(t *testing.T) { baseContent := []byte(` agent: agents/triage.md @@ -1970,6 +2153,46 @@ func TestResolveBaseScripts_RejectsAbsoluteForgeValidationLoop(t *testing.T) { assert.Contains(t, err.Error(), "forge.gitlab.validation_loop.script") } +func TestResolveBaseScripts_RejectsTraversalInValidationLoopSchema(t *testing.T) { + base := &Harness{ + ValidationLoop: &ValidationLoop{ + Schema: "../../../etc/shadow", + }, + } + _, err := resolveBaseScripts(context.Background(), base, "https://example.com/harness/triage.yaml#sha256=abc", nil, ComposeOpts{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain path traversal") + assert.Contains(t, err.Error(), "validation_loop.schema") +} + +func TestResolveBaseScripts_RejectsTraversalInForgeValidationLoopSchema(t *testing.T) { + base := &Harness{ + Forge: map[string]*ForgeConfig{ + "github": { + ValidationLoop: &ValidationLoop{ + Schema: "../escape.json", + }, + }, + }, + } + _, err := resolveBaseScripts(context.Background(), base, "https://example.com/harness/triage.yaml#sha256=abc", nil, ComposeOpts{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain path traversal") + assert.Contains(t, err.Error(), "forge.github.validation_loop.schema") +} + +func TestResolveBaseScripts_RejectsAbsoluteValidationLoopSchema(t *testing.T) { + base := &Harness{ + ValidationLoop: &ValidationLoop{ + Schema: "/etc/schema.json", + }, + } + _, err := resolveBaseScripts(context.Background(), base, "https://example.com/harness/triage.yaml#sha256=abc", nil, ComposeOpts{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be a relative path, not an absolute path") + assert.Contains(t, err.Error(), "validation_loop.schema") +} + func TestResolveBaseScripts_RejectsNullBytes(t *testing.T) { base := &Harness{PreScript: "scripts/pre\x00.sh"} _, err := resolveBaseScripts(context.Background(), base, "https://example.com/harness/triage.yaml#sha256=abc", nil, ComposeOpts{}) diff --git a/internal/harness/forge.go b/internal/harness/forge.go index 81fda34616..3cdde07a5e 100644 --- a/internal/harness/forge.go +++ b/internal/harness/forge.go @@ -70,6 +70,9 @@ func (h *Harness) validateForge() error { if IsURL(fc.ValidationLoop.Script) { return fmt.Errorf("forge.%s.validation_loop.script must be a local path, not a URL", key) } + if fc.ValidationLoop.Schema != "" && IsURL(fc.ValidationLoop.Schema) { + return fmt.Errorf("forge.%s.validation_loop.schema must be a local path, not a URL", key) + } } } return nil diff --git a/internal/harness/forge_test.go b/internal/harness/forge_test.go index 3a815cce3a..c9cfa31bc4 100644 --- a/internal/harness/forge_test.go +++ b/internal/harness/forge_test.go @@ -303,6 +303,25 @@ func TestValidate_ForgeScriptURL(t *testing.T) { assert.Contains(t, err.Error(), "forge.github.validation_loop.script must be a local path") }) + t.Run("validation_loop.schema URL", func(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + Role: "test", + Forge: map[string]*ForgeConfig{ + "github": { + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "https://evil.com/schema.json", + MaxIterations: 1, + }, + }, + }, + } + err := h.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "forge.github.validation_loop.schema must be a local path") + }) + t.Run("validation_loop missing script", func(t *testing.T) { h := &Harness{ Agent: "agents/test.md", diff --git a/internal/harness/harness.go b/internal/harness/harness.go index 62af46f3d2..9a952abee0 100644 --- a/internal/harness/harness.go +++ b/internal/harness/harness.go @@ -187,6 +187,7 @@ type APIServer struct { // ValidationLoop configures a deterministic validation step after the agent exits. type ValidationLoop struct { Script string `yaml:"script"` + Schema string `yaml:"schema,omitempty"` MaxIterations int `yaml:"max_iterations"` FeedbackMode string `yaml:"feedback_mode,omitempty"` } @@ -525,6 +526,11 @@ func (h *Harness) ResolveRelativeTo(baseDir string) error { if h.ValidationLoop.Script, err = resolve("validation_loop.script", h.ValidationLoop.Script); err != nil { return err } + if h.ValidationLoop.Schema != "" && !strings.Contains(h.ValidationLoop.Schema, "${") { + if h.ValidationLoop.Schema, err = resolve("validation_loop.schema", h.ValidationLoop.Schema); err != nil { + return err + } + } } return nil } @@ -557,6 +563,11 @@ func (h *Harness) ValidateRunnerEnvWith(lookup func(string) (string, bool)) erro return err } } + if h.ValidationLoop != nil && h.ValidationLoop.Schema != "" { + if err := checkVarRefs("validation_loop.schema", h.ValidationLoop.Schema); err != nil { + return err + } + } if h.Env != nil { for k, v := range h.Env.Runner { if err := checkVarRefs(fmt.Sprintf("env.runner[%s]", k), v); err != nil { @@ -637,6 +648,11 @@ func (h *Harness) ValidateFilesExist() error { if err := check("validation_loop.script", h.ValidationLoop.Script); err != nil { return err } + if h.ValidationLoop.Schema != "" && !strings.Contains(h.ValidationLoop.Schema, "${") { + if err := check("validation_loop.schema", h.ValidationLoop.Schema); err != nil { + return err + } + } } return nil } @@ -726,6 +742,9 @@ func (h *Harness) ValidateResourceTypes() error { if h.ValidationLoop != nil && h.ValidationLoop.Script != "" && IsURL(h.ValidationLoop.Script) { return fmt.Errorf("validation_loop.script must be a local path, not a URL") } + if h.ValidationLoop != nil && h.ValidationLoop.Schema != "" && IsURL(h.ValidationLoop.Schema) { + return fmt.Errorf("validation_loop.schema must be a local path, not a URL") + } for i, hf := range h.HostFiles { if IsURL(hf.Src) { return fmt.Errorf("host_files[%d].src must be a local path, not a URL", i) diff --git a/internal/harness/harness_test.go b/internal/harness/harness_test.go index 236db54c56..f405c5ca07 100644 --- a/internal/harness/harness_test.go +++ b/internal/harness/harness_test.go @@ -190,6 +190,47 @@ func TestResolveRelativeTo_HostFiles(t *testing.T) { assert.Equal(t, "/absolute/path/file.txt", h.HostFiles[2].Src) } +func TestResolveRelativeTo_ValidationLoopSchema(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "schemas/result.schema.json", + }, + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + assert.Equal(t, "/base/dir/schemas/result.schema.json", h.ValidationLoop.Schema) +} + +func TestResolveRelativeTo_ValidationLoopSchemaVarSkipped(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "${FULLSEND_DIR}/schemas/result.schema.json", + }, + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + assert.Equal(t, "${FULLSEND_DIR}/schemas/result.schema.json", h.ValidationLoop.Schema) +} + +func TestResolveRelativeTo_ValidationLoopSchemaTraversalRejected(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "../../etc/shadow.json", + }, + } + err := h.ResolveRelativeTo("/base/dir") + require.Error(t, err) + assert.Contains(t, err.Error(), "resolves outside fullsend directory") +} + func TestResolveRelativeTo_AbsolutePathsUnchanged(t *testing.T) { h := &Harness{ Agent: "/absolute/path/agent.md", @@ -492,6 +533,40 @@ func TestValidateRunnerEnvWith_EnvAllSet(t *testing.T) { require.NoError(t, err) } +func TestValidateRunnerEnvWith_ChecksValidationLoopSchema(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + Role: "test", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "${MISSING_DIR}/schemas/result.json", + }, + } + lookup := func(key string) (string, bool) { return "", false } + err := h.ValidateRunnerEnvWith(lookup) + require.Error(t, err) + assert.Contains(t, err.Error(), "MISSING_DIR") + assert.Contains(t, err.Error(), "validation_loop.schema") +} + +func TestValidateRunnerEnvWith_ValidationLoopSchemaVarSet(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + Role: "test", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "${FULLSEND_DIR}/schemas/result.json", + }, + } + lookup := func(key string) (string, bool) { + if key == "FULLSEND_DIR" { + return "/opt/fullsend", true + } + return "", false + } + require.NoError(t, h.ValidateRunnerEnvWith(lookup)) +} + func TestValidateRunnerEnvWith_NilEnvNoError(t *testing.T) { h := &Harness{Agent: "agents/test.md", Role: "test"} err := h.ValidateRunnerEnvWith(func(string) (string, bool) { return "", false }) @@ -662,6 +737,42 @@ func TestValidateFilesExist_SkipsVarPaths(t *testing.T) { require.NoError(t, h.ValidateFilesExist()) } +func TestValidateFilesExist_SkipsSchemaVarPaths(t *testing.T) { + dir := t.TempDir() + agentFile := filepath.Join(dir, "agent.md") + require.NoError(t, os.WriteFile(agentFile, []byte("agent"), 0o644)) + scriptFile := filepath.Join(dir, "validate.sh") + require.NoError(t, os.WriteFile(scriptFile, []byte("#!/bin/bash"), 0o755)) + + h := &Harness{ + Agent: agentFile, + ValidationLoop: &ValidationLoop{ + Script: scriptFile, + Schema: "${FULLSEND_DIR}/schemas/result.schema.json", + }, + } + require.NoError(t, h.ValidateFilesExist()) +} + +func TestValidateFilesExist_MissingSchema(t *testing.T) { + dir := t.TempDir() + agentFile := filepath.Join(dir, "agent.md") + require.NoError(t, os.WriteFile(agentFile, []byte("agent"), 0o644)) + scriptFile := filepath.Join(dir, "validate.sh") + require.NoError(t, os.WriteFile(scriptFile, []byte("#!/bin/bash"), 0o755)) + + h := &Harness{ + Agent: agentFile, + ValidationLoop: &ValidationLoop{ + Script: scriptFile, + Schema: "/nonexistent/schema.json", + }, + } + err := h.ValidateFilesExist() + require.Error(t, err) + assert.Contains(t, err.Error(), "validation_loop.schema") +} + func TestValidate_PluginNameValid(t *testing.T) { h := &Harness{ Agent: "agents/test.md", @@ -1021,6 +1132,32 @@ func TestValidateResourceTypes(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "agent_input must be a local path") }) + + t.Run("URL in validation_loop.schema", func(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "https://example.com/schemas/result.json", + MaxIterations: 1, + }, + } + err := h.ValidateResourceTypes() + require.Error(t, err) + assert.Contains(t, err.Error(), "validation_loop.schema must be a local path") + }) + + t.Run("local validation_loop.schema accepted", func(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + Schema: "schemas/result.schema.json", + MaxIterations: 1, + }, + } + require.NoError(t, h.ValidateResourceTypes()) + }) } func TestHasURLReferences(t *testing.T) {