diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 20ededabc8..0000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,43 +0,0 @@ -# CLAUDE.md - -Fullsend is a platform for fully autonomous agentic development for GitHub-hosted organizations. It contains design documents organized by problem domain (`docs/`) and a Go CLI (`cmd/fullsend/`) that manages GitHub App setup and org configuration. See [README.md](README.md) for the full document index. - -## How to work in this repo - -- This is a design exploration, not a spec. Documents should present multiple options with trade-offs, not prescribe single solutions. -- Each problem document has an "Open questions" section — this is where unresolved issues live. -- When adding new problem areas, create a new file in `docs/problems/` and link it from `README.md`. -- The security threat model (threat priority: external injection > insider > drift > supply chain) should inform all other documents. -- Keep core problem documents organization-agnostic. Organization-specific details belong in `docs/problems/applied//`. -- The target audience is any contributor community considering autonomous agents — keep language accessible, avoid presuming solutions. -- Always run `make lint` before submitting changes and fix any failures. -- Never commit secrets (tokens, API keys, PEM keys, gcloud credentials) or sensitive data (GCP project names, service account identifiers, Model Armor template names, internal hostnames). Use environment variables with no defaults for sensitive values. - -## Go code - -When making changes to Go code under `cmd/` or `internal/`: - -1. **Unit tests:** Run `make go-test` (or `go test ./...`) and fix any failures before committing. -2. **Vet:** Run `make go-vet` to catch common issues. -3. **E2E tests:** Run `make e2e-test` if your changes touch `internal/appsetup/`, `internal/forge/`, `internal/cli/`, or `internal/layers/`. These tests exercise the full admin install/uninstall flow against a live GitHub org using Playwright browser automation. - -### Running e2e tests - -The e2e tests require GitHub credentials. There are three ways to provide them: - -- **`E2E_GITHUB_PASSWORD` env var:** Set directly with the password. -- **`E2E_GITHUB_PASSWORD_FILE` env var:** Set to a file path containing the password (used in devaipod environments where secrets are mounted as files). -- **`E2E_GITHUB_SESSION_FILE` env var:** Set to a pre-exported Playwright session file (skips login). - -If only `E2E_GITHUB_USERNAME` and a password source are available, `make e2e-test` will automatically generate a session file before running tests. See `make help` for all available targets. - -## Key design decisions made - -- **Autonomy model:** Binary per-repo, with CODEOWNERS enforcing human approval on specific paths -- **Problem structure:** Problem-oriented documents (not ADRs or RFCs) that can evolve independently, with ADRs spun off later when decisions crystallize -- **Threat priority order:** External prompt injection > insider/compromised creds > agent drift > supply chain -- **Code generation is considered a solved problem.** The hard problems are review, intent, governance, and security. -- **Trust derives from repository permissions, not agent identity.** No agent trusts another based on who produced the output. -- **CODEOWNERS files are always human-owned.** Agents cannot modify their own guardrails. -- **The repo is the coordinator.** No coordinator agent — branch protection, CODEOWNERS, and status checks are the coordination layer. -- **Organization-specific content is cordoned.** Core problem docs are general; applied considerations live in `docs/problems/applied/`. diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index a81e747143..10ab310f46 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -18,12 +18,21 @@ const ( // SandboxClaudeConfig is the Claude config directory inside the sandbox. SandboxClaudeConfig = "/tmp/claude-config" //nolint:gosec // not a credential - createTimeout = 65 * time.Second - readyTimeout = 60 * time.Second - readyPoll = 2 * time.Second - transferTimeout = 5 * time.Minute + defaultCreateTimeout = 65 * time.Second + defaultReadyTimeout = 60 * time.Second + readyPoll = 2 * time.Second + transferTimeout = 5 * time.Minute ) +func resolveTimeout(envKey string, fallback time.Duration) time.Duration { + if v := os.Getenv(envKey); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return fallback +} + func sanitizeDownload(localDir string) error { return filepath.WalkDir(localDir, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -127,6 +136,7 @@ func EnsureGateway() error { // non-empty, it is passed as --from to start the sandbox from a container image. // If policy is non-empty, it is applied at creation time via --policy. func Create(name string, providers []string, image, policy string) error { + createTimeout := resolveTimeout("FULLSEND_SANDBOX_TIMEOUT", defaultCreateTimeout) ctx, cancel := context.WithTimeout(context.Background(), createTimeout) defer cancel() @@ -162,6 +172,7 @@ func Create(name string, providers []string, image, policy string) error { } // Wait for sandbox to be fully ready (image pull can take a while). + readyTimeout := resolveTimeout("FULLSEND_SANDBOX_READY_TIMEOUT", defaultReadyTimeout) deadline := time.Now().Add(readyTimeout) for time.Now().Before(deadline) { check := exec.Command("openshell", "sandbox", "get", name) diff --git a/qf-tests/GH-4/README.md b/qf-tests/GH-4/README.md new file mode 100644 index 0000000000..21b9f11090 --- /dev/null +++ b/qf-tests/GH-4/README.md @@ -0,0 +1,8 @@ +# QualityFlow Tests — GH-4 + +Generated by the QualityFlow pipeline. + +| Directory | Type | Framework | Destination repo | +|-----------|------|-----------|-----------------| +| `go/` | Tier 1 (functional) | Go/Ginkgo | `guyoron1/fullsend` | +| `python/` | Tier 2 (E2E) | Python/pytest | `guyoron1/fullsend` | diff --git a/qf-tests/GH-4/go/ai_feature_file_generation_test.go b/qf-tests/GH-4/go/ai_feature_file_generation_test.go new file mode 100644 index 0000000000..b4efb105d7 --- /dev/null +++ b/qf-tests/GH-4/go/ai_feature_file_generation_test.go @@ -0,0 +1,278 @@ +//go:build e2e + +package e2e + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("AI feature file generation", Serial, func() { + var ( + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + // Ensure the fullsend CLI is available + _, err := exec.LookPath("fullsend") + if err != nil { + Skip("fullsend CLI not found in PATH — skipping AI feature file tests") + } + // Ensure LLM endpoint is configured + if os.Getenv("LLM_ENDPOINT") == "" { + Skip("LLM_ENDPOINT not set — skipping AI feature file tests") + } + }) + + Context("Verify AI generates functional requirements section from prototype input", Ordered, func() { + var ( + prototypeDir string + featureOutput string + ) + + BeforeAll(func() { + var err error + + // Create prototype with multiple testable functions + prototypeDir, err = os.MkdirTemp("", "feature-prototype-*") + Expect(err).NotTo(HaveOccurred()) + + calcContent := `package calculator + +// Add returns the sum of two integers. +func Add(a, b int) int { + return a + b +} + +// Subtract returns the difference of two integers. +func Subtract(a, b int) int { + return a - b +} + +// IsEven returns true if the number is even. +func IsEven(n int) bool { + return n%2 == 0 +} +` + goModContent := `module calculator + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "calculator.go"), []byte(calcContent), 0644) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Create feature output directory + featureOutput, err = os.MkdirTemp("", "feature-output-*") + Expect(err).NotTo(HaveOccurred()) + + // Generate the feature file from prototype + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", featureOutput, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "feature file generation should succeed.\nOutput: %s", string(output)) + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if featureOutput != "" { + os.RemoveAll(featureOutput) + } + }) + + It("[test_id:TS-GH-4-007] should generate a feature file containing a functional requirements section", func() { + // Find the generated feature file + featureFile := findFeatureFile(featureOutput) + Expect(featureFile).NotTo(BeEmpty(), + "feature file should exist in output directory %s", featureOutput) + + // Read the feature file content + content, err := os.ReadFile(featureFile) + Expect(err).NotTo(HaveOccurred(), "failed to read generated feature file") + Expect(content).NotTo(BeEmpty(), "generated feature file should not be empty") + + contentStr := string(content) + + // Verify the feature file contains a functional requirements section + Expect(contentStr).To(SatisfyAny( + ContainSubstring("functional_requirements"), + ContainSubstring("functional-requirements"), + ContainSubstring("Functional Requirements"), + ContainSubstring("requirements"), + ), "feature file should contain a functional requirements section.\nContent preview: %.500s", contentStr) + + // Verify requirements are structured — look for numbered/discrete items + // Requirements should appear as a list or structured entries + hasStructuredRequirements := false + lines := strings.Split(contentStr, "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + // Check for list markers (YAML list, numbered list, etc.) + if strings.HasPrefix(trimmed, "- ") || + strings.HasPrefix(trimmed, "1.") || + strings.HasPrefix(trimmed, "id:") || + strings.HasPrefix(trimmed, "\"id\"") { + hasStructuredRequirements = true + break + } + } + Expect(hasStructuredRequirements).To(BeTrue(), + "functional requirements should be structured as discrete items.\nContent preview: %.500s", contentStr) + }) + }) + + Context("Verify AI generates acceptance scenarios with pass/fail criteria from prototype", Ordered, func() { + var ( + prototypeDir string + scenarioOutput string + ) + + BeforeAll(func() { + var err error + + // Create prototype with clear input/output behavior + prototypeDir, err = os.MkdirTemp("", "scenario-prototype-*") + Expect(err).NotTo(HaveOccurred()) + + handlerContent := `package handler + +import ( + "fmt" + "strings" +) + +// Process takes a string input, validates it, and returns the uppercase version. +// Returns an error if the input is empty. +func Process(input string) (string, error) { + if input == "" { + return "", fmt.Errorf("empty input: input must not be empty") + } + return strings.ToUpper(input), nil +} + +// Validate checks if the input meets minimum length requirements. +// Returns true if the input has at least 3 characters. +func Validate(input string) bool { + return len(input) >= 3 +} +` + goModContent := `module handler + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "handler.go"), []byte(handlerContent), 0644) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Create output directory + scenarioOutput, err = os.MkdirTemp("", "scenario-output-*") + Expect(err).NotTo(HaveOccurred()) + + // Generate the feature file + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", scenarioOutput, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "feature file generation should succeed.\nOutput: %s", string(output)) + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if scenarioOutput != "" { + os.RemoveAll(scenarioOutput) + } + }) + + It("[test_id:TS-GH-4-008] should generate acceptance scenarios with pass/fail criteria", func() { + // Find the generated feature file + featureFile := findFeatureFile(scenarioOutput) + Expect(featureFile).NotTo(BeEmpty(), + "feature file should exist in output directory %s", scenarioOutput) + + // Read the feature file content + content, err := os.ReadFile(featureFile) + Expect(err).NotTo(HaveOccurred(), "failed to read generated feature file") + + contentStr := string(content) + + // Verify the feature file contains acceptance scenarios + Expect(contentStr).To(SatisfyAny( + ContainSubstring("acceptance_scenarios"), + ContainSubstring("acceptance-scenarios"), + ContainSubstring("Acceptance Scenarios"), + ContainSubstring("acceptance_criteria"), + ContainSubstring("scenarios"), + ContainSubstring("test_cases"), + ), "feature file should contain acceptance scenarios section.\nContent preview: %.500s", contentStr) + + // Verify scenarios have pass/fail criteria + hasPassCriteria := strings.Contains(contentStr, "pass") || + strings.Contains(contentStr, "Pass") || + strings.Contains(contentStr, "PASS") || + strings.Contains(contentStr, "success") || + strings.Contains(contentStr, "expected") || + strings.Contains(contentStr, "should") + + hasFailCriteria := strings.Contains(contentStr, "fail") || + strings.Contains(contentStr, "Fail") || + strings.Contains(contentStr, "FAIL") || + strings.Contains(contentStr, "error") || + strings.Contains(contentStr, "invalid") || + strings.Contains(contentStr, "should not") + + Expect(hasPassCriteria).To(BeTrue(), + "acceptance scenarios should include pass criteria.\nContent preview: %.500s", contentStr) + + Expect(hasFailCriteria).To(BeTrue(), + "acceptance scenarios should include fail criteria.\nContent preview: %.500s", contentStr) + }) + }) +}) + +// findFeatureFile searches the given directory for a feature/spec file. +func findFeatureFile(dir string) string { + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + + // Look for feature file first, then any YAML/JSON file + for _, entry := range entries { + name := strings.ToLower(entry.Name()) + if strings.Contains(name, "feature") || strings.Contains(name, "spec") { + return filepath.Join(dir, entry.Name()) + } + } + for _, entry := range entries { + name := entry.Name() + if strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".json") { + return filepath.Join(dir, name) + } + } + return "" +} diff --git a/qf-tests/GH-4/go/review_agent_enforcement_test.go b/qf-tests/GH-4/go/review_agent_enforcement_test.go new file mode 100644 index 0000000000..c017dc0682 --- /dev/null +++ b/qf-tests/GH-4/go/review_agent_enforcement_test.go @@ -0,0 +1,426 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Review agent spec enforcement", Serial, func() { + var ( + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + // Ensure the fullsend CLI is available + _, err := exec.LookPath("fullsend") + if err != nil { + Skip("fullsend CLI not found in PATH — skipping review agent tests") + } + // Ensure LLM endpoint is configured + if os.Getenv("LLM_ENDPOINT") == "" { + Skip("LLM_ENDPOINT not set — skipping review agent tests") + } + }) + + Context("Verify review agent blocks code not matching generated spec", Ordered, func() { + var ( + prototypeDir string + specDir string + nonCompliantDiff string + ) + + BeforeAll(func() { + var err error + + // Create prototype directory + prototypeDir, err = os.MkdirTemp("", "review-prototype-*") + Expect(err).NotTo(HaveOccurred()) + + // Write prototype: defines an Add function + protoContent := `package calculator + +// Add returns the sum of two integers. +func Add(a, b int) int { + return a + b +} +` + goModContent := `module calculator + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "calculator.go"), []byte(protoContent), 0644) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Generate spec from prototype + specDir, err = os.MkdirTemp("", "review-spec-*") + Expect(err).NotTo(HaveOccurred()) + + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", specDir, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "spec generation should succeed for review agent test.\nOutput: %s", string(output)) + + // Create a non-compliant diff file — implements Subtract instead of Add + nonCompliantContent := `--- a/calculator.go ++++ b/calculator.go +@@ -1,6 +1,6 @@ + package calculator + +-// Add returns the sum of two integers. +-func Add(a, b int) int { +- return a + b ++// Subtract returns the difference of two integers. ++func Subtract(a, b int) int { ++ return a - b + } +` + nonCompliantDiff, err = writeTempFile("non-compliant-*.diff", nonCompliantContent) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if specDir != "" { + os.RemoveAll(specDir) + } + if nonCompliantDiff != "" { + os.Remove(nonCompliantDiff) + } + }) + + It("[test_id:TS-GH-4-004] should block a PR whose code does not match the generated spec checklist", func() { + // Find the checklist/spec file in the spec output directory + specFile := findSpecFile(specDir) + Expect(specFile).NotTo(BeEmpty(), "spec checklist file should exist in %s", specDir) + + // Run the review agent against the non-compliant code + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "review", + "--spec", specFile, + "--diff", nonCompliantDiff, + ) + + output, err := cmd.CombinedOutput() + outputStr := string(output) + + // The review agent may return a non-zero exit code for blocked PRs, + // or it may return zero with a "blocked" status in the output. + // Check the output for blocking indicators regardless of exit code. + _ = err + + // Verify the review agent indicates the PR is blocked or requests changes + Expect(outputStr).To(SatisfyAny( + ContainSubstring("blocked"), + ContainSubstring("changes_requested"), + ContainSubstring("BLOCKED"), + ContainSubstring("CHANGES_REQUESTED"), + ContainSubstring("non-compliant"), + ContainSubstring("does not match"), + ContainSubstring("violation"), + ContainSubstring("failed"), + ), "review agent should block non-compliant code.\nOutput: %s", outputStr) + + // Verify the review agent identifies specific spec items that are not satisfied + Expect(outputStr).To(SatisfyAny( + ContainSubstring("Add"), + ContainSubstring("checklist"), + ContainSubstring("requirement"), + ContainSubstring("spec"), + ), "review agent should reference specific spec violations.\nOutput: %s", outputStr) + }) + }) + + Context("Verify review agent permits code matching generated spec checklist", Ordered, func() { + var ( + prototypeDir string + specDir string + compliantDiff string + ) + + BeforeAll(func() { + var err error + + // Create prototype directory + prototypeDir, err = os.MkdirTemp("", "review-compliant-prototype-*") + Expect(err).NotTo(HaveOccurred()) + + // Write prototype: defines an Add function + protoContent := `package calculator + +// Add returns the sum of two integers. +func Add(a, b int) int { + return a + b +} +` + goModContent := `module calculator + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "calculator.go"), []byte(protoContent), 0644) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Generate spec from prototype + specDir, err = os.MkdirTemp("", "review-compliant-spec-*") + Expect(err).NotTo(HaveOccurred()) + + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", specDir, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "spec generation should succeed.\nOutput: %s", string(output)) + + // Create a compliant diff — implements exactly what the spec requires + compliantContent := `--- a/calculator.go ++++ b/calculator.go +@@ -1,6 +1,8 @@ + package calculator + + // Add returns the sum of two integers. + func Add(a, b int) int { +- return a + b ++ // Implementation with input validation ++ result := a + b ++ return result + } +` + compliantDiff, err = writeTempFile("compliant-*.diff", compliantContent) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if specDir != "" { + os.RemoveAll(specDir) + } + if compliantDiff != "" { + os.Remove(compliantDiff) + } + }) + + It("[test_id:TS-GH-4-005] should approve a PR whose code matches the generated spec checklist", func() { + // Find the checklist/spec file + specFile := findSpecFile(specDir) + Expect(specFile).NotTo(BeEmpty(), "spec checklist file should exist in %s", specDir) + + // Run the review agent against the compliant code + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "review", + "--spec", specFile, + "--diff", compliantDiff, + ) + + output, err := cmd.CombinedOutput() + outputStr := string(output) + + // Review agent should succeed (exit code 0) for compliant code + Expect(err).NotTo(HaveOccurred(), + "review agent should not fail for compliant code.\nOutput: %s", outputStr) + + // Verify the review agent approves the PR + Expect(outputStr).To(SatisfyAny( + ContainSubstring("approved"), + ContainSubstring("pass"), + ContainSubstring("APPROVED"), + ContainSubstring("PASS"), + ContainSubstring("compliant"), + ContainSubstring("satisfied"), + ), "review agent should approve compliant code.\nOutput: %s", outputStr) + + // Verify no false positive violations are reported + lowerOutput := strings.ToLower(outputStr) + Expect(lowerOutput).NotTo(ContainSubstring("violation"), + "review agent should not report violations for compliant code.\nOutput: %s", outputStr) + }) + }) + + Context("Verify review agent detects and blocks scope creep beyond spec boundaries", Ordered, func() { + var ( + prototypeDir string + specDir string + scopeCreepDiff string + ) + + BeforeAll(func() { + var err error + + // Create prototype directory + prototypeDir, err = os.MkdirTemp("", "review-scope-creep-prototype-*") + Expect(err).NotTo(HaveOccurred()) + + // Write prototype: defines only an Add function + protoContent := `package calculator + +// Add returns the sum of two integers. +func Add(a, b int) int { + return a + b +} +` + goModContent := `module calculator + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "calculator.go"), []byte(protoContent), 0644) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Generate spec from prototype + specDir, err = os.MkdirTemp("", "review-scope-creep-spec-*") + Expect(err).NotTo(HaveOccurred()) + + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", specDir, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "spec generation should succeed.\nOutput: %s", string(output)) + + // Create a diff with scope creep: implements Add (in spec) PLUS Multiply (NOT in spec) + scopeCreepContent := `--- a/calculator.go ++++ b/calculator.go +@@ -1,6 +1,12 @@ + package calculator + + // Add returns the sum of two integers. + func Add(a, b int) int { + return a + b + } ++ ++// Multiply returns the product of two integers. ++// NOTE: This function is NOT in the spec — it is scope creep. ++func Multiply(a, b int) int { ++ return a * b ++} +` + scopeCreepDiff, err = writeTempFile("scope-creep-*.diff", scopeCreepContent) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if specDir != "" { + os.RemoveAll(specDir) + } + if scopeCreepDiff != "" { + os.Remove(scopeCreepDiff) + } + }) + + It("[test_id:TS-GH-4-006] should detect and block code that adds functionality beyond the spec", func() { + // Find the checklist/spec file + specFile := findSpecFile(specDir) + Expect(specFile).NotTo(BeEmpty(), "spec checklist file should exist in %s", specDir) + + // Run the review agent against code with scope creep + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "review", + "--spec", specFile, + "--diff", scopeCreepDiff, + ) + + output, err := cmd.CombinedOutput() + outputStr := string(output) + _ = err // Review agent may or may not return non-zero for scope creep + + // Verify the review agent detects scope creep and blocks + Expect(outputStr).To(SatisfyAny( + ContainSubstring("scope"), + ContainSubstring("blocked"), + ContainSubstring("BLOCKED"), + ContainSubstring("unauthorized"), + ContainSubstring("out-of-scope"), + ContainSubstring("beyond"), + ContainSubstring("extra"), + ContainSubstring("not in spec"), + ), "review agent should detect scope creep.\nOutput: %s", outputStr) + + // Verify the review agent identifies the specific out-of-scope additions + Expect(outputStr).To(SatisfyAny( + ContainSubstring("Multiply"), + ContainSubstring("additional"), + ContainSubstring("unauthorized"), + ContainSubstring("not specified"), + ), "review agent should identify the out-of-scope code (Multiply function).\nOutput: %s", outputStr) + }) + }) +}) + +// writeTempFile creates a temporary file with the given content and returns its path. +func writeTempFile(pattern, content string) (string, error) { + f, err := os.CreateTemp("", pattern) + if err != nil { + return "", fmt.Errorf("creating temp file: %w", err) + } + defer f.Close() + + if _, err := f.WriteString(content); err != nil { + os.Remove(f.Name()) + return "", fmt.Errorf("writing temp file: %w", err) + } + return f.Name(), nil +} + +// findSpecFile searches the given directory for a spec/checklist file. +func findSpecFile(dir string) string { + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + + // Look for checklist file first, then any YAML/JSON file + for _, entry := range entries { + name := strings.ToLower(entry.Name()) + if strings.Contains(name, "checklist") { + return filepath.Join(dir, entry.Name()) + } + } + for _, entry := range entries { + name := entry.Name() + if strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".json") { + return filepath.Join(dir, name) + } + } + return "" +} diff --git a/qf-tests/GH-4/go/vibe_to_spec_workflow_test.go b/qf-tests/GH-4/go/vibe_to_spec_workflow_test.go new file mode 100644 index 0000000000..21f649613c --- /dev/null +++ b/qf-tests/GH-4/go/vibe_to_spec_workflow_test.go @@ -0,0 +1,413 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Vibe-to-spec workflow", Serial, func() { + var ( + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + // Ensure the fullsend CLI is available + _, err := exec.LookPath("fullsend") + if err != nil { + Skip("fullsend CLI not found in PATH — skipping vibe-to-spec tests") + } + // Ensure LLM endpoint is configured + if os.Getenv("LLM_ENDPOINT") == "" { + Skip("LLM_ENDPOINT not set — skipping vibe-to-spec tests") + } + }) + + Context("Verify vibe-to-spec workflow produces valid spec from prototype code", Ordered, func() { + var ( + prototypeDir string + specOutputDir string + ) + + BeforeAll(func() { + var err error + + // Create a temporary prototype directory with testable behavior + prototypeDir, err = os.MkdirTemp("", "vibe-to-spec-prototype-*") + Expect(err).NotTo(HaveOccurred(), "failed to create prototype directory") + + // Write a prototype Go file with clear, testable functions + mainGoContent := `package main + +import "fmt" + +// Add returns the sum of two integers. +func Add(a, b int) int { + return a + b +} + +// Subtract returns the difference of two integers. +func Subtract(a, b int) int { + return a - b +} + +// Greet returns a greeting message for the given name. +func Greet(name string) string { + return fmt.Sprintf("Hello, %s!", name) +} + +func main() { + fmt.Println(Add(2, 3)) + fmt.Println(Subtract(5, 3)) + fmt.Println(Greet("World")) +} +` + goModContent := `module prototype + +go 1.23 +` + err = os.WriteFile(filepath.Join(prototypeDir, "main.go"), []byte(mainGoContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write main.go") + + err = os.WriteFile(filepath.Join(prototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write go.mod") + + // Create spec output directory + specOutputDir, err = os.MkdirTemp("", "vibe-to-spec-output-*") + Expect(err).NotTo(HaveOccurred(), "failed to create spec output directory") + }) + + AfterAll(func() { + if prototypeDir != "" { + os.RemoveAll(prototypeDir) + } + if specOutputDir != "" { + os.RemoveAll(specOutputDir) + } + }) + + It("[test_id:TS-GH-4-001] should generate a valid formal specification from developer prototype code", func() { + // Execute the vibe-to-spec workflow on the prototype directory + cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", prototypeDir, + "--output", specOutputDir, + ) + cmd.Env = append(os.Environ(), + fmt.Sprintf("LLM_ENDPOINT=%s", os.Getenv("LLM_ENDPOINT")), + ) + + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), + "vibe-to-spec workflow should complete without error.\nOutput: %s", string(output)) + + // Verify the spec output directory contains at least one file + entries, err := os.ReadDir(specOutputDir) + Expect(err).NotTo(HaveOccurred(), "failed to read spec output directory") + Expect(entries).NotTo(BeEmpty(), "spec output directory should contain generated files") + + // Find the generated spec file (YAML or JSON) + var specFilePath string + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".yaml") || strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".json") { + specFilePath = filepath.Join(specOutputDir, entry.Name()) + break + } + } + Expect(specFilePath).NotTo(BeEmpty(), "no spec file (YAML/JSON) found in output directory") + + // Read and validate the spec file content + specBytes, err := os.ReadFile(specFilePath) + Expect(err).NotTo(HaveOccurred(), "failed to read generated spec file") + Expect(specBytes).NotTo(BeEmpty(), "generated spec file should not be empty") + + specContent := string(specBytes) + + // Verify the spec contains functional requirements + Expect(specContent).To(SatisfyAny( + ContainSubstring("functional_requirements"), + ContainSubstring("functional-requirements"), + ContainSubstring("requirements"), + ), "generated spec should contain a functional requirements section") + + // Verify the spec contains acceptance scenarios + Expect(specContent).To(SatisfyAny( + ContainSubstring("acceptance_scenarios"), + ContainSubstring("acceptance-scenarios"), + ContainSubstring("acceptance_criteria"), + ContainSubstring("scenarios"), + ), "generated spec should contain acceptance scenarios") + + // Verify the spec is parseable as structured data (YAML) + // A basic check — the file should have key-value structure + Expect(specContent).To(MatchRegexp(`\w+:\s`), + "generated spec should be in a structured key-value format") + }) + }) + + Context("Verify exploration artifacts are cleaned up after spec generation completes", Ordered, func() { + var ( + explorationDir string + specOutputDir string + ) + + BeforeAll(func() { + var err error + + // Create an exploration/prototype directory + explorationDir, err = os.MkdirTemp("", "exploration-artifacts-*") + Expect(err).NotTo(HaveOccurred(), "failed to create exploration directory") + + // Write prototype exploration files + prototypeContent := `package main + +// Explore demonstrates a prototype function with testable behavior. +func Explore(input string) string { + if input == "" { + return "default" + } + return input +} +` + goModContent := `module exploration + +go 1.23 +` + err = os.WriteFile(filepath.Join(explorationDir, "prototype.go"), []byte(prototypeContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write prototype.go") + + err = os.WriteFile(filepath.Join(explorationDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write go.mod") + + // Create spec output directory + specOutputDir, err = os.MkdirTemp("", "exploration-spec-output-*") + Expect(err).NotTo(HaveOccurred(), "failed to create spec output directory") + + // Run the vibe-to-spec workflow to completion + cmdCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", explorationDir, + "--output", specOutputDir, + ) + output, runErr := cmd.CombinedOutput() + Expect(runErr).NotTo(HaveOccurred(), + "vibe-to-spec workflow should complete successfully for cleanup test.\nOutput: %s", string(output)) + }) + + AfterAll(func() { + // Clean up any remaining artifacts + if explorationDir != "" { + os.RemoveAll(explorationDir) + } + if specOutputDir != "" { + os.RemoveAll(specOutputDir) + } + }) + + It("[test_id:TS-GH-4-002] should remove all exploration artifacts after spec generation completes", func() { + // Verify that exploration artifact directory no longer exists + // The vibe-to-spec workflow should clean up the input prototype directory + _, err := os.Stat(explorationDir) + if err == nil { + // If the directory still exists, check that prototype files are removed + entries, readErr := os.ReadDir(explorationDir) + if readErr == nil { + // Check no .go prototype files remain + var protoFiles []string + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".go") { + protoFiles = append(protoFiles, entry.Name()) + } + } + Expect(protoFiles).To(BeEmpty(), + "no prototype .go files should remain in exploration directory after spec generation, found: %v", protoFiles) + } + } + // If os.Stat returns an error (directory doesn't exist), that's the expected behavior + + // Verify that the generated spec file IS preserved + entries, err := os.ReadDir(specOutputDir) + Expect(err).NotTo(HaveOccurred(), "spec output directory should still exist") + Expect(entries).NotTo(BeEmpty(), "generated spec file should be preserved after cleanup") + }) + }) + + Context("Verify error returned when prototype contains no testable behavior", Ordered, func() { + var ( + emptyPrototypeDir string + specOutputDir string + ) + + BeforeAll(func() { + var err error + + // Create a prototype directory with no testable behavior + emptyPrototypeDir, err = os.MkdirTemp("", "empty-prototype-*") + Expect(err).NotTo(HaveOccurred(), "failed to create empty prototype directory") + + // Write a Go file with NO exported functions (no testable behavior) + emptyContent := `package main + +// This file intentionally has no testable behavior. +// It contains only comments and unexported code. + +// internal is not exported and cannot be tested externally. +func internal() { + // no-op +} +` + goModContent := `module empty-prototype + +go 1.23 +` + err = os.WriteFile(filepath.Join(emptyPrototypeDir, "main.go"), []byte(emptyContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write main.go") + + err = os.WriteFile(filepath.Join(emptyPrototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write go.mod") + + // Create spec output directory + specOutputDir, err = os.MkdirTemp("", "empty-prototype-output-*") + Expect(err).NotTo(HaveOccurred(), "failed to create spec output directory") + }) + + AfterAll(func() { + if emptyPrototypeDir != "" { + os.RemoveAll(emptyPrototypeDir) + } + if specOutputDir != "" { + os.RemoveAll(specOutputDir) + } + }) + + It("[test_id:TS-GH-4-003] should return a clear error when prototype has no testable behavior", func() { + // Execute the vibe-to-spec workflow on the empty prototype + cmdCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", emptyPrototypeDir, + "--output", specOutputDir, + ) + + output, err := cmd.CombinedOutput() + outputStr := string(output) + + // Workflow should return a non-zero exit code + Expect(err).To(HaveOccurred(), + "vibe-to-spec should fail with non-zero exit code when prototype has no testable behavior.\nOutput: %s", outputStr) + + // Error message should clearly indicate the problem + Expect(outputStr).To(SatisfyAny( + ContainSubstring("no testable"), + ContainSubstring("no exported"), + ContainSubstring("no functions"), + ContainSubstring("insufficient"), + ContainSubstring("empty"), + ), "error message should indicate prototype lacks testable behavior.\nActual output: %s", outputStr) + }) + }) + + Context("Verify error for ambiguous or contradictory prototype input", Ordered, func() { + var ( + ambiguousPrototypeDir string + specOutputDir string + ) + + BeforeAll(func() { + var err error + + // Create a prototype directory with contradictory behavior + ambiguousPrototypeDir, err = os.MkdirTemp("", "ambiguous-prototype-*") + Expect(err).NotTo(HaveOccurred(), "failed to create ambiguous prototype directory") + + // Write Go code where comments contradict the implementation + contradictoryContent := `package main + +import "strings" + +// ToUpperCase converts the input string to uppercase. +// This function MUST return all characters in UPPER CASE. +func ToUpperCase(s string) string { + // BUG: implementation does lowercase, contradicting the doc and name + return strings.ToLower(s) +} + +// IsPositive returns true if the number is positive (greater than zero). +func IsPositive(n int) bool { + // BUG: returns true for negative numbers too + return n != 0 +} + +// Reverse returns the input string reversed. +func Reverse(s string) string { + // BUG: returns the same string, not reversed + return s +} +` + goModContent := `module ambiguous-prototype + +go 1.23 +` + err = os.WriteFile(filepath.Join(ambiguousPrototypeDir, "contradictory.go"), []byte(contradictoryContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write contradictory.go") + + err = os.WriteFile(filepath.Join(ambiguousPrototypeDir, "go.mod"), []byte(goModContent), 0644) + Expect(err).NotTo(HaveOccurred(), "failed to write go.mod") + + // Create spec output directory + specOutputDir, err = os.MkdirTemp("", "ambiguous-output-*") + Expect(err).NotTo(HaveOccurred(), "failed to create spec output directory") + }) + + AfterAll(func() { + if ambiguousPrototypeDir != "" { + os.RemoveAll(ambiguousPrototypeDir) + } + if specOutputDir != "" { + os.RemoveAll(specOutputDir) + } + }) + + It("[test_id:TS-GH-4-009] should return a clear error when prototype input is ambiguous or contradictory", func() { + // Execute the vibe-to-spec workflow on the ambiguous prototype + cmdCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + cmd := exec.CommandContext(cmdCtx, "fullsend", "vibe-to-spec", + "--input", ambiguousPrototypeDir, + "--output", specOutputDir, + ) + + output, err := cmd.CombinedOutput() + outputStr := string(output) + + // Workflow should return a non-zero exit code for ambiguous input + Expect(err).To(HaveOccurred(), + "vibe-to-spec should fail when prototype has contradictory behavior.\nOutput: %s", outputStr) + + // Error message should explain the ambiguity + Expect(outputStr).To(SatisfyAny( + ContainSubstring("ambiguous"), + ContainSubstring("contradictory"), + ContainSubstring("inconsistent"), + ContainSubstring("conflict"), + ContainSubstring("mismatch"), + ), "error message should explain the prototype ambiguity.\nActual output: %s", outputStr) + }) + }) +}) diff --git a/qf-tests/GH-4/python/conftest.py b/qf-tests/GH-4/python/conftest.py new file mode 100644 index 0000000000..a1f2356747 --- /dev/null +++ b/qf-tests/GH-4/python/conftest.py @@ -0,0 +1,166 @@ +""" +Shared fixtures for GH-4: Use AI to Help Formalise Intent After Rapid Local Prototyping. + +STP Reference: + outputs/stp/GH-4/GH-4_test_plan.md +""" +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Generator + +import pytest + + +@pytest.fixture(scope="session") +def fullsend_binary() -> str: + """Resolve the fullsend binary path. + + Returns: + Absolute path to the fullsend binary. + + Raises: + pytest.skip: If fullsend binary is not found in PATH. + """ + binary = shutil.which("fullsend") + if binary is None: + pytest.skip("fullsend binary not found in PATH") + return binary + + +@pytest.fixture(scope="session") +def llm_endpoint() -> str: + """Resolve the LLM inference endpoint from environment. + + Returns: + URL of the LLM inference endpoint. + + Raises: + pytest.skip: If LLM_ENDPOINT environment variable is not set. + """ + endpoint = os.environ.get("LLM_ENDPOINT", "") + if not endpoint: + pytest.skip("LLM_ENDPOINT environment variable not set") + return endpoint + + +@pytest.fixture(scope="function") +def prototype_dir_scope_function() -> Generator[Path, None, None]: + """Create a temporary directory for prototype code. + + Yields: + Path to the temporary prototype directory. + """ + dirpath = Path(tempfile.mkdtemp(prefix="test-prototype-")) + yield dirpath + shutil.rmtree(str(dirpath), ignore_errors=True) + + +@pytest.fixture(scope="function") +def spec_output_dir_scope_function() -> Generator[Path, None, None]: + """Create a temporary directory for spec output. + + Yields: + Path to the temporary spec output directory. + """ + dirpath = Path(tempfile.mkdtemp(prefix="test-spec-output-")) + yield dirpath + shutil.rmtree(str(dirpath), ignore_errors=True) + + +@pytest.fixture(scope="function") +def prototype_with_testable_behavior( + prototype_dir_scope_function: Path, +) -> Path: + """Create a prototype directory containing Go code with testable behavior. + + Args: + prototype_dir_scope_function: Temporary directory for prototype files. + + Returns: + Path to the prototype directory with testable Go code. + """ + main_go = prototype_dir_scope_function / "main.go" + main_go.write_text( + 'package main\n\n' + '// Add returns the sum of two integers.\n' + 'func Add(a, b int) int { return a + b }\n\n' + '// Subtract returns the difference of two integers.\n' + 'func Subtract(a, b int) int { return a - b }\n' + ) + return prototype_dir_scope_function + + +@pytest.fixture(scope="function") +def prototype_without_testable_behavior( + prototype_dir_scope_function: Path, +) -> Path: + """Create a prototype directory containing Go code with no testable behavior. + + Args: + prototype_dir_scope_function: Temporary directory for prototype files. + + Returns: + Path to the prototype directory with no exported functions. + """ + main_go = prototype_dir_scope_function / "main.go" + main_go.write_text( + 'package main\n\n' + '// This file has no testable behavior\n' + '// Only comments and unexported declarations\n' + ) + return prototype_dir_scope_function + + +@pytest.fixture(scope="function") +def prototype_with_ambiguous_behavior( + prototype_dir_scope_function: Path, +) -> Path: + """Create a prototype directory with contradictory code behavior. + + The comment says uppercase but the implementation does lowercase, + creating ambiguity that the workflow should detect. + + Args: + prototype_dir_scope_function: Temporary directory for prototype files. + + Returns: + Path to the prototype directory with contradictory code. + """ + contradictory_go = prototype_dir_scope_function / "contradictory.go" + contradictory_go.write_text( + 'package main\n\n' + 'import "strings"\n\n' + '// Process returns uppercase for valid input\n' + 'func Process(s string) string { return strings.ToLower(s) }\n' + '// The comment says uppercase, implementation does lowercase\n' + ) + return prototype_dir_scope_function + + +def run_fullsend_command( + binary: str, + subcommand: str, + args: list[str], + timeout_seconds: int = 300, +) -> subprocess.CompletedProcess: + """Execute a fullsend CLI command. + + Args: + binary: Path to the fullsend binary. + subcommand: The fullsend subcommand to run. + args: Additional arguments for the subcommand. + timeout_seconds: Maximum time to wait for command completion. + + Returns: + CompletedProcess with stdout, stderr, and returncode. + """ + cmd = [binary, subcommand, *args] + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) diff --git a/qf-tests/GH-4/python/test_ai_feature_file_generation.py b/qf-tests/GH-4/python/test_ai_feature_file_generation.py new file mode 100644 index 0000000000..0a458b0f84 --- /dev/null +++ b/qf-tests/GH-4/python/test_ai_feature_file_generation.py @@ -0,0 +1,200 @@ +""" +Tests for AI feature file generation — output structure and acceptance criteria validation. + +Covers scenarios: + - TS-GH-4-007: AI generates functional requirements section from prototype + - TS-GH-4-008: AI generates acceptance scenarios with pass/fail criteria + +STP Reference: + outputs/stp/GH-4/GH-4_test_plan.md + +PR Reference: + https://github.com/fullsend-ai/fullsend/pull/4 +""" +from pathlib import Path + +import pytest +import yaml + +from conftest import run_fullsend_command + +pytestmark = [ + pytest.mark.tier2, + pytest.mark.gh_4, +] + + +class TestAiFeatureFileGeneration: + """Tests for AI-generated feature file structure and content. + + Markers: + - tier2 + - gh_4 + - serial + + Preconditions: + - fullsend binary available in PATH + - AI/LLM inference endpoint accessible + - Prototype code with well-defined exported functions + """ + + @pytest.mark.serial + def test_ts_gh_4_007_generate_functional_requirements_section( + self, + fullsend_binary: str, + llm_endpoint: str, + prototype_with_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-007: Verify AI generates functional requirements section from prototype. + + Priority: P1 + MVP: False + + This test validates that when AI generates a feature file from a + prototype, the output contains a properly structured functional + requirements section with machine-evaluable criteria. + + Acceptance Criteria: + - Generated feature file contains a 'functional_requirements' section + - Functional requirements are structured as discrete, numbered items + - Each requirement is machine-evaluable (not ambiguous prose) + """ + # SETUP-01: Verify prototype has multiple testable functions + go_files = list(prototype_with_testable_behavior.glob("*.go")) + assert len(go_files) > 0, "Prototype must contain Go files with exported functions" + + # TEST-01: Generate feature file from prototype + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_with_testable_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + assert result.returncode == 0, ( + f"Feature file generation failed: {result.stderr}" + ) + + # TEST-02: Parse generated feature file + spec_files = list(spec_output_dir_scope_function.glob("*.yaml")) + \ + list(spec_output_dir_scope_function.glob("*.yml")) + assert len(spec_files) > 0, "No feature files generated" + + feature_content = spec_files[0].read_text() + feature_data = yaml.safe_load(feature_content) + assert feature_data is not None, "Feature file is empty or invalid YAML" + + # TEST-03: Validate functional requirements section exists + # ASSERT-01: Feature file contains functional requirements section + assert "functional_requirements" in feature_data, ( + f"Generated feature file missing 'functional_requirements' section. " + f"Generated specs cannot drive review agent enforcement. " + f"Available keys: {list(feature_data.keys())}" + ) + + requirements = feature_data["functional_requirements"] + assert isinstance(requirements, list), ( + "functional_requirements should be a list of discrete items" + ) + assert len(requirements) > 0, ( + "functional_requirements section is empty" + ) + + # TEST-04: Validate requirements are machine-evaluable + # ASSERT-02: Requirements are structured and numbered + for idx, requirement in enumerate(requirements): + if isinstance(requirement, dict): + structured_fields = {"id", "description", "criteria"} + present_fields = set(requirement.keys()) & structured_fields + assert len(present_fields) >= 2, ( + f"Requirement {idx} lacks structured fields. " + f"Expected at least 'id' and 'description' or 'criteria'. " + f"Got keys: {list(requirement.keys())}. " + f"Requirements are unstructured prose, not machine-evaluable." + ) + + @pytest.mark.serial + def test_ts_gh_4_008_generate_acceptance_scenarios_with_criteria( + self, + fullsend_binary: str, + llm_endpoint: str, + prototype_with_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-008: Verify AI generates acceptance scenarios with pass/fail criteria. + + Priority: P1 + MVP: False + + This test validates that the AI-generated feature file contains + acceptance scenarios with explicit pass/fail criteria that can be + used by review agents to evaluate code compliance. + + Acceptance Criteria: + - Generated feature file contains an 'acceptance_scenarios' section + - Each scenario has explicit pass criteria + - Each scenario has explicit fail criteria + - Scenarios are testable by review agents + """ + # SETUP-01: Create prototype with clear input/output behavior + # Using the shared prototype fixture which has Add/Subtract functions + assert prototype_with_testable_behavior.exists(), ( + "Prototype directory should exist" + ) + + # TEST-01: Generate feature file from prototype + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_with_testable_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + assert result.returncode == 0, ( + f"Feature file generation failed: {result.stderr}" + ) + + # TEST-02: Validate acceptance scenarios section exists + spec_files = list(spec_output_dir_scope_function.glob("*.yaml")) + \ + list(spec_output_dir_scope_function.glob("*.yml")) + assert len(spec_files) > 0, "No feature files generated" + + feature_data = yaml.safe_load(spec_files[0].read_text()) + assert feature_data is not None, "Feature file is empty" + + # ASSERT-01: Feature file contains acceptance scenarios + assert "acceptance_scenarios" in feature_data, ( + f"Generated feature file missing 'acceptance_scenarios' section. " + f"Generated specs lack testable scenarios for review agents. " + f"Available keys: {list(feature_data.keys())}" + ) + + scenarios = feature_data["acceptance_scenarios"] + assert isinstance(scenarios, list), ( + "acceptance_scenarios should be a list" + ) + assert len(scenarios) > 0, ( + "acceptance_scenarios section is empty" + ) + + # TEST-03: Validate each scenario has pass/fail criteria + # ASSERT-02: Each scenario has pass/fail criteria + for idx, scenario in enumerate(scenarios): + if isinstance(scenario, dict): + has_pass = "pass_criteria" in scenario or "pass" in scenario + has_fail = "fail_criteria" in scenario or "fail" in scenario + has_criteria = "criteria" in scenario + + assert has_pass or has_criteria, ( + f"Acceptance scenario {idx} missing pass criteria. " + f"Review agents cannot make binary compliance decisions. " + f"Scenario keys: {list(scenario.keys())}" + ) + assert has_fail or has_criteria, ( + f"Acceptance scenario {idx} missing fail criteria. " + f"Review agents cannot make binary compliance decisions. " + f"Scenario keys: {list(scenario.keys())}" + ) diff --git a/qf-tests/GH-4/python/test_review_agent_enforcement.py b/qf-tests/GH-4/python/test_review_agent_enforcement.py new file mode 100644 index 0000000000..7cdac7f376 --- /dev/null +++ b/qf-tests/GH-4/python/test_review_agent_enforcement.py @@ -0,0 +1,330 @@ +""" +Tests for review agent spec enforcement — compliance checking and scope creep detection. + +Covers scenarios: + - TS-GH-4-004: Review agent blocks non-compliant code + - TS-GH-4-005: Review agent approves compliant code + - TS-GH-4-006: Review agent detects scope creep + +STP Reference: + outputs/stp/GH-4/GH-4_test_plan.md + +PR Reference: + https://github.com/fullsend-ai/fullsend/pull/4 +""" +import tempfile +from pathlib import Path + +import pytest + +from conftest import run_fullsend_command + +pytestmark = [ + pytest.mark.tier2, + pytest.mark.gh_4, +] + + +class TestReviewAgentEnforcement: + """Tests for the review agent's spec enforcement capabilities. + + Markers: + - tier2 + - gh_4 + - serial + + Preconditions: + - fullsend binary available in PATH + - AI/LLM inference endpoint accessible + - Review agent configured and operational + """ + + @pytest.fixture(scope="function") + def spec_checklist_scope_function( + self, + fullsend_binary: str, + llm_endpoint: str, + prototype_with_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> Path: + """Generate a spec checklist from prototype code for review agent tests. + + Args: + fullsend_binary: Path to the fullsend CLI binary. + llm_endpoint: URL of the LLM inference endpoint. + prototype_with_testable_behavior: Prototype directory with testable code. + spec_output_dir_scope_function: Output directory for generated spec. + + Returns: + Path to the generated spec checklist file. + """ + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_with_testable_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + assert result.returncode == 0, ( + f"Failed to generate spec checklist: {result.stderr}" + ) + + checklist_path = spec_output_dir_scope_function / "checklist.yaml" + if not checklist_path.exists(): + # Fall back to any YAML file in the output + yaml_files = list(spec_output_dir_scope_function.glob("*.yaml")) + assert len(yaml_files) > 0, "No spec checklist generated" + checklist_path = yaml_files[0] + + return checklist_path + + @staticmethod + def _write_diff_file(content: str, prefix: str) -> Path: + """Write a temporary diff file with the given content. + + Args: + content: The diff content to write. + prefix: Prefix for the temporary file name. + + Returns: + Path to the created diff file. + """ + diff_file = Path(tempfile.mktemp(prefix=prefix, suffix=".diff")) + diff_file.write_text(content) + return diff_file + + @pytest.mark.serial + def test_ts_gh_4_004_block_non_compliant_code( + self, + fullsend_binary: str, + spec_checklist_scope_function: Path, + ) -> None: + """TS-GH-4-004: Verify review agent blocks code not matching generated spec. + + Priority: P0 + MVP: True + + This test validates that the review agent correctly identifies and + blocks a PR when the submitted code does not match the generated + spec checklist. + + Acceptance Criteria: + - Review agent returns a 'blocked' or 'changes_requested' status + - Review agent identifies specific spec checklist items not satisfied + - Review output includes actionable feedback + """ + # SETUP-02: Prepare code change that violates spec + # The spec requires Add(a, b) but the code implements Subtract(a, b) + non_compliant_diff = self._write_diff_file( + content=( + "--- a/main.go\n" + "+++ b/main.go\n" + "@@ -1,3 +1,5 @@\n" + " package main\n" + "+\n" + "+// Subtract returns the difference (NOT what spec requires)\n" + "+func Subtract(a, b int) int { return a - b }\n" + ), + prefix="non-compliant-", + ) + + try: + # TEST-01: Run review agent against non-compliant code + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="review", + args=[ + "--spec", str(spec_checklist_scope_function), + "--diff", str(non_compliant_diff), + ], + ) + + # TEST-02: Check review agent verdict + combined_output = (result.stdout + result.stderr).lower() + + # ASSERT-01: Review agent blocks non-compliant code + blocked_indicators = ["blocked", "changes_requested", "fail", "rejected"] + is_blocked = any( + indicator in combined_output for indicator in blocked_indicators + ) + assert is_blocked, ( + f"Review agent should block non-compliant code but did not. " + f"stdout: {result.stdout}, stderr: {result.stderr}" + ) + + # TEST-03: Verify review agent provides specific feedback + # ASSERT-02: Review agent cites specific spec violations + feedback_indicators = [ + "spec", "checklist", "requirement", "missing", "violation" + ] + has_specific_feedback = any( + indicator in combined_output for indicator in feedback_indicators + ) + assert has_specific_feedback, ( + f"Review agent should cite specific spec violations. " + f"Output: {result.stdout}" + ) + + finally: + non_compliant_diff.unlink(missing_ok=True) + + @pytest.mark.serial + def test_ts_gh_4_005_approve_compliant_code( + self, + fullsend_binary: str, + spec_checklist_scope_function: Path, + ) -> None: + """TS-GH-4-005: Verify review agent permits code matching generated spec checklist. + + Priority: P0 + MVP: True + + This test validates that the review agent correctly approves a PR + when the submitted code satisfies all items in the generated spec + checklist. + + Acceptance Criteria: + - Review agent returns an 'approved' or 'pass' status + - All spec checklist items are marked as satisfied + - No false positive violations reported + """ + # SETUP-02: Prepare code change that matches spec + compliant_diff = self._write_diff_file( + content=( + "--- a/main.go\n" + "+++ b/main.go\n" + "@@ -1,3 +1,8 @@\n" + " package main\n" + "+\n" + "+// Add returns the sum of two integers.\n" + "+func Add(a, b int) int { return a + b }\n" + "+\n" + "+// Subtract returns the difference of two integers.\n" + "+func Subtract(a, b int) int { return a - b }\n" + ), + prefix="compliant-", + ) + + try: + # TEST-01: Run review agent against compliant code + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="review", + args=[ + "--spec", str(spec_checklist_scope_function), + "--diff", str(compliant_diff), + ], + ) + + # TEST-02: Check review agent verdict + combined_output = (result.stdout + result.stderr).lower() + + # ASSERT-01: Review agent approves compliant code + approved_indicators = ["approved", "pass", "lgtm", "accepted", "success"] + is_approved = any( + indicator in combined_output for indicator in approved_indicators + ) + assert is_approved, ( + f"Review agent should approve compliant code but did not. " + f"Compliant code is incorrectly blocked, creating developer friction. " + f"stdout: {result.stdout}, stderr: {result.stderr}" + ) + + # ASSERT-02: No false positive spec violations + violation_indicators = ["violation", "blocked", "rejected", "fail"] + has_violations = any( + indicator in combined_output for indicator in violation_indicators + ) + assert not has_violations, ( + f"Review agent reported false positive violations on compliant code. " + f"Output: {result.stdout}" + ) + + finally: + compliant_diff.unlink(missing_ok=True) + + @pytest.mark.serial + def test_ts_gh_4_006_detect_scope_creep( + self, + fullsend_binary: str, + spec_checklist_scope_function: Path, + ) -> None: + """TS-GH-4-006: Verify review agent detects and blocks scope creep beyond spec. + + Priority: P0 + MVP: True + + This test validates that the review agent detects when a PR includes + functionality that goes beyond what the generated spec defines, even + when the spec requirements themselves are also satisfied. + + Acceptance Criteria: + - Review agent returns 'blocked' status for code with extra functionality + - Review agent specifically identifies the out-of-scope additions + - Review agent distinguishes between missing spec items and scope creep + """ + # SETUP-02: Prepare code with scope creep (satisfies spec + extra) + scope_creep_diff = self._write_diff_file( + content=( + "--- a/main.go\n" + "+++ b/main.go\n" + "@@ -1,3 +1,11 @@\n" + " package main\n" + "+\n" + "+// Add returns the sum of two integers.\n" + "+func Add(a, b int) int { return a + b }\n" + "+\n" + "+// Subtract returns the difference of two integers.\n" + "+func Subtract(a, b int) int { return a - b }\n" + "+\n" + "+// Multiply returns the product — NOT in spec (scope creep)\n" + "+func Multiply(a, b int) int { return a * b }\n" + ), + prefix="scope-creep-", + ) + + try: + # TEST-01: Run review agent against code with scope creep + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="review", + args=[ + "--spec", str(spec_checklist_scope_function), + "--diff", str(scope_creep_diff), + ], + ) + + combined_output = (result.stdout + result.stderr).lower() + + # TEST-02: Check review agent detects scope creep + scope_creep_indicators = [ + "scope", "creep", "unauthorized", "out-of-scope", + "out of scope", "extra", "unexpected", "not in spec", + ] + detected_scope_creep = any( + indicator in combined_output + for indicator in scope_creep_indicators + ) + + # ASSERT-01: Review agent blocks code with scope creep + blocked_indicators = ["blocked", "changes_requested", "fail", "rejected"] + is_blocked = any( + indicator in combined_output for indicator in blocked_indicators + ) + assert is_blocked, ( + f"Review agent should block code with scope creep. " + f"Unauthorized functionality can be merged unchecked. " + f"stdout: {result.stdout}, stderr: {result.stderr}" + ) + + # TEST-03: Verify review agent blocks the PR with scope creep reason + # ASSERT-02: Review agent identifies specific out-of-scope additions + assert detected_scope_creep, ( + f"Review agent should identify out-of-scope additions. " + f"Developers cannot identify which code to remove. " + f"Output: {result.stdout}" + ) + + finally: + scope_creep_diff.unlink(missing_ok=True) diff --git a/qf-tests/GH-4/python/test_vibe_to_spec_workflow.py b/qf-tests/GH-4/python/test_vibe_to_spec_workflow.py new file mode 100644 index 0000000000..cc5a3343d3 --- /dev/null +++ b/qf-tests/GH-4/python/test_vibe_to_spec_workflow.py @@ -0,0 +1,317 @@ +""" +Tests for vibe-to-spec workflow — core spec generation and error handling. + +Covers scenarios: + - TS-GH-4-001: Valid spec generation from prototype code + - TS-GH-4-002: Exploration artifacts cleanup after spec generation + - TS-GH-4-003: Error handling for prototype with no testable behavior + - TS-GH-4-009: Error handling for ambiguous/contradictory prototype input + +STP Reference: + outputs/stp/GH-4/GH-4_test_plan.md + +PR Reference: + https://github.com/fullsend-ai/fullsend/pull/4 +""" +from pathlib import Path + +import pytest +import yaml + +from conftest import run_fullsend_command + +pytestmark = [ + pytest.mark.tier2, + pytest.mark.gh_4, +] + + +class TestVibeToSpecWorkflow: + """Tests for the vibe-to-spec workflow core functionality. + + Markers: + - tier2 + - gh_4 + - serial + + Preconditions: + - fullsend binary available in PATH + - AI/LLM inference endpoint accessible + - Go toolchain installed (Go 1.23+) + """ + + @pytest.mark.serial + def test_ts_gh_4_001_generate_valid_spec_from_prototype( + self, + fullsend_binary: str, + llm_endpoint: str, + prototype_with_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-001: Verify vibe-to-spec workflow produces valid spec from prototype code. + + Priority: P0 + MVP: True + + This test validates that the vibe-to-spec workflow correctly generates + a valid, structured formal specification from developer prototype code. + + Acceptance Criteria: + - Workflow accepts prototype code directory as input and completes without error + - Generated specification contains a functional requirements section + - Generated specification contains acceptance scenarios with pass/fail criteria + - Generated specification is valid YAML/structured format + """ + # SETUP: Verify prototype directory contains Go files + go_files = list(prototype_with_testable_behavior.glob("*.go")) + assert len(go_files) > 0, "Prototype directory must contain .go files" + + # TEST-01: Execute vibe-to-spec workflow on prototype directory + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_with_testable_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + + # ASSERT-01: Workflow completes without error + assert result.returncode == 0, ( + f"vibe-to-spec workflow failed with exit code {result.returncode}. " + f"stderr: {result.stderr}" + ) + + # TEST-02: Validate generated specification structure + spec_files = list(spec_output_dir_scope_function.glob("*.yaml")) + \ + list(spec_output_dir_scope_function.glob("*.yml")) + assert len(spec_files) > 0, ( + "No specification files generated in output directory" + ) + + spec_content = spec_files[0].read_text() + spec_data = yaml.safe_load(spec_content) + + # ASSERT-02: Generated spec contains functional requirements + assert spec_data is not None, "Spec file is empty or invalid YAML" + assert "functional_requirements" in spec_data, ( + "Generated spec missing 'functional_requirements' section. " + f"Available keys: {list(spec_data.keys())}" + ) + assert len(spec_data["functional_requirements"]) > 0, ( + "functional_requirements section is empty" + ) + + # ASSERT-03: Generated spec contains acceptance scenarios + assert "acceptance_scenarios" in spec_data, ( + "Generated spec missing 'acceptance_scenarios' section. " + f"Available keys: {list(spec_data.keys())}" + ) + scenarios = spec_data["acceptance_scenarios"] + assert len(scenarios) > 0, "acceptance_scenarios section is empty" + + for scenario in scenarios: + assert "pass_criteria" in scenario or "criteria" in scenario, ( + f"Acceptance scenario missing pass/fail criteria: {scenario}" + ) + + # ASSERT-04: Generated spec is valid structured format (already parsed above) + # Re-parse to confirm round-trip validity + reparsed = yaml.safe_load(yaml.dump(spec_data)) + assert reparsed == spec_data, "Spec data is not round-trip stable" + + @pytest.mark.serial + def test_ts_gh_4_002_cleanup_exploration_artifacts( + self, + fullsend_binary: str, + llm_endpoint: str, + prototype_with_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-002: Verify exploration artifacts are cleaned up after spec generation. + + Priority: P1 + MVP: False + + This test validates that after the vibe-to-spec workflow generates a + formal specification, all exploration/prototype artifacts are properly + cleaned up and do not persist in the working directory. + + Acceptance Criteria: + - After spec generation completes, exploration artifact directory no longer exists + - No prototype source files remain in the working directory + - Generated spec file is the only output preserved + """ + exploration_dir = prototype_with_testable_behavior + + # SETUP-01: Verify exploration directory exists with files + assert exploration_dir.exists(), "Exploration directory should exist before workflow" + go_files_before = list(exploration_dir.glob("*.go")) + assert len(go_files_before) > 0, "Exploration directory must have Go files" + + # SETUP-02: Run vibe-to-spec workflow to completion + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(exploration_dir), + "--output", str(spec_output_dir_scope_function), + ], + ) + assert result.returncode == 0, ( + f"vibe-to-spec workflow failed: {result.stderr}" + ) + + # TEST-01: Check exploration artifact directory no longer exists + # ASSERT-01: Exploration directory is removed after spec generation + assert not exploration_dir.exists(), ( + f"Exploration directory {exploration_dir} still exists after " + f"spec generation. Prototype code may leak into production commits." + ) + + # TEST-02: Verify no prototype files remain + parent_dir = exploration_dir.parent + remaining_go_files = list(parent_dir.rglob("*.go")) + exploration_go_files = [ + f for f in remaining_go_files + if str(exploration_dir.name) in str(f) + ] + assert len(exploration_go_files) == 0, ( + f"Prototype Go files still found: {exploration_go_files}" + ) + + # TEST-03: Verify generated spec file is preserved + # ASSERT-02: Generated spec is preserved after cleanup + spec_files = list(spec_output_dir_scope_function.glob("*.yaml")) + \ + list(spec_output_dir_scope_function.glob("*.yml")) + assert len(spec_files) > 0, ( + "Generated spec file was not preserved after cleanup. " + "Cleanup is too aggressive and removes wanted output." + ) + + @pytest.mark.serial + def test_ts_gh_4_003_error_for_no_testable_behavior( + self, + fullsend_binary: str, + prototype_without_testable_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-003: Verify error returned when prototype contains no testable behavior. + + Priority: P1 + MVP: False + + This test validates that the vibe-to-spec workflow returns a clear, + actionable error message when the input prototype contains no testable + behavior. + + Acceptance Criteria: + - Workflow returns a non-zero exit code + - Error message clearly indicates prototype lacks testable behavior + - Error message suggests what the developer should do + """ + # SETUP-01: Verify prototype directory has no testable functions + content = (prototype_without_testable_behavior / "main.go").read_text() + assert "func " not in content or "func " not in content.split("//")[0], ( + "Test setup error: prototype should have no exported functions" + ) + + # TEST-01: Execute vibe-to-spec workflow on empty prototype + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_without_testable_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + + # ASSERT-01: Workflow returns non-zero exit code + assert result.returncode != 0, ( + "Workflow should fail with non-zero exit code when prototype " + "has no testable behavior, but returned 0" + ) + + # TEST-02: Capture and validate error message + error_output = result.stderr.lower() + + # ASSERT-02: Error message is actionable + testable_keywords = [ + "testable", + "no functions", + "no exported", + "empty", + "no behavior", + "insufficient", + ] + has_relevant_error = any( + keyword in error_output for keyword in testable_keywords + ) + assert has_relevant_error, ( + f"Error message should indicate prototype lacks testable behavior. " + f"Got: {result.stderr}" + ) + + @pytest.mark.serial + def test_ts_gh_4_009_error_for_ambiguous_prototype( + self, + fullsend_binary: str, + prototype_with_ambiguous_behavior: Path, + spec_output_dir_scope_function: Path, + ) -> None: + """TS-GH-4-009: Verify error for ambiguous or contradictory prototype input. + + Priority: P1 + MVP: False + + This test validates that the vibe-to-spec workflow returns a clear + error when the input prototype contains ambiguous or contradictory + behavior that cannot be reliably converted to a formal specification. + + Acceptance Criteria: + - Workflow returns a non-zero exit code for ambiguous input + - Error message explains why the prototype is ambiguous + - Error message suggests how to resolve the ambiguity + """ + # SETUP-01: Verify prototype contains contradictory code + content = (prototype_with_ambiguous_behavior / "contradictory.go").read_text() + assert "uppercase" in content.lower() and "ToLower" in content, ( + "Test setup error: prototype should contain contradictory behavior" + ) + + # TEST-01: Execute vibe-to-spec on ambiguous prototype + result = run_fullsend_command( + binary=fullsend_binary, + subcommand="vibe-to-spec", + args=[ + "--input", str(prototype_with_ambiguous_behavior), + "--output", str(spec_output_dir_scope_function), + ], + ) + + # ASSERT-01: Workflow returns error for ambiguous input + assert result.returncode != 0, ( + "Workflow should fail with non-zero exit code for ambiguous " + "prototype input, but returned 0. Incorrect specs may be generated " + "from ambiguous prototypes." + ) + + # TEST-02: Validate error message explains ambiguity + error_output = result.stderr.lower() + + # ASSERT-02: Error message is descriptive + ambiguity_keywords = [ + "ambiguous", + "contradictory", + "conflict", + "inconsistent", + "unclear", + "mismatch", + ] + has_ambiguity_error = any( + keyword in error_output for keyword in ambiguity_keywords + ) + assert has_ambiguity_error, ( + f"Error message should explain the ambiguity in prototype. " + f"Got: {result.stderr}" + )