diff --git a/.github/workflows/ai-generate-tests.yml b/.github/workflows/ai-generate-tests.yml new file mode 100644 index 000000000..932d141c2 --- /dev/null +++ b/.github/workflows/ai-generate-tests.yml @@ -0,0 +1,308 @@ +name: AI Generate PHP Unit Tests + +on: + workflow_dispatch: # Manual trigger only + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate-ai-tests: + name: Generate AI-Powered Unit Tests + runs-on: ubuntu-22.04 + env: + WP_CORE_DIR: "/tmp/wordpress/src" + WP_TESTS_DIR: "/tmp/wordpress/tests/phpunit" + + steps: + - name: Install SVN + run: sudo apt-get install subversion -y + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup PHP + uses: woocommerce/grow/prepare-php@actions-v1 + with: + php-version: '8.2' + + - name: Prepare MySQL + uses: woocommerce/grow/prepare-mysql@actions-v1 + + - name: Install WP tests + run: | + chmod +x ./bin/install-wp-tests.sh + ./bin/install-wp-tests.sh wordpress_test root root localhost latest latest + + - name: Install dependencies + run: composer install + + - name: Generate coverage report + run: | + mkdir -p reports/coverage + vendor/bin/phpunit --coverage-html reports/coverage --coverage-clover reports/coverage.xml + + - name: Setup Python for Llama + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install Llama CPP Python + run: | + pip install llama-cpp-python + pip install requests + + - name: Download Llama Model + run: | + mkdir -p models + curl -L https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q4_K_M.gguf -o models/codellama.gguf + + - name: Create test generator script + run: | + mkdir -p bin + cat > bin/generate_llama_tests.py << 'EOL' + #!/usr/bin/env python3 + + import sys + import json + from llama_cpp import Llama + import os + import re + + def load_class_content(file_path): + """Load the PHP class content from a file.""" + with open(file_path, 'r') as f: + return f.read() + + def load_existing_test_content(test_file): + """Load existing test file content if it exists.""" + if os.path.exists(test_file): + with open(test_file, 'r') as f: + return f.read() + return None + + def extract_existing_test_methods(test_content): + """Extract existing test method names from test content.""" + if not test_content: + return set() + + pattern = r'public\s+function\s+(test\w+)\s*\(' + return set(re.findall(pattern, test_content)) + + def merge_test_content(existing_content, new_content): + """Merge existing test content with new test content.""" + if not existing_content: + return new_content + + namespace_pattern = r'namespace\s+[^;]+;' + use_statements_pattern = r'use\s+[^;]+;' + class_def_pattern = r'class\s+\w+Test\s+extends\s+TestCase\s*{' + + existing_namespace = re.search(namespace_pattern, existing_content) + existing_use_statements = re.findall(use_statements_pattern, existing_content) + existing_class_def = re.search(class_def_pattern, existing_content) + + new_methods_pattern = r'(public\s+function\s+test\w+\s*\([^{]*{[^}]*})' + new_methods = re.findall(new_methods_pattern, new_content, re.DOTALL) + + existing_methods = extract_existing_test_methods(existing_content) + + merged_content = [] + + if existing_namespace: + merged_content.append(existing_content[:existing_namespace.end()]) + else: + namespace_match = re.search(namespace_pattern, new_content) + if namespace_match: + merged_content.append(namespace_match.group(0)) + + new_use_statements = re.findall(use_statements_pattern, new_content) + all_use_statements = list(set(existing_use_statements + new_use_statements)) + merged_content.extend(all_use_statements) + + if existing_class_def: + merged_content.append('\n' + existing_content[existing_class_def.start():existing_class_def.end()] + '\n') + else: + class_def_match = re.search(class_def_pattern, new_content) + if class_def_match: + merged_content.append('\n' + class_def_match.group(0) + '\n') + + existing_methods_pattern = r'(public\s+function\s+test\w+\s*\([^{]*{[^}]*})' + existing_method_blocks = re.findall(existing_methods_pattern, existing_content, re.DOTALL) + merged_content.extend([' ' + method.strip() + '\n' for method in existing_method_blocks]) + + for method in new_methods: + method_name = re.search(r'function\s+(test\w+)', method) + if method_name and method_name.group(1) not in existing_methods: + merged_content.append(' ' + method.strip() + '\n') + + merged_content.append('}') + + return '\n'.join(merged_content) + + def generate_test_with_llama(llm, class_content, class_name, existing_test_methods): + """Generate test cases using the Llama model.""" + existing_methods_str = '\n'.join(existing_test_methods) if existing_test_methods else "No existing test methods" + + prompt = f""" + Given the following PHP class and its existing test methods, generate additional PHPUnit test cases for its public methods. + Do not duplicate existing test methods, only generate new ones that test different scenarios. + + PHP Class: + {class_content} + + Existing Test Methods: + {existing_methods_str} + + Generate a complete PHPUnit test class that: + 1. Has proper namespace and use statements + 2. Extends TestCase + 3. Has meaningful test methods with assertions + 4. Includes test cases for different scenarios not covered by existing tests + 5. Uses proper PHPUnit assertions + 6. Handles dependencies and mocking where necessary + + The test class should follow this format: + ") + sys.exit(1) + + class_file = sys.argv[1] + class_name = sys.argv[2] + + model_path = "models/codellama.gguf" + if not os.path.exists(model_path): + print(f"Error: Model file not found at {model_path}") + sys.exit(1) + + llm = Llama( + model_path=model_path, + n_ctx=2048, + n_threads=4 + ) + + class_content = load_class_content(class_file) + + test_dir = "tests/Unit" + os.makedirs(test_dir, exist_ok=True) + + test_file = f"{test_dir}/{class_name}Test.php" + + existing_content = load_existing_test_content(test_file) + existing_test_methods = extract_existing_test_methods(existing_content) if existing_content else set() + + new_test_code = generate_test_with_llama(llm, class_content, class_name, existing_test_methods) + + final_test_code = merge_test_content(existing_content, new_test_code) + + with open(test_file, 'w') as f: + f.write(final_test_code) + + print(json.dumps({ + "test_file": test_file, + "class_name": class_name, + "existing_tests": len(existing_test_methods), + "new_tests": len(extract_existing_test_methods(new_test_code)) - len(existing_test_methods) + })) + + if __name__ == "__main__": + main() + EOL + chmod +x bin/generate_llama_tests.py + + - name: Analyze untested classes + id: analyze + run: | + echo "xpath('//class') as \$class) { + \$metrics = \$class->metrics; + \$elements = (float)\$metrics['elements']; + if (\$elements > 0) { + \$coverage = (float)\$metrics['coveredelements'] / \$elements * 100; + if (\$coverage < 80 && (\$lowest_coverage === null || \$coverage < \$lowest_coverage)) { + \$lowest_coverage = \$coverage; + \$class_to_test = [ + 'class' => (string)\$class['name'], + 'file' => (string)\$class['filename'], + 'coverage' => round(\$coverage, 2) + ]; + } + } + } + + if (\$class_to_test) { + echo \"class_name={$class_to_test['class']}\" >> \$GITHUB_OUTPUT; + echo \"file_path={$class_to_test['file']}\" >> \$GITHUB_OUTPUT; + echo \"coverage={$class_to_test['coverage']}\" >> \$GITHUB_OUTPUT; + } else { + echo \"No classes found with coverage below 80%\\n\"; + exit(78); # Skip PR creation if no classes need testing + }" > analyze_coverage.php + + php analyze_coverage.php + + - name: Generate AI Tests + id: generate_tests + run: | + chmod +x ./tests/tools/generate_llama_tests.py + result=$(python3 ./tests/tools/generate_llama_tests.py "${{ steps.analyze.outputs.file_path }}" "${{ steps.analyze.outputs.class_name }}") + echo "test_info=$result" >> $GITHUB_OUTPUT + + - name: Cleanup temporary files + run: | + rm -f analyze_coverage.php + rm -rf reports/ + rm -rf models/ + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "test: Add AI-generated unit tests for ${{ steps.analyze.outputs.class_name }}" + title: "Add AI-generated unit tests for class with ${{ steps.analyze.outputs.coverage }}% coverage" + body: | + This PR adds AI-generated unit tests using CodeLlama for the class with the lowest coverage. + + Class: `${{ steps.analyze.outputs.class_name }}` + Current Coverage: ${{ steps.analyze.outputs.coverage }}% + + The tests were generated using the CodeLlama model and include assertions and different test scenarios. + Please review the generated tests and adjust them as needed. + branch: feature/ai-generate-unit-tests + base: main + labels: | + automated + tests + needs-review \ No newline at end of file diff --git a/.github/workflows/generate-tests.yml b/.github/workflows/generate-tests.yml new file mode 100644 index 000000000..933235935 --- /dev/null +++ b/.github/workflows/generate-tests.yml @@ -0,0 +1,130 @@ +name: Generate PHP Unit Tests + +on: + workflow_dispatch: # Manual trigger only + +jobs: + generate-tests: + name: Generate Missing Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + tools: phpunit, phpcov + coverage: xdebug + + - name: Install dependencies + run: composer install + + - name: Generate coverage report + run: | + mkdir -p coverage + vendor/bin/phpunit --coverage-html coverage --coverage-clover coverage.xml + + - name: Check current coverage + id: coverage + run: | + COVERAGE=$(php -r ' + $xml = new SimpleXMLElement(file_get_contents("coverage.xml")); + $metrics = $xml->xpath("//metrics")[0]; + $coverage = ($metrics["coveredelements"] / $metrics["elements"]) * 100; + echo number_format($coverage, 2); + ') + echo "total_coverage=$COVERAGE" >> $GITHUB_OUTPUT + echo "Current coverage: $COVERAGE%" + if (( $(echo "$COVERAGE >= 80" | bc -l) )); then + echo "Coverage goal achieved! No need to generate more tests." + exit 78 # Special exit code to skip PR creation + fi + + - name: Analyze untested classes + id: analyze + if: steps.coverage.outcome != 'skipped' + run: | + echo "xpath('//class') as \$class) { + \$metrics = \$class->metrics; + if ((float)\$metrics['elements'] > 0 && (float)\$metrics['coveredelements'] / (float)\$metrics['elements'] < 0.8) { + \$untested[] = [ + 'class' => (string)\$class['name'], + 'file' => (string)\$class['filename'], + 'coverage' => round((float)\$metrics['coveredelements'] / (float)\$metrics['elements'] * 100, 2) + ]; + } + } + + foreach (\$untested as \$class) { + \$reflection = new ReflectionClass(\$class['class']); + \$methods = \$reflection->getMethods(ReflectionMethod::IS_PUBLIC); + + \$testFile = 'tests/Unit/' . str_replace('.php', 'Test.php', basename(\$class['file'])); + \$testClass = basename(\$class['class']) . 'Test'; + + if (!file_exists(dirname(\$testFile))) { + mkdir(dirname(\$testFile), 0777, true); + } + + \$template = 'isConstructor()) { + \$template .= ' + public function test' . ucfirst(\$method->getName()) . '() + { + \$this->markTestIncomplete(\'Auto-generated test for ' . \$method->getName() . '\'); + }'; + } + } + + \$template .= ' + }'; + + file_put_contents(\$testFile, \$template); + echo \"Generated test file: {\$testFile}\\n\"; + }" > generate_tests.php + + php generate_tests.php > test_files.txt + + - name: Create Pull Request + if: steps.coverage.outcome != 'skipped' + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "test: Add generated unit tests to improve coverage" + title: "Add generated unit tests (Current coverage: ${{ steps.coverage.outputs.total_coverage }}%)" + body: | + This PR adds automatically generated unit test stubs for classes with low coverage. + + Current total coverage: ${{ steps.coverage.outputs.total_coverage }}% + Target coverage: 80% + + Generated test files: + ``` + $(cat test_files.txt) + ``` + + Please review and implement the actual test assertions for these generated test methods. + branch: feature/generate-unit-tests + base: main + labels: | + automated + tests + needs-review \ No newline at end of file