Repository navigation
Add autofix support to plugins-doc-up-to-date rule #475
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
9fae9a0
86507d6
c33f4a2
814fa01
e616512
3a2dcd2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,18 +4,20 @@ | |
|
|
||
| import subprocess | ||
| from pathlib import Path | ||
| from typing import List | ||
| from typing import Any, List | ||
|
|
||
| try: | ||
| from src.rule import Rule, RuleViolation, Severity | ||
| from src.rule import Rule, AutofixConfidence, AutofixResult, RuleViolation, Severity | ||
| from src.context import RepositoryContext | ||
| except ImportError: | ||
| from skillsaw import Rule, RuleViolation, Severity, RepositoryContext | ||
| from skillsaw import Rule, AutofixConfidence, AutofixResult, RuleViolation, Severity, RepositoryContext | ||
|
|
||
|
|
||
| class PluginsDocUpToDateRule(Rule): | ||
| """Check that PLUGINS.md and docs/data.json are up-to-date by running 'make update'""" | ||
|
|
||
| autofix_confidence = AutofixConfidence.SAFE | ||
|
|
||
| @property | ||
| def rule_id(self) -> str: | ||
| return "plugins-doc-up-to-date" | ||
|
|
@@ -27,10 +29,49 @@ def description(self) -> str: | |
| def default_severity(self) -> Severity: | ||
| return Severity.ERROR | ||
|
|
||
| def _run_generators(self, context: RepositoryContext): | ||
| """Run doc generation scripts. Returns (original_plugins_md, generated_plugins_md, | ||
| original_data_json, generated_data_json) or raises on failure.""" | ||
| plugins_md_path = context.root_path / "PLUGINS.md" | ||
| data_json_path = context.root_path / "docs" / "data.json" | ||
| script_path = context.root_path / "scripts" / "generate_plugin_docs.py" | ||
|
|
||
| original_plugins_md = plugins_md_path.read_text() | ||
| original_data_json = data_json_path.read_text() if data_json_path.exists() else None | ||
|
|
||
| result = subprocess.run( | ||
| ["python3", str(script_path)], | ||
| cwd=str(context.root_path), | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=30, | ||
| ) | ||
| if result.returncode != 0: | ||
| raise RuntimeError(f"generate_plugin_docs.py failed: {result.stderr}") | ||
|
|
||
| website_script_path = context.root_path / "scripts" / "build-website.py" | ||
| if website_script_path.exists(): | ||
| result = subprocess.run( | ||
| ["python3", str(website_script_path)], | ||
| cwd=str(context.root_path), | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=30, | ||
| ) | ||
| if result.returncode != 0: | ||
| raise RuntimeError(f"build-website.py failed: {result.stderr}") | ||
|
|
||
| generated_plugins_md = plugins_md_path.read_text() | ||
| generated_data_json = data_json_path.read_text() if data_json_path.exists() else None | ||
|
|
||
| return ( | ||
| original_plugins_md, generated_plugins_md, | ||
| original_data_json, generated_data_json, | ||
| ) | ||
|
|
||
| def check(self, context: RepositoryContext) -> List[RuleViolation]: | ||
| violations = [] | ||
|
|
||
| # Only check marketplace repos | ||
| if not context.has_marketplace(): | ||
| return violations | ||
|
|
||
|
|
@@ -40,95 +81,98 @@ def check(self, context: RepositoryContext) -> List[RuleViolation]: | |
| if not plugins_md_path.exists(): | ||
| return violations | ||
|
|
||
| # Check if generate_plugin_docs.py script exists | ||
| script_path = context.root_path / "scripts" / "generate_plugin_docs.py" | ||
| if not script_path.exists(): | ||
| return violations | ||
|
|
||
| try: | ||
| # Read current content of files to check | ||
| original_plugins_md = plugins_md_path.read_text() | ||
| original_data_json = data_json_path.read_text() if data_json_path.exists() else None | ||
|
|
||
| # Run the docs generation script | ||
| result = subprocess.run( | ||
| ["python3", str(script_path)], | ||
| cwd=str(context.root_path), | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=30 | ||
| ) | ||
| ( | ||
| original_plugins_md, generated_plugins_md, | ||
| original_data_json, generated_data_json, | ||
| ) = self._run_generators(context) | ||
|
|
||
| if result.returncode != 0: | ||
| violations.append( | ||
| self.violation( | ||
| f"'make update' failed: {result.stderr}", | ||
| file_path=plugins_md_path | ||
| ) | ||
| ) | ||
| return violations | ||
|
|
||
| # Also run build-website.py if it exists | ||
| website_script_path = context.root_path / "scripts" / "build-website.py" | ||
| if website_script_path.exists(): | ||
| result = subprocess.run( | ||
| ["python3", str(website_script_path)], | ||
| cwd=str(context.root_path), | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=30 | ||
| ) | ||
|
|
||
| if result.returncode != 0: | ||
| violations.append( | ||
| self.violation( | ||
| f"build-website.py failed: {result.stderr}", | ||
| file_path=data_json_path if data_json_path.exists() else plugins_md_path | ||
| ) | ||
| ) | ||
| return violations | ||
|
|
||
| # Check if PLUGINS.md changed | ||
| generated_plugins_md = plugins_md_path.read_text() | ||
| if original_plugins_md != generated_plugins_md: | ||
| # Restore original content | ||
| plugins_md_path.write_text(original_plugins_md) | ||
|
|
||
| violations.append( | ||
| self.violation( | ||
| "PLUGINS.md is out of sync with plugin metadata. Run 'make update' to update.", | ||
| file_path=plugins_md_path | ||
| file_path=plugins_md_path, | ||
| ) | ||
| ) | ||
|
|
||
| # Check if docs/data.json changed | ||
| if data_json_path.exists(): | ||
| generated_data_json = data_json_path.read_text() | ||
| if original_data_json != generated_data_json: | ||
| # Restore original content | ||
| if original_data_json is not None: | ||
| data_json_path.write_text(original_data_json) | ||
|
|
||
| violations.append( | ||
| self.violation( | ||
| "docs/data.json is out of sync with plugin metadata. Run 'make update' to update.", | ||
| file_path=data_json_path | ||
| ) | ||
| if data_json_path.exists() and original_data_json != generated_data_json: | ||
| if original_data_json is not None: | ||
| data_json_path.write_text(original_data_json) | ||
| violations.append( | ||
| self.violation( | ||
| "docs/data.json is out of sync with plugin metadata. Run 'make update' to update.", | ||
| file_path=data_json_path, | ||
| ) | ||
| ) | ||
|
|
||
| except subprocess.TimeoutExpired: | ||
| violations.append( | ||
| self.violation( | ||
| "'make update' timed out", | ||
| file_path=plugins_md_path | ||
| ) | ||
| self.violation("'make update' timed out", file_path=plugins_md_path) | ||
| ) | ||
| except Exception as e: | ||
| violations.append( | ||
| self.violation( | ||
| f"Error checking files up-to-date status: {e}", | ||
| file_path=plugins_md_path | ||
| file_path=plugins_md_path, | ||
| ) | ||
| ) | ||
|
|
||
| return violations | ||
|
|
||
| def fix( | ||
| self, | ||
| context: RepositoryContext, | ||
| violations: List[RuleViolation], | ||
| *, | ||
| provider: Any = None, | ||
| ) -> List[AutofixResult]: | ||
| results: List[AutofixResult] = [] | ||
| if not violations: | ||
| return results | ||
|
|
||
| plugins_md_path = context.root_path / "PLUGINS.md" | ||
| data_json_path = context.root_path / "docs" / "data.json" | ||
|
|
||
| try: | ||
| ( | ||
| original_plugins_md, generated_plugins_md, | ||
| original_data_json, generated_data_json, | ||
| ) = self._run_generators(context) | ||
| except Exception: | ||
| return results | ||
|
Comment on lines
+141
to
+147
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: #!/bin/bash
# Look for how AutofixResult is consumed and whether the framework writes fixed_content to disk.
fd -t f 'rule.py' .skillsaw skillsaw src 2>/dev/null
rg -nP -C5 '\bAutofixResult\b' --type=py
rg -nP -C5 '\bfixed_content\b|\bapply_fix\b|\bapply\s*\(' --type=py
# Also check other rules in this repo to see the established fix() contract.
fd -t f '_rule.py|rule.py' .skillsawRepository: openshift-eng/ai-helpers Length of output: 196 🏁 Script executed: # First, check the size and read the plugindocs_rule.py file
wc -l .skillsaw/plugindocs_rule.py
head -150 .skillsaw/plugindocs_rule.py | tail -50Repository: openshift-eng/ai-helpers Length of output: 1853 🏁 Script executed: # Read the full check() method to understand the restore logic
rg -n "def check" .skillsaw/plugindocs_rule.py -A 30Repository: openshift-eng/ai-helpers Length of output: 1261 🏁 Script executed: # Read the full fix() method to understand current behavior
rg -n "def fix" .skillsaw/plugindocs_rule.py -A 30Repository: openshift-eng/ai-helpers Length of output: 1297 🏁 Script executed: # Look at the other rule file to understand the established pattern
cat -n .skillsaw/promptfoo_budget_rule.py | head -80Repository: openshift-eng/ai-helpers Length of output: 3100 🏁 Script executed: # Search for AutofixResult definition and usage
rg -n "class AutofixResult|AutofixResult\(" --type=py -B2 -A5Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Look for how the autofix framework handles results
rg -n "def.*fix|AutofixResult" --type=py | head -20Repository: openshift-eng/ai-helpers Length of output: 845 🏁 Script executed: # Search for AutofixResult definition and imports
rg -n "AutofixResult|AutofixConfidence" --type=py -B2 -A3 | head -50Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Get the full fix() method and what comes after
sed -n '127,178p' .skillsaw/plugindocs_rule.pyRepository: openshift-eng/ai-helpers Length of output: 2062 🏁 Script executed: # Search for imports at the top of the file
head -40 .skillsaw/plugindocs_rule.pyRepository: openshift-eng/ai-helpers Length of output: 1557 🏁 Script executed: # Look for how AutofixResult is used elsewhere in the repo
rg -n "AutofixResult\(" --type=py -B1 -A2Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Search for AutofixResult definition
rg -n "class AutofixResult|def.*AutofixResult" --type=py -B2 -A10Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Look for where AutofixResult is imported from and how it's consumed
rg -n "from.*AutofixResult|import.*AutofixResult" --type=pyRepository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Search for how the framework processes/applies AutofixResult
rg -n "fixed_content|apply.*fix|_apply" --type=py | head -30Repository: openshift-eng/ai-helpers Length of output: 351 🏁 Script executed: # Check if there's a base Rule class or interface that documents the fix() contract
fd -name "*.py" -path "*/skillsaw/*" | head -20Repository: openshift-eng/ai-helpers Length of output: 299 🏁 Script executed: # Look at the complete _run_generators method to understand what it does
sed -n '40,70p' .skillsaw/plugindocs_rule.pyRepository: openshift-eng/ai-helpers Length of output: 1260 🏁 Script executed: # Look for src/rule.py or skillsaw module in the repo
find . -type f -name "rule.py" -o -name "__init__.py" | grep -E "src|skillsaw" | head -20Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Check what's imported from skillsaw - look for the package
rg -n "from skillsaw|import skillsaw" --type=py | head -10Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Search the repo for where AutofixResult is actually consumed/applied
rg -n "\.fixed_content|result\." --type=py | grep -i fix | head -20Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Look for the full file to understand the complete implementation
wc -l .skillsaw/plugindocs_rule.py
cat -n .skillsaw/plugindocs_rule.py | tail -30Repository: openshift-eng/ai-helpers Length of output: 1664 🏁 Script executed: # Check if there are any git references or docs about the autofix contract
find . -type f -name "*.md" -o -name "*.txt" | xargs grep -l "AutofixResult\|autofix" 2>/dev/null | head -5Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Check the git log for this file to understand intent
git log --oneline .skillsaw/plugindocs_rule.py 2>/dev/null | head -10Repository: openshift-eng/ai-helpers Length of output: 127 🏁 Script executed: # Check if there are any comments or TODOs in the file
grep -n "TODO\|FIXME\|XXX\|NOTE" .skillsaw/plugindocs_rule.pyRepository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Look at the complete check() method once more to see the full restoration pattern
sed -n '72,126p' .skillsaw/plugindocs_rule.pyRepository: openshift-eng/ai-helpers Length of output: 2079 🏁 Script executed: # Search for other uses of AutofixResult in the repo to understand the pattern
rg -n "AutofixResult\|fixed_content\|original_content" --type=py -B2 -A4 | head -80Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Check if promptfoo_budget_rule.py has a fix() method
rg -n "def fix" .skillsaw/promptfoo_budget_rule.py -A 20Repository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Look for any other custom rules
fd -t f "_rule.py\|_rule.py" .skillsawRepository: openshift-eng/ai-helpers Length of output: 50 🏁 Script executed: # Check git diff for the commit that added autofix support
git show 9fae9a0 --stat 2>/dev/nullRepository: openshift-eng/ai-helpers Length of output: 33320 🏁 Script executed: # Check if there's documentation or README about how fix() should work
find . -name "README*" -o -name "*.md" | xargs grep -l "fix\|autofix" 2>/dev/null | head -5Repository: openshift-eng/ai-helpers Length of output: 204 🏁 Script executed: # Check if skillsaw is a package we can inspect
python3 -c "import sys; sys.path.insert(0, '.'); from skillsaw import AutofixResult; help(AutofixResult)" 2>/dev/null || echo "skillsaw not importable"Repository: openshift-eng/ai-helpers Length of output: 92 Address exception handling and clarify the autofix contract for file mutation. Two issues in this block:
🧰 Tools🪛 Ruff (0.15.12)[warning] 146-146: Do not catch blind exception: (BLE001) 🤖 Prompt for AI Agents |
||
|
|
||
| plugins_md_violations = [v for v in violations if v.file_path == plugins_md_path] | ||
| data_json_violations = [v for v in violations if v.file_path == data_json_path] | ||
|
|
||
| if plugins_md_violations and original_plugins_md != generated_plugins_md: | ||
| results.append( | ||
| AutofixResult( | ||
| rule_id=self.rule_id, | ||
| file_path=plugins_md_path, | ||
| confidence=AutofixConfidence.SAFE, | ||
| original_content=original_plugins_md, | ||
| fixed_content=generated_plugins_md, | ||
| description="Regenerated PLUGINS.md from plugin metadata", | ||
| violations_fixed=plugins_md_violations, | ||
| ) | ||
| ) | ||
|
|
||
| if data_json_violations and original_data_json != generated_data_json: | ||
| results.append( | ||
| AutofixResult( | ||
| rule_id=self.rule_id, | ||
| file_path=data_json_path, | ||
| confidence=AutofixConfidence.SAFE, | ||
| original_content=original_data_json or "", | ||
| fixed_content=generated_data_json or "", | ||
| description="Regenerated docs/data.json from plugin metadata", | ||
| violations_fixed=data_json_violations, | ||
| ) | ||
| ) | ||
|
|
||
| return results | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Newly-generated
docs/data.jsonis left in the working tree when it didn't exist before.If
data_json_pathdoes not exist prior to running the generators,original_data_jsonisNone. After_run_generators()runs, the generator may create the file, so line 103'sdata_json_path.exists()is true and a violation is appended — but the restoration at line 104 is gated onoriginal_data_json is not None, so the newly-created file is left on disk. This makescheck()mutate the working tree, which violates the intent of a read-only check and can leak into commits or confuse later runs of the rule.🛠️ Proposed fix: unlink the file if it didn't exist originally
if data_json_path.exists() and original_data_json != generated_data_json: if original_data_json is not None: data_json_path.write_text(original_data_json) + else: + data_json_path.unlink() violations.append( self.violation( "docs/data.json is out of sync with plugin metadata. Run 'make update' to update.", file_path=data_json_path, ) )📝 Committable suggestion
🤖 Prompt for AI Agents