Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude-plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"name": "ci",
"source": "./plugins/ci",
"description": "A plugin to work with OpenShift CI and analyze Prow job results",
"version": "0.0.40"
"version": "0.0.41"
},
{
"name": "teams",
Expand Down
182 changes: 113 additions & 69 deletions .skillsaw/plugindocs_rule.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@

import subprocess
from pathlib import Path
from typing import List
from typing import Any, List

try:
from src.rule import Rule, RuleViolation, Severity
from src.rule import Rule, AutofixConfidence, AutofixResult, RuleViolation, Severity
from src.context import RepositoryContext
except ImportError:
from skillsaw import Rule, RuleViolation, Severity, RepositoryContext
from skillsaw import Rule, AutofixConfidence, AutofixResult, RuleViolation, Severity, RepositoryContext


class PluginsDocUpToDateRule(Rule):
"""Check that PLUGINS.md and docs/data.json are up-to-date by running 'make update'"""

autofix_confidence = AutofixConfidence.SAFE

@property
def rule_id(self) -> str:
return "plugins-doc-up-to-date"
Expand All @@ -27,10 +29,49 @@ def description(self) -> str:
def default_severity(self) -> Severity:
return Severity.ERROR

def _run_generators(self, context: RepositoryContext):
"""Run doc generation scripts. Returns (original_plugins_md, generated_plugins_md,
original_data_json, generated_data_json) or raises on failure."""
plugins_md_path = context.root_path / "PLUGINS.md"
data_json_path = context.root_path / "docs" / "data.json"
script_path = context.root_path / "scripts" / "generate_plugin_docs.py"

original_plugins_md = plugins_md_path.read_text()
original_data_json = data_json_path.read_text() if data_json_path.exists() else None

result = subprocess.run(
["python3", str(script_path)],
cwd=str(context.root_path),
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0:
raise RuntimeError(f"generate_plugin_docs.py failed: {result.stderr}")

website_script_path = context.root_path / "scripts" / "build-website.py"
if website_script_path.exists():
result = subprocess.run(
["python3", str(website_script_path)],
cwd=str(context.root_path),
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0:
raise RuntimeError(f"build-website.py failed: {result.stderr}")

generated_plugins_md = plugins_md_path.read_text()
generated_data_json = data_json_path.read_text() if data_json_path.exists() else None

return (
original_plugins_md, generated_plugins_md,
original_data_json, generated_data_json,
)

def check(self, context: RepositoryContext) -> List[RuleViolation]:
violations = []

# Only check marketplace repos
if not context.has_marketplace():
return violations

Expand All @@ -40,95 +81,98 @@ def check(self, context: RepositoryContext) -> List[RuleViolation]:
if not plugins_md_path.exists():
return violations

# Check if generate_plugin_docs.py script exists
script_path = context.root_path / "scripts" / "generate_plugin_docs.py"
if not script_path.exists():
return violations

try:
# Read current content of files to check
original_plugins_md = plugins_md_path.read_text()
original_data_json = data_json_path.read_text() if data_json_path.exists() else None

# Run the docs generation script
result = subprocess.run(
["python3", str(script_path)],
cwd=str(context.root_path),
capture_output=True,
text=True,
timeout=30
)
(
original_plugins_md, generated_plugins_md,
original_data_json, generated_data_json,
) = self._run_generators(context)

if result.returncode != 0:
violations.append(
self.violation(
f"'make update' failed: {result.stderr}",
file_path=plugins_md_path
)
)
return violations

# Also run build-website.py if it exists
website_script_path = context.root_path / "scripts" / "build-website.py"
if website_script_path.exists():
result = subprocess.run(
["python3", str(website_script_path)],
cwd=str(context.root_path),
capture_output=True,
text=True,
timeout=30
)

if result.returncode != 0:
violations.append(
self.violation(
f"build-website.py failed: {result.stderr}",
file_path=data_json_path if data_json_path.exists() else plugins_md_path
)
)
return violations

# Check if PLUGINS.md changed
generated_plugins_md = plugins_md_path.read_text()
if original_plugins_md != generated_plugins_md:
# Restore original content
plugins_md_path.write_text(original_plugins_md)

violations.append(
self.violation(
"PLUGINS.md is out of sync with plugin metadata. Run 'make update' to update.",
file_path=plugins_md_path
file_path=plugins_md_path,
)
)

# Check if docs/data.json changed
if data_json_path.exists():
generated_data_json = data_json_path.read_text()
if original_data_json != generated_data_json:
# Restore original content
if original_data_json is not None:
data_json_path.write_text(original_data_json)

violations.append(
self.violation(
"docs/data.json is out of sync with plugin metadata. Run 'make update' to update.",
file_path=data_json_path
)
if data_json_path.exists() and original_data_json != generated_data_json:
if original_data_json is not None:
data_json_path.write_text(original_data_json)
violations.append(
self.violation(
"docs/data.json is out of sync with plugin metadata. Run 'make update' to update.",
file_path=data_json_path,
)
)
Comment on lines +103 to +111

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

Newly-generated docs/data.json is left in the working tree when it didn't exist before.

If data_json_path does not exist prior to running the generators, original_data_json is None. After _run_generators() runs, the generator may create the file, so line 103's data_json_path.exists() is true and a violation is appended — but the restoration at line 104 is gated on original_data_json is not None, so the newly-created file is left on disk. This makes check() mutate the working tree, which violates the intent of a read-only check and can leak into commits or confuse later runs of the rule.

🛠️ Proposed fix: unlink the file if it didn't exist originally
             if data_json_path.exists() and original_data_json != generated_data_json:
                 if original_data_json is not None:
                     data_json_path.write_text(original_data_json)
+                else:
+                    data_json_path.unlink()
                 violations.append(
                     self.violation(
                         "docs/data.json is out of sync with plugin metadata. Run 'make update' to update.",
                         file_path=data_json_path,
                     )
                 )
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if data_json_path.exists() and original_data_json != generated_data_json:
if original_data_json is not None:
data_json_path.write_text(original_data_json)
violations.append(
self.violation(
"docs/data.json is out of sync with plugin metadata. Run 'make update' to update.",
file_path=data_json_path,
)
)
if data_json_path.exists() and original_data_json != generated_data_json:
if original_data_json is not None:
data_json_path.write_text(original_data_json)
else:
data_json_path.unlink()
violations.append(
self.violation(
"docs/data.json is out of sync with plugin metadata. Run 'make update' to update.",
file_path=data_json_path,
)
)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In @.skillsaw/plugindocs_rule.py around lines 103 - 111, The check currently
leaves a newly-created docs/data.json on disk when original_data_json is None;
update the post-generator cleanup in the method that calls _run_generators() to
remove the file if it didn't exist originally: when data_json_path.exists() and
original_data_json != generated_data_json, if original_data_json is None call
unlink/delete on data_json_path (otherwise restore original_data_json as before)
and then append the same violation; reference data_json_path,
original_data_json, generated_data_json, and _run_generators() in your change.


except subprocess.TimeoutExpired:
violations.append(
self.violation(
"'make update' timed out",
file_path=plugins_md_path
)
self.violation("'make update' timed out", file_path=plugins_md_path)
)
except Exception as e:
violations.append(
self.violation(
f"Error checking files up-to-date status: {e}",
file_path=plugins_md_path
file_path=plugins_md_path,
)
)

return violations

def fix(
self,
context: RepositoryContext,
violations: List[RuleViolation],
*,
provider: Any = None,
) -> List[AutofixResult]:
results: List[AutofixResult] = []
if not violations:
return results

plugins_md_path = context.root_path / "PLUGINS.md"
data_json_path = context.root_path / "docs" / "data.json"

try:
(
original_plugins_md, generated_plugins_md,
original_data_json, generated_data_json,
) = self._run_generators(context)
except Exception:
return results
Comment on lines +141 to +147

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | 🏗️ Heavy lift

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Look for how AutofixResult is consumed and whether the framework writes fixed_content to disk.
fd -t f 'rule.py' .skillsaw skillsaw src 2>/dev/null
rg -nP -C5 '\bAutofixResult\b' --type=py
rg -nP -C5 '\bfixed_content\b|\bapply_fix\b|\bapply\s*\(' --type=py
# Also check other rules in this repo to see the established fix() contract.
fd -t f '_rule.py|rule.py' .skillsaw

Repository: openshift-eng/ai-helpers

Length of output: 196


🏁 Script executed:

# First, check the size and read the plugindocs_rule.py file
wc -l .skillsaw/plugindocs_rule.py
head -150 .skillsaw/plugindocs_rule.py | tail -50

Repository: openshift-eng/ai-helpers

Length of output: 1853


🏁 Script executed:

# Read the full check() method to understand the restore logic
rg -n "def check" .skillsaw/plugindocs_rule.py -A 30

Repository: openshift-eng/ai-helpers

Length of output: 1261


🏁 Script executed:

# Read the full fix() method to understand current behavior
rg -n "def fix" .skillsaw/plugindocs_rule.py -A 30

Repository: openshift-eng/ai-helpers

Length of output: 1297


🏁 Script executed:

# Look at the other rule file to understand the established pattern
cat -n .skillsaw/promptfoo_budget_rule.py | head -80

Repository: openshift-eng/ai-helpers

Length of output: 3100


🏁 Script executed:

# Search for AutofixResult definition and usage
rg -n "class AutofixResult|AutofixResult\(" --type=py -B2 -A5

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Look for how the autofix framework handles results
rg -n "def.*fix|AutofixResult" --type=py | head -20

Repository: openshift-eng/ai-helpers

Length of output: 845


🏁 Script executed:

# Search for AutofixResult definition and imports
rg -n "AutofixResult|AutofixConfidence" --type=py -B2 -A3 | head -50

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Get the full fix() method and what comes after
sed -n '127,178p' .skillsaw/plugindocs_rule.py

Repository: openshift-eng/ai-helpers

Length of output: 2062


🏁 Script executed:

# Search for imports at the top of the file
head -40 .skillsaw/plugindocs_rule.py

Repository: openshift-eng/ai-helpers

Length of output: 1557


🏁 Script executed:

# Look for how AutofixResult is used elsewhere in the repo
rg -n "AutofixResult\(" --type=py -B1 -A2

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Search for AutofixResult definition
rg -n "class AutofixResult|def.*AutofixResult" --type=py -B2 -A10

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Look for where AutofixResult is imported from and how it's consumed
rg -n "from.*AutofixResult|import.*AutofixResult" --type=py

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Search for how the framework processes/applies AutofixResult
rg -n "fixed_content|apply.*fix|_apply" --type=py | head -30

Repository: openshift-eng/ai-helpers

Length of output: 351


🏁 Script executed:

# Check if there's a base Rule class or interface that documents the fix() contract
fd -name "*.py" -path "*/skillsaw/*" | head -20

Repository: openshift-eng/ai-helpers

Length of output: 299


🏁 Script executed:

# Look at the complete _run_generators method to understand what it does
sed -n '40,70p' .skillsaw/plugindocs_rule.py

Repository: openshift-eng/ai-helpers

Length of output: 1260


🏁 Script executed:

# Look for src/rule.py or skillsaw module in the repo
find . -type f -name "rule.py" -o -name "__init__.py" | grep -E "src|skillsaw" | head -20

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Check what's imported from skillsaw - look for the package
rg -n "from skillsaw|import skillsaw" --type=py | head -10

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Search the repo for where AutofixResult is actually consumed/applied
rg -n "\.fixed_content|result\." --type=py | grep -i fix | head -20

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Look for the full file to understand the complete implementation
wc -l .skillsaw/plugindocs_rule.py
cat -n .skillsaw/plugindocs_rule.py | tail -30

Repository: openshift-eng/ai-helpers

Length of output: 1664


🏁 Script executed:

# Check if there are any git references or docs about the autofix contract
find . -type f -name "*.md" -o -name "*.txt" | xargs grep -l "AutofixResult\|autofix" 2>/dev/null | head -5

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Check the git log for this file to understand intent
git log --oneline .skillsaw/plugindocs_rule.py 2>/dev/null | head -10

Repository: openshift-eng/ai-helpers

Length of output: 127


🏁 Script executed:

# Check if there are any comments or TODOs in the file
grep -n "TODO\|FIXME\|XXX\|NOTE" .skillsaw/plugindocs_rule.py

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Look at the complete check() method once more to see the full restoration pattern
sed -n '72,126p' .skillsaw/plugindocs_rule.py

Repository: openshift-eng/ai-helpers

Length of output: 2079


🏁 Script executed:

# Search for other uses of AutofixResult in the repo to understand the pattern
rg -n "AutofixResult\|fixed_content\|original_content" --type=py -B2 -A4 | head -80

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Check if promptfoo_budget_rule.py has a fix() method
rg -n "def fix" .skillsaw/promptfoo_budget_rule.py -A 20

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Look for any other custom rules
fd -t f "_rule.py\|_rule.py" .skillsaw

Repository: openshift-eng/ai-helpers

Length of output: 50


🏁 Script executed:

# Check git diff for the commit that added autofix support
git show 9fae9a0 --stat 2>/dev/null

Repository: openshift-eng/ai-helpers

Length of output: 33320


🏁 Script executed:

# Check if there's documentation or README about how fix() should work
find . -name "README*" -o -name "*.md" | xargs grep -l "fix\|autofix" 2>/dev/null | head -5

Repository: openshift-eng/ai-helpers

Length of output: 204


🏁 Script executed:

# Check if skillsaw is a package we can inspect
python3 -c "import sys; sys.path.insert(0, '.'); from skillsaw import AutofixResult; help(AutofixResult)" 2>/dev/null || echo "skillsaw not importable"

Repository: openshift-eng/ai-helpers

Length of output: 92


Address exception handling and clarify the autofix contract for file mutation.

Two issues in this block:

  1. except Exception at line 146 silently returns empty results with no logging, making failed regeneration indistinguishable from "nothing to fix" to the caller. This matches Ruff BLE001. Log the exception or return a sentinel (e.g., confidence=UNSAFE) so the autofix runner can surface the error.

  2. fix() leaves PLUGINS.md and docs/data.json mutated on disk as side effects, while also providing original_content/fixed_content in AutofixResult. Unlike check(), which restores originals after generation, fix() does not. Clarify whether the autofix framework applies fixed_content to disk (in which case fix() should restore originals and not mutate files) or whether fix() is expected to write directly (in which case omit the redundant original_content/fixed_content fields from AutofixResult). Align this method's behavior with check()'s restore-on-mismatch semantics.

🧰 Tools
🪛 Ruff (0.15.12)

[warning] 146-146: Do not catch blind exception: Exception

(BLE001)

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In @.skillsaw/plugindocs_rule.py around lines 141 - 147, The try/except around
self._run_generators currently swallows all exceptions and returns results;
change it to catch Exception as e and either log the exception via the module
logger or return an AutofixResult sentinel (e.g., with confidence=UNSAFE) so
callers can distinguish failures from "no fixes"; also decide and implement
consistent file-mutation semantics for fix(): either (A) treat fix() like
check() by restoring original files after producing AutofixResult (so keep
original_content/fixed_content fields and ensure PLUGINS.md and docs/data.json
are restored after generation), or (B) treat fix() as the authoritative writer
(have fix() write the files and remove original_content/fixed_content from
AutofixResult); update the fix() implementation accordingly and document the
chosen behavior so it aligns with check(). Ensure you reference and modify
_run_generators, fix(), check(), and AutofixResult logic when making these
changes.


plugins_md_violations = [v for v in violations if v.file_path == plugins_md_path]
data_json_violations = [v for v in violations if v.file_path == data_json_path]

if plugins_md_violations and original_plugins_md != generated_plugins_md:
results.append(
AutofixResult(
rule_id=self.rule_id,
file_path=plugins_md_path,
confidence=AutofixConfidence.SAFE,
original_content=original_plugins_md,
fixed_content=generated_plugins_md,
description="Regenerated PLUGINS.md from plugin metadata",
violations_fixed=plugins_md_violations,
)
)

if data_json_violations and original_data_json != generated_data_json:
results.append(
AutofixResult(
rule_id=self.rule_id,
file_path=data_json_path,
confidence=AutofixConfidence.SAFE,
original_content=original_data_json or "",
fixed_content=generated_data_json or "",
description="Regenerated docs/data.json from plugin metadata",
violations_fixed=data_json_violations,
)
)

return results
7 changes: 6 additions & 1 deletion docs/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,11 @@
"id": "analyze-payload",
"name": "Analyze Payload"
},
{
"description": "Build a hermetic eval archive from an original Claude payload agent session \u2014 extracts all tool call responses, downloads GCS artifacts, and saves reference outputs",
"id": "archive-payload-result",
"name": "Archive Payload Result"
},
{
"description": "Fetch JIRA issue details including status, assignee, comments, and progress classification",
"id": "fetch-jira-issue",
Expand Down Expand Up @@ -612,7 +617,7 @@
"name": "Trigger Payload Job"
}
],
"version": "0.0.40"
"version": "0.0.41"
},
{
"commands": [
Expand Down
Loading