Appeng 4903/phase 1 validation

.gitignore

@@ -9,8 +9,10 @@
 .cursor/
 .claude/
 
-# Plan review suggestions (working documents, not deliverables)
+# Review and planning working documents (not deliverables)
 Docs/plan_suggestions/
+Docs/commit_reviews/
+Docs/plan_reviews/
 
 # Python
 __pycache__/

pyproject.toml

@@ -33,7 +33,7 @@ target-version = "py311"
 line-length = 100
 
 [tool.ruff.lint]
-select = ["E", "F", "I", "W"]
+select = ["E", "F", "I", "UP", "W"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["skillsevalflow"]

scripts/validate.py

@@ -0,0 +1,150 @@
+"""Validate a skill submission directory against the submission contract.
+
+Checks:
+  1. instruction.md exists and is non-empty
+  2. skills/ directory contains SKILL.md (canonical name for agent recognition)
+  3. tests/test_outputs.py compiles
+  4. tests/llm_judge.py compiles if present
+  5. metadata.yaml passes Pydantic schema validation
+  6. supportive/ total size < 50 MB
+
+Exit codes: 0 = pass, 1 = validation failure (structured JSON on stdout).
+"""
+
+import argparse
+import ast
+import json
+import logging
+import sys
+from pathlib import Path
+
+import yaml
+from pydantic import ValidationError
+
+from skillsevalflow.schemas import SubmissionMetadata
+
+logger = logging.getLogger(__name__)
+
+MAX_SUPPORTIVE_SIZE_BYTES = 50 * 1024 * 1024  # 50 MB
+
+
+def _check_instruction_md(submission_dir: Path) -> list[str]:
+    instruction = submission_dir / "instruction.md"
+    if not instruction.is_file():
+        return ["instruction.md is missing"]
+    if not instruction.read_text().strip():
+        return ["instruction.md is empty"]
+    return []
+
+
+def _check_skills_dir(submission_dir: Path) -> list[str]:
+    """Validate skills/ contains SKILL.md — the canonical filename agents auto-recognize."""
+    skills_dir = submission_dir / "skills"
+    if not skills_dir.is_dir():
+        return ["skills/ directory is missing"]
+    skill_file = skills_dir / "SKILL.md"
+    if not skill_file.is_file():
+        return ["skills/SKILL.md is missing (must be exactly 'SKILL.md')"]
+    if not skill_file.read_text().strip():
+        return ["skills/SKILL.md is empty"]
+    return []
+
+
+def _check_py_compiles(file_path: Path) -> list[str]:
+    if not file_path.is_file():
+        return [f"{file_path.name} is missing"]
+    try:
+        source = file_path.read_text()
+        ast.parse(source, filename=str(file_path))
+    except SyntaxError as exc:
+        return [f"{file_path.name} does not compile: {exc}"]
+    return []
+
+
+def _check_metadata_yaml(submission_dir: Path) -> tuple[list[str], SubmissionMetadata | None]:
+    metadata_path = submission_dir / "metadata.yaml"
+    if not metadata_path.is_file():
+        return ["metadata.yaml is missing"], None
+    try:
+        raw = yaml.safe_load(metadata_path.read_text())
+    except yaml.YAMLError as exc:
+        return [f"metadata.yaml is not valid YAML: {exc}"], None
+    if not isinstance(raw, dict):
+        return ["metadata.yaml must contain a YAML mapping"], None
+    try:
+        model = SubmissionMetadata(**raw)
+    except ValidationError as exc:
+        errors = [
+            f"metadata.yaml validation: {e['msg']} ({'.'.join(str(loc) for loc in e['loc'])})"
+            for e in exc.errors()
+        ]
+        return errors, None
+    return [], model
+
+
+def _check_supportive_size(submission_dir: Path) -> list[str]:
+    supportive_dir = submission_dir / "supportive"
+    if not supportive_dir.is_dir():
+        return []
+    total_size = sum(f.stat().st_size for f in supportive_dir.rglob("*") if f.is_file())
+    if total_size > MAX_SUPPORTIVE_SIZE_BYTES:
+        size_mb = total_size / (1024 * 1024)
+        return [f"supportive/ exceeds 50 MB limit ({size_mb:.1f} MB)"]
+    return []
+
+
+def validate_submission(submission_dir: Path) -> list[str]:
+    """Run all validation checks and return a list of error strings (empty = valid)."""
+    logger.info("Validating submission: %s", submission_dir)
+    errors: list[str] = []
+
+    # Parse metadata first — generation_mode determines which files are required.
+    metadata_errors, metadata = _check_metadata_yaml(submission_dir)
+    errors.extend(metadata_errors)
+
+    # Currently only "manual" mode is implemented. In manual mode the submitter
+    # provides instruction.md and test_outputs.py. When "ai" mode is built
+    # (Phase 7), those files will be generated by the pipeline and these checks
+    # should be skipped for ai-mode submissions.
+    is_manual = metadata is None or metadata.generation_mode == "manual"
+
+    errors.extend(_check_skills_dir(submission_dir))
+
+    if is_manual:
+        errors.extend(_check_instruction_md(submission_dir))
+        errors.extend(_check_py_compiles(submission_dir / "tests" / "test_outputs.py"))
+    else:
+        logger.info("AI generation mode — skipping instruction.md and test_outputs.py checks")
+
+    llm_judge = submission_dir / "tests" / "llm_judge.py"
+    if llm_judge.is_file():
+        errors.extend(_check_py_compiles(llm_judge))
+
+    errors.extend(_check_supportive_size(submission_dir))
+
+    if errors:
+        logger.warning("Validation failed with %d error(s)", len(errors))
+    else:
+        logger.info("Validation passed")
+    return errors
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Validate a skill submission directory")
+    parser.add_argument("submission_dir", type=Path, help="Path to the submission directory")
+    args = parser.parse_args(argv)
+
+    submission_dir: Path = args.submission_dir
+    if not submission_dir.is_dir():
+        result = {"valid": False, "errors": [f"Not a directory: {submission_dir}"]}
+        print(json.dumps(result, indent=2))
+        return 1
+
+    errors = validate_submission(submission_dir)
+    result = {"valid": len(errors) == 0, "errors": errors}
+    print(json.dumps(result, indent=2))
+    return 0 if result["valid"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillsevalflow/schemas.py

@@ -0,0 +1,78 @@
+"""Pydantic models for skill submission metadata validation.
+
+The schema defines the structure of metadata.yaml files that accompany
+skill submissions. The schema_version field tracks the format version
+so the pipeline can handle older submissions gracefully when the schema
+evolves (e.g., new fields added, defaults changed).
+
+Current schema version: 1.0
+"""
+
+import re
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+CURRENT_SCHEMA_VERSION = "1.0"
+
+_SCHEMA_VERSION_RE = re.compile(r"\d+\.\d+")
+
+
+class GenerationMode(StrEnum):
+    MANUAL = "manual"
+    AI = "ai"
+
+
+class SubmissionMetadata(BaseModel):
+    """Schema for metadata.yaml in a skill submission directory.
+
+    Only 'name' is required. All other fields have sensible defaults so
+    that a minimal metadata.yaml can be as simple as:
+
+        name: my-skill
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    schema_version: str = Field(
+        default=CURRENT_SCHEMA_VERSION,
+        description=(
+            "Format version of this metadata file. Defaults to the current "
+            "version. The pipeline uses this to detect older submissions and "
+            "apply any necessary migration or compatibility logic."
+        ),
+    )
+
+    @field_validator("schema_version")
+    @classmethod
+    def _validate_schema_version(cls, v: str) -> str:
+        if not _SCHEMA_VERSION_RE.fullmatch(v):
+            raise ValueError("schema_version must be in 'MAJOR.MINOR' format (e.g. '1.0')")
+        return v
+
+    name: str = Field(min_length=1, description="Skill name, must be non-empty")
+    description: str | None = Field(default=None, description="Brief description of the skill")
+    persona: str | None = Field(
+        default=None,
+        description="Target persona (e.g. rh-sre, rh-developer). Used as category in Harbor.",
+    )
+    version: str = Field(default="0.1.0", min_length=1, description="Skill version string")
+    author: str | None = Field(default=None, description="Author or team name")
+    tags: list[str] | None = Field(default=None, description="Optional classification tags")
+    generation_mode: GenerationMode = Field(
+        default=GenerationMode.MANUAL,
+        description=(
+            "Whether the submission includes hand-written tests (manual) or "
+            "expects the pipeline to generate instruction/tests from the skill (ai). "
+            "AI mode is not yet implemented; defaults to manual."
+        ),
+    )
+
+    # Harbor timeout and resource configuration (all optional with defaults)
+    agent_timeout_sec: float = Field(default=600.0, gt=0, description="Agent solving timeout")
+    agent_setup_timeout_sec: float = Field(default=600.0, gt=0, description="Agent install timeout")
+    verifier_timeout_sec: float = Field(default=120.0, gt=0, description="Test runner timeout")
+    build_timeout_sec: float = Field(default=600.0, gt=0, description="Image build timeout")
+    cpus: int = Field(default=1, gt=0, description="CPU cores for trial container")
+    memory_mb: int = Field(default=2048, gt=0, description="Memory in MB for trial container")
+    storage_mb: int = Field(default=10240, gt=0, description="Storage in MB for trial container")