Repository navigation
Appeng 4903/phase 1 validation #1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
be980a8
feat: add submission schema and validation script
GuyZivRH 61cfabe
fix: address review feedback on schema and validation
GuyZivRH 71c8460
refactor: make metadata fields optional, add timeout/resource config
GuyZivRH a708aac
fix: use underscores in gitignore folder names for consistency
GuyZivRH File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| """Validate a skill submission directory against the submission contract. | ||
|
|
||
| Checks: | ||
| 1. instruction.md exists and is non-empty | ||
| 2. skills/ directory contains SKILL.md (canonical name for agent recognition) | ||
| 3. tests/test_outputs.py compiles | ||
| 4. tests/llm_judge.py compiles if present | ||
| 5. metadata.yaml passes Pydantic schema validation | ||
| 6. supportive/ total size < 50 MB | ||
|
|
||
| Exit codes: 0 = pass, 1 = validation failure (structured JSON on stdout). | ||
| """ | ||
|
|
||
| import argparse | ||
| import ast | ||
| import json | ||
| import logging | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| import yaml | ||
| from pydantic import ValidationError | ||
|
|
||
| from skillsevalflow.schemas import SubmissionMetadata | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| MAX_SUPPORTIVE_SIZE_BYTES = 50 * 1024 * 1024 # 50 MB | ||
|
|
||
|
|
||
| def _check_instruction_md(submission_dir: Path) -> list[str]: | ||
| instruction = submission_dir / "instruction.md" | ||
| if not instruction.is_file(): | ||
| return ["instruction.md is missing"] | ||
| if not instruction.read_text().strip(): | ||
| return ["instruction.md is empty"] | ||
| return [] | ||
|
|
||
|
|
||
| def _check_skills_dir(submission_dir: Path) -> list[str]: | ||
| """Validate skills/ contains SKILL.md — the canonical filename agents auto-recognize.""" | ||
| skills_dir = submission_dir / "skills" | ||
| if not skills_dir.is_dir(): | ||
| return ["skills/ directory is missing"] | ||
| skill_file = skills_dir / "SKILL.md" | ||
| if not skill_file.is_file(): | ||
| return ["skills/SKILL.md is missing (must be exactly 'SKILL.md')"] | ||
| if not skill_file.read_text().strip(): | ||
| return ["skills/SKILL.md is empty"] | ||
| return [] | ||
|
|
||
|
|
||
| def _check_py_compiles(file_path: Path) -> list[str]: | ||
| if not file_path.is_file(): | ||
| return [f"{file_path.name} is missing"] | ||
| try: | ||
| source = file_path.read_text() | ||
| ast.parse(source, filename=str(file_path)) | ||
| except SyntaxError as exc: | ||
| return [f"{file_path.name} does not compile: {exc}"] | ||
| return [] | ||
|
|
||
|
|
||
| def _check_metadata_yaml(submission_dir: Path) -> tuple[list[str], SubmissionMetadata | None]: | ||
| metadata_path = submission_dir / "metadata.yaml" | ||
| if not metadata_path.is_file(): | ||
| return ["metadata.yaml is missing"], None | ||
| try: | ||
| raw = yaml.safe_load(metadata_path.read_text()) | ||
| except yaml.YAMLError as exc: | ||
| return [f"metadata.yaml is not valid YAML: {exc}"], None | ||
| if not isinstance(raw, dict): | ||
| return ["metadata.yaml must contain a YAML mapping"], None | ||
| try: | ||
| model = SubmissionMetadata(**raw) | ||
| except ValidationError as exc: | ||
| errors = [ | ||
| f"metadata.yaml validation: {e['msg']} ({'.'.join(str(loc) for loc in e['loc'])})" | ||
| for e in exc.errors() | ||
| ] | ||
| return errors, None | ||
| return [], model | ||
|
|
||
|
|
||
| def _check_supportive_size(submission_dir: Path) -> list[str]: | ||
| supportive_dir = submission_dir / "supportive" | ||
| if not supportive_dir.is_dir(): | ||
| return [] | ||
| total_size = sum(f.stat().st_size for f in supportive_dir.rglob("*") if f.is_file()) | ||
| if total_size > MAX_SUPPORTIVE_SIZE_BYTES: | ||
| size_mb = total_size / (1024 * 1024) | ||
| return [f"supportive/ exceeds 50 MB limit ({size_mb:.1f} MB)"] | ||
| return [] | ||
|
|
||
|
|
||
| def validate_submission(submission_dir: Path) -> list[str]: | ||
| """Run all validation checks and return a list of error strings (empty = valid).""" | ||
| logger.info("Validating submission: %s", submission_dir) | ||
| errors: list[str] = [] | ||
|
|
||
| # Parse metadata first — generation_mode determines which files are required. | ||
| metadata_errors, metadata = _check_metadata_yaml(submission_dir) | ||
| errors.extend(metadata_errors) | ||
|
|
||
| # Currently only "manual" mode is implemented. In manual mode the submitter | ||
| # provides instruction.md and test_outputs.py. When "ai" mode is built | ||
| # (Phase 7), those files will be generated by the pipeline and these checks | ||
| # should be skipped for ai-mode submissions. | ||
| is_manual = metadata is None or metadata.generation_mode == "manual" | ||
|
|
||
| errors.extend(_check_skills_dir(submission_dir)) | ||
|
|
||
| if is_manual: | ||
| errors.extend(_check_instruction_md(submission_dir)) | ||
| errors.extend(_check_py_compiles(submission_dir / "tests" / "test_outputs.py")) | ||
| else: | ||
| logger.info("AI generation mode — skipping instruction.md and test_outputs.py checks") | ||
|
|
||
| llm_judge = submission_dir / "tests" / "llm_judge.py" | ||
| if llm_judge.is_file(): | ||
| errors.extend(_check_py_compiles(llm_judge)) | ||
|
|
||
| errors.extend(_check_supportive_size(submission_dir)) | ||
|
|
||
| if errors: | ||
| logger.warning("Validation failed with %d error(s)", len(errors)) | ||
| else: | ||
| logger.info("Validation passed") | ||
| return errors | ||
|
|
||
|
|
||
| def main(argv: list[str] | None = None) -> int: | ||
| parser = argparse.ArgumentParser(description="Validate a skill submission directory") | ||
| parser.add_argument("submission_dir", type=Path, help="Path to the submission directory") | ||
| args = parser.parse_args(argv) | ||
|
|
||
| submission_dir: Path = args.submission_dir | ||
| if not submission_dir.is_dir(): | ||
| result = {"valid": False, "errors": [f"Not a directory: {submission_dir}"]} | ||
| print(json.dumps(result, indent=2)) | ||
| return 1 | ||
|
|
||
| errors = validate_submission(submission_dir) | ||
| result = {"valid": len(errors) == 0, "errors": errors} | ||
| print(json.dumps(result, indent=2)) | ||
| return 0 if result["valid"] else 1 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| sys.exit(main()) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| """Pydantic models for skill submission metadata validation. | ||
|
|
||
| The schema defines the structure of metadata.yaml files that accompany | ||
| skill submissions. The schema_version field tracks the format version | ||
| so the pipeline can handle older submissions gracefully when the schema | ||
| evolves (e.g., new fields added, defaults changed). | ||
|
|
||
| Current schema version: 1.0 | ||
| """ | ||
|
|
||
| import re | ||
| from enum import StrEnum | ||
|
|
||
| from pydantic import BaseModel, ConfigDict, Field, field_validator | ||
|
|
||
| CURRENT_SCHEMA_VERSION = "1.0" | ||
|
|
||
| _SCHEMA_VERSION_RE = re.compile(r"\d+\.\d+") | ||
|
|
||
|
|
||
| class GenerationMode(StrEnum): | ||
| MANUAL = "manual" | ||
| AI = "ai" | ||
|
|
||
|
|
||
| class SubmissionMetadata(BaseModel): | ||
| """Schema for metadata.yaml in a skill submission directory. | ||
|
|
||
| Only 'name' is required. All other fields have sensible defaults so | ||
| that a minimal metadata.yaml can be as simple as: | ||
|
|
||
| name: my-skill | ||
| """ | ||
|
|
||
| model_config = ConfigDict(extra="forbid") | ||
|
|
||
| schema_version: str = Field( | ||
| default=CURRENT_SCHEMA_VERSION, | ||
| description=( | ||
| "Format version of this metadata file. Defaults to the current " | ||
| "version. The pipeline uses this to detect older submissions and " | ||
| "apply any necessary migration or compatibility logic." | ||
| ), | ||
| ) | ||
|
|
||
| @field_validator("schema_version") | ||
| @classmethod | ||
| def _validate_schema_version(cls, v: str) -> str: | ||
| if not _SCHEMA_VERSION_RE.fullmatch(v): | ||
| raise ValueError("schema_version must be in 'MAJOR.MINOR' format (e.g. '1.0')") | ||
| return v | ||
|
|
||
| name: str = Field(min_length=1, description="Skill name, must be non-empty") | ||
| description: str | None = Field(default=None, description="Brief description of the skill") | ||
| persona: str | None = Field( | ||
| default=None, | ||
| description="Target persona (e.g. rh-sre, rh-developer). Used as category in Harbor.", | ||
| ) | ||
| version: str = Field(default="0.1.0", min_length=1, description="Skill version string") | ||
| author: str | None = Field(default=None, description="Author or team name") | ||
| tags: list[str] | None = Field(default=None, description="Optional classification tags") | ||
| generation_mode: GenerationMode = Field( | ||
| default=GenerationMode.MANUAL, | ||
| description=( | ||
| "Whether the submission includes hand-written tests (manual) or " | ||
| "expects the pipeline to generate instruction/tests from the skill (ai). " | ||
| "AI mode is not yet implemented; defaults to manual." | ||
| ), | ||
| ) | ||
|
|
||
| # Harbor timeout and resource configuration (all optional with defaults) | ||
| agent_timeout_sec: float = Field(default=600.0, gt=0, description="Agent solving timeout") | ||
| agent_setup_timeout_sec: float = Field(default=600.0, gt=0, description="Agent install timeout") | ||
| verifier_timeout_sec: float = Field(default=120.0, gt=0, description="Test runner timeout") | ||
| build_timeout_sec: float = Field(default=600.0, gt=0, description="Image build timeout") | ||
| cpus: int = Field(default=1, gt=0, description="CPU cores for trial container") | ||
| memory_mb: int = Field(default=2048, gt=0, description="Memory in MB for trial container") | ||
| storage_mb: int = Field(default=10240, gt=0, description="Storage in MB for trial container") |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can we keep consistent naming (e.g. always with or w/o the "_" separator)?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
sure, np
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
a708aac