Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ ai-harness-scorecard assess . --format json

## What It Checks

Five categories, 31 checks, each grounded in published research:
Five categories, 32 checks, each grounded in published research:

### 1. Architectural Documentation (20%)
Architecture docs, agent instructions, ADRs, module boundary constraints, API documentation.
Architecture docs, agent instructions, harness docs, ADRs, module boundary constraints, API documentation.
Comment thread
coderabbitai[bot] marked this conversation as resolved.

### 2. Mechanical Constraints (25%)
CI pipeline, linter/formatter enforcement, type safety, dependency auditing, conventional commits, unsafe code policies.
Expand Down Expand Up @@ -66,7 +66,7 @@ Good foundation. Some gaps in enforcement or feedback loops.
┌──────────────────────────┬────────┬───────┬────────┐
│ Category │ Weight │ Score │ Checks │
├──────────────────────────┼────────┼───────┼────────┤
│ Architectural Docs │ 20% │ 60% │ 3/5
│ Architectural Docs │ 20% │ 60% │ 3/6
│ Mechanical Constraints │ 25% │ 91% │ 6/7 │
│ Testing & Stability │ 25% │ 72% │ 5/8 │
│ Review & Drift │ 15% │ 60% │ 3/6 │
Expand Down
60 changes: 60 additions & 0 deletions src/ai_harness_scorecard/checks/documentation.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from __future__ import annotations

import re
from typing import TYPE_CHECKING

from .base import BaseCheck
Expand Down Expand Up @@ -66,6 +67,64 @@ def run(self, context: RepoContext) -> CheckResult:
)


class HarnessDocsCheck(BaseCheck):
check_id = "documentation.harness_docs"
name = "Harness Documentation"
description = "Quality pipeline, CI stages, or quality gates documented for contributors"
max_points = 2.0
source = "Morris 2026 - harness engineering"

DOCUMENTATION_FILES = [
"contributing.md",
"docs/*.md",
"docs/*.rst",
"doc/*.md",
"doc/*.rst",
]

PIPELINE_PATTERNS = [
r"\bci\s+(pipeline|stages?|workflow)\b",
r"\bquality\s+gates?\b",
r"\bpre-commit\b",
r"\bdevelopment\s+workflow\b",
r"how\s+to\s+add\s+(a\s+)?(new\s+)?(check|quality\s+gate|ci\s+job)",
r"run\s+in\s+ci\s+and\s+must\s+pass",
]

COMMENTED_CI_PATTERN = r"(?m)^\s*#.*\b(ci|quality|check|lint|test|type|gate|workflow)\b"

def run(self, context: RepoContext) -> CheckResult:
for pattern in self.PIPELINE_PATTERNS:
found = context.search_any_file(self.DOCUMENTATION_FILES, pattern)
if found:
return self.pass_result(f"Quality pipeline documented in {found}")

contributing = context.has_file("contributing.md")
if contributing:
return self.partial_result(
1.0,
f"Found {contributing}, but no documented quality pipeline",
"Document CI stages, quality gates, or how to add a new quality check.",
)

if context.ci_configs and re.search(
self.COMMENTED_CI_PATTERN,
context.ci_raw_content(),
re.IGNORECASE,
):
return self.partial_result(
1.0,
"CI config comments mention quality checks",
"Move CI stage and quality gate guidance into CONTRIBUTING.md or docs/.",
)

return self.fail_result(
"No quality pipeline documentation found",
"Document the quality pipeline, CI stages, quality gates, or how to add a "
"new check in CONTRIBUTING.md or docs/.",
)


class ADRPresenceCheck(BaseCheck):
check_id = "adr_presence"
name = "Architecture Decision Records"
Expand Down Expand Up @@ -189,6 +248,7 @@ def run(self, context: RepoContext) -> CheckResult:
DOCUMENTATION_CHECKS: list[BaseCheck] = [
ArchitectureDocCheck(),
AgentInstructionsCheck(),
HarnessDocsCheck(),
ADRPresenceCheck(),
ModuleBoundaryDocsCheck(),
APIContractsCheck(),
Expand Down
91 changes: 88 additions & 3 deletions tests/test_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@

from typing import TYPE_CHECKING

if TYPE_CHECKING:
from pathlib import Path

import pytest
from hypothesis import HealthCheck, given, settings
from hypothesis import strategies as st

from ai_harness_scorecard.repo_context import RepoContext

if TYPE_CHECKING:
from pathlib import Path


def _build_context(tmp_path: Path, files: dict[str, str] | None = None) -> RepoContext:
"""Build a RepoContext from a tmp_path with optional files."""
Expand Down Expand Up @@ -290,6 +292,89 @@ def test_fail_without_any(self, tmp_path: Path) -> None:
assert not result.passed


class TestHarnessDocsCheck:
def test_harness_docs_pass(self, tmp_path: Path) -> None:
from ai_harness_scorecard.checks.documentation import HarnessDocsCheck

context = _build_context(
tmp_path,
{
"docs/development.md": (
"# Development\n\n"
"## CI pipeline\n\n"
"The quality gates run lint, type checks, security scans, and tests. "
"To add a new check, update the CI workflow and document the new gate here."
)
},
)
result = HarnessDocsCheck().run(context)
assert result.check_id == "documentation.harness_docs"
assert result.passed
assert result.score == pytest.approx(2.0)
assert "quality pipeline" in result.evidence.lower()

def test_harness_docs_pass_partial(self, tmp_path: Path) -> None:
from ai_harness_scorecard.checks.documentation import HarnessDocsCheck

context = _build_context(tmp_path, {"CONTRIBUTING.md": "# Contributing\n\nWelcome."})
result = HarnessDocsCheck().run(context)
assert result.passed
assert result.score == pytest.approx(1.0)
assert "contributing.md" in result.evidence.lower()

def test_harness_docs_fail(self, tmp_path: Path) -> None:
from ai_harness_scorecard.checks.documentation import HarnessDocsCheck

context = _build_context(tmp_path, {"README.md": "# Project"})
result = HarnessDocsCheck().run(context)
assert not result.passed
assert result.score == pytest.approx(0.0)
assert "quality pipeline" in result.remediation.lower()

@given(
words=st.sampled_from(
[
("ci", "pipeline"),
("ci", "workflow"),
("quality", "gate"),
("quality", "gates"),
("development", "workflow"),
("run", "in", "ci", "and", "must", "pass"),
]
),
separator=st.sampled_from([" ", " ", "\t"]),
uppercase=st.booleans(),
)
@settings(suppress_health_check=[HealthCheck.function_scoped_fixture])
def test_harness_docs_pass_variants(
self,
tmp_path: Path,
words: tuple[str, ...],
separator: str,
uppercase: bool,
) -> None:
from ai_harness_scorecard.checks.documentation import HarnessDocsCheck

phrase = separator.join(words)
if uppercase:
phrase = phrase.upper()

context = _build_context(
tmp_path,
{
"docs/development.md": (
"# Development\n\n"
f"## {phrase}\n\n"
"Document the checks contributors run before merging changes."
)
},
)
result = HarnessDocsCheck().run(context)

assert result.passed
assert result.score == pytest.approx(2.0)


class TestLinterEnforcementCheck:
@pytest.mark.parametrize(
("files", "expected_score", "evidence_substring"),
Expand Down