Files
planning-with-files/tests/test_template_transparency.py

210 lines
8.2 KiB
Python

"""Transparency and runtime-authority checks for shipped planning templates."""
from __future__ import annotations
import os
import re
import subprocess
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
CANONICAL_TEMPLATES = REPO_ROOT / "skills" / "planning-with-files" / "templates"
ROOT_TEMPLATES = REPO_ROOT / "templates"
INIT_SCRIPTS = (
REPO_ROOT / "skills" / "planning-with-files" / "scripts" / "init-session.sh",
REPO_ROOT / "scripts" / "init-session.sh",
)
def tracked_install_facing_templates() -> tuple[Path, ...]:
"""Return every tracked Markdown template shipped on an install surface."""
result = subprocess.run(
["git", "ls-files", "-z"],
cwd=REPO_ROOT,
check=True,
capture_output=True,
)
paths: list[Path] = []
for raw in result.stdout.split(b"\0"):
if not raw:
continue
relative = raw.decode("utf-8").replace("\\", "/")
if relative.endswith(".md") and "templates" in relative.split("/"):
paths.append(REPO_ROOT / Path(relative))
return tuple(paths)
class TemplateTransparencyTests(unittest.TestCase):
def read_template(self, path: Path) -> str:
return path.read_text(encoding="utf-8")
def test_shipped_planning_templates_have_no_hidden_html_instructions(self) -> None:
paths = tracked_install_facing_templates()
self.assertGreater(len(paths), 50)
for path in paths:
with self.subTest(path=path):
body = self.read_template(path)
self.assertNotIn("<!--", body)
self.assertNotIn("-->", body)
self.assertNotIn("attention manipulation", body.lower())
def test_root_template_copies_match_canonical_sources(self) -> None:
for name in (
"analytics_findings.md",
"analytics_task_plan.md",
"findings.md",
"progress.md",
"task_plan.md",
):
with self.subTest(name=name):
self.assertEqual(
(CANONICAL_TEMPLATES / name).read_bytes(),
(ROOT_TEMPLATES / name).read_bytes(),
)
def test_auxiliary_template_headings_and_placeholders_are_preserved(self) -> None:
required_by_name = {
"findings.md": (
"## Requirements",
"## Research Findings",
"## Technical Decisions",
"## Issues Encountered",
"## Resources",
"## Visual/Browser Findings",
"| Decision | Rationale |",
"| Issue | Resolution |",
),
"progress.md": (
"## Session: [DATE]",
"### Phase 1: [Title]",
"### Phase 2: [Title]",
"## Test Results",
"## Error Log",
"## 5-Question Reboot Check",
"| Test | Input | Expected | Actual | Status |",
"| Timestamp | Error | Attempt | Resolution |",
),
"analytics_findings.md": (
"## Data Sources",
"## Hypothesis Log",
"## Query Results",
"## Statistical Findings",
"## Technical Decisions",
"## Issues Encountered",
"## Resources",
"## Visual/Browser Findings",
"| Source | Location | Size | Key Fields | Quality Notes |",
"| Hypothesis | Test Method | Result | Confidence |",
"| Test | p-value | Effect Size | Conclusion |",
),
"analytics_task_plan.md": (
"[One sentence describing the analytical objective]",
"## Next Step",
"[The single next analytical action. Update whenever phase status changes.]",
"## Current Phase",
"## Phases",
"## Hypotheses",
"## Decisions Made",
"## Errors Encountered",
"## Notes",
"1. [Hypothesis to test]",
"| Error | Attempt | Resolution |",
),
}
for name, required_tokens in required_by_name.items():
body = self.read_template(CANONICAL_TEMPLATES / name)
for token in required_tokens:
with self.subTest(name=name, token=token):
self.assertIn(token, body)
analytics_plan = self.read_template(CANONICAL_TEMPLATES / "analytics_task_plan.md")
self.assertEqual(
4,
sum(line.startswith("### Phase ") for line in analytics_plan.splitlines()),
)
self.assertEqual(4, analytics_plan.count("- **Status:**"))
self.assertEqual(1, analytics_plan.count("- **Status:** in_progress"))
self.assertEqual(3, analytics_plan.count("- **Status:** pending"))
def test_shipped_task_plan_templates_have_a_nonempty_next_step_section(self) -> None:
analytics_paths = [
path
for path in tracked_install_facing_templates()
if path.name == "analytics_task_plan.md"
]
self.assertTrue(analytics_paths, "no shipped analytics task plan templates found")
paths = [CANONICAL_TEMPLATES / "task_plan.md", *analytics_paths]
for path in paths:
with self.subTest(path=path):
body = self.read_template(path)
section = re.search(
r"(?ms)^## Next Step\s*\n(?P<body>.*?)(?=^## |\Z)",
body,
)
self.assertIsNotNone(section, f"{path} is missing ## Next Step")
assert section is not None
self.assertTrue(
section.group("body").strip(),
f"{path} has an empty ## Next Step section",
)
def test_autonomous_template_names_the_real_gate_authority(self) -> None:
body = self.read_template(CANONICAL_TEMPLATES / "task_plan_autonomous.md")
for required in (
".mode",
"phase state",
"Stop hook state",
"stop block cap",
"ledger progress",
"The gate never executes commands declared in this plan.",
):
with self.subTest(required=required):
self.assertIn(required, body)
def test_autonomous_template_does_not_advertise_unsupported_gate_fields(self) -> None:
body = self.read_template(CANONICAL_TEMPLATES / "task_plan_autonomous.md")
for unsupported in (
"## Run Contract",
"## Model Routing",
"**Owner:**",
"**DependsOn:**",
"**AcceptanceCheck:**",
):
with self.subTest(unsupported=unsupported):
self.assertNotIn(unsupported, body)
def test_template_phase_and_status_contract_is_preserved(self) -> None:
for name in ("task_plan.md", "task_plan_autonomous.md"):
with self.subTest(name=name):
body = self.read_template(CANONICAL_TEMPLATES / name)
self.assertEqual(5, sum(line.startswith("### Phase ") for line in body.splitlines()))
self.assertEqual(5, body.count("- **Status:**"))
self.assertEqual(1, body.count("- **Status:** in_progress"))
self.assertEqual(4, body.count("- **Status:** pending"))
def test_generated_default_plans_contain_no_hidden_instructions(self) -> None:
for script in INIT_SCRIPTS:
with self.subTest(script=script), tempfile.TemporaryDirectory() as tmp:
env = os.environ.copy()
env.pop("PLAN_ID", None)
result = subprocess.run(
["sh", str(script)],
cwd=tmp,
text=True,
encoding="utf-8",
capture_output=True,
env=env,
check=False,
)
self.assertEqual(0, result.returncode, result.stderr)
body = (Path(tmp) / "task_plan.md").read_text(encoding="utf-8")
self.assertNotIn("<!--", body)
self.assertNotIn("attention manipulation", body.lower())
if __name__ == "__main__":
unittest.main()