tests/mutation_proof.py

#!/usr/bin/env python3 """Prove each fix is load-bearing by breaking it and watching the suite go red. A green suite only means something if it can go red. This reverts each fix to the behaviour it replaced, reruns the contract tests, and requires FAILURE. ⚠️ THE MUTATION MUST PROVE IT APPLIED. An earlier round of this patched text that was not in the file at all: the "broken" run and the "working" run produced an identical checksum, and only that identity gave the game away. So every mutation asserts its anchor exists BEFORE replacing, and the run aborts if it does not - a mutation that silently changed nothing would report the suite as unbreakable, which is the most flattering possible lie. """ from __future__ import annotations import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent BIN = ROOT / "bin" TESTS = ROOT / "tests" / "test_hook_contract.py" WIRING = ROOT / "tests" / "test_install_wiring.py" # Spelled out once so the mutation and the real file cannot drift apart. EXEC_OLD = 'entry = {"type": "command", "command": sys.executable,\n "args": [hook, arg], "timeout": timeout}' EXEC_NEW = 'entry = {"type": "command",\n "command": f\'"{sys.executable}" "{hook}" {arg}\', "timeout": timeout}' MUTATIONS = [ ("PreCompact goes back to the JSON envelope the harness rejects", BIN / "hook.py", ' print(text)\n', ' print(json.dumps({"hookSpecificOutput": {"hookEventName": "PreCompact",\n' ' "additionalContext": text}}))\n', "pre-compact does NOT emit the JSON hook envelope"), # Reverting only the elif does NOT reproduce the incident: compact would still be # routed to its carry slot and would still never touch the handoff queue. The # original code had no gate at all - it took a handoff whichever door it came in # by - so the mutation has to remove BOTH halves or it proves the wrong thing. ("SessionStart stops looking at which door it came in by (the 13:35 leak)", BIN / "hook.py", [(' if source == "compact":', ' if False:'), (' elif source in ("startup", ""):', ' elif True:')], None, "compact SessionStart does NOT consume another session"), ("rule mining goes back to trusting every quoted span in a summary", BIN / "brief.py", "_user_voiced(out[\"carried\"])", "out[\"carried\"]", "a quoted DOCSTRING is not promoted"), # The installer defects. A Windows check that a Linux developer cannot break is # not a check, so each of these puts the Windows-broken form back. ("hooks go back to the one-string form PowerShell refuses to run", ROOT / "install.py", [(EXEC_OLD, EXEC_NEW)], None, "shell-less exec form", WIRING), ("SessionEnd goes back to running in the background", ROOT / "install.py", [(' put("SessionEnd", None, "session-end", 60)', ' put("SessionEnd", None, "session-end", 60, is_async=True)')], None, "SessionEnd is NOT async", WIRING), ("hook de-duplication goes back to matching a word in the path", ROOT / "install.py", [(' return "hook.py" in blob and any(e in blob for e in EVENTS)', ' return "hook.py" in blob and "recall" in blob.lower()')], None, "renamed folder still does not double", WIRING), ("the request log goes back to being uncapped 'intent'", BIN / "brief.py", [(" ask_lines = [f\"- {a}\" for a in asks[-ASK_KEEP:]]", " ask_lines = [f\"- {a}\" for a in asks]")], None, "capped instead of handing the window back"), # BOTH halves, deliberately. Stripping the task-notification span and rejecting # leftover markup are two independent defences against the same thing, and # removing either one alone leaves the test green - which is what defence in depth # is supposed to look like. Mutating only one would have "proved" a fix that was # actually being covered by its neighbour, so this removes the pair. ("BOTH task-notification defences removed - harness text counts as user speech", BIN / "brief.py", [(' ("<task-notification>", "</task-notification>"),', ""), (' if "</" in piece or "<note>" in piece or "<summary>" in piece:\n' ' continue', "")], None, "task-notification does not become a standing order"), ("the word-count floor goes back to four, dropping real three-word rules", BIN / "brief.py", [(" if len(piece.split()) < 3:", " if len(piece.split()) < 4:")], None, "THREE-word absolute still survives"), ("take goes back to picking one when several are waiting", BIN / "handoff.py", ' if len(cands) > 1:', ' if False:', "two candidates: consumes NEITHER"), ] def run_suite(which=None) -> tuple[int, str]: r = subprocess.run([sys.executable, str(which or TESTS)], capture_output=True, text=True) return r.returncode, r.stdout def main() -> int: base_rc, _ = run_suite() wire_rc, _ = run_suite(WIRING) if base_rc != 0 or wire_rc != 0: print("baseline is already RED - fix that before mutating. Nothing proved.") return 1 print("baseline: green\n") bad = 0 for entry in MUTATIONS: label, path, old, new, expect = entry[:5] suite = entry[5] if len(entry) > 5 else None edits = old if isinstance(old, list) else [(old, new)] # BYTES, NOT TEXT. Reading and writing this as text rewrites every line # ending to the platform's own on Windows, so simply RUNNING this harness # permanently changed four files in the package and its checksum with them - # the very number the README tells people to verify. A tool that alters the # thing it is measuring is not a measurement. src = path.read_bytes() mutated = src for o, n in edits: o, n = o.encode("utf-8"), n.encode("utf-8") if mutated.count(o) != 1: print(f"ABORT [{label}]: anchor appears {mutated.count(o)} times in " f"{path.name}: {o[:60]!r}") print(" The mutation did NOT apply. Do not read the result below as anything.") return 1 mutated = mutated.replace(o, n, 1) if mutated == src: print(f"ABORT [{label}]: file unchanged after mutation. Nothing was proved.") return 1 try: path.write_bytes(mutated) rc, out = run_suite(suite) named = [l for l in out.splitlines() if l.strip().startswith("FAIL")] hit = any(expect in l for l in named) if rc != 0 and hit: print(f" ok RED as required: {label}") print(f" -> {[l.strip()[5:][:70] for l in named][:3]}") else: bad += 1 print(f" FAIL suite survived: {label}") print(f" expected a failure naming: {expect}") print(f" got rc={rc}, failures={[l.strip()[5:][:60] for l in named][:3]}") finally: path.write_bytes(src) rc, _ = run_suite() rc2, _ = run_suite(WIRING) rc = rc or rc2 print(f"\nrestored baseline: {'green' if rc == 0 else 'RED - RESTORE FAILED'}") if rc != 0: return 1 print(f"---- {len(MUTATIONS) - bad}/{len(MUTATIONS)} fixes proved load-bearing ----") return 1 if bad else 0 if __name__ == "__main__": raise SystemExit(main())