build.py

#!/usr/bin/env python3 """ build.py - rebuild this package and get the same bytes we published. WHY THIS SHIPS INSIDE THE PACKAGE --------------------------------- The page asks you to trust that the tarball you downloaded contains the source we published. Do not. Run this instead: python3 build.py It rebuilds the archive from the files around it and prints a sha256. If that matches the checksum on the page, you have proved the artifact is exactly this source - WITHOUT TRUSTING US AT ALL. A build script only we possessed would prove nothing, so it is in the tarball: the thing you downloaded contains the thing that rebuilds it. WHAT MAKES A BUILD REPRODUCIBLE - none of it is the code -------------------------------------------------------- Two builds of identical files still differ unless the archive METADATA is pinned: · FILE ORDER filesystem order varies by machine; we sort by name · MTIMES pinned to a fixed release date, never now(). If the timestamp were the moment of building, nobody could ever reproduce it and this whole feature would be theatre · OWNER / GROUP 0/0 with empty names, not whoever happened to build it · PERMISSIONS normalised: 0755 for directories, 0644 for files · GZIP HEADER gzip embeds a timestamp of its own by default; pinned to 0 NO NETWORK, BY CONSTRUCTION --------------------------- python3 and the standard library. No pip install, no downloads, no toolchain fetch. The package promises that grepping it for a URL returns nothing; a build script that fetched something would put one straight back. """ from __future__ import annotations import gzip import hashlib import io import sys import tarfile from pathlib import Path # The release date. A REAL FIXED FACT - deliberately not now(), because a build # stamped with its own build time can never be reproduced by anyone else. SOURCE_DATE_EPOCH = 1787270400 # 2026-08-20 00:00:00 UTC ARCHIVE_ROOT = "friday-recall" # Anything here is build output or local state, never source. EXCLUDE_NAMES = {"__pycache__", ".git", ".DS_Store"} # Matched against the END OF THE FILE NAME, not against Path.suffix. # # This list used to be compared with p.suffix, and it contained ".tar.gz". Path.suffix # for "thing.tar.gz" is ".gz" - so that entry could never match anything, and the check # quietly passed every previous release tarball straight into the next one. A build # ended up carrying a superseded build inside it, and its checksum depended on which # leftover files happened to be sitting in the directory. Nobody rebuilding from clean # source could have reproduced it. # # The lesson is the one this project keeps relearning: an exclusion that never fires # looks exactly like an exclusion that works. # # ".sha256" is here for the same reason as ".tar.gz". The checksum file published # beside a release lives in this directory too, and being plain text it sailed past # every archive check - so a build quietly packaged the checksum of the build before # it. Harmless-looking, and exactly the shape of the bug above: the artifact's # contents depending on which leftovers happened to be lying around. EXCLUDE_ENDS = (".pyc", ".tar.gz", ".tgz", ".zip", ".db", ".db-wal", ".db-shm", ".log", ".sha256", ".bak", ".swp", ".orig", ".rej") def sources(root: Path) -> list[Path]: out = [] for p in root.rglob("*"): rel_parts = p.relative_to(root).parts # DOT-DIRECTORIES ARE EXCLUDED, not just dot-files. # # The old test was `p.name.startswith(".")`, which only looks at the LAST # component. A tool directory like a test-runner cache is not a dot-file, but # the ordinary file inside it is not either - so `.somecache/README.md` sailed # through and got packaged. That file happened to contain a URL, which would # have broken the package's central promise, and because it only appears after # someone runs the tests, the resulting tarball had a different checksum than # the same source built on a clean tree. A stranger following our own # instructions - run the tests, then rebuild - would have got a mismatch and # concluded, reasonably, that we lied about the build being reproducible. if any(part in EXCLUDE_NAMES or part.startswith(".") for part in rel_parts): continue if p.is_file() and not p.name.endswith(EXCLUDE_ENDS): out.append(p) # SORT BY THE PATH AS TEXT, NOT BY THE PATH OBJECT. # # Comparing Path objects uses the platform's own idea of order, and on Windows # that is case-folded: README.md sorts BEFORE bin/ on Linux and AFTER it on # Windows. Same files, same bytes, different order in the archive - so the same # source produced two different checksums depending on which machine built it, # and the one promise this package makes to a stranger is "check the sha # yourself". Comparing the forward-slash string compares code points, which is # the same answer everywhere. out.sort(key=lambda q: q.relative_to(root).as_posix()) # THE GUARD IS WRITTEN OUT, NOT DERIVED FROM THE LIST IT IS GUARDING. # # This check reads its own literals on purpose. Built from EXCLUDE_ENDS it would # only ever restate whatever that tuple already said - it could not fail, and a # check that cannot fail looks exactly like one that works. The bug it exists to # catch was a broken entry IN that tuple, so a guard derived from the tuple would # have been broken in the same way and stayed silent. # # Also checks by magic bytes, not just by name, so a release renamed to something # innocent still cannot ride along inside the next one. _ARCHIVE_MAGIC = (b"\x1f\x8b", b"PK\x03\x04", b"BZh", b"\xfd7zXZ") smuggled = [] for f in out: if f.name.endswith((".tar.gz", ".tgz", ".zip", ".whl", ".gz", ".bz2", ".xz")): smuggled.append(f.name); continue try: head = f.open("rb").read(4) except Exception: continue if any(head.startswith(m) for m in _ARCHIVE_MAGIC): smuggled.append(f"{f.name} (an archive, whatever it is named)") if smuggled: raise SystemExit(f"BUILD REFUSED: build output would be packaged as source: {smuggled}") # Sorted by the path INSIDE the archive, so the order cannot depend on where # the tree happens to live on this machine. return sorted(out, key=lambda f: str(f.relative_to(root)).replace("\\", "/")) def build(root: Path) -> bytes: raw = io.BytesIO() with tarfile.open(fileobj=raw, mode="w", format=tarfile.GNU_FORMAT) as tar: seen_dirs: set[str] = set() for f in sources(root): rel = str(f.relative_to(root)).replace("\\", "/") # emit parent directories explicitly, also normalised parts = rel.split("/")[:-1] for i in range(len(parts)): d = "/".join(parts[: i + 1]) if d in seen_dirs: continue seen_dirs.add(d) di = tarfile.TarInfo(f"{ARCHIVE_ROOT}/{d}") di.type = tarfile.DIRTYPE di.mode = 0o755 di.mtime = SOURCE_DATE_EPOCH di.uid = di.gid = 0 di.uname = di.gname = "" tar.addfile(di) data = f.read_bytes() ti = tarfile.TarInfo(f"{ARCHIVE_ROOT}/{rel}") ti.size = len(data) ti.mode = 0o644 ti.mtime = SOURCE_DATE_EPOCH ti.uid = ti.gid = 0 ti.uname = ti.gname = "" ti.type = tarfile.REGTYPE tar.addfile(ti, io.BytesIO(data)) # gzip writes its own timestamp unless told otherwise; pin it to 0. out = io.BytesIO() with gzip.GzipFile(fileobj=out, mode="wb", compresslevel=9, mtime=0) as gz: gz.write(raw.getvalue()) return out.getvalue() def main() -> int: root = Path(__file__).resolve().parent files = sources(root) blob = build(root) sha = hashlib.sha256(blob).hexdigest() out = root / f"friday-recall-standalone-{sha[:12]}.tar.gz" if "--no-write" not in sys.argv: out.write_bytes(blob) print(f" files {len(files)}") print(f" size {len(blob):,} bytes") print(f" sha256 {sha}") if "--no-write" not in sys.argv: print(f" written {out.name}") print() print(" If that sha256 matches the one published beside the download, the") print(" tarball you have is exactly this source. You did not have to trust us.") return 0 if __name__ == "__main__": sys.exit(main())