bin/inspect_store.py

""" inspect_store.py - say what is inside a leftover store, before asking about it. WHY THIS IS THE WHOLE POINT --------------------------- Uninstalling a memory tool does not remove what it harvested. The program goes; its copy of your work stays. Most people do not know that, and nobody deletes a folder because a tool suggested it. They act on the second line: this database holds 1,204 sessions and 25 passwords or keys in plain text it has been on your disk since 12 July That is the informed part of informed consent. We can say it truthfully because the redactor already recognises those patterns, so the number is one a person can check by eye in their own file. TWO RULES, both absolute: · COUNT AND KIND, NEVER THE VALUE. This prints "9 chat tokens". It does not print a token. A tool that displayed the secrets it found in order to warn you about them would be the joke it deserves to be. · READ-ONLY. Everything here opens the file read-only. Inspecting a stranger's database must not be able to change it. """ from __future__ import annotations import sqlite3 import sys from datetime import datetime, timezone from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from redact import scrub # noqa: E402 # Plain words for what was found. The rule ids are ours; these are for a person who # just wants to know whether it matters. PLAIN = { "private-key": "private keys", "anthropic-key": "API keys", "openai-key": "API keys", "api-key-sk": "API keys", "google-key": "API keys", "github-token": "code-host tokens", "slack-token": "chat tokens", "telegram-bot": "chat tokens", "aws-akid": "cloud keys", "jwt": "session tokens", "bearer": "authorisation headers", "basic": "authorisation headers", "apikey-header": "authorisation headers", "url-cred": "web addresses with logins in them", "assigned-secret": "passwords", "cloudflare": "cloud keys", } # Text columns worth scanning, per table. Anything not listed is ids and timestamps. SCAN = { "observations": ("text", "title", "subtitle", "facts", "narrative", "concepts"), "session_summaries": ("request", "investigated", "learned", "completed", "next_steps", "notes"), "user_prompts": ("prompt_text",), "sdk_sessions": ("user_prompt", "custom_title"), } def inspect(db: Path, sample_limit: int = 0) -> dict: """Count what is in there. Never returns a secret, only kinds and counts.""" con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) con.row_factory = sqlite3.Row names = {r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")} rows = {} kinds: dict[str, int] = {} scanned = 0 for table, cols in SCAN.items(): if table not in names: continue have = {r[1] for r in con.execute(f"PRAGMA table_info({table})")} use = [c for c in cols if c in have] rows[table] = con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] if not use: continue q = f"SELECT {','.join(use)} FROM {table}" if sample_limit: q += f" LIMIT {int(sample_limit)}" for r in con.execute(q): for v in r: if isinstance(v, str) and v: scanned += 1 _, n, labels = scrub(v) for lab in labels: kinds[lab] = kinds.get(lab, 0) + 1 con.close() st = db.stat() plain: dict[str, int] = {} for lab, n in kinds.items(): plain[PLAIN.get(lab, lab)] = plain.get(PLAIN.get(lab, lab), 0) + n return { "path": db, "size": st.st_size, "since": datetime.fromtimestamp(st.st_mtime, timezone.utc), "rows": rows, "secrets_total": sum(kinds.values()), "secrets_by_kind": dict(sorted(plain.items(), key=lambda kv: -kv[1])), "fields_scanned": scanned, } def sentence(info: dict) -> list[str]: """The two lines a person actually decides on.""" sessions = info["rows"].get("sdk_sessions") or 0 obs = info["rows"].get("observations") or 0 n = info["secrets_total"] body = f"{sessions:,} sessions and {obs:,} entries" out = [] if n: out.append(f" This database holds {body}, and {n} password" f"{'s' if n != 1 else ''} or key{'s' if n != 1 else ''} in plain text.") for kind, c in info["secrets_by_kind"].items(): out.append(f" {c:>4} {kind}") else: out.append(f" This database holds {body}. The redactor found no secrets in it.") out.append(f" It has been on your disk since {info['since'].strftime('%d %B %Y')}" f" and is {info['size']/1e6:.1f} MB.") if n: out.append(" The kinds are listed; the values are not shown and never will be.") return out if __name__ == "__main__": if len(sys.argv) < 2: print("usage: inspect_store.py <database>", file=sys.stderr) raise SystemExit(2) info = inspect(Path(sys.argv[1])) print("\n".join(sentence(info)))