What to Log So Future-You Knows What Worked

import json, os, time
from dataclasses import dataclass, asdict
from typing import Optional

LOG_PATH = "bot_actions.jsonl"

@dataclass
class ActionLog:
    ts: float
    action_type: str
    rung: str
    stakes: str
    outcome: str
    human_override: Optional[str]  # "approved" | "edited" | "canceled" | None
    tokens_in: int
    tokens_out: int
    latency_ms: int
    cost_usd: float

def log(entry: ActionLog):
    with open(LOG_PATH, "a") as f:
        f.write(json.dumps(asdict(entry)) + "\n")

def read_log():
    if not os.path.exists(LOG_PATH):
        return []
    with open(LOG_PATH) as f:
        return [json.loads(line) for line in f]

# Simulate a few actions
for i in range(10):
    log(ActionLog(
        ts=time.time() + i,
        action_type="draft_reply",
        rung="NOTIFY",
        stakes="medium",
        outcome="clean" if i != 3 else "user_override",
        human_override="edited" if i == 3 else None,
        tokens_in=1200, tokens_out=250,
        latency_ms=1800,
        cost_usd=0.0042,
    ))

# The one query that matters: what's the human-override rate per rung?
rows = read_log()
from collections import Counter, defaultdict
overrides = defaultdict(lambda: {"total": 0, "overridden": 0})
for r in rows:
    key = (r["action_type"], r["rung"])
    overrides[key]["total"] += 1
    if r["human_override"] is not None:
        overrides[key]["overridden"] += 1

for (a, rung), s in sorted(overrides.items()):
    pct = 100 * s["overridden"] / s["total"] if s["total"] else 0
    print(f"{a:20s} rung={rung:6s} overrides={pct:4.1f}% ({s['overridden']}/{s['total']})")

# Rule of thumb: if AUTO gets >5% overridden, downgrade to NOTIFY.