scripts/kalshi_eval_set.py

"""Build the Kalshi transfer evaluation set from a rule that is written down.

    uv run python scripts/kalshi_eval_set.py --out data/kalshi_score_778_rebuilt.json

## Why this exists

`data/kalshi_score_778.json` has no builder anywhere in this repository. Its
metadata states a filter -- *"judgment categories only; numeric/threshold titles
removed; lifetime>=7d; one market per event"* -- but no committed code implements
it, so the population behind every Kalshi transfer number on three public model
cards cannot be rebuilt or checked. §9.1 of the technical report discloses this;
this script is the promise being kept.

`polymarket_eval_set.py` is the precedent, including its discipline: **the rule
below was written before this was run and was not adjusted afterwards.** No
stage of it was chosen by looking at what it does to the row count or to any
model comparison.

## The rule, stated in full

The artifact's four clauses, each made executable, plus two the artifact's own
`filter` string omits and which are needed for the rule to determine a set at
all. Those two are marked ADDED.

1. **Judgment categories only.** A judgment question resolves on a human or
   institutional decision, or on a world event a forecaster reasons about. It is
   NOT a price or index level, a sporting result, a physical measurement, or a
   mechanical count of utterances. On Kalshi's own 18-category series taxonomy
   that admits: Politics, Elections, Economics, Science and Technology, World,
   Companies, Health, Social, Transportation, Education, Entertainment, Exotics.
   It excludes: Sports, Financials, Crypto, Commodities, Climate and Weather,
   Mentions.

   Entertainment is admitted on principle -- an award outcome is a judgment --
   and its numeric titles are removed by clause 2 rather than by category.

2. **Numeric / threshold titles removed.** Any title expressing a threshold on a
   continuous quantity. Matched on the title, not the category, because a
   judgment category still carries "above $X" markets.

3. **Lifetime >= 7 days**, measured `settlement_ts - open_time`. `docs/11`'s
   domain prior: time to resolution is the property that has mattered twice, and
   a market open for under a week is a nowcast.

4. **One market per event.** Deterministic tiebreak: the market with the LONGEST
   lifetime, ties broken by ticker ascending. Stated because "one per event"
   alone does not say which one, and any unstated tiebreak is a silent
   researcher degree of freedom.

5. **ADDED -- multivariate combo markets excluded.** `docs/10` records that
   `/markets` has no default for `mve_filter`, so unfiltered pulls return
   multi-leg parlays whose titles are the legs comma-joined and whose timestamps
   track generation rather than events. They are not forecasting questions. The
   artifact's filter string does not mention them; without excluding them the
   rule admits 27M rows of noise.

6. **ADDED -- a resolution window.** The artifact's filter string names none, yet
   the shipped set spans 2026-06-20 to 2026-08-27, so a window was applied and
   not recorded. `--resolved-from` / `--resolved-to` make it explicit and
   default to the shipped set's own span so the comparison is like-for-like.

**Clauses 5 and 6 are the finding, not an implementation detail.** The stated
filter under-determines the set: two choices that materially change the
population were never written down.
"""

from __future__ import annotations

import argparse
import gzip
import hashlib
import json
import re
import sys
from collections.abc import Sequence
from datetime import datetime

JUDGMENT = {
    "Politics", "Elections", "Economics", "Science and Technology", "World",
    "Companies", "Health", "Social", "Transportation", "Education",
    "Entertainment", "Exotics",
}
MECHANICAL = {
    "Sports", "Financials", "Crypto", "Commodities", "Climate and Weather",
    "Mentions",
}

# Clause 2. Thresholds on a continuous quantity, however phrased.
NUMERIC_TITLE = re.compile(
    r"(above|below|over|under|at least|at most|more than|less than|greater than|"
    r"exceed|reach|between)\s+[\$€£]?[\d,]+(\.\d+)?"
    r"|[\$€£][\d,]+(\.\d+)?"
    r"|\b\d+(\.\d+)?\s*(%|percent|bps|basis points)"
    r"|\bhow many\b|\bnumber of\b",
    re.IGNORECASE,
)


def parse_ts(s: str | None) -> datetime | None:
    if not s:
        return None
    try:
        return datetime.fromisoformat(s.replace("Z", "+00:00"))
    except ValueError:
        return None


def series_of(event_ticker: str) -> str:
    """Kalshi event tickers are SERIES-SUFFIX. The series is the part before the
    first hyphen; a ticker with no hyphen is its own series."""
    return event_ticker.split("-", 1)[0] if event_ticker else ""


def main(argv: Sequence[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--sweep", default="data/kalshi/sweep-20260824.jsonl.gz")
    ap.add_argument("--categories", default="data/kalshi/series_category.json")
    ap.add_argument("--out", default="data/kalshi_score_778_rebuilt.json")
    ap.add_argument("--resolved-from", default="2026-06-20")
    ap.add_argument("--resolved-to", default="2026-08-28")
    ap.add_argument("--min-lifetime-days", type=int, default=7)
    ap.add_argument("--compare", default="data/kalshi_score_778.json")
    args = ap.parse_args(argv)

    cats: dict[str, str] = json.load(open(args.categories, encoding="utf-8"))
    lo = parse_ts(args.resolved_from + "T00:00:00+00:00")
    hi = parse_ts(args.resolved_to + "T00:00:00+00:00")

    drop = {
        "not_finalized": 0, "mve": 0, "no_event": 0, "unknown_series": 0,
        "mechanical_category": 0, "numeric_title": 0, "outside_window": 0,
        "short_lifetime": 0, "bad_timestamps": 0, "no_result": 0,
    }
    best: dict[str, dict] = {}
    seen = 0

    op = gzip.open if args.sweep.endswith(".gz") else open
    with op(args.sweep, "rt", encoding="utf-8") as h:
        for line in h:
            if not line.strip():
                continue
            seen += 1
            r = json.loads(line)

            if r.get("status") != "finalized":
                drop["not_finalized"] += 1; continue
            if r.get("mve_collection_ticker"):
                drop["mve"] += 1; continue
            ev = r.get("event_ticker") or ""
            if not ev:
                drop["no_event"] += 1; continue
            cat = cats.get(series_of(ev))
            if cat is None:
                drop["unknown_series"] += 1; continue
            if cat in MECHANICAL or cat not in JUDGMENT:
                drop["mechanical_category"] += 1; continue
            title = r.get("title") or ""
            if NUMERIC_TITLE.search(title):
                drop["numeric_title"] += 1; continue
            res = r.get("result")
            if res not in ("yes", "no"):
                drop["no_result"] += 1; continue

            opened = parse_ts(r.get("open_time"))
            settled = parse_ts(r.get("settlement_ts"))
            if opened is None or settled is None:
                drop["bad_timestamps"] += 1; continue
            if not (lo <= settled < hi):
                drop["outside_window"] += 1; continue
            life = (settled - opened).days
            if life < args.min_lifetime_days:
                drop["short_lifetime"] += 1; continue

            row = {
                # keyed on the MARKET ticker, not the event ticker: the shipped
                # set uses kalshi:KXNEWGLENN-262-JUL where the event is
                # KXNEWGLENN-262. Keying on the event made 642 genuine matches
                # look like 30 and nearly produced a wrong finding about a
                # public artifact.
                "question_id": f"kalshi:{r.get('ticker') or ev}",
                "text": title,
                "open_date": opened.isoformat(),
                "scheduled_resolve_date": (parse_ts(r.get("close_time")) or settled).isoformat(),
                "resolved_at": settled.isoformat(),
                "outcome": 1 if res == "yes" else 0,
                "quarter": 0,
                "event_ticker": ev,
                "category": cat,
                "_lifetime": life,
                "_ticker": r.get("ticker") or "",
            }
            # clause 4: longest lifetime wins, ties by ticker ascending
            cur = best.get(ev)
            if cur is None or (row["_lifetime"] > cur["_lifetime"]) or (
                row["_lifetime"] == cur["_lifetime"] and row["_ticker"] < cur["_ticker"]
            ):
                best[ev] = row

    rows = sorted(best.values(), key=lambda r: r["question_id"])
    for r in rows:
        r.pop("_lifetime", None); r.pop("_ticker", None)

    print(f"scanned {seen:,} sweep rows")
    print("dropped:")
    for k, v in sorted(drop.items(), key=lambda kv: -kv[1]):
        print(f"  {v:>10,}  {k}")
    print(f"\nkept {len(rows):,} rows, one per event")

    payload = {
        "content_hash": hashlib.sha256(
            json.dumps([r["question_id"] for r in rows], sort_keys=True).encode()
        ).hexdigest(),
        "n": len(rows),
        "split": "transfer_test",
        "platform": "kalshi",
        "builder": "scripts/kalshi_eval_set.py",
        "filter": ("judgment categories only; numeric/threshold titles removed; "
                   f"lifetime>={args.min_lifetime_days}d; one market per event "
                   "(longest lifetime, ties by ticker); MVE combos excluded; "
                   f"resolved in [{args.resolved_from}, {args.resolved_to})"),
        "judgment_categories": sorted(JUDGMENT),
        "mechanical_categories": sorted(MECHANICAL),
        "dropped": drop,
        "questions": rows,
    }
    with open(args.out, "w", encoding="utf-8") as h:
        json.dump(payload, h, indent=1, sort_keys=True)
    print(f"wrote {args.out}")

    if args.compare:
        try:
            old = json.load(open(args.compare, encoding="utf-8"))
        except FileNotFoundError:
            return 0
        a = {q["question_id"] for q in old["questions"]}
        b = {r["question_id"] for r in rows}
        print(f"\n== against the shipped set ({args.compare}) ==")
        print(f"  shipped {len(a):,}   rebuilt {len(b):,}   overlap {len(a & b):,}")
        print(f"  in shipped only: {len(a - b):,}")
        print(f"  in rebuilt only: {len(b - a):,}")
        if a == b:
            print("  IDENTICAL -- the stated rule reproduces the shipped set.")
        else:
            print("  NOT identical. The stated filter does not determine the set;")
            print("  see clauses 5 and 6 in this script's docstring.")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())