#!/usr/bin/env python3
"""How much of "insider trading" data is actual trading? — reproducible
Form 4 composition study.

Every transaction leg on a Form 4 declares what KIND of event it reports via
the SEC's one-letter transaction codes: P and S are open-market purchases and
sales, A is a grant from the issuer, M an option exercise, F shares withheld
to cover tax, G a gift, and so on. Feeds and headlines routinely present all
of it as "insider trading" — but a grant hitting a vesting schedule and a CEO
buying stock with their own money on the open market are different events.

This script measures, over a fixed filed-date window, the actual composition:

  1. Legs per transaction code — counts, non-derivative vs derivative table,
     acquired vs disposed.
  2. As-filed dollar value per code (shares x price, where a positive price
     is filed), and the share of that value carried by open-market P/S legs.
  3. The headline: what fraction of Form 4 activity, by legs and by valued
     dollars, is open-market trading at all.

This is a data-infrastructure measurement, not investment advice and not a
characterization of any filer: it counts filings by their own declared codes.
A price on a non-open-market leg (an option's exercise price, a plan price)
is not a market valuation, and this script never treats it as one — by-value
comparisons are meaningful mainly for the open-market codes, and the page
rendering these results says so.

Reproducible: the window bounds are fixed arguments, and the API's `since=`/
`until=` filters make a fixed window return the same population on any tier,
any day you run it. The default one-month window is ~120 requests (free
tier: 10/min, so expect ~15 minutes; paid tiers are faster).

Usage:
    export FILINGPULSE_API_KEY=fp_...       # free key: filingpulse.io/signup.html
    python form4_transaction_codes_study.py --since 2026-09-01 \
        --until 2026-09-30 --out results.json

Optional coverage cross-check (no key needed for EDGAR, but the SEC asks
for a contact identity in the User-Agent; set your own):
    export EDGAR_CONTACT="you@example.com"
    python form4_transaction_codes_study.py ... --verify-coverage

Only the Python standard library is used.
"""
import argparse
import json
import math
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from collections import Counter
from datetime import date, timedelta

BASE_URL = os.environ.get("FILINGPULSE_BASE_URL", "https://api.filingpulse.io")
API_KEY = os.environ.get("FILINGPULSE_API_KEY")

ISO_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")

# SEC Form 4 Table I/II transaction codes (General Instructions to Form 4;
# 17 CFR 240.16b-3 et seq. give several their regulatory homes). Captions
# here are short readings, not legal definitions.
CODE_MEANINGS = {
    "P": "Open-market or private purchase",
    "S": "Open-market or private sale",
    "A": "Grant / award from issuer",
    "M": "Exercise or conversion of derivative",
    "F": "Shares withheld for tax on vesting/exercise",
    "D": "Disposition back to the issuer",
    "G": "Bona fide gift",
    "C": "Conversion of derivative security",
    "X": "Exercise of in-the-money derivative",
    "J": "Other acquisition or disposition (see footnotes)",
    "W": "Acquisition or disposition by will / laws of descent",
    "Z": "Deposit into or withdrawal from voting trust",
    "I": "Discretionary transaction per 16b-3(f)",
    "E": "Expiration of short derivative position",
    "H": "Expiration of long derivative position",
    "O": "Exercise of out-of-the-money derivative",
    "L": "Small acquisition per 16a-6",
    "U": "Tender of shares in change-of-control",
    "K": "Equity swap or similar instrument",
    "V": "Voluntarily-reported earlier transaction",
}


def _ssl_context():
    """Prefer certifi's roots when it happens to be installed — some machines
    carry a stale OS trust store that rejects current Let's Encrypt chains,
    which would make a healthy API look down. Falls back to the OS store;
    certifi is never required."""
    try:
        import ssl

        import certifi
        return ssl.create_default_context(cafile=certifi.where())
    except ImportError:
        return None


_SSL_CTX = _ssl_context()


def parse_num(s):
    """As-filed numeric string -> float, else None. Shares and prices are
    strings exactly as filed; malformed values are data, and this script
    counts them instead of guessing."""
    if not isinstance(s, str) or not s.strip():
        return None
    try:
        v = float(s)
    except ValueError:
        return None
    if not math.isfinite(v):
        return None
    return v


def api_get(path: str, params: dict) -> dict:
    """GET with the key header; honors 429 Retry-After so the free tier just
    runs slower instead of failing, and retries transient network errors —
    a multi-minute run will eventually meet a dropped connection, and that
    is not a reason to lose the whole fetch."""
    qs = "&".join(f"{k}={urllib.parse.quote(str(v))}" for k, v in params.items())
    url = f"{BASE_URL}{path}?{qs}"
    req = urllib.request.Request(url, headers={"X-API-Key": API_KEY or ""})
    for attempt in range(8):
        try:
            with urllib.request.urlopen(req, timeout=60,
                                        context=_SSL_CTX) as resp:
                return json.load(resp)
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = int(e.headers.get("Retry-After", "10"))
                print(f"  rate limited — waiting {wait}s", file=sys.stderr)
                time.sleep(max(1, wait))
                continue
            if e.code >= 500:  # transient server blip — retry, don't die
                wait = 10 * (attempt + 1)
                print(f"  API {e.code} — retrying in {wait}s",
                      file=sys.stderr)
                time.sleep(wait)
                continue
            body = e.read().decode("utf-8", "replace")[:200]
            sys.exit(f"API error {e.code} on {url}: {body}")
        except (OSError, TimeoutError) as e:  # timeouts, resets, DNS blips
            wait = 5 * (attempt + 1)
            print(f"  network error ({e}) — retrying in {wait}s",
                  file=sys.stderr)
            time.sleep(wait)
    sys.exit("giving up after repeated failures")


def fetch_window(since: date, until: date):
    """Walk the window one filed-date at a time (since=D&until=D), paging
    inside each day. Daily volume stays far below the offset cap, so every
    filing is reachable — and the per-day Form 4 counts double as the
    coverage table. No form_type filter: forms 3/5 arriving in the window
    are fetched and counted by name rather than invisibly excluded."""
    filings, daily = {}, []
    d = since
    while d <= until:
        day = d.isoformat()
        got, total, offset = 0, None, 0
        form4_family = 0
        while total is None or got < total:
            page = api_get("/v1/insider-trades",
                           {"since": day, "until": day, "limit": 200,
                            "offset": offset})
            total = page["total"]
            for obj in page["data"]:
                filings[obj["accession"]] = obj
                if obj.get("form_type") in ("4", "4/A"):
                    form4_family += 1
            got += len(page["data"])
            offset += 200
            if not page["data"]:
                break
            time.sleep(0.05)
        # "filed" is the Form 4/4A count — the figure the EDGAR coverage
        # check compares; the all-forms total is kept alongside
        daily.append({"date": day, "filed": form4_family,
                      "all_ownership_forms": total or 0})
        print(f"  {day}: {total or 0} filings "
              f"({form4_family} Form 4/4A)", file=sys.stderr)
        d += timedelta(days=1)
    return filings, daily


def edgar_index_counts(start: date, end: date, contact: str):
    """Distinct Form 4 (+4/A) accessions per filed date from EDGAR's own
    daily master indexes — the ground truth the API's per-day counts are
    checked against.

    Three EDGAR facts the naive version of this check gets wrong:
    - EDGAR serves 403 (not 404) for never-published days (weekends /
      holidays); those come back as None.
    - A day's index lists everything DISSEMINATED that day, and the row's
      own Date Filed column can differ — re-disseminated corrections keep
      their original filed date. Membership in the index is NOT "filed
      that day"; rows are bucketed by their own Date Filed column, since
      that is the population a filed_date window returns.
    - Dissemination day and Date Filed routinely differ by one (a filing
      can surface only in the NEXT day's index under the prior day's
      date), so a single day's index is not the whole population for that
      date. The sweep therefore accumulates Date-Filed buckets across
      every index in the range, with a few buffer days on each side."""
    buckets = {}          # filed date iso -> set of accessions
    published = set()     # days whose index exists
    buffer = timedelta(days=4)
    d = start - buffer
    sweep_end = min(end + buffer, date.today())
    while d <= sweep_end:
        qtr = (d.month - 1) // 3 + 1
        day_compact = d.strftime("%Y%m%d")
        url = (f"https://www.sec.gov/Archives/edgar/daily-index/{d.year}/"
               f"QTR{qtr}/master.{day_compact}.idx")
        req = urllib.request.Request(
            url,
            headers={"User-Agent": f"form4-txn-codes-study ({contact})"})
        text = None
        for attempt in range(6):
            try:
                with urllib.request.urlopen(req, timeout=60,
                                            context=_SSL_CTX) as resp:
                    text = resp.read().decode("latin-1")
                break
            except urllib.error.HTTPError as e:
                if e.code in (403, 404):
                    break  # no index published that day
                if e.code == 429 or e.code >= 500:
                    if attempt == 5:
                        sys.exit(f"EDGAR kept answering {e.code} for {url}")
                    wait = 20 * (attempt + 1)
                    print(f"  EDGAR {e.code} — backing off {wait}s",
                          file=sys.stderr)
                    time.sleep(wait)
                    continue
                raise
            except (OSError, TimeoutError) as e:
                if attempt == 5:
                    raise
                wait = 5 * (attempt + 1)
                print(f"  network error ({e}) — retrying in {wait}s",
                      file=sys.stderr)
                time.sleep(wait)
        if text is not None:
            published.add(d.isoformat())
            for line in text.splitlines():
                parts = line.split("|")
                if len(parts) == 5 and parts[2] in ("4", "4/A"):
                    df = parts[3]
                    if len(df) == 8 and df.isdigit():
                        iso = f"{df[:4]}-{df[4:6]}-{df[6:8]}"
                        acc = parts[4].rsplit("/", 1)[-1].removesuffix(".txt")
                        buckets.setdefault(iso, set()).add(acc)
        print(f"  EDGAR index {d.isoformat()}: "
              f"{'ok' if text is not None else 'not published'}",
              file=sys.stderr)
        time.sleep(1.1)  # SEC fair access: stay well under their rate cap
        d += timedelta(days=1)
    counts = {}
    d = start
    while d <= end:
        iso = d.isoformat()
        n = len(buckets.get(iso, ()))
        # an unpublished index day with nothing dated to it anywhere in the
        # sweep is unchecked (weekend/holiday), not zero
        counts[iso] = n if (iso in published or n) else None
        d += timedelta(days=1)
    return counts


def percentile(sorted_vals, p):
    """Nearest-rank percentile — deterministic, no interpolation."""
    if not sorted_vals:
        return None
    k = max(1, -(-len(sorted_vals) * p // 100))  # ceil without floats
    return sorted_vals[int(k) - 1]


def analyze(filings: dict, daily: list, window: dict) -> dict:
    by_form = Counter(f.get("form_type") or "?" for f in filings.values())
    originals = [f for f in filings.values() if f.get("form_type") == "4"]

    holdings_only = 0
    legs_total = legs_nd = legs_d = 0
    legs_uncoded = legs_priced = legs_valued = 0
    shares_unparseable = 0
    by_code = {}

    for f in originals:
        if f.get("holdings_only") or not (f.get("transactions") or []):
            holdings_only += 1
            continue
        acc = f.get("accession")
        for t in f.get("transactions") or []:
            legs_total += 1
            deriv = bool(t.get("is_derivative"))
            if deriv:
                legs_d += 1
            else:
                legs_nd += 1
            code = (t.get("transaction_code") or "").strip().upper()
            if not code:
                code = "?"
                legs_uncoded += 1
            shares = parse_num(t.get("shares"))
            price = parse_num(t.get("price_per_share"))
            if shares is None:
                shares_unparseable += 1
            priced = price is not None and price > 0
            if priced:
                legs_priced += 1
            value = (shares * price
                     if priced and shares is not None and shares > 0
                     else None)
            if value is not None:
                legs_valued += 1
            row = by_code.setdefault(code, {
                "legs": 0, "filings": set(),
                "non_derivative": 0, "derivative": 0,
                "acquired": 0, "disposed": 0, "ad_other": 0,
                "priced_legs": 0, "valued_legs": 0,
                "values": [],
            })
            row["legs"] += 1
            row["filings"].add(acc)
            row["non_derivative" if not deriv else "derivative"] += 1
            ad = (t.get("acquired_disposed") or "").strip().upper()
            if ad == "A":
                row["acquired"] += 1
            elif ad == "D":
                row["disposed"] += 1
            else:
                row["ad_other"] += 1
            if priced:
                row["priced_legs"] += 1
            if value is not None:
                row["valued_legs"] += 1
                row["values"].append(value)

    # open-market figures before folding (P/S never fold: they are the
    # subject of the study, and both clear the fold floor in any real month)
    def om_side(code):
        row = by_code.get(code)
        if not row:
            return {"legs": 0, "filings": 0, "non_derivative": 0,
                    "valued_legs": 0, "value_usd": 0,
                    "median_leg_value_usd": None}
        vals = sorted(row["values"])
        return {
            "legs": row["legs"],
            "filings": len(row["filings"]),
            "non_derivative": row["non_derivative"],
            "valued_legs": row["valued_legs"],
            "value_usd": round(sum(vals)),
            "median_leg_value_usd": (round(percentile(vals, 50))
                                     if vals else None),
        }

    total_value = round(sum(sum(r["values"]) for r in by_code.values()))

    # fold tiny codes so the table reads as signal, not noise
    FOLD_FLOOR = 25
    code_table = {}
    for code, row in by_code.items():
        key = code if row["legs"] >= FOLD_FLOOR else "other"
        tgt = code_table.setdefault(key, {
            "legs": 0, "filings": set(),
            "non_derivative": 0, "derivative": 0,
            "acquired": 0, "disposed": 0, "ad_other": 0,
            "priced_legs": 0, "valued_legs": 0, "values": [],
            "folded_codes": [],
        })
        for k in ("legs", "non_derivative", "derivative", "acquired",
                  "disposed", "ad_other", "priced_legs", "valued_legs"):
            tgt[k] += row[k]
        tgt["filings"] |= row["filings"]
        tgt["values"] += row["values"]
        if key == "other" and code != "other":
            tgt["folded_codes"].append(code)

    rendered = {}
    for code, row in sorted(code_table.items(),
                            key=lambda kv: -kv[1]["legs"]):
        vals = sorted(row["values"])
        meaning = (CODE_MEANINGS.get(code)
                   if code != "other"
                   else "All codes with fewer than "
                        f"{FOLD_FLOOR} legs in the window ("
                        + ", ".join(sorted(row["folded_codes"])) + ")")
        rendered[code] = {
            "meaning": meaning or "Unknown / not in the Form 4 code table",
            "legs": row["legs"],
            "filings": len(row["filings"]),
            "non_derivative": row["non_derivative"],
            "derivative": row["derivative"],
            "acquired": row["acquired"],
            "disposed": row["disposed"],
            "ad_other": row["ad_other"],
            "priced_legs": row["priced_legs"],
            "valued_legs": row["valued_legs"],
            "value_usd": round(sum(vals)),
            "median_leg_value_usd": (round(percentile(vals, 50))
                                     if vals else None),
        }

    p_om, s_om = om_side("P"), om_side("S")
    om_legs = p_om["legs"] + s_om["legs"]
    om_value = p_om["value_usd"] + s_om["value_usd"]

    return {
        "study": "form4_transaction_codes",
        "version": 1,
        "window": window,
        "fetched": {
            "total_ownership_filings": len(filings),
            "form_4": by_form.get("4", 0),
            "form_4a_amendments_excluded": by_form.get("4/A", 0),
            "forms_3_5_excluded": sum(
                by_form.get(ft, 0) for ft in ("3", "3/A", "5", "5/A")),
            "forms_3_5_by_type": {
                ft: by_form.get(ft, 0) for ft in ("3", "3/A", "5", "5/A")
                if by_form.get(ft, 0)},
            "other_form_types": sum(
                n for ft, n in by_form.items()
                if ft not in ("3", "3/A", "4", "4/A", "5", "5/A")),
        },
        "filings": {
            "analyzed_form4": len(originals) - holdings_only,
            "holdings_only_excluded": holdings_only,
        },
        "legs": {
            "total": legs_total,
            "non_derivative": legs_nd,
            "derivative": legs_d,
            "uncoded": legs_uncoded,
            "priced": legs_priced,
            "valued": legs_valued,
            "shares_unparseable": shares_unparseable,
        },
        "value": {
            "total_valued_usd": total_value,
        },
        "by_code": rendered,
        "open_market": {
            "legs": om_legs,
            "value_usd": om_value,
            "P": p_om,
            "S": s_om,
        },
        "daily": daily,
        "generated_by": "form4_transaction_codes_study.py v1",
    }


def main():
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--since", default="2026-09-01",
                    help="window start (filings with filed_date from this "
                         "date)")
    ap.add_argument("--until", default="2026-09-30",
                    help="window end (inclusive)")
    ap.add_argument("--out", default="form4_transaction_codes_results.json")
    ap.add_argument("--verify-coverage", action="store_true",
                    help="cross-check per-day counts against EDGAR's daily "
                         "master index (needs EDGAR_CONTACT set)")
    args = ap.parse_args()

    if not API_KEY:
        sys.exit("set FILINGPULSE_API_KEY (free key: "
                 "https://filingpulse.io/signup.html)")
    if not ISO_RE.match(args.since) or not ISO_RE.match(args.until):
        sys.exit("--since/--until must be YYYY-MM-DD")
    since = date.fromisoformat(args.since)
    until = date.fromisoformat(args.until)
    if until < since:
        sys.exit("--since must be on or before --until")

    print(f"fetching ownership filings {since} .. {until}", file=sys.stderr)
    filings, daily = fetch_window(since, until)

    if args.verify_coverage:
        contact = os.environ.get("EDGAR_CONTACT")
        if not contact:
            sys.exit("--verify-coverage needs EDGAR_CONTACT set (the SEC "
                     "asks for a contact identity in the User-Agent)")
        print("verifying per-day counts against EDGAR daily indexes",
              file=sys.stderr)
        index_counts = edgar_index_counts(since, until, contact)
        for row in daily:
            row["edgar_index"] = index_counts.get(row["date"])
            row["match"] = (None if row["edgar_index"] is None
                            else row["filed"] == row["edgar_index"])

    results = analyze(filings, daily,
                      {"since": since.isoformat(),
                       "until": until.isoformat()})
    with open(args.out, "w", encoding="utf-8") as fh:
        json.dump(results, fh, indent=2)
        fh.write("\n")

    fe = results["fetched"]
    legs = results["legs"]
    om = results["open_market"]
    print(f"\n{fe['total_ownership_filings']} filings in window "
          f"({fe['form_4']} Form 4, {fe['form_4a_amendments_excluded']} 4/A "
          f"excluded, {fe['forms_3_5_excluded']} Forms 3/5 excluded)")
    if legs["total"]:
        print(f"{legs['total']} legs: open-market P+S = {om['legs']} "
              f"({om['legs'] / legs['total']:.1%} of legs)")
    if results["value"]["total_valued_usd"]:
        print(f"valued dollars: P+S ${om['value_usd']:,} of "
              f"${results['value']['total_valued_usd']:,} "
              f"({om['value_usd'] / results['value']['total_valued_usd']:.1%})")
    top = list(results["by_code"].items())[:6]
    print("top codes by legs: " + ", ".join(
        f"{code} ({rec['legs']:,})" for code, rec in top))
    if args.verify_coverage:
        checked = [r for r in daily if r.get("match") is not None]
        matched = sum(1 for r in checked if r["match"])
        print(f"coverage vs EDGAR daily index: {matched}/{len(checked)} "
              f"days match exactly")
    print(f"full results -> {args.out}")


if __name__ == "__main__":
    main()
