"""Compare modeled congressional portfolios, performance and shared holdings.

https://vizualstocks.com/api/tutorials/congress-portfolios-python
Python 3.10+. Install requests. Set VIZUALSTOCKS_API_KEY in your environment.
Read-only requests; exports calculated analytics, never raw transaction records.
"""
import argparse
import csv
from datetime import datetime, timezone
from html import escape
from itertools import combinations
import json
import math
import os
from pathlib import Path
import re
import time

import requests

BASE_URL = "https://api.vizualstocks.com/v1"
DEFAULT_MEMBERS = ["Nancy Pelosi", "Michael McCaul", "Ro Khanna"]
WINDOWS = ["YTD", "1Y", "2Y", "3Y", "MAX"]
COLORS = ["#68b7ab", "#7c9ece", "#a393bd"]


def make_session():
    key = os.environ.get("VIZUALSTOCKS_API_KEY", "").strip()
    if not key:
        raise ValueError("Set VIZUALSTOCKS_API_KEY in this terminal first.")
    session = requests.Session()
    session.headers.update({"X-API-Key": key, "Accept": "application/json"})
    return session


def api_get(session, path, **params):
    """Retry transient GET failures, without following credential-bearing redirects."""
    for attempt in range(4):
        try:
            response = session.get(BASE_URL + path, params=params,
                                   timeout=(10, 60), allow_redirects=False)
        except requests.RequestException:
            if attempt == 3:
                raise ValueError("Could not reach the API. Check your connection and retry.") from None
            time.sleep(2 ** attempt)
            continue
        if response.status_code in (429, 502, 503, 504) and attempt < 3:
            delay = response.headers.get("Retry-After", "")
            time.sleep(min(30, int(delay)) if delay.isdigit() else 2 ** attempt)
            continue
        if response.status_code != 200:
            hints = {401: "Check your API key.", 403: "This member or ticker is not available on your plan.",
                     404: "The requested modeled portfolio was not found.", 429: "Quota or rate limit reached; retry later."}
            raise ValueError(f"API HTTP {response.status_code}: {hints.get(response.status_code, 'Request could not be completed.')} ({path})")
        try:
            payload = response.json()
        except ValueError:
            raise ValueError("The API returned invalid JSON.") from None
        if not isinstance(payload, dict):
            raise ValueError("Expected a JSON object from the API.")
        return payload


def number(value):
    return value if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None


def select_members(directory, requested):
    selected = []
    for query in requested:
        matches = [row for row in directory.get("members", [])
                   if query.casefold() in (str(row["member"].get("memberId", "")).casefold(),
                                           str(row["member"].get("name", "")).casefold())]
        if len(matches) != 1 or not matches[0].get("portfolioAvailable"):
            raise ValueError(f"No unique available portfolio for {query!r}. Run --list-members and use its memberId.")
        identity = matches[0]["member"]
        if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", identity["memberId"]):
            raise ValueError("The directory returned an invalid memberId.")
        if identity["memberId"] not in [member["memberId"] for member in selected]:
            selected.append(identity)
    return selected


def normalized_holdings(portfolio):
    """Aggregate by ticker before measuring concentration or counting holders."""
    grouped = {}
    for holding in portfolio.get("holdings", []):
        ticker = str(holding.get("ticker", "")).strip().upper()
        if not ticker:
            continue
        grouped.setdefault(ticker, []).append(holding)
    result = []
    for ticker, positions in sorted(grouped.items()):
        row = {"memberId": portfolio["member"]["memberId"], "member": portfolio["member"]["name"],
               "party": portfolio["member"].get("party"), "ticker": ticker,
               "generatedAtUtc": portfolio.get("generatedAtUtc")}
        for field in ("portfolioWeightPercent", "estimatedValue", "estimatedShares"):
            values = [number(position.get(field)) for position in positions]
            row[field] = round(sum(values), 4) if all(value is not None for value in values) else None
        result.append(row)
    return result


def concentration(holdings):
    weights = [number(row.get("portfolioWeightPercent")) for row in holdings]
    known = [weight for weight in weights if weight is not None]
    complete = bool(weights) and len(known) == len(weights) and all(weight >= 0 for weight in known)
    return {"holdingCount": len(holdings), "holdingsWithoutWeight": len(weights) - len(known),
            "top5WeightPercent": round(sum(sorted(known, reverse=True)[:5]), 2) if complete else None}


def overlap_rows(portfolios):
    """Counts describe selected members, not the entire congressional universe."""
    tickers = {}
    for portfolio in portfolios:
        member = portfolio["member"]
        for holding in normalized_holdings(portfolio):
            tickers.setdefault(holding["ticker"], {})[member["memberId"]] = member
    rows = []
    for ticker, holders in tickers.items():
        parties = {str(member.get("party", "")).casefold() for member in holders.values()}
        rows.append({"ticker": ticker, "selectedHolderCount": len(holders),
                     "selectedMemberCount": len(portfolios),
                     "democratAndRepublican": bool(parties & {"d", "democrat", "democratic"}) and bool(parties & {"r", "republican"}),
                     "members": "; ".join(sorted(member["name"] for member in holders.values()))})
    return sorted(rows, key=lambda row: (-row["selectedHolderCount"], row["ticker"]))


def pairwise_rows(portfolios):
    result = []
    for left, right in combinations(portfolios, 2):
        a = {row["ticker"] for row in normalized_holdings(left)}
        b = {row["ticker"] for row in normalized_holdings(right)}
        union = a | b
        result.append({"memberA": left["member"]["name"], "memberB": right["member"]["name"],
                       "sharedTickerCount": len(a & b), "unionTickerCount": len(union),
                       "jaccardPercent": round(100 * len(a & b) / len(union), 2) if union else None})
    return result


def build_analysis(portfolios, performances, leaderboard, holders, window):
    if leaderboard.get("selectedWindow") != window:
        raise ValueError("The API returned a different leaderboard window than requested.")
    holdings = [row for portfolio in portfolios for row in normalized_holdings(portfolio)]
    results = {item["member"]["memberId"]: item for item in performances}
    members = []
    for portfolio in portfolios:
        identity = portfolio["member"]
        performance = results[identity["memberId"]]
        if performance.get("selectedWindow") != window:
            raise ValueError("The API returned a different performance window than requested.")
        coverage = performance.get("coverage") or {}
        row = {"memberId": identity["memberId"], "member": identity["name"], "party": identity.get("party"),
               **concentration([holding for holding in holdings if holding["memberId"] == identity["memberId"]]),
               "window": window, "periodStartDate": performance.get("periodStartDate"),
               "periodEndDate": performance.get("periodEndDate"),
               "portfolioReturnPercent": number(performance.get("portfolioReturnPercent")),
               "benchmarkReturnPercent": number(performance.get("benchmarkReturnPercent")),
               "excessReturnPercentagePoints": number(performance.get("excessReturnPercent")),
               "pricedCoveragePercent": number(coverage.get("coveragePercent")),
               "hasCompleteWindow": coverage.get("hasCompleteWindow") is True,
               "portfolioGeneratedAtUtc": portfolio.get("generatedAtUtc"),
               "performanceGeneratedAtUtc": performance.get("generatedAtUtc")}
        members.append(row)
    dates = {(row["periodStartDate"], row["periodEndDate"]) for row in members}
    comparable = bool(members) and len(dates) == 1 and all(
        row["hasCompleteWindow"] and row["periodStartDate"] and row["periodEndDate"] and
        row["portfolioReturnPercent"] is not None and row["benchmarkReturnPercent"] is not None for row in members)
    rankings = [{"rank": row.get("rank"), "member": row["member"]["name"],
                 "party": row["member"].get("party"), "window": window,
                 "portfolioReturnPercent": number(row.get("portfolioReturnPercent")),
                 "benchmarkReturnPercent": number(row.get("benchmarkReturnPercent")),
                 "excessReturnPercentagePoints": number(row.get("excessReturnPercent")),
                 "pricedCoveragePercent": number((row.get("coverage") or {}).get("coveragePercent")),
                 "hasCompleteWindow": (row.get("coverage") or {}).get("hasCompleteWindow"),
                 "coverageScope": (leaderboard.get("scope") or {}).get("coverageScope")}
                for row in leaderboard.get("members", [])]
    watchlist, watchlist_summary = [], []
    for payload in holders:
        scope = payload.get("scope") or {}
        unique = {holder["member"]["memberId"] for holder in payload.get("holders", [])}
        watchlist_summary.append({"ticker": payload["ticker"], "holderCount": len(unique),
                                  "memberUniverseCount": scope.get("memberUniverseCount"),
                                  "coverageScope": scope.get("coverageScope"), "generatedAtUtc": payload.get("generatedAtUtc")})
        for holder in payload.get("holders", []):
            watchlist.append({"ticker": payload["ticker"], "memberId": holder["member"]["memberId"],
                              "member": holder["member"]["name"], "party": holder["member"].get("party"),
                              "portfolioWeightPercent": number(holder.get("portfolioWeightPercent")),
                              "coverageScope": scope.get("coverageScope"), "generatedAtUtc": payload.get("generatedAtUtc")})
    return {"members": members, "holdings": holdings, "overlap": overlap_rows(portfolios),
            "pairwise": pairwise_rows(portfolios), "leaderboard": rankings, "watchlist": watchlist,
            "watchlist_summary": watchlist_summary, "comparable": comparable}


def display(value):
    if value is None:
        return "—"
    if isinstance(value, bool):
        return "Yes" if value else "No"
    return f"{value:,.2f}" if isinstance(value, float) else str(value)


def bars_chart(rows, title, suffix="%"):
    """Responsive HTML bars; labels stay readable and losses use a real zero axis."""
    rows = [(str(label), number(value), color) for label, value, color in rows if number(value) is not None]
    if not rows:
        return '<p class="muted">No numeric observations returned for this chart.</p>'
    low, high = min(0, min(value for _, value, _ in rows)), max(0, max(value for _, value, _ in rows))
    span = high - low or 1
    x = lambda value: (value - low) / span * 100
    pieces = [f'<div class="report-chart" role="group" aria-label="{escape(title, quote=True)}">']
    for label, value, color in rows:
        pieces.append(f'<div class="bar-row"><span class="bar-label">{escape(label)}</span>'
                      f'<div class="bar-track" aria-hidden="true"><i class="bar-zero" style="left:{x(0)}%"></i>'
                      f'<i class="bar-fill" style="left:{min(x(0), x(value))}%;width:{abs(x(value)-x(0))}%;background:{color}"></i></div>'
                      f'<strong>{value:,.2f}{suffix}</strong></div>')
    return "".join(pieces) + "</div>"


def table(rows, columns):
    if not rows:
        return '<p class="muted">No matching rows in this scope.</p>'
    header = "".join(f"<th>{escape(label)}</th>" for _, label in columns)
    body = "".join("<tr>" + "".join(f"<td>{escape(display(row.get(key)))}</td>" for key, _ in columns) + "</tr>" for row in rows)
    return f'<div class="table"><table><thead><tr>{header}</tr></thead><tbody>{body}</tbody></table></div>'


def csv_cell(value):
    # Prevent spreadsheet formula execution when an API text field is opened in Excel.
    if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@")):
        return "'" + value
    return "" if value is None else value


CSV_FIELDS = {
    "members": ["memberId", "member", "party", "holdingCount", "holdingsWithoutWeight", "top5WeightPercent", "window", "periodStartDate", "periodEndDate", "portfolioReturnPercent", "benchmarkReturnPercent", "excessReturnPercentagePoints", "pricedCoveragePercent", "hasCompleteWindow", "portfolioGeneratedAtUtc", "performanceGeneratedAtUtc"],
    "holdings": ["memberId", "member", "party", "ticker", "portfolioWeightPercent", "estimatedValue", "estimatedShares", "generatedAtUtc"],
    "overlap": ["ticker", "selectedHolderCount", "selectedMemberCount", "democratAndRepublican", "members"],
    "pairwise": ["memberA", "memberB", "sharedTickerCount", "unionTickerCount", "jaccardPercent"],
    "leaderboard": ["rank", "member", "party", "window", "portfolioReturnPercent", "benchmarkReturnPercent", "excessReturnPercentagePoints", "pricedCoveragePercent", "hasCompleteWindow", "coverageScope"],
    "watchlist": ["ticker", "memberId", "member", "party", "portfolioWeightPercent", "coverageScope", "generatedAtUtc"],
    "watchlist_summary": ["ticker", "holderCount", "memberUniverseCount", "coverageScope", "generatedAtUtc"],
}


def write_report(analysis, metadata, destination):
    destination = Path(destination)
    destination.mkdir(parents=True, exist_ok=False)
    for name, fields in CSV_FIELDS.items():
        with (destination / f"{name}.csv").open("x", newline="", encoding="utf-8-sig") as file:
            writer = csv.DictWriter(file, fieldnames=fields)
            writer.writeheader()
            writer.writerows({field: csv_cell(row.get(field)) for field in fields} for row in analysis[name])
    with (destination / "metadata.json").open("x", encoding="utf-8") as file:
        json.dump(metadata, file, indent=2, allow_nan=False)
    parts = ["<h1>Congressional portfolio research</h1>",
             f'<p class="muted">Modeled holdings and returns · retrieved {escape(metadata["retrievedAtUtc"])} · {escape(metadata["window"])} performance</p>',
             "<h2>Holdings and concentration</h2>"]
    for index, member in enumerate(analysis["members"]):
        holdings = [row for row in analysis["holdings"] if row["memberId"] == member["memberId"]]
        top = sorted((row for row in holdings if number(row["portfolioWeightPercent"]) is not None), key=lambda row: row["portfolioWeightPercent"], reverse=True)[:5]
        parts.extend([f'<section><h3>{escape(member["member"])}</h3>',
                      f'<p>{member["holdingCount"]} modeled holdings · top-five weight: {display(member["top5WeightPercent"])}{"%" if member["top5WeightPercent"] is not None else ""} · {member["holdingsWithoutWeight"]} without a weight</p>',
                      bars_chart([(row["ticker"], row["portfolioWeightPercent"], COLORS[index % 3]) for row in top], "Largest available holding weights"), "</section>"])
    parts.extend(["<h2>Performance versus benchmark</h2>",
                  "<p>Modeled results, not a post-disclosure copy-trading backtest. Priced coverage and complete-window coverage measure different things.</p>"])
    if analysis["comparable"]:
        rows = [(row["member"], row["portfolioReturnPercent"], COLORS[index % 3]) for index, row in enumerate(analysis["members"])]
        # Keep each benchmark attached to its member; do not assume identical values.
        rows += [(row["member"] + " · benchmark", row["benchmarkReturnPercent"], "#65758c") for row in analysis["members"]]
        parts.append(bars_chart(rows, "Same-window portfolio and benchmark returns"))
    else:
        parts.append("<p>Combined chart omitted: dates, returns, or complete-window coverage do not match. Inspect the individual results below.</p>")
    parts.append(table(analysis["members"], [("member", "Member"), ("periodStartDate", "Start"), ("periodEndDate", "End"), ("portfolioReturnPercent", "Return %"), ("benchmarkReturnPercent", "Benchmark %"), ("excessReturnPercentagePoints", "Excess (pp)"), ("pricedCoveragePercent", "Priced coverage %"), ("hasCompleteWindow", "Complete window")]))
    parts.extend(["<h2>Explore the leaderboard</h2>", f'<p>Scope: {escape(str(metadata["leaderboardScope"]))}. First 10 returned ranks; the CSV contains every returned rank.</p>',
                  table(analysis["leaderboard"][:10], [("rank", "Rank"), ("member", "Member"), ("portfolioReturnPercent", "Return %"), ("pricedCoveragePercent", "Priced coverage %"), ("hasCompleteWindow", "Complete window")]),
                  "<h2>Your watchlist</h2><p>All returned holders within your API scope, not only the members selected above.</p>",
                  table(analysis["watchlist_summary"], [("ticker", "Ticker"), ("holderCount", "Modeled holders"), ("memberUniverseCount", "Members in scope"), ("coverageScope", "Scope")]),
                  table(analysis["watchlist"], [("ticker", "Ticker"), ("member", "Member"), ("party", "Party"), ("portfolioWeightPercent", "Portfolio weight %")]),
                  "<h2>Shared holdings</h2><p>Overlap among selected members only. Counts are distinct members, not transactions.</p>",
                  table([row for row in analysis["overlap"] if row["selectedHolderCount"] >= 2], [("ticker", "Ticker"), ("selectedHolderCount", "Selected holders"), ("democratAndRepublican", "Both D and R"), ("members", "Members")]),
                  table(analysis["pairwise"], [("memberA", "Member A"), ("memberB", "Member B"), ("sharedTickerCount", "Shared tickers"), ("jaccardPercent", "Shared / union %")]),
                  "<h2>Downloads</h2><p>CSV files keep all rows; blank cells mean missing, not zero.</p><ul>"])
    parts += [f'<li><a href="{name}.csv">{name}.csv</a></li>' for name in CSV_FIELDS]
    parts.append('</ul><p><a href="metadata.json">Retrieval metadata and scope</a></p>')
    html = '''<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Congressional portfolio research</title><style>
body{margin:0;background:#0c1119;color:#edf3fb;font:15px/1.7 system-ui,sans-serif}main{max-width:1050px;margin:auto;padding:32px 20px}h1,h2,h3{line-height:1.3}h2{margin-top:45px}p,.muted{color:#aab9cc}section{background:#121b29;border:1px solid #2a384c;border-radius:12px;padding:20px;margin:20px 0}svg{display:block;width:100%;max-width:760px}a{color:#7dd7c3}.table{overflow-x:auto;margin:20px 0}table{width:100%;border-collapse:collapse;font-size:13px}th,td{text-align:left;padding:10px;border-bottom:1px solid #2a384c}th{color:#7dd7c3}td{vertical-align:top}th,td{min-width:75px}h1{font-size:30px}
.report-chart{display:grid;gap:14px;margin:24px 0}.bar-row{display:grid;grid-template-columns:190px minmax(0,1fr) 72px;align-items:center;gap:16px;font-size:13px}.bar-label{overflow-wrap:anywhere}.bar-row strong{text-align:right;font-size:13px}.bar-track{height:24px;position:relative;background:#182336;border-radius:3px}.bar-zero{position:absolute;width:1px;height:100%;background:#60718a}.bar-fill{position:absolute;height:100%;border-radius:3px}@media(max-width:600px){.bar-row{grid-template-columns:1fr 64px;gap:6px 12px}.bar-label{grid-column:1/-1}.bar-track{height:20px}.report-chart{gap:18px}}
</style><main>''' + "".join(parts) + "</main></html>"
    with (destination / "report.html").open("x", encoding="utf-8") as file:
        file.write(html)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--list-members", action="store_true", help="Show directory IDs and sandbox availability, then exit")
    parser.add_argument("--members", nargs="+", default=DEFAULT_MEMBERS, help="Exact member names (quoted) or directory IDs")
    parser.add_argument("--window", type=str.upper, choices=WINDOWS, default="1Y")
    parser.add_argument("--watchlist", nargs="*", default=["NVDA", "MSFT"], help="Tickers to query; use this flag alone to skip")
    parser.add_argument("--output-dir", default="congress-research", help="A NEW directory, never overwritten")
    args = parser.parse_args()
    try:
        tickers = list(dict.fromkeys(ticker.strip().upper() for ticker in args.watchlist))
        if any(not re.fullmatch(r"[A-Z0-9][A-Z0-9.-]{0,14}", ticker) for ticker in tickers):
            raise ValueError("Use valid stock tickers in --watchlist.")
        if len(args.members) > 20 or len(tickers) > 20:
            raise ValueError("Keep each report to at most 20 members and 20 watchlist tickers.")
        if not args.list_members and Path(args.output_dir).exists():
            raise ValueError("Choose a new --output-dir; existing exports are never overwritten.")
        with make_session() as session:
            directory = api_get(session, "/congress/members")
            if args.list_members:
                for item in directory.get("members", []):
                    member = item["member"]
                    print(f'{member["memberId"]:24} {member["name"]:28} sandbox={item.get("freeSandboxAvailable", False)}')
                return
            members = select_members(directory, args.members)
            portfolios, performances = [], []
            for member in members:
                path = f'/congress/members/{member["memberId"]}'
                portfolios.append(api_get(session, path + "/portfolio"))
                performances.append(api_get(session, path + "/performance", window=args.window))
            leaderboard = api_get(session, "/congress/leaderboard", window=args.window)
            holders = [api_get(session, f"/congress/stocks/{ticker}/holders") for ticker in tickers]
        analysis = build_analysis(portfolios, performances, leaderboard, holders, args.window)
        payloads = [directory, *portfolios, *performances, leaderboard, *holders]
        metadata = {"retrievedAtUtc": datetime.now(timezone.utc).isoformat(), "window": args.window,
                    "selectedMembers": members, "watchlist": tickers,
                    "leaderboardScope": leaderboard.get("scope"),
                    "leaderboardPeriod": {key: leaderboard.get(key) for key in ("periodStartDate", "periodEndDate")},
                    "generatedAtUtcValues": sorted({item["generatedAtUtc"] for item in payloads if item.get("generatedAtUtc")}),
                    "combinedPerformanceChart": analysis["comparable"],
                    "notes": ["Calculated portfolios, not official holdings or raw trade feeds.",
                              "Current weights do not attribute historical returns.",
                              "Overlap refers only to selected members; holders and leaderboard use the returned API scope."]}
        write_report(analysis, metadata, args.output_dir)
        print(f'Created {Path(args.output_dir).resolve() / "report.html"}')
        print("Open report.html in your browser. Seven CSV files and metadata.json are beside it.")
    except (ValueError, OSError, KeyError, TypeError) as error:
        parser.exit(1, f"Error: {error}\n")


if __name__ == "__main__":
    main()