"""Tables VI and VII of Reinstein & Snyder (JIE 2005).

Table VI: 'traditional methodology' (authors' TRADIT2.DO, cols 1-4).
Table VII: competitors' weekend box office, weekend-level (authors' MKTBOX2.DO, first two regressions).
SCREENS rescaled to thousands to match the paper's units.

Table VII here keeps the first movie of each weekend in file order. The authors' Stata
sort left that choice arbitrary; check_row_choice.py tries every possibility.

Run from the package root:  python3 replication/tables6_7.py data/movie.dta data/mktbox.txt
Writes replication/output/keycoefs_tables6_7.csv
"""
import sys
from pathlib import Path
import numpy as np
import pandas as pd
import statsmodels.api as sm

pd.set_option("display.width", 250)
pd.set_option("display.max_columns", 20)

movie_path, mkt_path = sys.argv[1], sys.argv[2]
raw = pd.read_stata(movie_path, convert_categoricals=False)
raw["weekend"] = raw.opendate + 6 - raw.openwday   # first Friday of the run (SAS date)

GENRE = "advent animated children comedy crime document drama fantasy noir horror musical mystery romance scifi thrill war western".split()
PROD = "disney fox sony mgm_ua gram orion univers warner paramnt".split()
YEARS = [f"year{i}" for i in range(2, 10)]
M11 = [f"month{i}" for i in range(1, 12)]


def stars(p):
    return "***" if p < .01 else "**" if p < .05 else "*" if p < .10 else ""


def design(df, xvars):
    X = df[xvars].astype(float)
    X = X.loc[:, X.std() > 0]
    X = sm.add_constant(X)
    keep = []
    for c in X.columns:
        if np.linalg.matrix_rank(X[keep + [c]].values) == len(keep) + 1:
            keep.append(c)
    return X[keep]


def fit(df, y, xvars, cluster=None):
    df = df.dropna(subset=[y] + xvars)
    X = design(df, xvars)
    if cluster is not None:
        return sm.OLS(df[y].astype(float), X).fit(cov_type="cluster", cov_kwds={"groups": df[cluster].values})
    return sm.OLS(df[y].astype(float), X).fit(cov_type="HC1")


def col(res, names):
    out = {n: (f"{res.params[n]:7.2f} ({res.bse[n]:.2f}){stars(res.pvalues[n])}" if n in res.params else "") for n in names}
    out["N"] = int(res.nobs)
    out["R2"] = round(res.rsquared, 2)
    return out


# ======================= Table VI =======================
d = raw[raw.pre != 1].copy()
d["lwk1cpi"] = np.log(d.wk1cpi)
d["screens"] = d.openscr / 1000
base = GENRE + PROD + YEARS + ["screens", "fourday"] + M11
m6 = {
    "(1) all": fit(d, "lwk1cpi", ["ou", "uu"] + base, "weekend"),
    "(2) all +MALTIN": fit(d, "lwk1cpi", ["ou", "uu", "maltin"] + base, "weekend"),
    "(3) DURING=1": fit(d[d.aft != 1], "lwk1cpi", ["ou", "uu"] + base, "weekend"),
    "(4) DURING=1 +MALTIN": fit(d[d.aft != 1], "lwk1cpi", ["ou", "uu", "maltin"] + base, "weekend"),
}
print("TABLE VI  traditional methodology, ln(OPENREV)")
print(pd.DataFrame({k: col(v, ["ou", "uu", "maltin", "screens", "fourday", "const"]) for k, v in m6.items()}))

# ======================= Table VII =======================
mk = pd.read_csv(mkt_path, sep=r"\s+", header=None, names=["x", "mktbox"])
# authors' merge: a market-box date x belongs to a weekend if x is the Friday or one of the 3 days before
wk = pd.Series(sorted(raw.weekend.unique()))
rows = []
for w in wk:
    hit = mk[(mk.x >= w - 3) & (mk.x <= w)]
    if len(hit) > 1:
        print("WARNING: several market dates for weekend", w)
    if len(hit):
        rows.append((w, hit.mktbox.iloc[0]))
wmap = dict(rows)
print(f"\nmktbox.txt rows: {len(mk)}; weekends in movie data: {len(wk)}; weekends matched: {len(wmap)}")

m = raw.copy()
m["mktbox"] = m.weekend.map(wmap)
m["mbcpi"] = m.mktbox * m.grosscpi / m.gross          # deflate with that movie's CPI factor
m = m[m.pre != 1].copy()
m["screens"] = m.openscr / 1000

# how much does the deflated market figure vary across movies in the same weekend? (row-order sensitivity)
spread = m.groupby("weekend").mbcpi.agg(lambda s: s.max() / s.min() - 1 if s.notna().all() else np.nan)
print(f"max within-weekend relative spread in deflated MKTREV: {spread.max():.4f}")

vars_ = ["dur", "oudur", "uudur", "ou", "uu", "screens", "maltin"]
first = m.sort_values("weekend", kind="stable").groupby("weekend").first()


def market(stat):
    agg = m.groupby("weekend")[vars_ + ["wk1cpi"]].agg(stat)
    out = first[["mbcpi", "fourday"] + YEARS + M11].join(agg)
    # NB: authors' code subtracts sum (col 1) or MEAN (col 2) of own opening revenue
    out["ratio"] = out.wk1cpi / out.mbcpi
    out = out[out.mbcpi.notna() & (out.ratio <= 1)]
    out["lresid"] = np.log(out.mbcpi - out.wk1cpi)
    return out


xv = ["dur", "oudur", "uudur", "ou", "uu", "fourday"] + M11 + YEARS + ["screens", "maltin"]
s, mn = market("sum"), market("mean")
m7 = {"(1) f=Sum": fit(s, "lresid", xv), "(2) f=Mean": fit(mn, "lresid", xv)}
print("\nTABLE VII  ln(MKTREV - SOPENREV), weekend level, HC1 SEs")
print(pd.DataFrame({k: col(v, ["dur", "oudur", "uudur", "ou", "uu", "maltin", "screens", "fourday", "const"]) for k, v in m7.items()}))
print(f"\nmean competitors' weekend revenue (sum version): ${(s.mbcpi - s.wk1cpi).mean()/1e6:.1f}m (paper: $48m)")

rows = []
for tab, models in (("VI", m6), ("VII", m7)):
    for k, r in models.items():
        for v in r.params.index:
            if v in ("ou", "uu", "dur", "oudur", "uudur", "maltin", "screens", "fourday", "const"):
                rows.append(dict(table=tab, model=k, var=v, b=r.params[v], se=r.bse[v], p=r.pvalues[v], N=int(r.nobs), r2=r.rsquared))
OUT = Path(__file__).resolve().parent / "output"
OUT.mkdir(exist_ok=True)
pd.DataFrame(rows).to_csv(OUT / "keycoefs_tables6_7.csv", index=False)