"""Derive every figure used on the site from the raw Eurostat snapshots.
Run:  python3 analysis/build_data.py    -> writes src/static/data/*.json and analysis/data/derived/*.csv
Only the standard library and numpy are needed."""
import json, math, csv
import numpy as np
from common import read_raw, EU27, NAMES, ROOT

OUT = ROOT.parent / "src" / "static" / "data"; OUT.mkdir(parents=True, exist_ok=True)
DER = ROOT / "data" / "derived"; DER.mkdir(parents=True, exist_ok=True)
RETRIEVED = "2026-11-14"

def r(x, n=2): return None if x is None else round(float(x), n)
def dump(name, obj):
    (OUT / f"{name}.json").write_text(json.dumps(obj, separators=(",", ":"), ensure_ascii=False), encoding="utf-8")
def write_csv(name, header, rows):
    with open(DER / f"{name}.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f); w.writerow(header); w.writerows(rows)

# ---------------------------------------------------------------- 1. Beveridge
def beveridge():
    mu, yu, U = read_raw("une_rt_a"); mv, yv, V = read_raw("jvs_a_rate_r2")
    years = [y for y in yu if y in yv]
    countries, rows = [], []
    for g in EU27 + ["EU27_2020"]:
        if g not in V: continue
        pts = []
        for y in years:
            u, fu = U[g][y]; v, fv = V[g][y]
            pts.append({"y": y, "u": u, "v": v, "f": (fu + fv) or None})
            rows.append([g, y, u, v, fu, fv])
        pre = [(p["u"], p["v"]) for p in pts if p["y"] <= 2019 and p["u"] and p["v"]]
        fit = None
        if g == "EU27_2020" and len(pre) >= 7:
            lu = np.log([a for a, _ in pre]); lv = np.log([b for _, b in pre])
            b, a = np.polyfit(lu, lv, 1)
            res = lv - (a + b * lu); r2 = 1 - res.var() / lv.var() if lv.var() > 0 else 0
            fit = {"a": r(a, 4), "b": r(b, 4), "r2": r(r2, 3), "n": len(pre),
                   "umin": min(a_ for a_, _ in pre), "umax": max(a_ for a_, _ in pre)}
        for p in pts:
            p["t"] = r(p["v"] / p["u"], 3) if p["u"] and p["v"] else None      # tightness v/u
        countries.append({"id": g, "name": NAMES[g], "pts": pts, "fit": fit})
    write_csv("beveridge", ["geo", "year", "unemployment_rate", "vacancy_rate", "flag_u", "flag_v"], rows)
    dump("beveridge", {"meta": {"retrieved": RETRIEVED, "sources": [mu, mv]}, "years": years, "countries": countries})
    return countries

# ---------------------------------------------------------------- 2. Mobility
def mobility():
    m1, years, MOV = read_raw("lfst_lmbpcita"); m2, _, MER = read_raw("lfst_lmbercita")
    m3, _, POP = read_raw("lfsa_pganws"); m4, _, NER = read_raw("lfsa_ergan")
    countries, rows = [], []
    for g in EU27:
        pts = []
        for y in years:
            mv, f1 = MOV[g][y]; pop, f3 = POP[g][y]; er_m, f2 = MER[g][y]; er_n, f4 = NER[g][y]
            share = 100 * mv / (mv + pop) if mv is not None and pop is not None else None
            gap = er_m - er_n if er_m is not None and er_n is not None else None
            lowrel = "u" in f1 or "u" in f2
            pts.append({"y": y, "abroad": mv, "home": pop, "share": r(share, 2), "erm": er_m, "ern": er_n,
                        "gap": r(gap, 1), "u": lowrel or None, "b": ("b" in (f1 + f3)) or None})
            rows.append([g, y, mv, pop, r(share, 2), er_m, er_n, r(gap, 1), f1, f2, f3, f4])
        countries.append({"id": g, "name": NAMES[g], "pts": pts})
    eu = []
    for y in years:
        mv = MOV["EU27_2020_FOR"][y][0]; pop = POP["EU27_2020"][y][0]
        eu.append({"y": y, "abroad": mv, "home": pop, "share": r(100 * mv / (mv + pop), 2) if mv else None,
                   "erm": MER["EU27_2020_FOR"][y][0], "ern": NER["EU27_2020"][y][0]})
    write_csv("mobility", ["citizenship", "year", "abroad_ths", "home_ths", "share_abroad_pct", "emp_rate_abroad",
                           "emp_rate_home", "gap_pp", "flag_abroad", "flag_er_abroad", "flag_home", "flag_er_home"], rows)
    dump("mobility", {"meta": {"retrieved": RETRIEVED, "sources": [m1, m2, m3, m4]}, "years": years, "countries": countries, "eu": eu})
    return countries, eu

# ---------------------------------------------------------------- 3. Labour costs
def labour_cost():
    m, years, LC = read_raw("lc_lci_lev", key_cols=2)
    countries, rows = [], []
    for g in EU27 + ["EU27_2020"]:
        pts = []
        for y in years:
            tot, f1 = LC[("D1_D4_MD5", g)][y]; wag, f2 = LC[("D11", g)][y]
            nw = 100 * (1 - wag / tot) if tot and wag else None
            pts.append({"y": y, "c": tot, "w": wag, "nw": r(nw, 1), "p": ("p" in f1) or None})
            rows.append([g, y, tot, wag, r(nw, 1), f1, f2])
        countries.append({"id": g, "name": NAMES[g], "pts": pts})
    stats = []
    for y in years:
        vals = [LC[("D1_D4_MD5", g)][y][0] for g in EU27]
        if any(v is None for v in vals): 
            stats.append({"y": y, "n": sum(v is not None for v in vals)}); continue
        lv = np.log(vals)
        stats.append({"y": y, "n": 27, "max": max(vals), "min": min(vals), "ratio": r(max(vals) / min(vals), 1),
                      "sdlog": r(lv.std(ddof=0), 3), "p90p10": r(np.percentile(vals, 90) / np.percentile(vals, 10), 2)})
    # beta-convergence 2012-2024: annualised growth on log initial level
    x = np.log([LC[("D1_D4_MD5", g)][2012][0] for g in EU27])
    yv = np.array([100 * ((LC[("D1_D4_MD5", g)][2024][0] / LC[("D1_D4_MD5", g)][2012][0]) ** (1 / 12) - 1) for g in EU27])
    b, a = np.polyfit(x, yv, 1); rr = np.corrcoef(x, yv)[0, 1]
    conv = {"from": 2012, "to": 2024, "slope": r(b, 3), "intercept": r(a, 3), "r": r(rr, 3), "r2": r(rr * rr, 3),
            "pts": [{"id": g, "name": NAMES[g], "c0": LC[("D1_D4_MD5", g)][2012][0], "g": r(v, 2)} for g, v in zip(EU27, yv)]}
    write_csv("labour_cost", ["geo", "year", "labour_cost_eur_h", "wages_eur_h", "non_wage_share_pct", "flag_total", "flag_wages"], rows)
    dump("labour_cost", {"meta": {"retrieved": RETRIEVED, "sources": [m]}, "years": years, "countries": countries, "stats": stats, "conv": conv})
    return countries, stats, conv

if __name__ == "__main__":
    bev = beveridge(); mob, eu = mobility(); lc, stats, conv = labour_cost()
    print("== Mobility share abroad ==")
    for c in sorted(mob, key=lambda c: -(c["pts"][-1]["share"] or 0)):
        p = {q["y"]: q for q in c["pts"]}
        print(f'{c["id"]} 2010={p[2010]["share"]} 2015={p[2015]["share"]} 2019={p[2019]["share"]} 2020={p[2020]["share"]} 2021={p[2021]["share"]} 2024={p[2024]["share"]} | gap 2010={p[2010]["gap"]} 2019={p[2019]["gap"]} 2024={p[2024]["gap"]} erm24={p[2024]["erm"]} ern24={p[2024]["ern"]}')
    print("EU", [(e["y"], e["share"], e["erm"], e["ern"]) for e in eu])
    print("== Labour cost =="); [print(s) for s in stats]; print({k: v for k, v in conv.items() if k != "pts"})
    for c in sorted(lc, key=lambda c: -c["pts"][7]["c"]):
        p = {q["y"]: q for q in c["pts"]}
        print(c["id"], p[2008]["c"], p[2012]["c"], p[2024]["c"], p[2025]["c"], "nw", p[2012]["nw"], p[2024]["nw"], "x", round(p[2024]["c"]/p[2012]["c"],2))