"""Re-download every Eurostat table used on the site and compare it with the stored snapshot.

    python3 analysis/fetch_eurostat.py            # report differences only
    python3 analysis/fetch_eurostat.py --update   # overwrite the snapshots, then run build_data.py and build.py

Needs outbound HTTPS to ec.europa.eu. Standard library only."""
import csv, io, sys, datetime, urllib.request
from common import RAW, EU27, parse_cell

API = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/{code}/{key}?format=TSV&startPeriod={a}&endPeriod={b}"
TABLES = {  # name: (SDMX key, first year, last year, number of key columns kept, extra geo codes)
    "une_rt_a":       ("A.Y15-74.PC_ACT.T.", 2010, 2025, 1, ["EU27_2020"]),
    "jvs_a_rate_r2":  ("A.B-S.TOTAL.AVG_A.", 2010, 2025, 1, ["EU27_2020"]),
    "lfst_lmbpcita":  ("A.THS_PER.EU27_2020_EFTA.Y20-64.", 2010, 2024, 1, ["EU27_2020_FOR"]),
    "lfst_lmbercita": ("A.PC.EU27_2020_EFTA.Y20-64.", 2010, 2024, 1, ["EU27_2020_FOR"]),
    "lfsa_pganws":    ("A.THS_PER.T.NAT.Y20-64.POP.", 2010, 2024, 1, ["EU27_2020"]),
    "lfsa_ergan":     ("A.PC.T.Y20-64.NAT.", 2010, 2024, 1, ["EU27_2020"]),
    "lc_lci_lev":     ("A.EUR.D1_D4_MD5+D11.B-S_X_O.", 2008, 2025, 2, ["EU27_2020"]),
}

def fetch(name):
    key, a, b, k, extra = TABLES[name]
    with urllib.request.urlopen(API.format(code=name, key=key, a=a, b=b), timeout=60) as r:
        text = r.read().decode("utf-8")
    rows = list(csv.reader(io.StringIO(text), delimiter="\t"))
    years = [c.strip() for c in rows[0][1:]]
    out = {}
    for r in rows[1:]:
        dims = r[0].split(","); geo = dims[-1]
        if geo not in EU27 + extra: continue
        rk = geo if k == 1 else (dims[2], geo)          # lc_lci_lev: (lcstruct, geo)
        out[rk] = {y: " ".join(c.replace("@C", "").split()).lower() for y, c in zip(years, r[1:])}
    return years, out

def stored(name):
    k = TABLES[name][3]
    lines = (RAW / f"{name}.csv").read_text(encoding="utf-8").splitlines()
    rows = list(csv.reader(lines[1:])); years = rows[0][k:]
    return lines[0], rows[0], {(r[0] if k == 1 else (r[0], r[1])): dict(zip(years, r[k:])) for r in rows[1:]}

def main(update=False):
    total = diff = 0
    for name in TABLES:
        meta, header, old = stored(name); k = TABLES[name][3]
        years, new = fetch(name)
        for rk, cells in old.items():
            for y, c in cells.items():
                total += 1
                a, b = parse_cell(c), parse_cell(new.get(rk, {}).get(y, ":") or ":")
                if a != b: diff += 1; print(f"DIFF {name} {rk} {y}: stored {c!r} / Eurostat now {new.get(rk, {}).get(y)!r}")
        if update:
            ys = header[k:]
            with open(RAW / f"{name}.csv", "w", newline="", encoding="utf-8") as f:
                f.write(meta.split("| retrieved")[0] + f"| retrieved {datetime.date.today()}\n")
                w = csv.writer(f); w.writerow(header)
                for rk in old: w.writerow(([rk] if k == 1 else list(rk)) + [new[rk].get(y, ":") for y in ys])
    print(f"compared {total} cells in {len(TABLES)} tables: {diff} differ" + (" (snapshots updated)" if update else ""))
    return diff

if __name__ == "__main__":
    update = "--update" in sys.argv
    sys.exit(1 if main(update) and not update else 0)