#!/usr/bin/env python3
"""One Cluster Lens pass (at most one per 50 min) and rebuild ./site.
Usage: python3 build.py [--force] [--offline] [--budget N]"""
import json
import os
import shutil
import sys
import time

import fetch
import lens
import render

HERE = os.path.dirname(os.path.abspath(__file__))
DATA, SITE = os.path.join(HERE, "data"), os.path.join(HERE, "site")
MIN_INTERVAL_S = 50 * 60
SRC_FILES = ["fetch.py", "lens.py", "render.py", "build.py", "hourly.sh", "README.md", "LICENSE",
             "allowlist.json", "tests/test_lens.py", "tests/test_review_line.py"]


def p(name):
    return os.path.join(DATA, name)


def _write(path, obj, compact=True):
    tmp = path + ".tmp"
    with open(tmp, "w") as f:
        if isinstance(obj, str):
            f.write(obj)
        else:
            json.dump(obj, f, separators=(",", ":") if compact else (",", ": "), indent=None if compact else 1)
    os.replace(tmp, path)


def review_line(bodies, index, shown):
    """Data-derived review line (forum comment 4386): independent-pair overlap baseline from the shown segment
    when it has enough answer text and independent pairs, else from the newest segment that does (labelled as carried over).
    Offline: uses cached text only, no fetches."""
    texts = {}
    for k, v in bodies.items():
        if v and isinstance(v.get("answer"), str):
            texts.setdefault(v.get("seg", fetch.SEGMENTS[0][0]), {})[k] = v
    order = [shown] + [sid for sid, _ in reversed(fetch.SEGMENTS) if sid != shown]
    for sid in order:
        ans = texts.get(sid) or {}
        eps = {v.get("epoch") for v in ans.values() if v.get("epoch") is not None}
        if len(ans) < lens.BASELINE_MIN_ANSWERS or len(eps) < 2:
            continue
        rows = lens.prepare(ans)
        seg_index = {k: v for k, v in index.items() if v.get("seg") == sid}
        tg = [g["members"] for g in lens.sync_groups(lens.sync_pairs(seg_index))]
        out = lens.independent_baseline(rows, tg)
        out.update({"segment": sid, "carried_over": sid != shown, "computed_at": render.utc(int(time.time())),
                    "shown_segment_answers_with_text": len(texts.get(shown) or {})})
        if out.get("line") is None or out.get("pairs_independent_sharing_a_gram", 0) < lens.BASELINE_MIN_PAIRS:
            continue
        return out
    return None


def main(argv):
    force, offline = "--force" in argv, "--offline" in argv
    budget = int(os.environ.get("CLUSTERS_BUDGET", "25"))
    if "--budget" in argv:
        budget = int(argv[argv.index("--budget") + 1])
    os.makedirs(DATA, exist_ok=True)
    now = int(time.time())
    state = fetch.load(p("state.json"), {})
    index = fetch.load(p("index.json"), {})
    bodies = fetch.load(p("bodies.json"), {})
    intros = fetch.load(p("intros.json"), [])
    if offline or (not force and now - state.get("ts", 0) < MIN_INTERVAL_S):
        print("no fetch this run (last pass %s, min interval %ds)" % (render.utc(state.get("ts")), MIN_INTERVAL_S))
    else:
        state["ts"] = now
        fetch.save(p("state.json"), state)  # stamp first so a crash still respects the interval
        errors = []
        try:
            added = fetch.update_index(index)
            fetch.save(p("index.json"), index)
        except Exception as e:  # keep the old index
            added, errors = 0, errors + ["index: %s" % type(e).__name__]
        try:
            intros = fetch.fetch_intros()
            fetch.save(p("intros.json"), intros)
        except Exception as e:
            errors.append("intros: %s" % type(e).__name__)
        got, missing, why = fetch.fill_bodies(index, bodies, budget=budget)
        fetch.save(p("bodies.json"), bodies)
        state.update({"last_added": added, "last_bodies": got, "missing": missing, "stop": why, "errors": errors})
        fetch.save(p("state.json"), state)
        print("pass at %s: index +%d (%d), bodies +%d (missing %d, %s), intros %d, errors %s" % (
            render.utc(now), added, len(index), got, missing, why, len(intros), errors or "none"))
    allow = fetch.load(os.path.join(HERE, "allowlist.json"), {"entries": []})
    fetch.tag_segments(index)
    fetch.tag_segments({k: v for k, v in bodies.items() if v})
    seg_counts = {}
    for v in index.values():
        seg_counts[v.get("seg")] = seg_counts.get(v.get("seg"), 0) + 1
    # show the newest segment that has answers; older segments stay frozen as their own data sets
    shown = next((sid for sid, _ in reversed(fetch.SEGMENTS) if seg_counts.get(sid)), fetch.SEGMENTS[0][0])
    first = shown == fetch.SEGMENTS[0][0]
    full_index = index
    index = {k: v for k, v in index.items() if v.get("seg") == shown}
    answers = {k: v for k, v in bodies.items() if v and v.get("seg", fetch.SEGMENTS[0][0]) == shown}
    shadow_file = "shadow.json" if first else "shadow_%s.json" % shown
    sk = "shadow_start" if first else "shadow_start_%s" % shown
    backtest = fetch.load(p("backtest.json"), {})
    review = review_line(bodies, full_index, shown)
    doc = render.build_doc(answers, index, intros, allow, now, fetch.load(p(shadow_file), []), backtest, review=review)
    doc["segment"] = {"shown": shown, "rows_by_segment": seg_counts, "segments": [{"id": s, "from_ts": t} for s, t in fetch.SEGMENTS],
                      "note": "Epoch numbers restart in each chain segment; this view is built from %s only. Other segments are kept "
                              "as their own data sets and never mixed into these clusters." % shown}
    # shadow: record each epoch once, after it has closed (an epoch closes when a later epoch has answers)
    eps = [e for e in doc["coverage"]["epochs"] if isinstance(e, int)]
    last_closed = max(eps) - 1 if eps else None
    shadow = fetch.load(p(shadow_file), [])
    if state.get(sk) is None and last_closed is not None:
        state[sk] = last_closed
        fetch.save(p("state.json"), state)
    if last_closed is not None and last_closed >= state.get(sk, 10 ** 9):
        shadow = render.update_shadow(shadow, doc, last_closed)
        fetch.save(p(shadow_file), shadow)
    doc["shadow"] = shadow
    if os.path.isdir(SITE):
        shutil.rmtree(SITE)
    os.makedirs(os.path.join(SITE, "src", "tests"))
    _write(os.path.join(SITE, "clusters.json"), doc)
    _write(os.path.join(SITE, "index.html"), render.render_html(doc))
    shutil.copy(os.path.join(HERE, "allowlist.json"), os.path.join(SITE, "allowlist.json"))
    # public answer text, one file per epoch, so anyone can recompute without re-crawling the node
    os.makedirs(os.path.join(SITE, "answers"))
    per = {}
    for t, a in answers.items():
        per.setdefault(str(a.get("epoch")), {})[t] = {"from": a["from"], "epoch": a.get("epoch"), "ts": a.get("ts"), "answer": a["answer"]}
    for e, d in per.items():
        _write(os.path.join(SITE, "answers", ("e%s.json" % e) if first else ("%s-e%s.json" % (shown, e))), dict(sorted(d.items())))
    for f in SRC_FILES:
        if os.path.exists(os.path.join(HERE, f)):
            shutil.copy(os.path.join(HERE, f), os.path.join(SITE, "src", f))
    _write(os.path.join(SITE, "src", "index.html"), "<!doctype html><meta charset=utf-8><title>Cluster Lens source</title><h1>Cluster Lens source (MIT)</h1><ul>%s</ul>" % "".join(
        "<li><a href='%s'>%s</a></li>" % (f, f) for f in SRC_FILES if os.path.exists(os.path.join(HERE, f))))
    S = doc["summary"]
    print("built site/: families %d (registry %d), accounts %d, answers %d, timing groups %d, intro groups %d, cleared %d; text %d/%d" % (
        S["families"], S["registry_families"], S["accounts_in_families"], S["answers_in_families"], S["timing_groups"],
        S["intro_groups"], S["cleared"], doc["coverage"]["answers_with_text"], doc["coverage"]["answers_indexed"]))
    me = "poa13YUqWLYxEv7rfXcJKWUAAJgaK3Vd5nYyHVH8mKwvia7B"  # EmBot-Lantern: no special treatment, just reported in the log
    inf = [f["id"] for f in doc["families"] if me in f["accounts"]] + ["timing" for g in doc["timing"]["groups"] if me in g["accounts"]]
    print("EmBot-Lantern in clusters: %s" % (inf or "none"))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))