#!/usr/bin/env python3
"""Study #7 (PREREG v3 s4): re-pull, once, every cell that is incomplete or errored.
A cell is complete when ids are within max(2, 0.5% of total_count) of total_count.
Writes raw/<step>/recheck/<same filename>; analyze_trend.py uses the recheck file when present.
Collects only. Reuses collect_trend.py's guarded request functions.
"""
import glob, importlib.util, json, os, sys

HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location("ct", os.path.join(HERE, "collect_trend.py"))
ct = importlib.util.module_from_spec(spec); spec.loader.exec_module(ct)


def incomplete(c):
    t = c.get("total_count")
    return c.get("error") or t is None or c.get("n_ids", 0) < t - max(2, 0.005 * t)


def main(step):
    d = os.path.join(ct.RAW, step); rd = os.path.join(d, "recheck"); os.makedirs(rd, exist_ok=True)
    todo = [fn for fn in sorted(glob.glob(os.path.join(d, "*.json"))) if incomplete(json.load(open(fn)))]
    for fn in todo:
        out = os.path.join(rd, os.path.basename(fn))
        if os.path.exists(out):
            continue
        q = json.load(open(fn))["q"]
        items, total, inc, page = {}, None, False, 0
        while page < 10:
            page += 1
            r = ct.gh_search(q, page, 100)
            if not r or "items" not in r or not r["items"]:
                break
            if total is None:
                total = r.get("total_count")
            inc = inc or bool(r.get("incomplete_results"))
            for it in r["items"]:
                items[it["id"]] = ct.slim(it)
            # A3 (2026-10-13): page like collect_trend.list_cell (stop at total, page 10 or an empty page). GitHub can
            # return a short page mid-pull without flagging incomplete_results; the old `< 100` stop truncated re-pulls.
            if len(items) >= (total or 0):
                break
        json.dump({"q": q, "total_count": total, "n_ids": len(items), "recheck": True, "incomplete_flag": inc,
                   "error": total is None, "items": list(items.values())}, open(out, "w"))
    print(step, "rechecked", len(todo))


if __name__ == "__main__":
    main(sys.argv[1])