#!/usr/bin/env python3
"""Study #7 (PREREG s5): save each sampled repo's README, repo metadata and root manifests for the classifiers.
api.github.com only (collect_trend.py host allowlist and request function). Fetches, never runs, any code.
Output: work/repos/<owner>__<repo>/{repo.json, README.md, <manifest>}; a 404 repo gets GONE.txt (= unreadable = no).
Then writes work/batches/*.json: N items (P1+P2 shuffled together) and D items, period hidden from classifiers.
"""
import base64, importlib.util, json, os, random, sys

HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location("ct", os.path.join(HERE, "collect_trend.py"))
ct = importlib.util.module_from_spec(spec); spec.loader.exec_module(ct)
WORK = os.path.join(HERE, "work")
MANIFESTS = ["package.json", "pyproject.toml", "requirements.txt", "setup.py", "go.mod", "Cargo.toml", "server.json", "smithery.yaml"]


def save_repo(full):
    d = os.path.join(WORK, "repos", full.replace("/", "__"))
    if os.path.exists(os.path.join(d, "DONE")):
        return
    os.makedirs(d, exist_ok=True)
    meta = ct.get(f"https://api.github.com/repos/{full}", gh=True)
    if not meta or meta.get("_http"):
        open(os.path.join(d, "GONE.txt"), "w").write(f"repo metadata not readable: {meta}\n")
        open(os.path.join(d, "DONE"), "w").write("gone"); return
    keep = {k: meta.get(k) for k in ("full_name", "description", "topics", "homepage", "created_at", "fork", "archived", "default_branch")}
    json.dump(keep, open(os.path.join(d, "repo.json"), "w"), indent=1)
    rd = ct.get(f"https://api.github.com/repos/{full}/readme", gh=True)
    if rd and rd.get("content"):
        open(os.path.join(d, "README.md"), "w").write(base64.b64decode(rd["content"]).decode("utf-8", "replace"))
    root = ct.get(f"https://api.github.com/repos/{full}/contents/", gh=True)
    names = [x.get("name") for x in root] if isinstance(root, list) else []
    open(os.path.join(d, "FILES.txt"), "w").write("\n".join(sorted(n for n in names if n)) + "\n")
    for m in MANIFESTS:
        if m in names:
            f = ct.get(f"https://api.github.com/repos/{full}/contents/{m}", gh=True)
            if f and f.get("content"):
                open(os.path.join(d, m), "w").write(base64.b64decode(f["content"]).decode("utf-8", "replace")[:20000])
    open(os.path.join(d, "DONE"), "w").write("ok")


def main():
    samples = {n: json.load(open(os.path.join(WORK, f"sample_{n}.json"))) for n in ("N_P1", "N_P2", "D_P1", "D_P2")}
    fulls = sorted({x["full_name"] for s in samples.values() for x in s})
    for i, full in enumerate(fulls):
        save_repo(full)
        if i % 50 == 0:
            print("fetched", i, "/", len(fulls), flush=True)
    os.makedirs(os.path.join(WORK, "batches"), exist_ok=True)
    rnd = random.Random(20261013)
    for kind, parts, nb in (("N", ("N_P1", "N_P2"), 6), ("D", ("D_P1", "D_P2"), 2)):
        items = sorted({x["full_name"] for p in parts for x in samples[p]})
        rnd.shuffle(items)
        for b in range(nb):
            json.dump(items[b::nb], open(os.path.join(WORK, "batches", f"{kind}{b + 1}.json"), "w"), indent=1)
    print("repos", len(fulls), "gone", sum(os.path.exists(os.path.join(WORK, "repos", f.replace("/", "__"), "GONE.txt")) for f in fulls))


if __name__ == "__main__":
    main()