#!/usr/bin/env python3
"""Study #7 collector (PREREG v2). COLLECTS ONLY: saves raw API responses, computes no trend.

Steps (run one at a time):
  n     numerator: social+mcp repos, every query cell listed by id, 2025-01..2026-09
  d     denominator: all mcp repos, listed by id, 2025-10..2026-09
  npm   npm cohort search + package metadata (time.created) + daily downloads

Search-cap rule (PREREG s4): each cell is listed by id; a cell over 1,000 results is split
(halving the time window, down to 1 hour) until ids collected == total_count. Every cell's
completeness is written to its file and to raw/<step>/CELLS.log.

Endpoint safety: host allowlist. No MCP endpoint, registry remote or package is ever contacted
or run. The GitHub token comes from `gh auth token` and is never printed.
"""
import calendar, datetime as dt, hashlib, json, os, re, subprocess, sys, time, urllib.parse, urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, "raw")
ALLOW = {"api.github.com", "registry.npmjs.org", "api.npmjs.org"}


def guard(url):
    u = urllib.parse.urlparse(url)
    if u.scheme != "https" or u.netloc not in ALLOW:
        raise SystemExit(f"BLOCKED host: {u.netloc}")


_TOKEN = None


def token():
    global _TOKEN
    if _TOKEN is None:
        _TOKEN = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True).stdout.strip()
    return _TOKEN


def get(url, gh=False, tries=5):
    guard(url)
    h = {"User-Agent": "blotato-research"}
    if gh:
        h["Authorization"] = f"Bearer {token()}"
        h["Accept"] = "application/vnd.github+json"
    for _ in range(tries):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers=h), timeout=60) as r:
                return json.loads(r.read().decode("utf-8", "replace"))
        except urllib.error.HTTPError as e:
            if e.code in (403, 429):
                time.sleep(65); continue
            if e.code in (404, 422):
                return {"_http": e.code}
            time.sleep(8)
        except Exception:
            time.sleep(8)
    return None


def gh_search(q, page=1, per_page=100):
    d = get("https://api.github.com/search/repositories?" + urllib.parse.urlencode({"q": q, "per_page": per_page, "page": page}), gh=True)
    time.sleep(2.1)  # search API: 30 requests/minute
    return d


# ---------- query set: study #6 collect.py GH_TERMS, X_NAME_QUERIES, A2, A3 (no star floor) ----------
GH_TERMS = {
    "x": ["twitter", "tweet", "x.com", "\"x api\"", "xurl"],
    "linkedin": ["linkedin"],
    "facebook": ["facebook", "\"fb page\"", "\"meta graph\""],
    "instagram": ["instagram", "\"ig graph\""],
    "threads": ["\"threads api\"", "threads.net", "\"meta threads\"", "threads"],
    "tiktok": ["tiktok"],
    "youtube": ["youtube"],
    "pinterest": ["pinterest"],
    "bluesky": ["bluesky", "bsky", "atproto", "\"at protocol\""],
    "reddit": ["reddit", "subreddit"],
}
EXTRA = [
    ("x", "x-mcp in:name"), ("x", "xmcp in:name"), ("x", "mcp-x in:name"),
    ("instagram", "insta mcp in:name"), ("instagram", "ig mcp in:name"), ("facebook", "fb mcp in:name"),
    ("generic", "\"social media\" mcp in:name,description,topics"), ("generic", "social-media mcp in:name"),
    ("generic", "\"social network\" mcp in:name,description,topics"), ("generic", "socials mcp in:name,description,topics"),
    ("generic", "\"social posts\" mcp in:name,description,topics"), ("generic", "\"social listening\" mcp in:name,description,topics"),
]
for _slug in ["twitter", "linkedin", "facebook", "instagram", "tiktok", "youtube", "pinterest", "bluesky", "reddit", "threads", "social-media"]:
    EXTRA.append((_slug, f"topic:model-context-protocol {_slug}"))


def n_queries():
    qs = [(p, f"{t} mcp in:name,description,topics") for p, terms in GH_TERMS.items() for t in terms]
    return qs + EXTRA


KEEP = ("id", "full_name", "name", "description", "topics", "created_at", "fork", "stargazers_count", "archived", "owner")


def slim(it):
    o = {k: it.get(k) for k in KEEP}
    o["owner"] = (it.get("owner") or {}).get("login")
    return o


def iso(t):
    return t.strftime("%Y-%m-%dT%H:%M:%SZ")


def list_cell(base, start, end, outdir, tag, log, depth=0):
    """List all ids for `base created:start..end`; split while total > 1000. Returns list of cell records."""
    q = f"{base} fork:false created:{iso(start)}..{iso(end)}"
    fn = os.path.join(outdir, f"{tag}_{hashlib.md5(q.encode()).hexdigest()[:12]}.json")
    if os.path.exists(fn):
        return
    head = gh_search(q, 1, 100)
    if not head or "items" not in head:
        json.dump({"q": q, "error": True}, open(fn, "w"))
        log.write(f"ERROR {q}\n"); log.flush(); return
    total = head.get("total_count") or 0
    if total > 1000 and (end - start) > dt.timedelta(hours=1):
        mid = start + (end - start) / 2
        mid = mid.replace(microsecond=0)
        list_cell(base, start, mid, outdir, tag, log, depth + 1)
        list_cell(base, mid + dt.timedelta(seconds=1), end, outdir, tag, log, depth + 1)
        return
    items = {it["id"]: slim(it) for it in head["items"]}
    page, incomplete = 1, bool(head.get("incomplete_results"))
    while len(items) < total and page < 10:
        page += 1
        d = gh_search(q, page, 100)
        if not d or "items" not in d or not d["items"]:
            break
        incomplete = incomplete or bool(d.get("incomplete_results"))
        for it in d["items"]:
            items[it["id"]] = slim(it)
    complete = len(items) >= total and not incomplete
    json.dump({"q": q, "total_count": total, "n_ids": len(items), "complete": complete, "incomplete_flag": incomplete,
               "items": list(items.values())}, open(fn, "w"))
    log.write(f"{'OK ' if complete else 'INC'} total={total} ids={len(items)} {q}\n"); log.flush()


def month_bounds(y, m):
    last = calendar.monthrange(y, m)[1]
    return dt.datetime(y, m, 1, 0, 0, 0), dt.datetime(y, m, last, 23, 59, 59)


def month_iter(start, end):
    y, m = start
    while (y, m) <= end:
        yield y, m
        m += 1
        if m == 13:
            y, m = y + 1, 1


def step_n():
    out = os.path.join(RAW, "n"); os.makedirs(out, exist_ok=True)
    log = open(os.path.join(out, "CELLS.log"), "a")
    for y, m in month_iter((2025, 1), (2026, 9)):
        a, b = month_bounds(y, m)
        for p, q in n_queries():
            list_cell(q, a, b, out, f"{y}-{m:02d}", log)
        print("n", y, m, flush=True)


def step_d():
    out = os.path.join(RAW, "d"); os.makedirs(out, exist_ok=True)
    log = open(os.path.join(out, "CELLS.log"), "a")
    for y, m in month_iter((2025, 10), (2026, 9)):
        a, b = month_bounds(y, m)
        list_cell("mcp in:name,description,topics", a, b, out, f"{y}-{m:02d}", log)
        print("d", y, m, flush=True)


NPM_TERMS = ["twitter", "tweet", "x", "linkedin", "facebook", "instagram", "threads", "tiktok", "youtube", "pinterest",
             "bluesky", "atproto", "reddit", "social media", "social"]


def step_npm():
    out = os.path.join(RAW, "npm"); os.makedirs(out, exist_ok=True)
    os.makedirs(os.path.join(out, "meta"), exist_ok=True); os.makedirs(os.path.join(out, "dl"), exist_ok=True)
    names = set()
    for t in NPM_TERMS:
        for frm in range(0, 2000, 250):
            fn = os.path.join(out, f"search_{hashlib.md5((t + str(frm)).encode()).hexdigest()[:10]}.json")
            if os.path.exists(fn):
                d = json.load(open(fn))
            else:
                d = get("https://registry.npmjs.org/-/v1/search?" + urllib.parse.urlencode({"text": f"keywords:mcp {t}", "size": 250, "from": frm}))
                json.dump({"term": t, "from": frm, "resp": d}, open(fn, "w")); time.sleep(0.5)
                d = {"resp": d}
            objs = ((d.get("resp") or {}).get("objects") or [])
            for o in objs:
                names.add(o["package"]["name"])
            if len(objs) < 250:
                break
    json.dump(sorted(names), open(os.path.join(out, "search_names.json"), "w"))
    for nm in sorted(names):
        fn = os.path.join(out, "meta", hashlib.md5(nm.encode()).hexdigest()[:12] + ".json")
        if not os.path.exists(fn):
            d = get("https://registry.npmjs.org/" + urllib.parse.quote(nm, safe="@"))
            keep = {k: (d or {}).get(k) for k in ("name", "description", "keywords", "time", "repository")}
            if isinstance(keep.get("time"), dict):
                keep["time"] = {"created": keep["time"].get("created"), "modified": keep["time"].get("modified")}
            json.dump(keep, open(fn, "w")); time.sleep(0.3)
    for nm in sorted(names):
        fn = os.path.join(out, "dl", hashlib.md5(nm.encode()).hexdigest()[:12] + ".json")
        if not os.path.exists(fn):
            d = get(f"https://api.npmjs.org/downloads/range/2025-10-01:2026-09-30/{urllib.parse.quote(nm, safe='@')}")
            json.dump({"package": nm, "resp": d}, open(fn, "w")); time.sleep(0.4)
    print("npm names", len(names))


if __name__ == "__main__":
    {"n": step_n, "d": step_d, "npm": step_npm}[sys.argv[1]]()