#!/usr/bin/env python3
"""Study #7 amendment A2 (2026-10-13): the D2 npm cohort, collected COMPLETELY. Supersedes `collect_trend.py npm`.

Why: `text=keywords:mcp <term>` does not filter by term. Every term reported the same total (79,385, all
packages tagged mcp) and the pull stopped at the top 2,000 by a relevance score that tracks downloads, so the
cohort was truncated and tilted toward packages popular today. That capped pull is kept in raw/npm_capped_v3/
and is never computed.

A2 cohort search: `text=keywords:mcp keywords:<tag>` (both tags required, a real filter). Every page is pulled
until collected names == the response's `total` for that tag; per-tag completeness goes to raw/npm/SEARCH.log.
Exits non-zero if any tag is incomplete after one full re-pull. Metadata and daily downloads are then pulled
exactly as collect_trend.py step_npm does (same hosts, same files, same date range), so analyze_trend.py
d2_gate reads them unchanged (created before 2025-10-01 + mcp text + platform/generic regex).

Collects only. Host allowlist and request functions are reused from collect_trend.py.
"""
import hashlib, importlib.util, json, os, sys, time, urllib.parse

HERE = os.path.dirname(os.path.abspath(__file__))
spec = importlib.util.spec_from_file_location("ct", os.path.join(HERE, "collect_trend.py"))
ct = importlib.util.module_from_spec(spec); spec.loader.exec_module(ct)

TAGS = [t.replace(" ", "-") for t in ct.NPM_TERMS]  # "social media" -> "social-media" (a tag has no spaces)
SIZE = 250


def pull_tag(tag, out):
    names, total, page_files = {}, None, []
    frm = 0
    while True:
        d = ct.get("https://registry.npmjs.org/-/v1/search?" + urllib.parse.urlencode({"text": f"keywords:mcp keywords:{tag}", "size": SIZE, "from": frm}))
        fn = os.path.join(out, f"search_{tag}_{frm}.json")
        json.dump({"tag": tag, "from": frm, "resp": d}, open(fn, "w")); time.sleep(0.5)
        objs = ((d or {}).get("objects") or [])
        total = (d or {}).get("total", total)
        for o in objs:
            names[o["package"]["name"]] = 1
        if not objs or len(objs) < SIZE or frm + SIZE >= (total or 0):
            break
        frm += SIZE
    return sorted(names), total


def main():
    out = os.path.join(ct.RAW, "npm"); os.makedirs(os.path.join(out, "meta"), exist_ok=True); os.makedirs(os.path.join(out, "dl"), exist_ok=True)
    log = open(os.path.join(out, "SEARCH.log"), "a")
    allnames, bad = set(), []
    for tag in TAGS:
        names, total = pull_tag(tag, out)
        if total is None or len(names) != total:
            names, total = pull_tag(tag, out)  # one full re-pull
        ok = total is not None and len(names) == total
        log.write(f"{'OK ' if ok else 'INC'} tag={tag} total={total} collected={len(names)}\n"); log.flush()
        if not ok:
            bad.append(tag)
        allnames.update(names)
    json.dump(sorted(allnames), open(os.path.join(out, "search_names.json"), "w"))
    log.write(f"UNION names={len(allnames)} incomplete_tags={bad}\n"); log.flush()
    if bad:
        print("INCOMPLETE tags:", bad); sys.exit(2)
    for nm in sorted(allnames):
        fn = os.path.join(out, "meta", hashlib.md5(nm.encode()).hexdigest()[:12] + ".json")
        if not os.path.exists(fn):
            d = ct.get("https://registry.npmjs.org/" + urllib.parse.quote(nm, safe="@"))
            keep = {k: (d or {}).get(k) for k in ("name", "description", "keywords", "time", "repository")}
            if isinstance(keep.get("time"), dict):
                keep["time"] = {"created": keep["time"].get("created"), "modified": keep["time"].get("modified")}
            json.dump(keep, open(fn, "w")); time.sleep(0.3)
    for nm in sorted(allnames):
        fn = os.path.join(out, "dl", hashlib.md5(nm.encode()).hexdigest()[:12] + ".json")
        if not os.path.exists(fn):
            d = ct.get(f"https://api.npmjs.org/downloads/range/2025-10-01:2026-09-30/{urllib.parse.quote(nm, safe='@')}")
            json.dump({"package": nm, "resp": d}, open(fn, "w")); time.sleep(0.4)
    print("npm A2 names", len(allnames))


if __name__ == "__main__":
    main()