"""Study #6 collection: social media MCP servers (frame = official MCP Registry + GitHub 5+ stars).

Desk data only. Contacts ONLY allowlisted hosts; every MCP server address is hard-denied.
Snapshot: registry dump pulled 2026-10-12 08:17 CT (raw/registry/official_all_latest.json);
GitHub metadata frozen at this run; last commit measured with until=PULL_UNTIL.

Usage: python3 -I collect.py search|candidates|fetch|all
"""
import json, os, re, sys, time, subprocess, urllib.parse, urllib.request, hashlib

HERE = os.path.dirname(os.path.abspath(__file__))
RAW = os.path.join(HERE, "raw")
PULL_DATE = "2026-10-12"
PULL_UNTIL = "2026-10-12T23:59:59Z"

ALLOW_HOSTS = {"api.github.com", "raw.githubusercontent.com", "registry.modelcontextprotocol.io"}
DENY = [r"open_mcp", r"^api\.x\.com/mcp", r"^ads-api\.x\.com/mcp", r"^mcp\.facebook\.com",
        r"^api\.pinterest\.com/mcp", r"^mcp\.blotato\.com"]
REMOTE_DENY = set()  # filled from registry remotes[].url host+path


def guard(url):
    p = urllib.parse.urlparse(url)
    hp = p.netloc + p.path
    if p.netloc not in ALLOW_HOSTS:
        raise SystemExit(f"BLOCKED (host not allowlisted): {url}")
    for d in DENY:
        if re.search(d, hp):
            raise SystemExit(f"BLOCKED (deny rule {d}): {url}")
    if any(hp.startswith(r) for r in REMOTE_DENY):
        raise SystemExit(f"BLOCKED (registry remote): {url}")
    return url


def gh(path, params=None):
    """GitHub REST via authenticated gh CLI. Returns parsed JSON or None on 404."""
    url = "https://api.github.com/" + path.lstrip("/")
    if params:
        url += "?" + urllib.parse.urlencode(params)
    guard(url)
    for attempt in range(5):
        r = subprocess.run(["gh", "api", "-H", "Accept: application/vnd.github+json", url.replace("https://api.github.com/", "")],
                           capture_output=True, text=True)
        if r.returncode == 0:
            return json.loads(r.stdout) if r.stdout.strip() else None
        err = r.stderr + r.stdout
        if "404" in err or "Not Found" in err:
            return None
        if "409" in err or "Git Repository is empty" in err:
            return {"__empty__": True}
        if "rate limit" in err.lower() or "403" in err or "429" in err:
            time.sleep(60 * (attempt + 1))
            continue
        time.sleep(3)
    raise RuntimeError(f"gh failed: {url}: {err[:300]}")


def raw_get(owner, repo, ref, path):
    url = f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{urllib.parse.quote(path)}"
    guard(url)
    try:
        with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "blotato-research"}), timeout=30) as r:
            return r.read().decode("utf-8", "replace")
    except Exception:
        return None


# ---------- platform matching ----------
PLATFORMS = {
    "x": r"\btwitter\b|\btweets?\b|\bx\.com\b|\bx api\b|\bx-mcp\b|\bxmcp\b|\bmcp-x\b|\bx \(formerly twitter\)|\bx/twitter\b|\bx \(twitter\)|\bxurl\b",
    "linkedin": r"linkedin",
    "facebook": r"\bfacebook\b|\bfb page|\bmeta graph",
    "instagram": r"instagram|\big graph",
    "threads": r"threads\.net|threads api|meta threads|threads by meta|\bthreads\b",
    "tiktok": r"tiktok",
    "youtube": r"youtube",
    "pinterest": r"pinterest",
    "bluesky": r"bluesky|\bbsky\b|atproto|at protocol",
    "reddit": r"reddit|subreddit",
}
GH_TERMS = {
    "x": ["twitter", "tweet", "x.com", "\"x api\"", "xurl"],
    "linkedin": ["linkedin"],
    "facebook": ["facebook", "\"fb page\"", "\"meta graph\""],
    "instagram": ["instagram", "\"ig graph\""],
    "threads": ["\"threads api\"", "threads.net", "\"meta threads\"", "threads"],
    "tiktok": ["tiktok"],
    "youtube": ["youtube"],
    "pinterest": ["pinterest"],
    "bluesky": ["bluesky", "bsky", "atproto"],
    "reddit": ["reddit", "subreddit"],
}
X_NAME_QUERIES = ["x-mcp in:name", "xmcp in:name", "mcp-x in:name"]
X_FALSE_STEMS = re.compile(r"x64dbg|xcode|xero|xmind|xmpp|xlsx|xml|xray|xterm|xbox", re.I)


def plats_in(text):
    t = (text or "")
    return sorted(k for k, rx in PLATFORMS.items() if re.search(rx, t, re.I))


def norm_repo(url):
    if not url:
        return None
    m = re.match(r"https?://(?:www\.)?github\.com/([^/\s]+)/([^/\s#?]+)", url.strip())
    if not m:
        return url.strip().rstrip("/").lower()
    return f"github.com/{m.group(1)}/{re.sub(r'\.git$', '', m.group(2))}".lower()


# ---------- steps ----------
def step_search():
    os.makedirs(f"{RAW}/github_search", exist_ok=True)
    queries = []
    for p, terms in GH_TERMS.items():
        for t in terms:
            queries.append((p, f"{t} mcp in:name,description stars:>=5 fork:false"))
            queries.append((p, f"topic:mcp-server {t} stars:>=5 fork:false"))
            queries.append((p, f"topic:mcp {t} stars:>=5 fork:false"))
    for q in X_NAME_QUERIES:
        queries.append(("x", f"{q} stars:>=5 fork:false"))
    log = []
    for p, q in queries:
        fn = f"{RAW}/github_search/{hashlib.md5(q.encode()).hexdigest()[:10]}.json"
        if os.path.exists(fn):
            log.append({"platform": p, "q": q, "file": os.path.basename(fn)}); continue
        items, total = [], None
        for page in range(1, 11):
            d = gh("search/repositories", {"q": q, "per_page": 100, "page": page})
            time.sleep(2.2)
            if not d:
                break
            total = d.get("total_count")
            items += d.get("items", [])
            if len(d.get("items", [])) < 100:
                break
        json.dump({"q": q, "platform": p, "total_count": total, "fetched": time.strftime("%Y-%m-%dT%H:%M:%S"), "items": items}, open(fn, "w"))
        log.append({"platform": p, "q": q, "file": os.path.basename(fn), "total_count": total, "n": len(items)})
        print(p, q, total, len(items), flush=True)
    json.dump(log, open(f"{RAW}/github_search/_index.json", "w"), indent=1)


def step_candidates():
    reg = json.load(open(f"{RAW}/registry/official_all_latest.json"))
    for e in reg:
        for r in e["server"].get("remotes", []) or []:
            u = urllib.parse.urlparse(r.get("url", ""))
            REMOTE_DENY.add(u.netloc + u.path)
    json.dump(sorted(REMOTE_DENY), open(f"{RAW}/registry/_remote_deny.json", "w"))
    cands = {}

    def add(key, **kw):
        c = cands.setdefault(key, {"key": key, "repo": None, "registry": [], "github_queries": [], "platform_hits": set(), "stars_search": None})
        for k, v in kw.items():
            if k == "registry": c["registry"].append(v)
            elif k == "q": c["github_queries"].append(v)
            elif k == "plats": c["platform_hits"].update(v)
            elif v is not None: c[k] = v

    # registry arm
    n_reg = 0
    for e in reg:
        s = e["server"]; meta = e.get("_meta", {}).get("io.modelcontextprotocol.registry/official", {})
        if meta.get("status") == "deleted":
            continue
        text = " ".join([s.get("name", ""), s.get("title", "") or "", s.get("description", "") or ""])
        ps = plats_in(text)
        if not ps:
            continue
        n_reg += 1
        repo = norm_repo((s.get("repository") or {}).get("url"))
        remote = ((s.get("remotes") or [{}])[0] or {}).get("url")
        key = repo or (("remote:" + urllib.parse.urlparse(remote).netloc + urllib.parse.urlparse(remote).path) if remote else "name:" + s["name"])
        add(key, repo=repo if repo and repo.startswith("github.com/") else None,
            registry={"name": s["name"], "title": s.get("title"), "description": s.get("description"), "version": s.get("version"),
                      "status": meta.get("status"), "publishedAt": meta.get("publishedAt"), "updatedAt": meta.get("updatedAt"),
                      "repository": s.get("repository"), "remotes": s.get("remotes"),
                      "packages": [{k: p.get(k) for k in ("registryType", "identifier", "version")} for p in (s.get("packages") or [])]},
            plats=ps)
        if repo and not repo.startswith("github.com/"):
            cands[key]["non_github_repo"] = repo
    # github arm
    idx = json.load(open(f"{RAW}/github_search/_index.json"))
    n_gh_raw = 0; dropped = []
    for row in idx:
        d = json.load(open(f"{RAW}/github_search/{row['file']}"))
        for it in d["items"]:
            n_gh_raw += 1
            if it.get("fork") or it.get("stargazers_count", 0) < 5:
                continue
            text = " ".join([it["name"], it.get("description") or "", " ".join(it.get("topics") or [])])
            if not re.search(r"mcp|model context protocol", text, re.I):
                dropped.append({"repo": it["full_name"], "why": "no mcp in name/description/topics", "q": d["q"]}); continue
            ps = plats_in(text)
            if row["platform"] == "x" and "x" not in ps:
                pass
            if row["platform"] == "x" and X_FALSE_STEMS.search(it["name"]) and "x" in ps and not re.search(r"twitter|tweet", text, re.I):
                ps = [p for p in ps if p != "x"]
            if not ps:
                dropped.append({"repo": it["full_name"], "why": "no platform term in name/description/topics", "q": d["q"]}); continue
            key = norm_repo(it["html_url"])
            add(key, repo=key, q=d["q"], plats=ps, stars_search=it["stargazers_count"])
    out = []
    for c in cands.values():
        c["platform_hits"] = sorted(c["platform_hits"])
        c["arm"] = "both" if c["registry"] and c["github_queries"] else ("registry" if c["registry"] else "github")
        c["github_queries"] = sorted(set(c["github_queries"]))
        out.append(c)
    out.sort(key=lambda c: c["key"])
    os.makedirs(f"{HERE}/work", exist_ok=True)
    json.dump(out, open(f"{HERE}/work/candidates.json", "w"), indent=1)
    json.dump(dropped, open(f"{HERE}/work/gh_dropped_by_filter.json", "w"), indent=1)
    from collections import Counter
    print("registry entries matched:", n_reg, "| gh raw items:", n_gh_raw, "| dropped by filter:", len(dropped))
    print("candidates:", len(out), Counter(c["arm"] for c in out))
    print("with github repo:", sum(1 for c in out if c["repo"]), "| no repo:", sum(1 for c in out if not c["repo"]))


BOT = re.compile(r"\[bot\]$|^dependabot|^renovate|^github-actions", re.I)
MANIFESTS = ["package.json", "pyproject.toml", "requirements.txt", "setup.py", "go.mod", "Cargo.toml"]


def fetch_one(c):
    _, owner, name = c["repo"].split("/")
    d = f"{RAW}/repos/{owner}__{name}"
    if os.path.exists(f"{d}/_done"):
        return 0
    os.makedirs(d, exist_ok=True)
    meta = gh(f"repos/{owner}/{name}")
    if not meta:
        json.dump({"missing": True}, open(f"{d}/repo.json", "w")); open(f"{d}/_done", "w").write(PULL_DATE); return 1
    json.dump(meta, open(f"{d}/repo.json", "w"))
    if meta.get("__empty__"):
        open(f"{d}/_done", "w").write(PULL_DATE); return 1
    own, nm = meta["full_name"].split("/")
    br = meta.get("default_branch") or "main"
    commits, last_human = [], None
    for page in (1, 2, 3):
        cs = gh(f"repos/{own}/{nm}/commits", {"sha": br, "until": PULL_UNTIL, "per_page": 100, "page": page})
        if not cs or isinstance(cs, dict):
            break
        batch = [{"sha": x["sha"], "author_login": (x.get("author") or {}).get("login"), "author_name": x["commit"]["author"]["name"],
                  "committer_date": x["commit"]["committer"]["date"]} for x in cs]
        commits += batch
        for x in batch:
            if not BOT.search(x["author_login"] or x["author_name"] or ""):
                last_human = x; break
        if last_human or len(cs) < 100:
            break
    json.dump({"until": PULL_UNTIL, "branch": br, "commits_seen": commits[:300], "last_non_bot": last_human}, open(f"{d}/commits.json", "w"))
    sub = None
    for r in c["registry"]:
        sub = ((r.get("repository") or {}).get("subfolder")) or sub
    rd = gh(f"repos/{own}/{nm}/readme" + (f"/{sub}" if sub else ""))
    if rd and rd.get("path"):
        txt = raw_get(own, nm, br, rd["path"])
        if txt is not None:
            open(f"{d}/README.md", "w").write(txt)
    for mf in MANIFESTS:
        txt = raw_get(own, nm, br, (sub + "/" if sub else "") + mf)
        if txt is not None and len(txt) < 400000:
            open(f"{d}/{mf}", "w").write(txt)
    open(f"{d}/_done", "w").write(PULL_DATE)
    return 1


def step_fetch():
    from concurrent.futures import ThreadPoolExecutor
    cands = json.load(open(f"{HERE}/work/candidates.json"))
    reg = json.load(open(f"{RAW}/registry/official_all_latest.json"))
    for e in reg:
        for r in e["server"].get("remotes", []) or []:
            u = urllib.parse.urlparse(r.get("url", ""))
            REMOTE_DENY.add(u.netloc + u.path)
    os.makedirs(f"{RAW}/repos", exist_ok=True)
    todo = [c for c in cands if c["repo"]]
    done = 0
    with ThreadPoolExecutor(8) as ex:
        for n in ex.map(fetch_one, todo):
            done += n
            if done and done % 50 == 0:
                print("fetched", done, flush=True)
    print("fetch complete", done)


GENERIC = re.compile(r"social[ -]media|social network|social posting|social publishing|social scheduling|social accounts|social platforms|cross-?post", re.I)
A2_QUERIES = ['"social media" mcp in:name,description stars:>=5 fork:false', 'social-media mcp in:name stars:>=5 fork:false',
              'topic:mcp-server social-media stars:>=5 fork:false', 'topic:mcp social-media stars:>=5 fork:false',
              '"social network" mcp in:name,description stars:>=5 fork:false']


def step_a2():
    """Amendment A2: generic social media servers (registry + GitHub), new keys only."""
    base = {c["key"] for c in json.load(open(f"{HERE}/work/candidates.json"))}
    reg = json.load(open(f"{RAW}/registry/official_all_latest.json"))
    new = {}
    for e in reg:
        s = e["server"]; meta = e.get("_meta", {}).get("io.modelcontextprotocol.registry/official", {})
        if meta.get("status") == "deleted":
            continue
        t = " ".join([s.get("name", ""), s.get("title", "") or "", s.get("description", "") or ""])
        if not GENERIC.search(t) or plats_in(t):
            continue
        repo = norm_repo((s.get("repository") or {}).get("url"))
        remote = ((s.get("remotes") or [{}])[0] or {}).get("url")
        key = repo or (("remote:" + urllib.parse.urlparse(remote).netloc + urllib.parse.urlparse(remote).path) if remote else "name:" + s["name"])
        if key in base:
            continue
        c = new.setdefault(key, {"key": key, "repo": repo if repo and repo.startswith("github.com/") else None, "registry": [], "github_queries": [], "platform_hits": [], "a2": True})
        c["registry"].append({"name": s["name"], "title": s.get("title"), "description": s.get("description"), "version": s.get("version"),
                              "status": meta.get("status"), "publishedAt": meta.get("publishedAt"), "updatedAt": meta.get("updatedAt"),
                              "repository": s.get("repository"), "remotes": s.get("remotes"),
                              "packages": [{k: p.get(k) for k in ("registryType", "identifier", "version")} for p in (s.get("packages") or [])]})
    os.makedirs(f"{RAW}/github_search", exist_ok=True)
    for q in A2_QUERIES:
        fn = f"{RAW}/github_search/a2_{hashlib.md5(q.encode()).hexdigest()[:10]}.json"
        if not os.path.exists(fn):
            items, total = [], None
            for page in range(1, 11):
                d = gh("search/repositories", {"q": q, "per_page": 100, "page": page}); time.sleep(2.2)
                if not d: break
                total = d.get("total_count"); items += d.get("items", [])
                if len(d.get("items", [])) < 100: break
            json.dump({"q": q, "platform": "generic", "total_count": total, "items": items}, open(fn, "w"))
        d = json.load(open(fn))
        print(q, d["total_count"], len(d["items"]))
        for it in d["items"]:
            if it.get("fork") or it.get("stargazers_count", 0) < 5:
                continue
            text = " ".join([it["name"], it.get("description") or "", " ".join(it.get("topics") or [])])
            if not re.search(r"mcp|model context protocol", text, re.I):
                continue
            key = norm_repo(it["html_url"])
            if key in base:
                continue
            c = new.setdefault(key, {"key": key, "repo": key, "registry": [], "github_queries": [], "platform_hits": [], "a2": True})
            c["github_queries"].append(d["q"])
    out = []
    for c in new.values():
        c["arm"] = "both" if c["registry"] and c["github_queries"] else ("registry" if c["registry"] else "github")
        out.append(c)
    out.sort(key=lambda c: c["key"])
    json.dump(out, open(f"{HERE}/work/candidates_a2.json", "w"), indent=1)
    from collections import Counter
    print("A2 new candidates:", len(out), Counter(c["arm"] for c in out), "with repo:", sum(1 for c in out if c["repo"]))


# ---------- Amendment A3 (2026-10-12): frame fixes found by the verifiers ----------
PLATFORMS_A3 = {
    "x": r"twitter|\btweets?\b|\bx\.com\b|\bx api\b|\bx-mcp\b|\bxmcp\b|\bmcp-x\b|\bx \(formerly twitter\)|\bx/twitter\b|\bx \(twitter\)|\bxurl\b|\bx search\b|\bsearch x\b|\bx posts?\b",
    "linkedin": r"linkedin",
    "facebook": r"facebook|\bfb\b|\bfb page|\bmeta graph",
    "instagram": r"instagram|\big\b|\binsta\b|\big graph",
    "threads": r"threads\.net|threads api|meta threads|threads by meta|\bthreads\b",
    "tiktok": r"tiktok|tik tok",
    "youtube": r"youtube|\byt\b",
    "pinterest": r"pinterest",
    "bluesky": r"bluesky|\bbsky\b|atproto|at protocol",
    "reddit": r"reddit|subreddit",
}
GENERIC_A3 = re.compile(r"social[ -_]?media|social network|\bsocials\b|\bsocial[ -](posts?|content|listening|channels?|scheduler|scheduling|accounts?|profiles?|apis?|inbox|analytics|data|presence|marketing|management|feeds?|networks?|platforms?|publishing|posting|intelligence|monitoring|signals?|engagement|automation)\b|cross-?post", re.I)


def plats_a3(text):
    return sorted(k for k, rx in PLATFORMS_A3.items() if re.search(rx, text or "", re.I))


def full_remote_key(s):
    remote = ((s.get("remotes") or [{}])[0] or {}).get("url")
    if not remote:
        return None
    u = urllib.parse.urlparse(remote)
    return "remote:" + u.netloc + u.path + (("?" + u.query) if u.query else "")


A3_GH = []
for _slug in ["twitter", "linkedin", "facebook", "instagram", "tiktok", "youtube", "pinterest", "bluesky", "reddit", "threads", "social-media"]:
    A3_GH.append(f"topic:model-context-protocol {_slug} stars:>=5 fork:false")
    A3_GH.append(f"topic:{_slug} mcp stars:>=5 fork:false")
for _q in ["insta mcp in:name", "ig mcp in:name", "fb mcp in:name", "\"at protocol\" mcp", "atproto mcp", "socials mcp", "\"social posts\" mcp", "\"social listening\" mcp",
           "topic:facebook-ads mcp", "topic:instagram-api mcp", "topic:twitter-api mcp", "topic:youtube-api mcp", "topic:reddit-api mcp", "topic:tiktok-api mcp", "topic:linkedin-api mcp"]:
    A3_GH.append(f"{_q} stars:>=5 fork:false")


def step_a3():
    cands = json.load(open(f"{HERE}/work/candidates.json"))
    a2 = json.load(open(f"{HERE}/work/candidates_a2.json"))
    known = {c["key"] for c in cands + a2}
    known_repos = {c["repo"] for c in cands + a2 if c["repo"]}
    known_regnames = {r["name"] for c in cands + a2 for r in c["registry"]}
    reg = json.load(open(f"{RAW}/registry/official_all_latest_full.json"))
    new = {}
    splits = []

    def regrow(s, meta):
        return {"name": s["name"], "title": s.get("title"), "description": s.get("description"), "version": s.get("version"),
                "status": meta.get("status"), "publishedAt": meta.get("publishedAt"), "updatedAt": meta.get("updatedAt"),
                "repository": s.get("repository"), "remotes": s.get("remotes"),
                "packages": [{k: p.get(k) for k in ("registryType", "identifier", "version")} for p in (s.get("packages") or [])]}

    # 1. split merged remote rows (no repo, several registry entries with different full remote URLs)
    for c in cands + a2:
        if c["repo"] or len(c["registry"]) < 2:
            continue
        by = {}
        for r in c["registry"]:
            fk = full_remote_key({"remotes": r.get("remotes")}) or "name:" + r["name"]
            by.setdefault(fk, []).append(r)
        if len(by) > 1:
            splits.append({"old_key": c["key"], "new_keys": sorted(by)})
            for fk, rs in by.items():
                new[fk] = {"key": fk, "repo": None, "registry": rs, "github_queries": [], "platform_hits": [], "a3": "split", "split_from": c["key"]}
    # 2. registry entries newly matched by the fixed patterns or the wider generic wording
    for e in reg:
        s = e["server"]; meta = e.get("_meta", {}).get("io.modelcontextprotocol.registry/official", {})
        if meta.get("status") == "deleted" or s["name"] in known_regnames:
            continue
        t = " ".join([s.get("name", ""), s.get("title", "") or "", s.get("description", "") or ""])
        if not (plats_a3(t) or GENERIC_A3.search(t)):
            continue
        repo = norm_repo((s.get("repository") or {}).get("url"))
        key = repo or full_remote_key(s) or "name:" + s["name"]
        if key in known or (repo and repo in known_repos):
            continue
        c = new.setdefault(key, {"key": key, "repo": repo if repo and repo.startswith("github.com/") else None, "registry": [], "github_queries": [], "platform_hits": plats_a3(t), "a3": "registry"})
        c["registry"].append(regrow(s, meta))
    # 3. GitHub: new queries + items the old filter dropped
    pool = []
    for q in A3_GH:
        fn = f"{RAW}/github_search/a3_{hashlib.md5(q.encode()).hexdigest()[:10]}.json"
        if not os.path.exists(fn):
            items, total = [], None
            for page in range(1, 11):
                d = gh("search/repositories", {"q": q, "per_page": 100, "page": page}); time.sleep(2.2)
                if not d: break
                total = d.get("total_count"); items += d.get("items", [])
                if len(d.get("items", [])) < 100: break
            json.dump({"q": q, "total_count": total, "items": items}, open(fn, "w"))
        d = json.load(open(fn)); print(q, d["total_count"], len(d["items"]), flush=True)
        pool += [(d["q"], it) for it in d["items"]]
    for f in os.listdir(f"{RAW}/github_search"):
        if f.endswith(".json") and not f.startswith(("a3_", "_")):
            d = json.load(open(f"{RAW}/github_search/{f}"))
            pool += [(d["q"], it) for it in d.get("items", [])]
    for q, it in pool:
        if it.get("fork") or it.get("stargazers_count", 0) < 5:
            continue
        text = " ".join([it["name"], it.get("description") or "", " ".join(it.get("topics") or [])])
        if not re.search(r"mcp|model[ -]context[ -]protocol", text, re.I):
            continue
        ps = plats_a3(text)
        if not ps and not GENERIC_A3.search(text):
            continue
        key = norm_repo(it["html_url"])
        if key in known:
            continue
        c = new.setdefault(key, {"key": key, "repo": key, "registry": [], "github_queries": [], "platform_hits": ps, "a3": "github"})
        if q not in c["github_queries"]:
            c["github_queries"].append(q)
    out = []
    for c in new.values():
        c["arm"] = "both" if c["registry"] and c["github_queries"] else ("registry" if c["registry"] else "github")
        out.append(c)
    out.sort(key=lambda c: c["key"])
    json.dump(out, open(f"{HERE}/work/candidates_a3.json", "w"), indent=1)
    json.dump(splits, open(f"{HERE}/work/a3_splits.json", "w"), indent=1)
    from collections import Counter
    print("A3 new candidates:", len(out), Counter(c["a3"] for c in out), "| splits:", len(splits), "| with repo:", sum(1 for c in out if c["repo"]))


if __name__ == "__main__":
    step = sys.argv[1] if len(sys.argv) > 1 else "all"
    if step in ("search", "all"): step_search()
    if step in ("candidates", "all"): step_candidates()
    if step in ("fetch", "all"): step_fetch()
    if step == "a2": step_a2()
    if step == "a3": step_a3()
    if step == "a2fetch":
        import builtins
        cands = json.load(open(f"{HERE}/work/candidates_a2.json"))
        json.dump(json.load(open(f"{HERE}/work/candidates.json")) , open(f"{HERE}/work/candidates_base_backup.json", "w"))
        from concurrent.futures import ThreadPoolExecutor
        reg = json.load(open(f"{RAW}/registry/official_all_latest.json"))
        for e in reg:
            for r in e["server"].get("remotes", []) or []:
                u = urllib.parse.urlparse(r.get("url", "")); REMOTE_DENY.add(u.netloc + u.path)
        with ThreadPoolExecutor(8) as ex: print("fetched", sum(ex.map(fetch_one, [c for c in cands if c["repo"]])))