#!/usr/bin/env python3
"""index_build.py — 生成 INDEX.json（机器可读库全貌）。

汇总：资源域、学科覆盖矩阵、抓取进度、各数据集记录数、license 分布、OSS 前缀、红线摘要。
注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
"""
import os, glob, json, datetime, collections

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# 与 sources/source.schema.json 的 subjects 枚举保持一致
SUBJECTS = [
    "chinese", "math", "english", "science", "morality",
    "physics", "chemistry", "biology", "history", "geography", "it",
    "ket", "pet", "fce", "amc8", "cspj",
]
STAGES = ["primary", "junior"]
# 可全自动抓取的 fetch_method（fetch_source.py 支持）
AUTO_METHODS = {"curl", "git-clone", "api"}
PRODUCT_SAFE = {"official-free", "mit", "cc-by", "cc-by-sa", "llm-generated"}


def load_sources():
    """返回 [(domain, source_dict, yaml_rel_path)]"""
    import yaml
    out = []
    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
        rel = os.path.relpath(path, os.path.join(REPO, "sources"))
        domain = os.path.dirname(rel) or "."
        try:
            doc = yaml.safe_load(open(path, encoding="utf-8"))
        except Exception:
            continue
        for s in (doc or {}).get("sources", []) or []:
            out.append((domain, s, rel))
    return out


def source_summary(rows):
    domains, lic = collections.Counter(), collections.Counter()
    for domain, s, _ in rows:
        domains[domain] += 1
        lic[s.get("license")] += 1
    return {
        "domains": dict(domains),
        "n_sources": len(rows),
        "license_breakdown": dict(lic),
    }


def fetch_progress(rows):
    """条目级抓取进度：planned / fetched / uploaded。"""
    st = collections.Counter()
    n_items = 0
    auto, manual = 0, 0
    for _, s, _ in rows:
        if s.get("fetch_method") in AUTO_METHODS:
            auto += 1
        else:
            manual += 1
        for it in s.get("items", []) or []:
            n_items += 1
            st[it.get("status") or "planned"] += 1
    return {
        "n_items": n_items,
        "by_status": dict(st),
        "auto_fetchable_sources": auto,
        "manual_sources": manual,
    }


def subject_coverage(rows):
    """学科 → 资源覆盖矩阵。每个学科给出源清单、学段、可自动抓取数、条目进度。"""
    cov = {}
    for subj in SUBJECTS:
        cov[subj] = {
            "n_sources": 0,
            "stages": [],
            "auto_fetchable": 0,
            "n_items": 0,
            "n_fetched": 0,
            "product_safe_sources": 0,
            "sources": [],
        }
    for domain, s, rel in rows:
        subs = s.get("subjects") or []
        auto = s.get("fetch_method") in AUTO_METHODS
        items = s.get("items", []) or []
        n_fetched = sum(1 for it in items if it.get("status") in ("fetched", "uploaded"))
        safe = s.get("license") in PRODUCT_SAFE
        entry = {
            "id": s.get("id"),
            "title": s.get("title"),
            "license": s.get("license"),
            "fetch_method": s.get("fetch_method"),
            "auto_fetchable": auto,
            "product_safe": safe,
            "stages": s.get("stages") or [],
            "n_items": len(items),
            "n_fetched": n_fetched,
            "yaml": rel,
        }
        for subj in subs:
            if subj not in cov:
                cov[subj] = {"n_sources": 0, "stages": [], "auto_fetchable": 0,
                             "n_items": 0, "n_fetched": 0, "product_safe_sources": 0, "sources": []}
            c = cov[subj]
            c["n_sources"] += 1
            c["auto_fetchable"] += 1 if auto else 0
            c["n_items"] += len(items)
            c["n_fetched"] += n_fetched
            c["product_safe_sources"] += 1 if safe else 0
            c["sources"].append(entry)
            for st in s.get("stages") or []:
                if st not in c["stages"]:
                    c["stages"].append(st)
    for c in cov.values():
        c["stages"] = [x for x in STAGES if x in c["stages"]]
    return cov


def gaps(cov):
    """缺口清单：完全无源 / 有源但一条都没抓下来 / 有源但无 product_safe。"""
    no_source = sorted([k for k, v in cov.items() if v["n_sources"] == 0])
    no_data = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["n_fetched"] == 0])
    no_safe = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["product_safe_sources"] == 0])
    return {
        "subjects_without_source": no_source,
        "subjects_without_fetched_data": no_data,
        "subjects_without_product_safe_source": no_safe,
    }


def structured_summary():
    datasets, lic = {}, collections.Counter()
    for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
        rel = os.path.relpath(path, os.path.join(REPO, "structured"))
        cnt = 0
        for line in open(path, encoding="utf-8"):
            line = line.strip()
            if not line:
                continue
            cnt += 1
            try:
                r = json.loads(line)
            except Exception:
                continue
            l = r.get("license")
            if l:
                lic[l] += 1
        datasets[rel] = cnt
    return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}


def main():
    rows = load_sources()
    cov = subject_coverage(rows)
    idx = {
        "generated_at": datetime.date.today().isoformat(),
        "repo": "edu-content",
        "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
        "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
        "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
        "product_safe_licenses": sorted(PRODUCT_SAFE),
        "sources": source_summary(rows),
        "fetch_progress": fetch_progress(rows),
        "subject_coverage": cov,
        "gaps": gaps(cov),
        "structured": structured_summary(),
        "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
    }
    out = os.path.join(REPO, "INDEX.json")
    json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
    fp = idx["fetch_progress"]
    g = idx["gaps"]
    print(f"已生成 {out}")
    print(f"  source {idx['sources']['n_sources']} 条 | 条目 {fp['n_items']} 个 | 状态 {fp['by_status']}")
    print(f"  可自动抓取源 {fp['auto_fetchable_sources']} / 需人工 {fp['manual_sources']}")
    print(f"  无源学科：{', '.join(g['subjects_without_source']) or '（无）'}")
    print(f"  有源但零数据：{', '.join(g['subjects_without_fetched_data']) or '（无）'}")
    print(f"  结构化记录 {idx['structured']['n_records']} 条")


if __name__ == "__main__":
    main()