#!/usr/bin/env python3
"""index_build.py — 生成 INDEX.json（机器可读库全貌）。

汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。
注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
"""
import os, glob, json, datetime, collections

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

def source_summary():
    import yaml
    domains, n_sources, lic = {}, 0, collections.Counter()
    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
        rel = os.path.relpath(path, os.path.join(REPO, "sources"))
        domain = os.path.dirname(rel)
        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
        except Exception: continue
        for s in (doc or {}).get("sources", []) or []:
            n_sources += 1
            lic[s.get("license")] += 1
            domains.setdefault(domain, 0)
            domains[domain] += 1
    return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)}

def structured_summary():
    datasets, lic = {}, collections.Counter()
    for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True):
        rel = os.path.relpath(path, os.path.join(REPO, "structured"))
        cnt = 0
        for line in open(path, encoding="utf-8"):
            line = line.strip()
            if not line: continue
            cnt += 1
            try: r = json.loads(line)
            except Exception: continue
            l = r.get("license")
            if l: lic[l] += 1
        datasets[rel] = cnt
    return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}

def main():
    idx = {
        "generated_at": datetime.date.today().isoformat(),
        "repo": "edu-content",
        "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
        "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
        "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
        "sources": source_summary(),
        "structured": structured_summary(),
        "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
    }
    out = os.path.join(REPO, "INDEX.json")
    json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
    print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录")

if __name__ == "__main__":
    main()