feat: 补 MIT 许可、自动抓取工具与学科覆盖元数据

登记 chinese-poetry / ck12 / phet，并让 source schema、lint 与 INDEX 能按学科和抓取进度消费。

Co-authored-by: Cursor <cursoragent@cursor.com>

tools/index_build.py

@@ -1,58 +1,185 @@
 #!/usr/bin/env python3
 """index_build.py — 生成 INDEX.json（机器可读库全貌）。
 
-汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。
+汇总：资源域、学科覆盖矩阵、抓取进度、各数据集记录数、license 分布、OSS 前缀、红线摘要。
 注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
 """
 import os, glob, json, datetime, collections
 
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-def source_summary():
+# 与 sources/source.schema.json 的 subjects 枚举保持一致
+SUBJECTS = [
+    "chinese", "math", "english", "science", "morality",
+    "physics", "chemistry", "biology", "history", "geography", "it",
+    "ket", "pet", "fce", "amc8", "cspj",
+]
+STAGES = ["primary", "junior"]
+# 可全自动抓取的 fetch_method（fetch_source.py 支持）
+AUTO_METHODS = {"curl", "git-clone", "api"}
+PRODUCT_SAFE = {"official-free", "mit", "cc-by", "cc-by-sa", "llm-generated"}
+
+
+def load_sources():
+    """返回 [(domain, source_dict, yaml_rel_path)]"""
     import yaml
-    domains, n_sources, lic = {}, 0, collections.Counter()
-    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
+    out = []
+    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
         rel = os.path.relpath(path, os.path.join(REPO, "sources"))
-        domain = os.path.dirname(rel)
-        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
-        except Exception: continue
+        domain = os.path.dirname(rel) or "."
+        try:
+            doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception:
+            continue
         for s in (doc or {}).get("sources", []) or []:
-            n_sources += 1
-            lic[s.get("license")] += 1
-            domains.setdefault(domain, 0)
-            domains[domain] += 1
-    return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)}
+            out.append((domain, s, rel))
+    return out
+
+
+def source_summary(rows):
+    domains, lic = collections.Counter(), collections.Counter()
+    for domain, s, _ in rows:
+        domains[domain] += 1
+        lic[s.get("license")] += 1
+    return {
+        "domains": dict(domains),
+        "n_sources": len(rows),
+        "license_breakdown": dict(lic),
+    }
+
+
+def fetch_progress(rows):
+    """条目级抓取进度：planned / fetched / uploaded。"""
+    st = collections.Counter()
+    n_items = 0
+    auto, manual = 0, 0
+    for _, s, _ in rows:
+        if s.get("fetch_method") in AUTO_METHODS:
+            auto += 1
+        else:
+            manual += 1
+        for it in s.get("items", []) or []:
+            n_items += 1
+            st[it.get("status") or "planned"] += 1
+    return {
+        "n_items": n_items,
+        "by_status": dict(st),
+        "auto_fetchable_sources": auto,
+        "manual_sources": manual,
+    }
+
+
+def subject_coverage(rows):
+    """学科 → 资源覆盖矩阵。每个学科给出源清单、学段、可自动抓取数、条目进度。"""
+    cov = {}
+    for subj in SUBJECTS:
+        cov[subj] = {
+            "n_sources": 0,
+            "stages": [],
+            "auto_fetchable": 0,
+            "n_items": 0,
+            "n_fetched": 0,
+            "product_safe_sources": 0,
+            "sources": [],
+        }
+    for domain, s, rel in rows:
+        subs = s.get("subjects") or []
+        auto = s.get("fetch_method") in AUTO_METHODS
+        items = s.get("items", []) or []
+        n_fetched = sum(1 for it in items if it.get("status") in ("fetched", "uploaded"))
+        safe = s.get("license") in PRODUCT_SAFE
+        entry = {
+            "id": s.get("id"),
+            "title": s.get("title"),
+            "license": s.get("license"),
+            "fetch_method": s.get("fetch_method"),
+            "auto_fetchable": auto,
+            "product_safe": safe,
+            "stages": s.get("stages") or [],
+            "n_items": len(items),
+            "n_fetched": n_fetched,
+            "yaml": rel,
+        }
+        for subj in subs:
+            if subj not in cov:
+                cov[subj] = {"n_sources": 0, "stages": [], "auto_fetchable": 0,
+                             "n_items": 0, "n_fetched": 0, "product_safe_sources": 0, "sources": []}
+            c = cov[subj]
+            c["n_sources"] += 1
+            c["auto_fetchable"] += 1 if auto else 0
+            c["n_items"] += len(items)
+            c["n_fetched"] += n_fetched
+            c["product_safe_sources"] += 1 if safe else 0
+            c["sources"].append(entry)
+            for st in s.get("stages") or []:
+                if st not in c["stages"]:
+                    c["stages"].append(st)
+    for c in cov.values():
+        c["stages"] = [x for x in STAGES if x in c["stages"]]
+    return cov
+
+
+def gaps(cov):
+    """缺口清单：完全无源 / 有源但一条都没抓下来 / 有源但无 product_safe。"""
+    no_source = sorted([k for k, v in cov.items() if v["n_sources"] == 0])
+    no_data = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["n_fetched"] == 0])
+    no_safe = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["product_safe_sources"] == 0])
+    return {
+        "subjects_without_source": no_source,
+        "subjects_without_fetched_data": no_data,
+        "subjects_without_product_safe_source": no_safe,
+    }
+
 
 def structured_summary():
     datasets, lic = {}, collections.Counter()
-    for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True):
+    for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
         rel = os.path.relpath(path, os.path.join(REPO, "structured"))
         cnt = 0
         for line in open(path, encoding="utf-8"):
             line = line.strip()
-            if not line: continue
+            if not line:
+                continue
             cnt += 1
-            try: r = json.loads(line)
-            except Exception: continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
             l = r.get("license")
-            if l: lic[l] += 1
+            if l:
+                lic[l] += 1
         datasets[rel] = cnt
     return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}
 
+
 def main():
+    rows = load_sources()
+    cov = subject_coverage(rows)
     idx = {
         "generated_at": datetime.date.today().isoformat(),
         "repo": "edu-content",
         "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
         "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
         "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
-        "sources": source_summary(),
+        "product_safe_licenses": sorted(PRODUCT_SAFE),
+        "sources": source_summary(rows),
+        "fetch_progress": fetch_progress(rows),
+        "subject_coverage": cov,
+        "gaps": gaps(cov),
         "structured": structured_summary(),
         "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
     }
     out = os.path.join(REPO, "INDEX.json")
     json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
-    print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录")
+    fp = idx["fetch_progress"]
+    g = idx["gaps"]
+    print(f"已生成 {out}")
+    print(f"  source {idx['sources']['n_sources']} 条 | 条目 {fp['n_items']} 个 | 状态 {fp['by_status']}")
+    print(f"  可自动抓取源 {fp['auto_fetchable_sources']} / 需人工 {fp['manual_sources']}")
+    print(f"  无源学科：{', '.join(g['subjects_without_source']) or '（无）'}")
+    print(f"  有源但零数据：{', '.join(g['subjects_without_fetched_data']) or '（无）'}")
+    print(f"  结构化记录 {idx['structured']['n_records']} 条")
+
 
 if __name__ == "__main__":
     main()