feat: 补 MIT 许可、自动抓取工具与学科覆盖元数据

登记 chinese-poetry / ck12 / phet，并让 source schema、lint 与 INDEX 能按学科和抓取进度消费。

Co-authored-by: Cursor <cursoragent@cursor.com>

tools/fetch_source.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""fetch_source.py — 按 sources/*.yaml 自动抓取 L2 原始数据到本地 staging。
+
+只处理**可自动化**的 fetch_method：
+  - curl       : 逐 item 下载单文件（item.url，或 source.url_template 渲染 {item_id}）
+  - git-clone  : 浅克隆一次仓库，再按 item.src_path 打包成 .tar.gz
+
+manual-download / manual-export / generated 会被跳过（需人工介入）。
+
+用法：
+  # 看有哪些可自动抓取（不下载）
+  python tools/fetch_source.py --list
+
+  # 试跑：只打印将要下载的 URL / 目标路径
+  python tools/fetch_source.py --source phet --dry-run
+
+  # 实际下载到 .staging/，并回填 sha256/size/fetch_date/status=fetched
+  python tools/fetch_source.py --source phet --apply
+  python tools/fetch_source.py --source chinese-poetry --apply
+  python tools/fetch_source.py --subject physics --apply     # 按学科批量
+  python tools/fetch_source.py --all-auto --apply            # 所有可自动化来源
+
+下载产物落在 <repo>/.staging/{source_id}/...（已 gitignore），随后用
+  python tools/oss_sync.py --raw-root .staging --apply
+上传到 OSS 并对齐 oss_key。
+
+注意：回填 YAML 走 yaml_patch（行级替换），**保留文件内注释**。
+"""
+import os, sys, glob, hashlib, datetime, argparse, subprocess, shutil, tarfile, urllib.request
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from yaml_patch import patch_item_fields, patch_source_fields
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STAGING = os.path.join(REPO, ".staging")
+CACHE = os.path.join(REPO, ".cache", "repos")
+AUTO_METHODS = {"curl", "git-clone"}
+UA = "edu-content-fetcher/1.0 (+https://git.aibeaver.cn/fengjunchen/edu-content)"
+
+
+def sha256_file(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def load_sources():
+    import yaml
+    out = []
+    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
+        try:
+            doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception as e:
+            print(f"WARN: {path} 解析失败: {e}")
+            continue
+        for s in (doc or {}).get("sources", []) or []:
+            out.append((path, s))
+    return out
+
+
+def item_url(source, item):
+    if item.get("url"):
+        return item["url"]
+    tpl = source.get("url_template")
+    if tpl:
+        return tpl.replace("{item_id}", item["id"])
+    return None
+
+
+def fetch_curl(source, item, dest_dir, dry_run):
+    url = item_url(source, item)
+    if not url:
+        return None, "无 url / url_template，跳过"
+    fname = os.path.basename(item.get("oss_key") or "") or f"{item['id']}"
+    dest = os.path.join(dest_dir, fname)
+    if dry_run:
+        return dest, f"[dry-run] {url} -> {dest}"
+    os.makedirs(dest_dir, exist_ok=True)
+    req = urllib.request.Request(url, headers={"User-Agent": UA})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as f:
+            shutil.copyfileobj(r, f)
+    except Exception as e:
+        return None, f"下载失败: {type(e).__name__}: {e}"
+    return dest, f"ok {os.path.getsize(dest)}B"
+
+
+def ensure_clone(source, dry_run):
+    """浅克隆到 .cache/repos/{id}，返回本地路径。"""
+    url = source.get("url")
+    if not url:
+        return None, "source.url 为空，无法 clone"
+    local = os.path.join(CACHE, source["id"])
+    if os.path.isdir(os.path.join(local, ".git")):
+        return local, "复用已有克隆"
+    if dry_run:
+        return local, f"[dry-run] git clone --depth 1 {url} {local}"
+    os.makedirs(os.path.dirname(local), exist_ok=True)
+    r = subprocess.run(["git", "clone", "--depth", "1", url, local],
+                       capture_output=True, text=True, timeout=1800)
+    if r.returncode != 0:
+        return None, f"clone 失败: {(r.stderr or '').strip()[:200]}"
+    return local, "克隆完成"
+
+
+def fetch_git_item(source, item, repo_dir, dest_dir, dry_run):
+    """按 item.src_path 从克隆仓库取内容。
+
+    目录 → 打包 .tar.gz；单文件 → 直接复制（除非 oss_key 显式要 .tar.gz）。
+    否则会产出「名叫 KET.json 其实是 tar.gz」的坑数据。
+    """
+    rel = item.get("src_path")
+    if not rel:
+        return None, "缺 src_path，跳过"
+    src = os.path.join(repo_dir, rel)
+    fname = os.path.basename(item.get("oss_key") or f"{item['id']}.tar.gz")
+    dest = os.path.join(dest_dir, fname)
+    want_tar = fname.endswith((".tar.gz", ".tgz"))
+    if dry_run:
+        action = "tar" if want_tar else "copy"
+        return dest, f"[dry-run] {action} {rel} -> {dest}"
+    if not os.path.exists(src):
+        return None, f"仓库内路径不存在: {rel}"
+    if os.path.isdir(src) and not want_tar:
+        return None, f"src_path 是目录但 oss_key 不是 .tar.gz（{fname}），请修正 oss_key"
+    os.makedirs(dest_dir, exist_ok=True)
+    if want_tar:
+        with tarfile.open(dest, "w:gz") as tf:
+            tf.add(src, arcname=rel)
+    else:
+        shutil.copy2(src, dest)
+    return dest, f"ok {os.path.getsize(dest)}B"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--source", help="按 source id 抓取")
+    ap.add_argument("--subject", help="按学科代码抓取（chinese/physics/science/...）")
+    ap.add_argument("--all-auto", action="store_true", help="所有 fetch_method 可自动化的来源")
+    ap.add_argument("--list", action="store_true", help="只列出可自动抓取的来源")
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--apply", action="store_true")
+    ap.add_argument("--limit", type=int, default=0, help="每个 source 最多抓 N 个条目（调试用）")
+    args = ap.parse_args()
+
+    all_sources = load_sources()
+
+    if args.list:
+        print(f"{'source':22} {'method':12} {'items':>5}  subjects")
+        print("-" * 70)
+        for path, s in all_sources:
+            mark = "" if s.get("fetch_method") in AUTO_METHODS else "  (需人工)"
+            print(f"{s['id']:22} {str(s.get('fetch_method')):12} "
+                  f"{len(s.get('items') or []):5}  {','.join(s.get('subjects') or [])}{mark}")
+        return
+
+    # 选择目标
+    targets = []
+    for path, s in all_sources:
+        if s.get("fetch_method") not in AUTO_METHODS:
+            continue
+        if args.source and s["id"] != args.source:
+            continue
+        if args.subject and args.subject not in (s.get("subjects") or []):
+            continue
+        if not (args.source or args.subject or args.all_auto):
+            continue
+        targets.append((path, s))
+
+    if not targets:
+        print("没有匹配的可自动抓取来源。用 --list 查看，或检查 --source/--subject 拼写。")
+        sys.exit(0 if (args.source or args.subject or args.all_auto) else 2)
+
+    dry = args.dry_run or not args.apply
+    today = datetime.date.today().isoformat()
+    total_ok = total_fail = 0
+
+    for path, s in targets:
+        sid = s["id"]
+        method = s["fetch_method"]
+        items = s.get("items") or []
+        if args.limit:
+            items = items[:args.limit]
+        print(f"\n=== {sid} ({method}) — {len(items)} 个条目 ===")
+        if not items:
+            print("  items 为空，无可抓取内容（该来源尚未登记具体条目）")
+            continue
+
+        dest_dir = os.path.join(STAGING, sid)
+        repo_dir = None
+        if method == "git-clone":
+            repo_dir, msg = ensure_clone(s, dry)
+            print(f"  clone: {msg}")
+            if repo_dir is None:
+                total_fail += len(items)
+                continue
+
+        ok_here = 0
+        for it in items:
+            if method == "curl":
+                dest, msg = fetch_curl(s, it, dest_dir, dry)
+            else:
+                dest, msg = fetch_git_item(s, it, repo_dir, dest_dir, dry)
+            print(f"  {it['id']:34} {msg}")
+            if dest is None:
+                total_fail += 1
+                continue
+            total_ok += 1
+            ok_here += 1
+            if not dry and os.path.isfile(dest):
+                patch_item_fields(path, sid, it["id"], {
+                    "sha256": sha256_file(dest),
+                    "size": os.path.getsize(dest),
+                    "fetch_date": today,
+                    "status": "fetched",
+                })
+        if not dry and ok_here:
+            patch_source_fields(path, sid, {"status": "fetched", "fetched_at": today})
+
+    print(f"\n合计：成功 {total_ok} | 失败/跳过 {total_fail}")
+    if dry:
+        print("[dry-run] 未实际下载。加 --apply 执行。")
+    else:
+        print(f"产物在 {STAGING}/，已回填 sha256/size 到 sources YAML（注释保留）。")
+        print("下一步上传 OSS： python tools/oss_sync.py --raw-root .staging --apply")
+
+
+if __name__ == "__main__":
+    main()

tools/index_build.py

@@ -1,58 +1,185 @@
 #!/usr/bin/env python3
 """index_build.py — 生成 INDEX.json（机器可读库全貌）。
 
-汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。
+汇总：资源域、学科覆盖矩阵、抓取进度、各数据集记录数、license 分布、OSS 前缀、红线摘要。
 注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
 """
 import os, glob, json, datetime, collections
 
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-def source_summary():
+# 与 sources/source.schema.json 的 subjects 枚举保持一致
+SUBJECTS = [
+    "chinese", "math", "english", "science", "morality",
+    "physics", "chemistry", "biology", "history", "geography", "it",
+    "ket", "pet", "fce", "amc8", "cspj",
+]
+STAGES = ["primary", "junior"]
+# 可全自动抓取的 fetch_method（fetch_source.py 支持）
+AUTO_METHODS = {"curl", "git-clone", "api"}
+PRODUCT_SAFE = {"official-free", "mit", "cc-by", "cc-by-sa", "llm-generated"}
+
+
+def load_sources():
+    """返回 [(domain, source_dict, yaml_rel_path)]"""
     import yaml
-    domains, n_sources, lic = {}, 0, collections.Counter()
-    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
+    out = []
+    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
         rel = os.path.relpath(path, os.path.join(REPO, "sources"))
-        domain = os.path.dirname(rel)
-        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
-        except Exception: continue
+        domain = os.path.dirname(rel) or "."
+        try:
+            doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception:
+            continue
         for s in (doc or {}).get("sources", []) or []:
-            n_sources += 1
-            lic[s.get("license")] += 1
-            domains.setdefault(domain, 0)
-            domains[domain] += 1
-    return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)}
+            out.append((domain, s, rel))
+    return out
+
+
+def source_summary(rows):
+    domains, lic = collections.Counter(), collections.Counter()
+    for domain, s, _ in rows:
+        domains[domain] += 1
+        lic[s.get("license")] += 1
+    return {
+        "domains": dict(domains),
+        "n_sources": len(rows),
+        "license_breakdown": dict(lic),
+    }
+
+
+def fetch_progress(rows):
+    """条目级抓取进度：planned / fetched / uploaded。"""
+    st = collections.Counter()
+    n_items = 0
+    auto, manual = 0, 0
+    for _, s, _ in rows:
+        if s.get("fetch_method") in AUTO_METHODS:
+            auto += 1
+        else:
+            manual += 1
+        for it in s.get("items", []) or []:
+            n_items += 1
+            st[it.get("status") or "planned"] += 1
+    return {
+        "n_items": n_items,
+        "by_status": dict(st),
+        "auto_fetchable_sources": auto,
+        "manual_sources": manual,
+    }
+
+
+def subject_coverage(rows):
+    """学科 → 资源覆盖矩阵。每个学科给出源清单、学段、可自动抓取数、条目进度。"""
+    cov = {}
+    for subj in SUBJECTS:
+        cov[subj] = {
+            "n_sources": 0,
+            "stages": [],
+            "auto_fetchable": 0,
+            "n_items": 0,
+            "n_fetched": 0,
+            "product_safe_sources": 0,
+            "sources": [],
+        }
+    for domain, s, rel in rows:
+        subs = s.get("subjects") or []
+        auto = s.get("fetch_method") in AUTO_METHODS
+        items = s.get("items", []) or []
+        n_fetched = sum(1 for it in items if it.get("status") in ("fetched", "uploaded"))
+        safe = s.get("license") in PRODUCT_SAFE
+        entry = {
+            "id": s.get("id"),
+            "title": s.get("title"),
+            "license": s.get("license"),
+            "fetch_method": s.get("fetch_method"),
+            "auto_fetchable": auto,
+            "product_safe": safe,
+            "stages": s.get("stages") or [],
+            "n_items": len(items),
+            "n_fetched": n_fetched,
+            "yaml": rel,
+        }
+        for subj in subs:
+            if subj not in cov:
+                cov[subj] = {"n_sources": 0, "stages": [], "auto_fetchable": 0,
+                             "n_items": 0, "n_fetched": 0, "product_safe_sources": 0, "sources": []}
+            c = cov[subj]
+            c["n_sources"] += 1
+            c["auto_fetchable"] += 1 if auto else 0
+            c["n_items"] += len(items)
+            c["n_fetched"] += n_fetched
+            c["product_safe_sources"] += 1 if safe else 0
+            c["sources"].append(entry)
+            for st in s.get("stages") or []:
+                if st not in c["stages"]:
+                    c["stages"].append(st)
+    for c in cov.values():
+        c["stages"] = [x for x in STAGES if x in c["stages"]]
+    return cov
+
+
+def gaps(cov):
+    """缺口清单：完全无源 / 有源但一条都没抓下来 / 有源但无 product_safe。"""
+    no_source = sorted([k for k, v in cov.items() if v["n_sources"] == 0])
+    no_data = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["n_fetched"] == 0])
+    no_safe = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["product_safe_sources"] == 0])
+    return {
+        "subjects_without_source": no_source,
+        "subjects_without_fetched_data": no_data,
+        "subjects_without_product_safe_source": no_safe,
+    }
+
 
 def structured_summary():
     datasets, lic = {}, collections.Counter()
-    for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True):
+    for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
         rel = os.path.relpath(path, os.path.join(REPO, "structured"))
         cnt = 0
         for line in open(path, encoding="utf-8"):
             line = line.strip()
-            if not line: continue
+            if not line:
+                continue
             cnt += 1
-            try: r = json.loads(line)
-            except Exception: continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
             l = r.get("license")
-            if l: lic[l] += 1
+            if l:
+                lic[l] += 1
         datasets[rel] = cnt
     return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}
 
+
 def main():
+    rows = load_sources()
+    cov = subject_coverage(rows)
     idx = {
         "generated_at": datetime.date.today().isoformat(),
         "repo": "edu-content",
         "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
         "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
         "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
-        "sources": source_summary(),
+        "product_safe_licenses": sorted(PRODUCT_SAFE),
+        "sources": source_summary(rows),
+        "fetch_progress": fetch_progress(rows),
+        "subject_coverage": cov,
+        "gaps": gaps(cov),
         "structured": structured_summary(),
         "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
     }
     out = os.path.join(REPO, "INDEX.json")
     json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
-    print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录")
+    fp = idx["fetch_progress"]
+    g = idx["gaps"]
+    print(f"已生成 {out}")
+    print(f"  source {idx['sources']['n_sources']} 条 | 条目 {fp['n_items']} 个 | 状态 {fp['by_status']}")
+    print(f"  可自动抓取源 {fp['auto_fetchable_sources']} / 需人工 {fp['manual_sources']}")
+    print(f"  无源学科：{', '.join(g['subjects_without_source']) or '（无）'}")
+    print(f"  有源但零数据：{', '.join(g['subjects_without_fetched_data']) or '（无）'}")
+    print(f"  结构化记录 {idx['structured']['n_records']} 条")
+
 
 if __name__ == "__main__":
     main()

tools/lint_sources.py

@@ -2,7 +2,7 @@
 """lint_sources.py — 校验 sources/ 资源清单合规。
 
 检查项：
-- 每个 source 必填字段、id kebab-case 唯一、license/type/status 枚举合法
+- 每个 source 必填字段、id kebab-case 唯一、license/type/status/subjects 枚举合法
 - status=fetched 的 item 必须有 oss_key + 64位 sha256
 - license=unknown 且 fetched_at 距今 > 30 天 → warning
 
@@ -11,9 +11,12 @@
 import sys, os, glob, re, datetime, hashlib
 
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
+LICENSES = {"official-free","mit","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
 TYPES = {"dataset","website","book","internal","tool"}
 STATUSES = {"planned","fetching","fetched","failed","archived"}
+SUBJECTS = {"chinese","math","english","science","morality","physics","chemistry",
+            "biology","history","geography","it","ket","pet","fce","amc8","cspj"}
+STAGES = {"primary","junior"}
 ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
 SHA_RE = re.compile(r"^[a-f0-9]{64}$")
 
@@ -42,6 +45,13 @@ def main():
             if s.get("type") not in TYPES: err(f"{sid}: type 非法 {s.get('type')}"); errors += 1
             if s.get("license") not in LICENSES: err(f"{sid}: license 非法 {s.get('license')}"); errors += 1
             if s.get("status") not in STATUSES: err(f"{sid}: status 非法 {s.get('status')}"); errors += 1
+            subs = s.get("subjects")
+            if not subs: err(f"{sid}: 缺必填 subjects（覆盖学科，见 LICENSES.md 同级约定）"); errors += 1
+            else:
+                for x in subs:
+                    if x not in SUBJECTS: err(f"{sid}: subjects 含非法值 {x}"); errors += 1
+            for x in (s.get("stages") or []):
+                if x not in STAGES: err(f"{sid}: stages 含非法值 {x}"); errors += 1
             if s.get("license") == "unknown" and s.get("fetched_at"):
                 try:
                     d = datetime.date.fromisoformat(s["fetched_at"])

tools/lint_structured.py

@@ -12,7 +12,7 @@
 import sys, os, glob, json
 
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
+LICENSES = {"official-free","mit","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
 
 def load_source_ids():
     import yaml

tools/oss_sync.py

@@ -16,6 +16,9 @@
 """
 import os, sys, glob, json, hashlib, datetime, argparse, yaml
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from yaml_patch import patch_item_fields, patch_source_fields
+
 REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SHA_RE = __import__("re").compile(r"^[a-f0-9]{64}$")
 
@@ -87,19 +90,16 @@ def main():
     for path, sid, iid, local, oss_key, sha, size in pending:
         b.put_object_from_file(oss_key, local)
         print(f"  uploaded {oss_key}")
-        # 回填
-        doc = yaml.safe_load(open(path, encoding="utf-8"))
-        for s in doc["sources"]:
-            if s.get("id") == sid:
-                for it in s.get("items", []):
-                    if it.get("id") == iid:
-                        it["oss_key"] = oss_key
-                        it["sha256"] = sha
-                        it["size"] = size
-                        it["fetch_date"] = today
-                        it["status"] = "fetched"
-        yaml.safe_dump(doc, open(path, "w", encoding="utf-8"), allow_unicode=True, sort_keys=False)
-    print("已回填 sha256/size 到 sources YAML。请 git commit。")
+        # 回填（行级替换，保留 YAML 注释；勿用 safe_dump 整file重写）
+        patch_item_fields(path, sid, iid, {
+            "oss_key": oss_key,
+            "sha256": sha,
+            "size": size,
+            "fetch_date": today,
+            "status": "fetched",
+        })
+        patch_source_fields(path, sid, {"status": "fetched", "fetched_at": today})
+    print("已回填 oss_key/sha256/size 到 sources YAML（注释保留）。请 git commit。")
 
 if __name__ == "__main__":
     main()

tools/stage_local.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""stage_local.py — 把「已在本机的原始文件」归位到 .staging 并回填 L1 元数据。
+
+适用场景：fetch_method 为 manual-download / manual-export 的来源（官方网站需人工点下载、
+需登录、或历史上已经下载过），文件已存在本地磁盘，但 sources/*.yaml 的 item 还挂在
+status: planned、sha256/size 为空。
+
+设计原则：**本机绝对路径只在命令行出现，绝不写进仓库**（换机器就失效，属脏数据）。
+仓库里只保留 oss_key / sha256 / size / status —— 与机器无关的事实。
+
+用法：
+  # 单个文件
+  python tools/stage_local.py --source cambridge-official --item a2-key-vocab \
+      --file "/abs/path/a2-key-vocabulary-list.pdf"
+
+  # 批量：从 TSV 读（source<TAB>item<TAB>abs_path），文件本身放仓库外或 .staging 里
+  python tools/stage_local.py --manifest /tmp/stage.tsv --apply
+
+  # 默认 dry-run，加 --apply 才真正复制 + 回填
+"""
+import os, sys, glob, shutil, hashlib, datetime, argparse
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from yaml_patch import patch_item_fields, patch_source_fields
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STAGING = os.path.join(REPO, ".staging")
+
+
+def sha256_file(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def load_index():
+    """返回 {(source_id, item_id): (yaml_path, source_dict, item_dict)}"""
+    import yaml
+    idx = {}
+    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
+        try:
+            doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception:
+            continue
+        for s in (doc or {}).get("sources", []) or []:
+            for it in s.get("items", []) or []:
+                idx[(s["id"], it["id"])] = (path, s, it)
+    return idx
+
+
+def stage_one(idx, sid, iid, src_file, apply_):
+    key = (sid, iid)
+    if key not in idx:
+        return False, f"sources 里找不到 {sid}/{iid}（先在 YAML 登记 item）"
+    if not os.path.isfile(src_file):
+        return False, f"本地文件不存在: {src_file}"
+    path, s, it = idx[key]
+    oss_key = it.get("oss_key")
+    if not oss_key:
+        return False, f"{sid}/{iid} 缺 oss_key，无法确定归位文件名"
+    fname = os.path.basename(oss_key.rstrip("/"))
+    if not fname or oss_key.endswith("/"):
+        return False, f"{sid}/{iid} 的 oss_key 是目录（{oss_key}），需改成具体文件名"
+    dest_dir = os.path.join(STAGING, sid)
+    dest = os.path.join(dest_dir, fname)
+    size = os.path.getsize(src_file)
+    if not apply_:
+        return True, f"[dry-run] {src_file} -> {os.path.relpath(dest, REPO)} ({size}B)"
+    os.makedirs(dest_dir, exist_ok=True)
+    if os.path.abspath(src_file) != os.path.abspath(dest):
+        shutil.copy2(src_file, dest)
+    sha = sha256_file(dest)
+    patch_item_fields(path, sid, iid, {
+        "sha256": sha,
+        "size": os.path.getsize(dest),
+        "fetch_date": datetime.date.today().isoformat(),
+        "status": "fetched",
+    })
+    patch_source_fields(path, sid, {"status": "fetched",
+                                    "fetched_at": datetime.date.today().isoformat()})
+    return True, f"ok {size}B sha256={sha[:12]}..."
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--source")
+    ap.add_argument("--item")
+    ap.add_argument("--file")
+    ap.add_argument("--manifest", help="TSV: source<TAB>item<TAB>abs_path，# 开头为注释")
+    ap.add_argument("--apply", action="store_true")
+    args = ap.parse_args()
+
+    jobs = []
+    if args.manifest:
+        for ln in open(args.manifest, encoding="utf-8-sig"):  # utf-8-sig 吃掉 BOM
+            ln = ln.lstrip("\ufeff").strip()
+            if not ln or ln.startswith("#"):
+                continue
+            parts = ln.split("\t")
+            if len(parts) < 3:
+                print(f"WARN 跳过格式错误行: {ln[:80]}")
+                continue
+            jobs.append((parts[0].strip(), parts[1].strip(), parts[2].strip()))
+    if args.source and args.item and args.file:
+        jobs.append((args.source, args.item, args.file))
+    if not jobs:
+        ap.error("需要 --source/--item/--file 或 --manifest")
+
+    idx = load_index()
+    ok = fail = 0
+    for sid, iid, f in jobs:
+        good, msg = stage_one(idx, sid, iid, f, args.apply)
+        print(f"  {sid}/{iid:28} {msg}")
+        ok += 1 if good else 0
+        fail += 0 if good else 1
+    print(f"\n合计：成功 {ok} | 失败 {fail}")
+    if not args.apply:
+        print("[dry-run] 未复制。加 --apply 执行。")
+    else:
+        print("已回填 sha256/size/status 到 sources YAML（注释保留）。")
+
+
+if __name__ == "__main__":
+    main()

tools/yaml_patch.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""yaml_patch.py — 对 sources/*.yaml 做**行级**字段回填，保留注释与原有格式。
+
+为什么需要它：
+`yaml.safe_load` + `yaml.safe_dump` 会把整个文件重写，**注释全部丢失**。
+sources/*.yaml 里的注释承载了关键信息（下载模板、许可解释、未纳入原因、运维提示），
+必须保住。因此所有自动回填（fetch_source.py / oss_sync.py）统一走本模块的行级替换。
+
+约定的缩进层级（与 sources/*.yaml 一致）：
+    sources:
+      - id: <source_id>        # 2 空格 + "- "
+        status: planned        # 4 空格
+        items:
+          - id: <item_id>      # 6 空格 + "- "
+            oss_key: ...       # 8 空格
+"""
+import re
+
+
+def _yaml_scalar(v):
+    """把 Python 值渲染成 YAML 标量。"""
+    if v is None:
+        return "null"
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    if isinstance(v, (int, float)):
+        return str(v)
+    s = str(v)
+    # 需要引号的情况：空串、含特殊起始字符、含冒号空格、纯数字样式
+    if s == "" or s[0] in "&*?|-<>=!%@`{[\"'" or ": " in s or s.endswith(":"):
+        return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"'
+    return s
+
+
+def patch_item_fields(path, source_id, item_id, fields):
+    """回填某 source 下某 item 的字段。返回 True 表示有改动。
+
+    fields: dict，如 {"sha256": "ab..", "size": 123, "status": "fetched"}
+    已存在的键 → 原地替换；不存在的键 → 追加到该 item 块末尾。
+    """
+    with open(path, encoding="utf-8") as f:
+        lines = f.read().split("\n")
+
+    src_re = re.compile(r"^  - id: (\S+)\s*$")
+    item_re = re.compile(r"^      - id: (\S+)\s*$")
+
+    # 1) 定位 source 块范围
+    s_start = s_end = None
+    for i, ln in enumerate(lines):
+        m = src_re.match(ln)
+        if m:
+            if m.group(1) == source_id and s_start is None:
+                s_start = i
+            elif s_start is not None and s_end is None:
+                s_end = i
+                break
+    if s_start is None:
+        return False
+    if s_end is None:
+        s_end = len(lines)
+
+    # 2) 在 source 块内定位 item 块范围
+    i_start = i_end = None
+    for i in range(s_start, s_end):
+        m = item_re.match(lines[i])
+        if m:
+            if m.group(1) == item_id and i_start is None:
+                i_start = i
+            elif i_start is not None and i_end is None:
+                i_end = i
+                break
+    if i_start is None:
+        return False
+    if i_end is None:
+        # item 块延伸到 source 块末尾；剔除尾部空行与注释行
+        j = s_end - 1
+        while j > i_start and (lines[j].strip() == "" or lines[j].lstrip().startswith("#")):
+            j -= 1
+        i_end = j + 1
+
+    # 3) 逐字段替换或追加
+    changed = False
+    for key, val in fields.items():
+        new_line = f"        {key}: {_yaml_scalar(val)}"
+        hit = None
+        for i in range(i_start, i_end):
+            if re.match(rf"^        {re.escape(key)}:", lines[i]):
+                hit = i
+                break
+        if hit is not None:
+            if lines[hit] != new_line:
+                lines[hit] = new_line
+                changed = True
+        else:
+            lines.insert(i_end, new_line)
+            i_end += 1
+            changed = True
+
+    if changed:
+        with open(path, "w", encoding="utf-8") as f:
+            f.write("\n".join(lines))
+    return changed
+
+
+def patch_source_fields(path, source_id, fields):
+    """回填 source 级字段（如 status / fetched_at）。返回 True 表示有改动。"""
+    with open(path, encoding="utf-8") as f:
+        lines = f.read().split("\n")
+
+    src_re = re.compile(r"^  - id: (\S+)\s*$")
+    s_start = s_end = None
+    for i, ln in enumerate(lines):
+        m = src_re.match(ln)
+        if m:
+            if m.group(1) == source_id and s_start is None:
+                s_start = i
+            elif s_start is not None and s_end is None:
+                s_end = i
+                break
+    if s_start is None:
+        return False
+    if s_end is None:
+        s_end = len(lines)
+
+    # items: 之前才是 source 级字段区
+    limit = s_end
+    for i in range(s_start, s_end):
+        if re.match(r"^    items:", lines[i]):
+            limit = i
+            break
+
+    changed = False
+    for key, val in fields.items():
+        new_line = f"    {key}: {_yaml_scalar(val)}"
+        hit = None
+        for i in range(s_start, limit):
+            if re.match(rf"^    {re.escape(key)}:", lines[i]):
+                hit = i
+                break
+        if hit is not None:
+            if lines[hit] != new_line:
+                lines[hit] = new_line
+                changed = True
+        else:
+            lines.insert(limit, new_line)
+            limit += 1
+            changed = True
+
+    if changed:
+        with open(path, "w", encoding="utf-8") as f:
+            f.write("\n".join(lines))
+    return changed