init: edu-content 内容库骨架（L1-L4 四层数据模型 + B类迁移 + lint/CI 工具）

tools/index_build.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""index_build.py — 生成 INDEX.json（机器可读库全貌）。
+
+汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。
+注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
+"""
+import os, glob, json, datetime, collections
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+def source_summary():
+    import yaml
+    domains, n_sources, lic = {}, 0, collections.Counter()
+    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
+        rel = os.path.relpath(path, os.path.join(REPO, "sources"))
+        domain = os.path.dirname(rel)
+        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception: continue
+        for s in (doc or {}).get("sources", []) or []:
+            n_sources += 1
+            lic[s.get("license")] += 1
+            domains.setdefault(domain, 0)
+            domains[domain] += 1
+    return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)}
+
+def structured_summary():
+    datasets, lic = {}, collections.Counter()
+    for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True):
+        rel = os.path.relpath(path, os.path.join(REPO, "structured"))
+        cnt = 0
+        for line in open(path, encoding="utf-8"):
+            line = line.strip()
+            if not line: continue
+            cnt += 1
+            try: r = json.loads(line)
+            except Exception: continue
+            l = r.get("license")
+            if l: lic[l] += 1
+        datasets[rel] = cnt
+    return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}
+
+def main():
+    idx = {
+        "generated_at": datetime.date.today().isoformat(),
+        "repo": "edu-content",
+        "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
+        "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
+        "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
+        "sources": source_summary(),
+        "structured": structured_summary(),
+        "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
+    }
+    out = os.path.join(REPO, "INDEX.json")
+    json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
+    print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录")
+
+if __name__ == "__main__":
+    main()

tools/lint_sources.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""lint_sources.py — 校验 sources/ 资源清单合规。
+
+检查项：
+- 每个 source 必填字段、id kebab-case 唯一、license/type/status 枚举合法
+- status=fetched 的 item 必须有 oss_key + 64位 sha256
+- license=unknown 且 fetched_at 距今 > 30 天 → warning
+
+退出码非 0 表示有 error（CI 应标红）。
+"""
+import sys, os, glob, re, datetime, hashlib
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
+TYPES = {"dataset","website","book","internal","tool"}
+STATUSES = {"planned","fetching","fetched","failed","archived"}
+ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
+SHA_RE = re.compile(r"^[a-f0-9]{64}$")
+
+def err(msg): print("ERROR: " + msg)
+def warn(msg): print("WARN:  " + msg)
+
+def main():
+    errors, warnings = 0, 0
+    seen_ids = {}
+    for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
+        import yaml
+        try:
+            doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception as e:
+            err(f"{path}: YAML 解析失败: {e}"); errors += 1; continue
+        if not doc or "sources" not in doc:
+            err(f"{path}: 缺少 sources 列表"); errors += 1; continue
+        for s in doc["sources"]:
+            sid = s.get("id")
+            if not sid: err(f"{path}: source 缺 id"); errors += 1; continue
+            if not ID_RE.match(sid): err(f"{sid}: id 非 kebab-case"); errors += 1
+            if sid in seen_ids: err(f"{sid}: id 重复 ({seen_ids[sid]} 与 {path})"); errors += 1
+            else: seen_ids[sid] = path
+            for f in ("title","publisher","type","license","status"):
+                if not s.get(f): err(f"{sid}: 缺必填 {f}"); errors += 1
+            if s.get("type") not in TYPES: err(f"{sid}: type 非法 {s.get('type')}"); errors += 1
+            if s.get("license") not in LICENSES: err(f"{sid}: license 非法 {s.get('license')}"); errors += 1
+            if s.get("status") not in STATUSES: err(f"{sid}: status 非法 {s.get('status')}"); errors += 1
+            if s.get("license") == "unknown" and s.get("fetched_at"):
+                try:
+                    d = datetime.date.fromisoformat(s["fetched_at"])
+                    if (datetime.date.today() - d).days > 30: warn(f"{sid}: license=unknown 已超 30 天，请核查"); warnings += 1
+                except Exception: pass
+            for it in s.get("items", []) or []:
+                iid = it.get("id")
+                if not ID_RE.match(iid or ""): err(f"{sid}/{iid}: item id 非 kebab-case"); errors += 1
+                if s.get("status") == "fetched" and it.get("status") == "fetched":
+                    if not it.get("oss_key"): err(f"{sid}/{iid}: fetched 但缺 oss_key"); errors += 1
+                    if not SHA_RE.match(str(it.get("sha256") or "")): err(f"{sid}/{iid}: sha256 非 64 位 hex"); errors += 1
+    print(f"\n校验完成：{len(seen_ids)} 条 source | {errors} error | {warnings} warn")
+    sys.exit(1 if errors else 0)
+
+if __name__ == "__main__":
+    main()

tools/lint_structured.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""lint_structured.py — 校验 structured/ 血缘与 license。
+
+检查项：
+- 每条 JSONL 记录必须有合法 schema（qb-1.1 或 vocab-1.1）
+- 记录的 source.source_id 必须存在于 sources/
+- license 必须是 LICENSES.md 枚举值
+- 孤儿数据（引用不存在的 source）即 error
+
+退出码非 0 表示有 error。
+"""
+import sys, os, glob, json
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
+
+def load_source_ids():
+    import yaml
+    ids = set()
+    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
+        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception: continue
+        for s in (doc or {}).get("sources", []) or []:
+            if s.get("id"): ids.add(s["id"])
+    return ids
+
+def err(msg): print("ERROR: " + msg); return 1
+
+def main():
+    source_ids = load_source_ids()
+    errors, n = 0, 0
+    for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
+        for i, line in enumerate(open(path, encoding="utf-8"), 1):
+            line = line.strip()
+            if not line: continue
+            try: r = json.loads(line)
+            except Exception as e: err(f"{path}:{i}: JSON 解析失败 {e}"); errors += 1; continue
+            n += 1
+            sid = (r.get("source") or {}).get("source_id")
+            if not sid: err(f"{path}:{i}: 缺 source.source_id"); errors += 1; continue
+            if sid not in source_ids: err(f"{path}:{i}: source_id '{sid}' 不在 sources/（孤儿数据）"); errors += 1
+            lic = r.get("license")
+            if lic and lic not in LICENSES: err(f"{path}:{i}: license 非法 '{lic}'"); errors += 1
+    print(f"\n校验完成：{n} 条结构化记录 | {errors} error")
+    sys.exit(1 if errors else 0)
+
+if __name__ == "__main__":
+    main()

tools/oss_sync.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""oss_sync.py — 上传 L2 原始文件到 OSS 并回填 sha256/size/fetch_date。
+
+用法：
+  python tools/oss_sync.py --raw-root <DIR> --dry-run      # 仅报告将要上传的文件
+  EDU_OSS_KEY_ID=... EDU_OSS_KEY_SECRET=... \\
+  python tools/oss_sync.py --raw-root <DIR> --apply        # 实际上传 + 回填 YAML
+
+约定：
+- 本地 DIR 结构：<raw-root>/{source-id}/{filename...}，与 OSS key 的
+  edu-raw/{k9|exams|internal}/{学科或考试}/{类目}/{source-id}/{filename} 对应。
+- 上传后自动把 oss_key / sha256 / size / fetch_date 写回对应 source YAML 的 item。
+
+凭据：从环境变量 EDU_OSS_KEY_ID / EDU_OSS_KEY_SECRET / EDU_OSS_BUCKET / EDU_OSS_ENDPOINT 读取，
+      绝不写死、不进 git。缺省 bucket=edu-content-hy，endpoint=https://oss-cn-heyuan.aliyuncs.com。
+"""
+import os, sys, glob, json, hashlib, datetime, argparse, yaml
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SHA_RE = __import__("re").compile(r"^[a-f0-9]{64}$")
+
+def sha256_file(p):
+    h = hashlib.sha256()
+    with open(p, "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+def load_sources():
+    import yaml
+    docs = []
+    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
+        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
+        except Exception: continue
+        docs.append((path, doc))
+    return docs
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--raw-root", required=True)
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--apply", action="store_true")
+    args = ap.parse_args()
+    if not os.path.isdir(args.raw_root):
+        print("raw-root 不存在:", args.raw_root); sys.exit(2)
+
+    docs = load_sources()
+    pending = []
+    for path, doc in docs:
+        for s in (doc or {}).get("sources", []) or []:
+            sid = s.get("id")
+            for it in s.get("items", []) or []:
+                # 本地文件：raw-root/{sid}/{filename}
+                local = os.path.join(args.raw_root, sid, os.path.basename(it.get("oss_key") or it.get("title") or it.get("id")))
+                if os.path.isfile(local):
+                    sha = sha256_file(local)
+                    oss_key = it.get("oss_key") or f"edu-raw/exams/_staging/{sid}/{os.path.basename(local)}"
+                    pending.append((path, sid, it.get("id"), local, oss_key, sha, os.path.getsize(local)))
+
+    if not pending:
+        print("无可上传文件（raw-root 下未匹配到 source item 对应的本地文件）。")
+        sys.exit(0)
+
+    print(f"待上传 {len(pending)} 个文件：")
+    for p, sid, iid, local, ok, sha, size in pending:
+        print(f"  {sid}/{iid}: {local} -> {ok} ({size}B, sha256={sha[:12]}...)")
+
+    if args.dry_run or not args.apply:
+        print("\n[dry-run] 未实际上传。加 --apply 执行。")
+        sys.exit(0)
+
+    # 实际上传
+    try:
+        import oss2
+    except Exception as e:
+        print("oss2 不可用，无法上传:", e); sys.exit(3)
+    key_id = os.environ.get("EDU_OSS_KEY_ID")
+    key_secret = os.environ.get("EDU_OSS_KEY_SECRET")
+    bucket = os.environ.get("EDU_OSS_BUCKET", "edu-content-hy")
+    endpoint = os.environ.get("EDU_OSS_ENDPOINT", "https://oss-cn-heyuan.aliyuncs.com")
+    if not (key_id and key_secret):
+        print("缺少 EDU_OSS_KEY_ID / EDU_OSS_KEY_SECRET 环境变量"); sys.exit(3)
+    auth = oss2.Auth(key_id, key_secret)
+    b = oss2.Bucket(auth, endpoint, bucket)
+    today = datetime.date.today().isoformat()
+    # 上传 + 回填 YAML
+    for path, sid, iid, local, oss_key, sha, size in pending:
+        b.put_object_from_file(oss_key, local)
+        print(f"  uploaded {oss_key}")
+        # 回填
+        doc = yaml.safe_load(open(path, encoding="utf-8"))
+        for s in doc["sources"]:
+            if s.get("id") == sid:
+                for it in s.get("items", []):
+                    if it.get("id") == iid:
+                        it["oss_key"] = oss_key
+                        it["sha256"] = sha
+                        it["size"] = size
+                        it["fetch_date"] = today
+                        it["status"] = "fetched"
+        yaml.safe_dump(doc, open(path, "w", encoding="utf-8"), allow_unicode=True, sort_keys=False)
+    print("已回填 sha256/size 到 sources YAML。请 git commit。")
+
+if __name__ == "__main__":
+    main()