init: edu-content 内容库骨架(L1-L4 四层数据模型 + B类迁移 + lint/CI 工具)

这个提交包含在:
2026-08-22 00:16:49 +08:00
当前提交 f18264111b
共有 79 个文件被更改,包括 7182 次插入 和 0 次删除

48
tools/lint_structured.py 普通文件
查看文件

@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""lint_structured.py — 校验 structured/ 血缘与 license。
检查项:
- 每条 JSONL 记录必须有合法 schema(qb-1.1 或 vocab-1.1)
- 记录的 source.source_id 必须存在于 sources/
- license 必须是 LICENSES.md 枚举值
- 孤儿数据(引用不存在的 source)即 error
退出码非 0 表示有 error。
"""
import sys, os, glob, json
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
def load_source_ids():
import yaml
ids = set()
for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
try: doc = yaml.safe_load(open(path, encoding="utf-8"))
except Exception: continue
for s in (doc or {}).get("sources", []) or []:
if s.get("id"): ids.add(s["id"])
return ids
def err(msg): print("ERROR: " + msg); return 1
def main():
source_ids = load_source_ids()
errors, n = 0, 0
for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
for i, line in enumerate(open(path, encoding="utf-8"), 1):
line = line.strip()
if not line: continue
try: r = json.loads(line)
except Exception as e: err(f"{path}:{i}: JSON 解析失败 {e}"); errors += 1; continue
n += 1
sid = (r.get("source") or {}).get("source_id")
if not sid: err(f"{path}:{i}: 缺 source.source_id"); errors += 1; continue
if sid not in source_ids: err(f"{path}:{i}: source_id '{sid}' 不在 sources/(孤儿数据)"); errors += 1
lic = r.get("license")
if lic and lic not in LICENSES: err(f"{path}:{i}: license 非法 '{lic}'"); errors += 1
print(f"\n校验完成:{n} 条结构化记录 | {errors} error")
sys.exit(1 if errors else 0)
if __name__ == "__main__":
main()