init: edu-content 内容库骨架(L1-L4 四层数据模型 + B类迁移 + lint/CI 工具)
这个提交包含在:
48
tools/lint_structured.py
普通文件
48
tools/lint_structured.py
普通文件
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lint_structured.py — 校验 structured/ 血缘与 license。
|
||||
|
||||
检查项:
|
||||
- 每条 JSONL 记录必须有合法 schema(qb-1.1 或 vocab-1.1)
|
||||
- 记录的 source.source_id 必须存在于 sources/
|
||||
- license 必须是 LICENSES.md 枚举值
|
||||
- 孤儿数据(引用不存在的 source)即 error
|
||||
|
||||
退出码非 0 表示有 error。
|
||||
"""
|
||||
import sys, os, glob, json
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
|
||||
|
||||
def load_source_ids():
|
||||
import yaml
|
||||
ids = set()
|
||||
for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
|
||||
try: doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||||
except Exception: continue
|
||||
for s in (doc or {}).get("sources", []) or []:
|
||||
if s.get("id"): ids.add(s["id"])
|
||||
return ids
|
||||
|
||||
def err(msg): print("ERROR: " + msg); return 1
|
||||
|
||||
def main():
|
||||
source_ids = load_source_ids()
|
||||
errors, n = 0, 0
|
||||
for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
|
||||
for i, line in enumerate(open(path, encoding="utf-8"), 1):
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
try: r = json.loads(line)
|
||||
except Exception as e: err(f"{path}:{i}: JSON 解析失败 {e}"); errors += 1; continue
|
||||
n += 1
|
||||
sid = (r.get("source") or {}).get("source_id")
|
||||
if not sid: err(f"{path}:{i}: 缺 source.source_id"); errors += 1; continue
|
||||
if sid not in source_ids: err(f"{path}:{i}: source_id '{sid}' 不在 sources/(孤儿数据)"); errors += 1
|
||||
lic = r.get("license")
|
||||
if lic and lic not in LICENSES: err(f"{path}:{i}: license 非法 '{lic}'"); errors += 1
|
||||
print(f"\n校验完成:{n} 条结构化记录 | {errors} error")
|
||||
sys.exit(1 if errors else 0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
在新工单中引用
屏蔽一个用户