49 行
1.9 KiB
Python
49 行
1.9 KiB
Python
#!/usr/bin/env python3
|
||
"""lint_structured.py — 校验 structured/ 血缘与 license。
|
||
|
||
检查项：
|
||
- 每条 JSONL 记录必须有合法 schema（qb-1.1 或 vocab-1.1）
|
||
- 记录的 source.source_id 必须存在于 sources/
|
||
- license 必须是 LICENSES.md 枚举值
|
||
- 孤儿数据（引用不存在的 source）即 error
|
||
|
||
退出码非 0 表示有 error。
|
||
"""
|
||
import sys, os, glob, json
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
|
||
|
||
def load_source_ids():
|
||
import yaml
|
||
ids = set()
|
||
for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
|
||
try: doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||
except Exception: continue
|
||
for s in (doc or {}).get("sources", []) or []:
|
||
if s.get("id"): ids.add(s["id"])
|
||
return ids
|
||
|
||
def err(msg): print("ERROR: " + msg); return 1
|
||
|
||
def main():
|
||
source_ids = load_source_ids()
|
||
errors, n = 0, 0
|
||
for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
|
||
for i, line in enumerate(open(path, encoding="utf-8"), 1):
|
||
line = line.strip()
|
||
if not line: continue
|
||
try: r = json.loads(line)
|
||
except Exception as e: err(f"{path}:{i}: JSON 解析失败 {e}"); errors += 1; continue
|
||
n += 1
|
||
sid = (r.get("source") or {}).get("source_id")
|
||
if not sid: err(f"{path}:{i}: 缺 source.source_id"); errors += 1; continue
|
||
if sid not in source_ids: err(f"{path}:{i}: source_id '{sid}' 不在 sources/（孤儿数据）"); errors += 1
|
||
lic = r.get("license")
|
||
if lic and lic not in LICENSES: err(f"{path}:{i}: license 非法 '{lic}'"); errors += 1
|
||
print(f"\n校验完成：{n} 条结构化记录 | {errors} error")
|
||
sys.exit(1 if errors else 0)
|
||
|
||
if __name__ == "__main__":
|
||
main()
|