Drjunchenfeng
a006c8c504
一些检查失败了
ci / lint (push) Failing after 52s
登记 chinese-poetry / ck12 / phet，并让 source schema、lint 与 INDEX 能按学科和抓取进度消费。 Co-authored-by: Cursor <cursoragent@cursor.com>
71 行
3.7 KiB
Python
71 行
3.7 KiB
Python
#!/usr/bin/env python3
|
||
"""lint_sources.py — 校验 sources/ 资源清单合规。
|
||
|
||
检查项：
|
||
- 每个 source 必填字段、id kebab-case 唯一、license/type/status/subjects 枚举合法
|
||
- status=fetched 的 item 必须有 oss_key + 64位 sha256
|
||
- license=unknown 且 fetched_at 距今 > 30 天 → warning
|
||
|
||
退出码非 0 表示有 error（CI 应标红）。
|
||
"""
|
||
import sys, os, glob, re, datetime, hashlib
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
LICENSES = {"official-free","mit","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"}
|
||
TYPES = {"dataset","website","book","internal","tool"}
|
||
STATUSES = {"planned","fetching","fetched","failed","archived"}
|
||
SUBJECTS = {"chinese","math","english","science","morality","physics","chemistry",
|
||
"biology","history","geography","it","ket","pet","fce","amc8","cspj"}
|
||
STAGES = {"primary","junior"}
|
||
ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||
SHA_RE = re.compile(r"^[a-f0-9]{64}$")
|
||
|
||
def err(msg): print("ERROR: " + msg)
|
||
def warn(msg): print("WARN: " + msg)
|
||
|
||
def main():
|
||
errors, warnings = 0, 0
|
||
seen_ids = {}
|
||
for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
|
||
import yaml
|
||
try:
|
||
doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||
except Exception as e:
|
||
err(f"{path}: YAML 解析失败: {e}"); errors += 1; continue
|
||
if not doc or "sources" not in doc:
|
||
err(f"{path}: 缺少 sources 列表"); errors += 1; continue
|
||
for s in doc["sources"]:
|
||
sid = s.get("id")
|
||
if not sid: err(f"{path}: source 缺 id"); errors += 1; continue
|
||
if not ID_RE.match(sid): err(f"{sid}: id 非 kebab-case"); errors += 1
|
||
if sid in seen_ids: err(f"{sid}: id 重复 ({seen_ids[sid]} 与 {path})"); errors += 1
|
||
else: seen_ids[sid] = path
|
||
for f in ("title","publisher","type","license","status"):
|
||
if not s.get(f): err(f"{sid}: 缺必填 {f}"); errors += 1
|
||
if s.get("type") not in TYPES: err(f"{sid}: type 非法 {s.get('type')}"); errors += 1
|
||
if s.get("license") not in LICENSES: err(f"{sid}: license 非法 {s.get('license')}"); errors += 1
|
||
if s.get("status") not in STATUSES: err(f"{sid}: status 非法 {s.get('status')}"); errors += 1
|
||
subs = s.get("subjects")
|
||
if not subs: err(f"{sid}: 缺必填 subjects（覆盖学科，见 LICENSES.md 同级约定）"); errors += 1
|
||
else:
|
||
for x in subs:
|
||
if x not in SUBJECTS: err(f"{sid}: subjects 含非法值 {x}"); errors += 1
|
||
for x in (s.get("stages") or []):
|
||
if x not in STAGES: err(f"{sid}: stages 含非法值 {x}"); errors += 1
|
||
if s.get("license") == "unknown" and s.get("fetched_at"):
|
||
try:
|
||
d = datetime.date.fromisoformat(s["fetched_at"])
|
||
if (datetime.date.today() - d).days > 30: warn(f"{sid}: license=unknown 已超 30 天，请核查"); warnings += 1
|
||
except Exception: pass
|
||
for it in s.get("items", []) or []:
|
||
iid = it.get("id")
|
||
if not ID_RE.match(iid or ""): err(f"{sid}/{iid}: item id 非 kebab-case"); errors += 1
|
||
if s.get("status") == "fetched" and it.get("status") == "fetched":
|
||
if not it.get("oss_key"): err(f"{sid}/{iid}: fetched 但缺 oss_key"); errors += 1
|
||
if not SHA_RE.match(str(it.get("sha256") or "")): err(f"{sid}/{iid}: sha256 非 64 位 hex"); errors += 1
|
||
print(f"\n校验完成：{len(seen_ids)} 条 source | {errors} error | {warnings} warn")
|
||
sys.exit(1 if errors else 0)
|
||
|
||
if __name__ == "__main__":
|
||
main()
|