Drjunchenfeng
a006c8c504
一些检查失败了
ci / lint (push) Failing after 52s
登记 chinese-poetry / ck12 / phet，并让 source schema、lint 与 INDEX 能按学科和抓取进度消费。 Co-authored-by: Cursor <cursoragent@cursor.com>
186 行
7.0 KiB
Python
186 行
7.0 KiB
Python
#!/usr/bin/env python3
|
||
"""index_build.py — 生成 INDEX.json（机器可读库全貌）。
|
||
|
||
汇总：资源域、学科覆盖矩阵、抓取进度、各数据集记录数、license 分布、OSS 前缀、红线摘要。
|
||
注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
|
||
"""
|
||
import os, glob, json, datetime, collections
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
# 与 sources/source.schema.json 的 subjects 枚举保持一致
|
||
SUBJECTS = [
|
||
"chinese", "math", "english", "science", "morality",
|
||
"physics", "chemistry", "biology", "history", "geography", "it",
|
||
"ket", "pet", "fce", "amc8", "cspj",
|
||
]
|
||
STAGES = ["primary", "junior"]
|
||
# 可全自动抓取的 fetch_method（fetch_source.py 支持）
|
||
AUTO_METHODS = {"curl", "git-clone", "api"}
|
||
PRODUCT_SAFE = {"official-free", "mit", "cc-by", "cc-by-sa", "llm-generated"}
|
||
|
||
|
||
def load_sources():
|
||
"""返回 [(domain, source_dict, yaml_rel_path)]"""
|
||
import yaml
|
||
out = []
|
||
for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
|
||
rel = os.path.relpath(path, os.path.join(REPO, "sources"))
|
||
domain = os.path.dirname(rel) or "."
|
||
try:
|
||
doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
for s in (doc or {}).get("sources", []) or []:
|
||
out.append((domain, s, rel))
|
||
return out
|
||
|
||
|
||
def source_summary(rows):
|
||
domains, lic = collections.Counter(), collections.Counter()
|
||
for domain, s, _ in rows:
|
||
domains[domain] += 1
|
||
lic[s.get("license")] += 1
|
||
return {
|
||
"domains": dict(domains),
|
||
"n_sources": len(rows),
|
||
"license_breakdown": dict(lic),
|
||
}
|
||
|
||
|
||
def fetch_progress(rows):
|
||
"""条目级抓取进度：planned / fetched / uploaded。"""
|
||
st = collections.Counter()
|
||
n_items = 0
|
||
auto, manual = 0, 0
|
||
for _, s, _ in rows:
|
||
if s.get("fetch_method") in AUTO_METHODS:
|
||
auto += 1
|
||
else:
|
||
manual += 1
|
||
for it in s.get("items", []) or []:
|
||
n_items += 1
|
||
st[it.get("status") or "planned"] += 1
|
||
return {
|
||
"n_items": n_items,
|
||
"by_status": dict(st),
|
||
"auto_fetchable_sources": auto,
|
||
"manual_sources": manual,
|
||
}
|
||
|
||
|
||
def subject_coverage(rows):
|
||
"""学科 → 资源覆盖矩阵。每个学科给出源清单、学段、可自动抓取数、条目进度。"""
|
||
cov = {}
|
||
for subj in SUBJECTS:
|
||
cov[subj] = {
|
||
"n_sources": 0,
|
||
"stages": [],
|
||
"auto_fetchable": 0,
|
||
"n_items": 0,
|
||
"n_fetched": 0,
|
||
"product_safe_sources": 0,
|
||
"sources": [],
|
||
}
|
||
for domain, s, rel in rows:
|
||
subs = s.get("subjects") or []
|
||
auto = s.get("fetch_method") in AUTO_METHODS
|
||
items = s.get("items", []) or []
|
||
n_fetched = sum(1 for it in items if it.get("status") in ("fetched", "uploaded"))
|
||
safe = s.get("license") in PRODUCT_SAFE
|
||
entry = {
|
||
"id": s.get("id"),
|
||
"title": s.get("title"),
|
||
"license": s.get("license"),
|
||
"fetch_method": s.get("fetch_method"),
|
||
"auto_fetchable": auto,
|
||
"product_safe": safe,
|
||
"stages": s.get("stages") or [],
|
||
"n_items": len(items),
|
||
"n_fetched": n_fetched,
|
||
"yaml": rel,
|
||
}
|
||
for subj in subs:
|
||
if subj not in cov:
|
||
cov[subj] = {"n_sources": 0, "stages": [], "auto_fetchable": 0,
|
||
"n_items": 0, "n_fetched": 0, "product_safe_sources": 0, "sources": []}
|
||
c = cov[subj]
|
||
c["n_sources"] += 1
|
||
c["auto_fetchable"] += 1 if auto else 0
|
||
c["n_items"] += len(items)
|
||
c["n_fetched"] += n_fetched
|
||
c["product_safe_sources"] += 1 if safe else 0
|
||
c["sources"].append(entry)
|
||
for st in s.get("stages") or []:
|
||
if st not in c["stages"]:
|
||
c["stages"].append(st)
|
||
for c in cov.values():
|
||
c["stages"] = [x for x in STAGES if x in c["stages"]]
|
||
return cov
|
||
|
||
|
||
def gaps(cov):
|
||
"""缺口清单：完全无源 / 有源但一条都没抓下来 / 有源但无 product_safe。"""
|
||
no_source = sorted([k for k, v in cov.items() if v["n_sources"] == 0])
|
||
no_data = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["n_fetched"] == 0])
|
||
no_safe = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["product_safe_sources"] == 0])
|
||
return {
|
||
"subjects_without_source": no_source,
|
||
"subjects_without_fetched_data": no_data,
|
||
"subjects_without_product_safe_source": no_safe,
|
||
}
|
||
|
||
|
||
def structured_summary():
|
||
datasets, lic = {}, collections.Counter()
|
||
for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)):
|
||
rel = os.path.relpath(path, os.path.join(REPO, "structured"))
|
||
cnt = 0
|
||
for line in open(path, encoding="utf-8"):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
cnt += 1
|
||
try:
|
||
r = json.loads(line)
|
||
except Exception:
|
||
continue
|
||
l = r.get("license")
|
||
if l:
|
||
lic[l] += 1
|
||
datasets[rel] = cnt
|
||
return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}
|
||
|
||
|
||
def main():
|
||
rows = load_sources()
|
||
cov = subject_coverage(rows)
|
||
idx = {
|
||
"generated_at": datetime.date.today().isoformat(),
|
||
"repo": "edu-content",
|
||
"model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
|
||
"oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
|
||
"redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
|
||
"product_safe_licenses": sorted(PRODUCT_SAFE),
|
||
"sources": source_summary(rows),
|
||
"fetch_progress": fetch_progress(rows),
|
||
"subject_coverage": cov,
|
||
"gaps": gaps(cov),
|
||
"structured": structured_summary(),
|
||
"entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
|
||
}
|
||
out = os.path.join(REPO, "INDEX.json")
|
||
json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||
fp = idx["fetch_progress"]
|
||
g = idx["gaps"]
|
||
print(f"已生成 {out}")
|
||
print(f" source {idx['sources']['n_sources']} 条 | 条目 {fp['n_items']} 个 | 状态 {fp['by_status']}")
|
||
print(f" 可自动抓取源 {fp['auto_fetchable_sources']} / 需人工 {fp['manual_sources']}")
|
||
print(f" 无源学科：{', '.join(g['subjects_without_source']) or '（无）'}")
|
||
print(f" 有源但零数据：{', '.join(g['subjects_without_fetched_data']) or '（无）'}")
|
||
print(f" 结构化记录 {idx['structured']['n_records']} 条")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|