59 行
2.6 KiB
Python
59 行
2.6 KiB
Python
#!/usr/bin/env python3
|
||
"""index_build.py — 生成 INDEX.json（机器可读库全貌）。
|
||
|
||
汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。
|
||
注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。
|
||
"""
|
||
import os, glob, json, datetime, collections
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
def source_summary():
|
||
import yaml
|
||
domains, n_sources, lic = {}, 0, collections.Counter()
|
||
for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
|
||
rel = os.path.relpath(path, os.path.join(REPO, "sources"))
|
||
domain = os.path.dirname(rel)
|
||
try: doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||
except Exception: continue
|
||
for s in (doc or {}).get("sources", []) or []:
|
||
n_sources += 1
|
||
lic[s.get("license")] += 1
|
||
domains.setdefault(domain, 0)
|
||
domains[domain] += 1
|
||
return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)}
|
||
|
||
def structured_summary():
|
||
datasets, lic = {}, collections.Counter()
|
||
for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True):
|
||
rel = os.path.relpath(path, os.path.join(REPO, "structured"))
|
||
cnt = 0
|
||
for line in open(path, encoding="utf-8"):
|
||
line = line.strip()
|
||
if not line: continue
|
||
cnt += 1
|
||
try: r = json.loads(line)
|
||
except Exception: continue
|
||
l = r.get("license")
|
||
if l: lic[l] += 1
|
||
datasets[rel] = cnt
|
||
return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)}
|
||
|
||
def main():
|
||
idx = {
|
||
"generated_at": datetime.date.today().isoformat(),
|
||
"repo": "edu-content",
|
||
"model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)",
|
||
"oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]},
|
||
"redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用",
|
||
"sources": source_summary(),
|
||
"structured": structured_summary(),
|
||
"entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"},
|
||
}
|
||
out = os.path.join(REPO, "INDEX.json")
|
||
json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2)
|
||
print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|