diff --git a/.gitignore b/.gitignore index 6e22234..97a92ef 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ __pycache__/ *.pyc .env +.staging/ +.cache/ diff --git a/INDEX.json b/INDEX.json index c08e1a9..73ffde2 100644 --- a/INDEX.json +++ b/INDEX.json @@ -11,27 +11,1115 @@ ] }, "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用", + "product_safe_licenses": [ + "cc-by", + "cc-by-sa", + "llm-generated", + "mit", + "official-free" + ], "sources": { "domains": { - "internal": 4, - "china": 8, - "exams": 4 + "china": 10, + "exams": 4, + "internal": 4 }, - "n_sources": 16, + "n_sources": 18, "license_breakdown": { - "internal-only": 4, - "official-free": 3, - "cc-nc-education": 6, + "cc-nc-education": 7, + "unknown": 2, + "mit": 1, "cc-by": 1, - "unknown": 2 + "official-free": 3, + "internal-only": 4 } }, + "fetch_progress": { + "n_items": 43, + "by_status": { + "planned": 22, + "fetched": 21 + }, + "auto_fetchable_sources": 5, + "manual_sources": 13 + }, + "subject_coverage": { + "chinese": { + "n_sources": 4, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 12, + "n_fetched": 0, + "product_safe_sources": 2, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "chinese-poetry", + "title": "chinese-poetry 最全中文诗歌古典文集数据库（37 万首）", + "license": "mit", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 12, + "n_fetched": 0, + "yaml": "china/chinese-poetry.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "math": { + "n_sources": 8, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 2, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "tal-scq5k", + "title": "TAL-SCQ5K 小学应用题数据集", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "ape210k", + "title": "Ape210K 小学应用题数据集（21万题）", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "primary" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "math23k", + "title": "Math23K 初中数学应用题数据集", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "cmath", + "title": "CMATH 中文小学数学数据集", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "amc-past", + "title": "AMC 8 历年真题（含答案）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "exams/amc-past.yaml" + } + ] + }, + "english": { + "n_sources": 5, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 6, + "n_fetched": 0, + "product_safe_sources": 2, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "cambridge-official", + "title": "剑桥官方 KET/PET 样卷、词表与听力音频", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 4, + "n_fetched": 0, + "yaml": "exams/cambridge-official.yaml" + }, + { + "id": "cepoc", + "title": "CEPOC 剑桥英语机读完形填空题库（KET/PET/FCE）", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 2, + "n_fetched": 0, + "yaml": "exams/cepoc.yaml" + } + ] + }, + "science": { + "n_sources": 4, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 21, + "n_fetched": 21, + "product_safe_sources": 2, + "sources": [ + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "ck12", + "title": "CK-12 FlexBooks — 科学 / 物理开放电子课本", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ck12.yaml" + }, + { + "id": "phet", + "title": "PhET 互动仿真 — 简体中文离线单文件（科学/物理）", + "license": "cc-by", + "fetch_method": "curl", + "auto_fetchable": true, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 21, + "n_fetched": 21, + "yaml": "china/phet.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "morality": { + "n_sources": 2, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "physics": { + "n_sources": 5, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 21, + "n_fetched": 21, + "product_safe_sources": 2, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "ck12", + "title": "CK-12 FlexBooks — 科学 / 物理开放电子课本", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ck12.yaml" + }, + { + "id": "phet", + "title": "PhET 互动仿真 — 简体中文离线单文件（科学/物理）", + "license": "cc-by", + "fetch_method": "curl", + "auto_fetchable": true, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 21, + "n_fetched": 21, + "yaml": "china/phet.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "chemistry": { + "n_sources": 4, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 21, + "n_fetched": 21, + "product_safe_sources": 2, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "phet", + "title": "PhET 互动仿真 — 简体中文离线单文件（科学/物理）", + "license": "cc-by", + "fetch_method": "curl", + "auto_fetchable": true, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 21, + "n_fetched": 21, + "yaml": "china/phet.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "biology": { + "n_sources": 3, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "history": { + "n_sources": 3, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "geography": { + "n_sources": 3, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + } + ] + }, + "it": { + "n_sources": 8, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 4, + "n_fetched": 0, + "product_safe_sources": 2, + "sources": [ + { + "id": "ceval", + "title": "C-Eval 中文综合评测集（含 K12 学科）", + "license": "cc-nc-education", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "web-papers", + "title": "第一试卷网 / 教习网等公开试卷（版权灰）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/ceval.yaml" + }, + { + "id": "smartedu", + "title": "国家中小学智慧教育平台（含组卷/课件导出）", + "license": "official-free", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "china/smartedu.yaml" + }, + { + "id": "ccf-csp", + "title": "CCF CSP-J/S 历年真题", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "exams/amc-past.yaml" + }, + { + "id": "qihang-v1", + "title": "启航班 V1 课件与素材（pptx/pdf/wav/png）", + "license": "internal-only", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 1, + "n_fetched": 0, + "yaml": "internal/qihang-v1.yaml" + }, + { + "id": "qihang-v2", + "title": "启航班 V2 课件与素材", + "license": "internal-only", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 1, + "n_fetched": 0, + "yaml": "internal/qihang-v2.yaml" + }, + { + "id": "qihang-v3", + "title": "启航班 V3 课程作品展示站（js/ts 代码工程）", + "license": "internal-only", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 1, + "n_fetched": 0, + "yaml": "internal/qihang-v3.yaml" + }, + { + "id": "zhizaoying-v1", + "title": "智造营 V1 录播课与课件（mp4/xlsx/pptx）", + "license": "internal-only", + "fetch_method": "manual-export", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 1, + "n_fetched": 0, + "yaml": "internal/zhizaoying-v1.yaml" + } + ] + }, + "ket": { + "n_sources": 2, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 6, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "cambridge-official", + "title": "剑桥官方 KET/PET 样卷、词表与听力音频", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 4, + "n_fetched": 0, + "yaml": "exams/cambridge-official.yaml" + }, + { + "id": "cepoc", + "title": "CEPOC 剑桥英语机读完形填空题库（KET/PET/FCE）", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 2, + "n_fetched": 0, + "yaml": "exams/cepoc.yaml" + } + ] + }, + "pet": { + "n_sources": 2, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 6, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "cambridge-official", + "title": "剑桥官方 KET/PET 样卷、词表与听力音频", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 4, + "n_fetched": 0, + "yaml": "exams/cambridge-official.yaml" + }, + { + "id": "cepoc", + "title": "CEPOC 剑桥英语机读完形填空题库（KET/PET/FCE）", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 2, + "n_fetched": 0, + "yaml": "exams/cepoc.yaml" + } + ] + }, + "fce": { + "n_sources": 2, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 1, + "n_items": 6, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "cambridge-official", + "title": "剑桥官方 KET/PET 样卷、词表与听力音频", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "primary", + "junior" + ], + "n_items": 4, + "n_fetched": 0, + "yaml": "exams/cambridge-official.yaml" + }, + { + "id": "cepoc", + "title": "CEPOC 剑桥英语机读完形填空题库（KET/PET/FCE）", + "license": "cc-nc-education", + "fetch_method": "git-clone", + "auto_fetchable": true, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 2, + "n_fetched": 0, + "yaml": "exams/cepoc.yaml" + } + ] + }, + "amc8": { + "n_sources": 1, + "stages": [ + "primary", + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 0, + "sources": [ + { + "id": "amc-past", + "title": "AMC 8 历年真题（含答案）", + "license": "unknown", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": false, + "stages": [ + "primary", + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "exams/amc-past.yaml" + } + ] + }, + "cspj": { + "n_sources": 1, + "stages": [ + "junior" + ], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 1, + "sources": [ + { + "id": "ccf-csp", + "title": "CCF CSP-J/S 历年真题", + "license": "official-free", + "fetch_method": "manual-download", + "auto_fetchable": false, + "product_safe": true, + "stages": [ + "junior" + ], + "n_items": 0, + "n_fetched": 0, + "yaml": "exams/amc-past.yaml" + } + ] + } + }, + "gaps": { + "subjects_without_source": [], + "subjects_without_fetched_data": [ + "amc8", + "biology", + "chinese", + "cspj", + "english", + "fce", + "geography", + "history", + "it", + "ket", + "math", + "morality", + "pet" + ], + "subjects_without_product_safe_source": [ + "amc8" + ] + }, "structured": { "datasets": { - "exams/pet/vocab/b1-preliminary-vocab.jsonl": 3164, - "exams/pet/questions/cepoc-b1-preliminary.jsonl": 21, + "exams/ket/questions/cepoc-a2-key.jsonl": 6, "exams/ket/vocab/a2-key-vocab.jsonl": 1738, - "exams/ket/questions/cepoc-a2-key.jsonl": 6 + "exams/pet/questions/cepoc-b1-preliminary.jsonl": 21, + "exams/pet/vocab/b1-preliminary-vocab.jsonl": 3164 }, "n_records": 4929, "license_breakdown": { diff --git a/LICENSES.md b/LICENSES.md index c61a5a0..8123b3b 100644 --- a/LICENSES.md +++ b/LICENSES.md @@ -2,11 +2,12 @@ 本文件是 `sources/`、`structured/`、`edu-derived` 所有 lint 的**唯一权威依据**。任何 source 记录的 `license`、L3 记录的 `license`、L4 资产的 `license_out` 必须是下表枚举值之一。 -## 枚举（8 值） +## 枚举（9 值） | license | 含义 | 可进 derived | 可商用 | 典型来源 | |---|---|---|---|---| | `official-free` | 官方免费发布，允许教学使用 | ✅ | 视条款 | 剑桥样卷/词表、CCF CSP 真题、国家平台导出（仅教学） | +| `mit` | MIT 开源许可，保留版权与许可声明即可 | ✅ | ✅ | chinese-poetry（古诗文）、chinese-xinhua | | `cc-by` | CC-BY，署名即可 | ✅ | ✅ | PhET | | `cc-by-sa` | CC-BY-SA 同享 | ✅ | ✅（同享） | ECDICT | | `cc-nc-education` | 教育/研究用途，非商业 | ✅ 但 `product_safe=false` | ❌ | CEPOC、C-Eval、Ape210K、Math23K、CMATH | @@ -17,18 +18,20 @@ ## 组合规则（lint 强制） -1. **L3 记录对外许可** = 其引用的所有 source license 的**最严格者**（取序：`restricted` > `internal-only` > `unknown` > `cc-nc-education` > `cc-by-sa`/`cc-by`/`official-free`/`llm-generated`）。 +1. **L3 记录对外许可** = 其引用的所有 source license 的**最严格者**（取序：`restricted` > `internal-only` > `unknown` > `cc-nc-education` > `cc-by-sa`/`cc-by`/`mit`/`official-free`/`llm-generated`）。 - 例：词库词条 = `official-free` 原词表 + `cc-by-sa` 释义 → 整条按 `cc-by-sa` 对待（须署名）。 - 例：gloss 含 `llm-generated` 部分 → 该字段按 `llm-generated` 处理，记录整体按最严格者。 2. **L4 资产红线**：`inputs` 中出现 `restricted` / `internal-only` / `unknown` → **lint 直接失败**（即"衍生层不得引用受限原始数据"的机器实现）。 -3. **`product_safe = true`** 当且仅当全部 inputs ∈ `{official-free, cc-by, cc-by-sa, llm-generated}`。 +3. **`product_safe = true`** 当且仅当全部 inputs ∈ `{official-free, mit, cc-by, cc-by-sa, llm-generated}`。 4. `sources/` 条目 `license` 必填且必须合法；`unknown` 状态超过 30 天的条目 lint 报 warning（督促核查）。 +5. **署名/声明义务**：`mit` 与 `cc-by`/`cc-by-sa` 均要求保留出处。派生产物对外发布时，须在页面或包内保留来源仓库名与许可声明（MIT 需附原 LICENSE 文本）。 ## 红线速记 - **restricted / internal-only 数据永不可进 derived，永不可对外产品使用。** 它们只作内部参考，存在 `edu-raw/` 受控归档。 - `cc-nc-education` 可在内部教研使用、可生成内部衍生资产，但 `product_safe=false`，不得用于对外收费产品。 -- 对外产品（自学平台公开内容）只可用 `official-free / cc-by / cc-by-sa / llm-generated` 且 `product_safe=true` 的资产。 +- 对外产品（自学平台公开内容）只可用 `official-free / mit / cc-by / cc-by-sa / llm-generated` 且 `product_safe=true` 的资产。 +- **公版文本（古诗文等）走 `mit`/`cc-by-sa` 的开源汇编仓库最安全**：文本本身已过版权期，汇编仓库的开源许可只要求署名，因此可对外商用。 ## 引用本文件 diff --git a/sources/china/ceval.yaml b/sources/china/ceval.yaml index c6c85e4..22d0995 100644 --- a/sources/china/ceval.yaml +++ b/sources/china/ceval.yaml @@ -8,17 +8,10 @@ sources: license_note: "⚠️ 仅限非商业研究/教育用途，禁止商用" fetch_method: manual-download status: planned + subjects: [chinese, math, english, physics, chemistry, biology, history, geography, it] + stages: [junior] items: [] - - id: phet - title: PhET 互动仿真（科学/数学） - publisher: University of Colorado Boulder - type: website - url: https://phet.colorado.edu/ - license: cc-by - license_note: "CC-BY-4.0，署名即可，可商用" - fetch_method: manual-download - status: planned - items: [] + # phet 已拆分至 sources/china/phet.yaml（科学/物理主力来源，含可自动下载的 items） - id: web-papers title: 第一试卷网 / 教习网等公开试卷（版权灰） publisher: 各第三方站点（非官方） @@ -28,4 +21,6 @@ sources: license_note: "版权状态不明，集中登记、默认按最严格处理；未经核查不得进 structured/derived" fetch_method: manual-download status: planned + subjects: [chinese, math, english, science, morality, physics, chemistry, biology, history, geography, it] + stages: [primary, junior] items: [] diff --git a/sources/china/chinese-poetry.yaml b/sources/china/chinese-poetry.yaml new file mode 100644 index 0000000..d88fd22 --- /dev/null +++ b/sources/china/chinese-poetry.yaml @@ -0,0 +1,152 @@ +# 语文 — 公版古诗文开源汇编 +# +# 说明：小学/初中语文的「课文、阅读理解、作文范文」几乎全部锁在教材与教辅中（restricted， +# 不可再分发）。语文唯一能合法规模化再分发的路径是「公版文本 + 开源汇编仓库」： +# 文本本体已过版权保护期（进入公有领域），汇编仓库以 MIT 开源，仅要求保留许可声明。 +# 因此本 source 是语文学科的主力来源，L4 的阅读理解题/作文素材由此衍生（自有 IP，版权干净）。 + +sources: + - id: chinese-poetry + title: chinese-poetry 最全中文诗歌古典文集数据库（37 万首） + publisher: chinese-poetry（GitHub 开源社区项目） + type: dataset + url: https://github.com/chinese-poetry/chinese-poetry + license: mit + license_note: >- + 仓库 LICENSE 为 MIT（已通过 GitHub API 核实 spdx_id=MIT）。文本本体为历代公版作品， + 已进入公有领域。再分发/商用均可，但须保留 MIT 许可声明与来源仓库署名。 + fetch_method: git-clone + status: planned + subjects: [chinese] + stages: [primary, junior] + fetched_at: null + items: + # —— 小学/初中核心（优先下载）—— + - id: quan-tang-shi + title: 全唐诗 + 全宋诗（含作者表） + oss_key: edu-raw/k9/chinese/chinese-poetry/quan-tang-shi.tar.gz + src_path: "全唐诗" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: >- + 仓库「全唐诗/」目录，320 个文件：poet.tang.*.json（唐诗约 5.7 万首）、 + poet.song.*.json（宋诗约 25 万首）、authors.tang.json / authors.song.json 作者表。 + 小学必背古诗与初中古诗文的主力来源。 + - id: song-ci + title: 宋词（含作者表） + oss_key: edu-raw/k9/chinese/chinese-poetry/song-ci.tar.gz + src_path: "宋词" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「宋词/」目录，ci.song.*.json 约 2.1 万首 + author.song.json。 + - id: shijing + title: 诗经 + oss_key: edu-raw/k9/chinese/chinese-poetry/shijing.tar.gz + src_path: "诗经" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「诗经/」目录。初中文言启蒙与「关雎」「蒹葭」等课内篇目。 + - id: lunyu + title: 论语 + oss_key: edu-raw/k9/chinese/chinese-poetry/lunyu.tar.gz + src_path: "论语" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「论语/」目录。小学「论语十二章」及初中文言文课内篇目。 + - id: mengxue + title: 蒙学（三字经/百家姓/千字文/弟子规/千家诗/古文观止/声律启蒙等） + oss_key: edu-raw/k9/chinese/chinese-poetry/mengxue.tar.gz + src_path: "蒙学" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: >- + 仓库「蒙学/」目录，13 部：baijiaxing、dizigui、qianziwen、qianjiashi、 + guwenguanzhi 等。小学低年级识字与国学启蒙的核心素材。 + - id: sishu-wujing + title: 四书五经（大学/中庸/孟子） + oss_key: edu-raw/k9/chinese/chinese-poetry/sishu-wujing.tar.gz + src_path: "四书五经" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「四书五经/」目录：daxue.json、zhongyong.json、mengzi.json（论语见 lunyu 条目）。 + - id: chuci + title: 楚辞 + oss_key: edu-raw/k9/chinese/chinese-poetry/chuci.tar.gz + src_path: "楚辞" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「楚辞/」目录。初中「离骚」节选等。 + # —— 拓展（次优先）—— + - id: yuanqu + title: 元曲 + oss_key: edu-raw/k9/chinese/chinese-poetry/yuanqu.tar.gz + src_path: "元曲" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「元曲/」目录。初中「天净沙·秋思」等散曲。 + - id: wudai-shici + title: 五代诗词 + oss_key: edu-raw/k9/chinese/chinese-poetry/wudai-shici.tar.gz + src_path: "五代诗词" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「五代诗词/」目录，含南唐二主词（李煜「虞美人」等课内篇目）。 + - id: nalanxingde + title: 纳兰性德词集 + oss_key: edu-raw/k9/chinese/chinese-poetry/nalanxingde.tar.gz + src_path: "纳兰性德" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「纳兰性德/」目录。初中「长相思」等。 + - id: caocao + title: 曹操诗集 + oss_key: edu-raw/k9/chinese/chinese-poetry/caocao.tar.gz + src_path: "曹操诗集" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「曹操诗集/」目录。初中「观沧海」「龟虽寿」。 + - id: youmengying + title: 幽梦影 + oss_key: edu-raw/k9/chinese/chinese-poetry/youmengying.tar.gz + src_path: "幽梦影" + sha256: null + size: null + media_type: application/gzip + fetch_date: null + status: planned + note: 仓库「幽梦影/」目录。小品文，作文素材向，非课内。 + # 未纳入：「御定全唐詩」（繁体，与 quan-tang-shi 内容重复）、「水墨唐诗」（配图素材， + # 如需可后续作为 L4 图片素材单列）、rank/ strains/ loader/（项目脚本与排行，非内容数据）。 diff --git a/sources/china/ck12.yaml b/sources/china/ck12.yaml new file mode 100644 index 0000000..b514548 --- /dev/null +++ b/sources/china/ck12.yaml @@ -0,0 +1,27 @@ +# 科学 / 物理 — CK-12 FlexBooks（次选，需人工导出） +# +# 定位：内容体系完整（美国 K-12 科学/物理电子课本），但**不可自动下载**——需网页交互 +# 或账号导出 PDF/ePub，且为英文，进 L3 前还要翻译改编。因此优先级低于 phet。 +# 只在 PhET 覆盖不到的知识点（如系统性的教材式讲解、章末习题）才考虑取用。 + +sources: + - id: ck12 + title: CK-12 FlexBooks — 科学 / 物理开放电子课本 + publisher: CK-12 Foundation + type: book + url: https://www.ck12.org/ + license: cc-nc-education + license_note: >- + CC BY-NC（署名-非商业）。允许改编与翻译，但**禁止商业使用**，因此 product_safe=false： + 可用于内部教研与非商业教学，不得进入对外收费产品。取用前须逐本核实当册许可页。 + fetch_method: manual-download + status: planned + subjects: [science, physics] + stages: [primary, junior] + fetched_at: null + items: [] + # items 待人工导出后登记。候选册目（英文）： + # - Middle School Physical Science / Middle School Physics（初中物理） + # - Middle School Life Science / Earth Science（初中科学） + # - Elementary School Science（小学科学） + # 导出流程：ck12.org 选册 → Download → PDF/ePub → 上传 OSS edu-raw/k9/{science|physics}/ck12/ diff --git a/sources/china/phet.yaml b/sources/china/phet.yaml new file mode 100644 index 0000000..cbf34f5 --- /dev/null +++ b/sources/china/phet.yaml @@ -0,0 +1,225 @@ +# 科学 / 物理 — PhET 互动仿真（可自动下载） +# +# 为什么是主力来源：CC-BY 4.0（署名即可商用再分发）+ 有官方简体中文版 + 单文件可直接 curl。 +# 这是科学/物理两科里**唯一既高质量又能全自动下载**的开放资源。 +# +# 下载模板（已实测可用，2026-08-22 全部返回 200）： +# https://phet.colorado.edu/sims/html/{item-id}/latest/{item-id}_zh_CN.html +# 每个仿真是一个自包含的离线 HTML5 单文件（内联全部 JS/资源），大小约 2–5 MB， +# 下载后可直接本地/内网打开，无需联网。本 source 21 个条目合计约 70 MB。 +# +# 注意：并非所有仿真都有 zh_CN 版本。已实测 404 的（geometric-optics-basics、 +# sound-waves、my-solar-system）未纳入；新增条目前请先 curl 探测再登记。 + +sources: + - id: phet + title: PhET 互动仿真 — 简体中文离线单文件（科学/物理） + publisher: University of Colorado Boulder + type: tool + url: https://phet.colorado.edu/ + license: cc-by + license_note: >- + CC-BY 4.0，署名即可，允许商用与再分发。对外发布须保留 "PhET Interactive Simulations, + University of Colorado Boulder" 署名与 CC-BY 链接。仿真为交互式工具（非题库）， + 适合嵌入课程页面作探究活动。 + url_template: https://phet.colorado.edu/sims/html/{item_id}/latest/{item_id}_zh_CN.html + fetch_method: curl + status: fetched + subjects: [science, physics, chemistry] + stages: [primary, junior] + fetched_at: 2026-08-22 + items: + # ===== 初中物理：力与运动 ===== + - id: forces-and-motion-basics + title: 力和运动基础 + oss_key: edu-raw/k9/physics/phet/forces-and-motion-basics_zh_CN.html + sha256: 2d5bb20f1243f89e0edc807461481904b61a98dee8438854563658515d800a16 + size: 4851186 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜力的合成、摩擦、牛顿定律入门。实测约 4.8 MB。 + - id: friction + title: 摩擦力 + oss_key: edu-raw/k9/physics/phet/friction_zh_CN.html + sha256: abe2f1ee0de0bb614ab8e00de29f2f6e1668ada97a067e343db306030f1b1320 + size: 2081436 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜摩擦生热的微观解释。实测约 2.1 MB。 + - id: balancing-act + title: 平衡（杠杆） + oss_key: edu-raw/k9/physics/phet/balancing-act_zh_CN.html + sha256: f8a725c4242c308ba6a6450fd79a93a98a0954c8a697bee88637f68f82148c86 + size: 3152716 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜杠杆平衡条件、力矩。实测约 3.2 MB。 + - id: hookes-law + title: 胡克定律 + oss_key: edu-raw/k9/physics/phet/hookes-law_zh_CN.html + sha256: 3ab65654e49500c721f6bdf7995ab48257b8b894042853731adc5d1e64361f26 + size: 2427596 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜弹力与形变的正比关系。 + - id: masses-and-springs-basics + title: 质量与弹簧（基础） + oss_key: edu-raw/k9/physics/phet/masses-and-springs-basics_zh_CN.html + sha256: e2802cc4ec2dccde90467fbbda7a47e8dbf37e4a5ce166209501aae38eb8347c + size: 1762060 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜弹簧测力计原理、简谐运动直观。 + - id: projectile-motion + title: 抛体运动 + oss_key: edu-raw/k9/physics/phet/projectile-motion_zh_CN.html + sha256: a331570b5d2a6225a26b078e25700a3098b7eb4d99a997b1ccfe04a4f7097943 + size: 4378652 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理（拓展）｜运动的合成与分解，衔接高中。 + - id: energy-skate-park-basics + title: 能量滑板公园（基础） + oss_key: edu-raw/k9/physics/phet/energy-skate-park-basics_zh_CN.html + sha256: 33704f185353838f97fde2482d970df456f6ce7c1d34fe487b0303c28c724eb7 + size: 4912386 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜动能/势能转化与机械能守恒。实测约 4.9 MB。 + # ===== 初中物理：压强与浮力 ===== + - id: density + title: 密度 + oss_key: edu-raw/k9/physics/phet/density_zh_CN.html + sha256: 1dd91f8c05a0e7839ab5e0a122755ed8808fc46fcd87bc0c08d7ead706f9ebd9 + size: 4846707 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜密度概念与沉浮判断。实测约 4.8 MB。 + - id: under-pressure + title: 压强（液体内部压强） + oss_key: edu-raw/k9/physics/phet/under-pressure_zh_CN.html + sha256: df9c145fd82f28157427e452da22f84188366960a258aac19c26fecd91106231 + size: 1234194 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜液体压强随深度/密度变化。 + # ===== 初中物理：电与磁 ===== + - id: circuit-construction-kit-dc + title: 直流电路搭建实验 + oss_key: edu-raw/k9/physics/phet/circuit-construction-kit-dc_zh_CN.html + sha256: 94efaabb68fc5a61b69a50b0b63953cb38fe5d4b801473fa3348e2833a640fe7 + size: 3509296 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜串并联、欧姆定律、电表读数。**电学最重要的一个仿真**。实测约 3.5 MB。 + - id: balloons-and-static-electricity + title: 气球与静电 + oss_key: edu-raw/k9/physics/phet/balloons-and-static-electricity_zh_CN.html + sha256: fde2657d9ba242fd153a5c4b7bd26a8624a38a25249251094d45af5c870bd09c + size: 3539205 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜电荷转移与摩擦起电。实测约 3.5 MB。 + - id: john-travoltage + title: 静电放电（John Travoltage） + oss_key: edu-raw/k9/physics/phet/john-travoltage_zh_CN.html + sha256: ceb1fb773cf93e1e42fab1784f6a72843ff02dda84d3128fac7562fbb767110b + size: 1918798 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜电荷积累与放电现象，趣味性强。 + - id: faradays-law + title: 法拉第电磁感应定律 + oss_key: edu-raw/k9/physics/phet/faradays-law_zh_CN.html + sha256: 9c3270c9e0d9485779c92400945be32b03ea6148c77b49790c96f55cabab68e8 + size: 2227465 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理（拓展）｜磁生电，衔接发电机原理。 + # ===== 初中物理：光与波 ===== + - id: bending-light + title: 光的折射 + oss_key: edu-raw/k9/physics/phet/bending-light_zh_CN.html + sha256: 7edcfa3f426b9241485e06c3e9ad132a38c9564e4e1eb2637362bf4c08c0116e + size: 3406402 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜反射/折射定律、色散。实测约 3.4 MB。 + - id: wave-on-a-string + title: 绳上的波 + oss_key: edu-raw/k9/physics/phet/wave-on-a-string_zh_CN.html + sha256: f7e197797642e04be1d7360681f65bf2df67377d60e53b22a996881801f06b8d + size: 2487284 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 初中物理｜波的产生与传播、频率/振幅/波速。实测约 2.5 MB。 + - id: color-vision + title: 视觉与颜色 + oss_key: edu-raw/k9/physics/phet/color-vision_zh_CN.html + sha256: 26466a0acb4515bbd9fda6d70495a9d713e5c6af8064f6a2e94ec6b4c868c9b8 + size: 5772634 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学 / 初中物理｜三原色与色觉形成。 + # ===== 小学 / 初中科学 ===== + - id: states-of-matter-basics + title: 物态变化（基础） + oss_key: edu-raw/k9/science/phet/states-of-matter-basics_zh_CN.html + sha256: e0b511b18f3d19ee3be0df01c694e1eddf10003aae453ed8ff3e07e8fe5a2fe0 + size: 1918612 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学 / 初中物理｜固液气三态与微观粒子模型。实测约 1.9 MB。 + - id: gravity-force-lab-basics + title: 万有引力实验（基础） + oss_key: edu-raw/k9/science/phet/gravity-force-lab-basics_zh_CN.html + sha256: 8e302da41a15a1d2d3280082dcf61c495b01b8f606efb4afafdbec328ccab0c5 + size: 2888393 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学 / 初中物理｜质量与距离如何影响引力。 + - id: gravity-and-orbits + title: 引力与轨道 + oss_key: edu-raw/k9/science/phet/gravity-and-orbits_zh_CN.html + sha256: 9c82f56dd60fe2be49036b9b2eedfa74d7fc36fac37f33148bcd82be361688fa + size: 3446870 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学（天文）｜日地月系统、四季与公转。 + - id: concentration + title: 溶液浓度 + oss_key: edu-raw/k9/science/phet/concentration_zh_CN.html + sha256: 8a8f72ee0281f6032492fe42669e5dac7cf1f9855342c0c0f67acd546a488de5 + size: 2621430 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学 / 初中化学｜溶解、浓度、饱和。 + - id: ph-scale-basics + title: pH 值（基础） + oss_key: edu-raw/k9/science/phet/ph-scale-basics_zh_CN.html + sha256: 04123b8ae4ba580533f6b784ab8eb26335e244fea71f0051a205b145644137bc + size: 2523039 + media_type: text/html + fetch_date: 2026-08-22 + status: fetched + note: 小学科学 / 初中化学｜酸碱性与常见物质 pH。 diff --git a/sources/china/smartedu.yaml b/sources/china/smartedu.yaml index 47f86a7..df031e1 100644 --- a/sources/china/smartedu.yaml +++ b/sources/china/smartedu.yaml @@ -8,6 +8,8 @@ sources: license_note: "官方平台，教学使用；具体导出内容须遵守平台服务条款" fetch_method: manual-export status: planned + subjects: [chinese, math, english, science, morality, physics, chemistry, biology, history, geography, it] + stages: [primary, junior] items: [] - id: tal-scq5k title: TAL-SCQ5K 小学应用题数据集 @@ -18,6 +20,8 @@ sources: license_note: "研究/教育用途，非商业（见仓库许可）" fetch_method: manual-download status: planned + subjects: [math] + stages: [primary, junior] items: [] - id: ape210k title: Ape210K 小学应用题数据集（21万题） @@ -28,6 +32,8 @@ sources: license_note: "研究/教育用途，非商业" fetch_method: git-clone status: planned + subjects: [math] + stages: [primary] items: [] - id: math23k title: Math23K 初中数学应用题数据集 @@ -38,6 +44,8 @@ sources: license_note: "研究/教育用途，非商业" fetch_method: git-clone status: planned + subjects: [math] + stages: [junior] items: [] - id: cmath title: CMATH 中文小学数学数据集 @@ -48,4 +56,6 @@ sources: license_note: "研究/教育用途，非商业" fetch_method: manual-download status: planned + subjects: [math] + stages: [primary] items: [] diff --git a/sources/exams/amc-past.yaml b/sources/exams/amc-past.yaml index caa8f64..f23e07a 100644 --- a/sources/exams/amc-past.yaml +++ b/sources/exams/amc-past.yaml @@ -8,6 +8,8 @@ sources: license_note: "AMC 真题版权归 MAA，公开传播受限；内部教研参考，禁止再分发/商用。待核查后定级" fetch_method: manual-download status: planned + subjects: [amc8, math] + stages: [primary, junior] items: [] - id: ccf-csp title: CCF CSP-J/S 历年真题 @@ -18,4 +20,6 @@ sources: license_note: "CCF 官方公开，允许教学使用" fetch_method: manual-download status: planned + subjects: [cspj, it] + stages: [junior] items: [] diff --git a/sources/exams/cambridge-official.yaml b/sources/exams/cambridge-official.yaml index 5a143d1..42664a5 100644 --- a/sources/exams/cambridge-official.yaml +++ b/sources/exams/cambridge-official.yaml @@ -8,38 +8,90 @@ sources: license_note: "官方免费发布，允许教学使用（样卷/词表/听力音频）" fetch_method: manual-download status: fetched - fetched_at: "2026-08-21" + subjects: [ket, pet, fce, english] + stages: [primary, junior] + fetched_at: 2026-08-22 items: - id: a2-key-vocab title: A2 Key 2020 词汇表 PDF oss_key: edu-raw/exams/ket/vocab/cambridge-official/a2-key-vocabulary-list.pdf - sha256: null - size: null + sha256: 2abe174d067cd17ffa061cfc4bbbc0e5a55185ac696a5ef8ba49372c753a3ad6 + size: 544296 media_type: application/pdf - fetch_date: "2026-08-21" - status: planned + fetch_date: 2026-08-22 + status: fetched note: "vocab JSONL (structured/exams/ket/vocab/) 由此清洗；待 oss_sync 上传" - id: b1-preliminary-vocab - title: B1 Preliminary 词汇表 PDF - oss_key: edu-raw/exams/pet/vocab/cambridge-official/b1-preliminary-vocabulary-list.pdf - sha256: null - size: null + title: B1 Preliminary 2020 词汇表 PDF + oss_key: edu-raw/exams/pet/vocab/cambridge-official/b1-preliminary-2020-vocabulary-list.pdf + sha256: 85b094be8c214e4da6cb710f0b5450a455ae010ccb4064c5578f68ef6aa139ad + size: 628829 media_type: application/pdf - fetch_date: "2026-08-21" - status: planned + fetch_date: 2026-08-22 + status: fetched + note: "B1 词表清洗源；直链域名用 cambridgeenglish.cn（.org 已 404）" + - id: b1-preliminary-vocab-booklet + title: B1 Preliminary 词汇手册 PDF（含释义/例句，较厚） + oss_key: edu-raw/exams/pet/vocab/cambridge-official/b1-preliminary-vocabulary-booklet.pdf + sha256: c4fae109ab5416c7bc68e6686fd9a69abf2428f73e5768ce248d810e612eef20 + size: 8202984 + media_type: application/pdf + fetch_date: 2026-08-22 + status: fetched - id: a2-key-listening title: A2 Key 听力音频 oss_key: edu-raw/exams/ket/audio/cambridge-official/a2-key-listening.mp3 - sha256: null - size: null + sha256: f50df0695c23148f0caac3cf059fb9028db6ee467e75b993ebb0e60af2f70654 + size: 11974884 media_type: audio/mpeg - fetch_date: "2026-08-21" - status: planned - - id: sample-papers - title: 官方样卷包（PDF/MP3/ZIP） - oss_key: edu-raw/exams/ket/sample_papers/cambridge-official/ - sha256: null - size: null + fetch_date: 2026-08-22 + status: fetched + - id: b1-preliminary-listening + title: B1 Preliminary 听力音频 + oss_key: edu-raw/exams/pet/audio/cambridge-official/b1-preliminary-listening.mp3 + sha256: 8d2cd238fc9b3fe6342467a0f288a98dfb4481c5239d7bd8539f6d9b4bc76090 + size: 26401678 + media_type: audio/mpeg + fetch_date: 2026-08-22 + status: fetched + # 样卷：oss_key 必须是具体文件（目录形式无法单文件上传/校验） + - id: a2-key-sample-tests-zip + title: A2 Key for Schools 官方样卷包 ZIP + oss_key: edu-raw/exams/ket/sample_papers/cambridge-official/a2-key-for-schools-sample-tests.zip + sha256: 349d83c36718e597322588895a5dfce59e319f773263b3a60d96e0650f784e07 + size: 24264467 media_type: application/zip - fetch_date: "2026-08-21" - status: planned + fetch_date: 2026-08-22 + status: fetched + - id: b1-preliminary-sample-tests-zip + title: B1 Preliminary for Schools 官方样卷包 ZIP + oss_key: edu-raw/exams/pet/sample_papers/cambridge-official/b1-preliminary-for-schools-sample-tests.zip + sha256: 3e11b30226c492d4dab168b60d8607d19131751e4e465e2e5b20e98c217cc077 + size: 23143382 + media_type: application/zip + fetch_date: 2026-08-22 + status: fetched + - id: a2-key-handbook + title: A2 Key for Schools 考官手册 PDF + oss_key: edu-raw/exams/ket/sample_papers/cambridge-official/a2-key-handbook.pdf + sha256: ebd74b6103c0a06276c6459f5eb947531b49f365bc4df2d52fc7bec85bbd9f0c + size: 7860476 + media_type: application/pdf + fetch_date: 2026-08-22 + status: fetched + - id: b1-preliminary-handbook + title: B1 Preliminary for Schools 考官手册 PDF + oss_key: edu-raw/exams/pet/sample_papers/cambridge-official/b1-preliminary-handbook.pdf + sha256: fd47cbd4bf647a9188b4e913eafe8a63eb904e4babd7f89fc8abc1ebd47e5d49 + size: 9073455 + media_type: application/pdf + fetch_date: 2026-08-22 + status: fetched + - id: score-conversion + title: 分数换算表 PDF（原始分 → Cambridge Scale） + oss_key: edu-raw/exams/ket/sample_papers/cambridge-official/score-conversion.pdf + sha256: 63e371b4f1dc6d51069a443461af3f3811dac5599ce0f879f867e7521f5d9330 + size: 695401 + media_type: application/pdf + fetch_date: 2026-08-22 + status: fetched diff --git a/sources/exams/cepoc.yaml b/sources/exams/cepoc.yaml index d1240d2..cda3011 100644 --- a/sources/exams/cepoc.yaml +++ b/sources/exams/cepoc.yaml @@ -8,23 +8,27 @@ sources: license_note: "研究/教育用途，非商业（见仓库 licence.txt）" fetch_method: git-clone status: fetched + subjects: [ket, pet, fce, english] + stages: [primary, junior] fetched_at: "2026-08-21" items: - id: cepoc-ket-json - title: KET.json（6套68空） + title: KET.json（6套完形） oss_key: edu-raw/exams/ket/questions/cepoc/KET.json + src_path: KET.json sha256: null size: null media_type: application/json fetch_date: "2026-08-21" status: planned - note: "latin1 编码；清洗为 qb-1.1 落 structured/exams/ket/questions/cepoc-a2-key.jsonl" + note: "latin1 编码 JSONL；清洗为 qb-1.1 落 structured/exams/ket/questions/cepoc-a2-key.jsonl" - id: cepoc-b1-json - title: B1 Preliminary.json（含 FCE 36套360空） - oss_key: edu-raw/exams/pet/questions/cepoc/B1-Preliminary.json + title: PET.json（21套完形，即 B1 Preliminary） + oss_key: edu-raw/exams/pet/questions/cepoc/PET.json + src_path: PET.json sha256: null size: null media_type: application/json fetch_date: "2026-08-21" status: planned - note: "清洗为 qb-1.1 落 structured/exams/pet/questions/cepoc-b1-preliminary.jsonl" + note: "repo 实际文件名为 PET.json（此前误登记为 B1-Preliminary.json）；清洗落 structured/exams/pet/questions/cepoc-b1-preliminary.jsonl" diff --git a/sources/internal/qihang-v1.yaml b/sources/internal/qihang-v1.yaml index fdf4dbf..f475f59 100644 --- a/sources/internal/qihang-v1.yaml +++ b/sources/internal/qihang-v1.yaml @@ -8,6 +8,8 @@ sources: license_note: "自制课件，仅内部参考，禁止再分发/商用" fetch_method: manual-export status: archived + subjects: [it] + stages: [primary, junior] items: - id: qihang-v1-all title: 启航班V1 整包归档 diff --git a/sources/internal/qihang-v2.yaml b/sources/internal/qihang-v2.yaml index f18af02..7ad961f 100644 --- a/sources/internal/qihang-v2.yaml +++ b/sources/internal/qihang-v2.yaml @@ -8,6 +8,8 @@ sources: license_note: "自制课件，仅内部参考，禁止再分发/商用" fetch_method: manual-export status: archived + subjects: [it] + stages: [primary, junior] items: - id: qihang-v2-all title: 启航班V2 整包归档 diff --git a/sources/internal/qihang-v3.yaml b/sources/internal/qihang-v3.yaml index ff68670..14a458f 100644 --- a/sources/internal/qihang-v3.yaml +++ b/sources/internal/qihang-v3.yaml @@ -8,6 +8,8 @@ sources: license_note: "自制工程，仅内部参考；若持续演进可未来拆独立 repo" fetch_method: manual-export status: archived + subjects: [it] + stages: [primary, junior] items: - id: qihang-v3-all title: 启航班V3 整包归档 diff --git a/sources/internal/zhizaoying-v1.yaml b/sources/internal/zhizaoying-v1.yaml index a6fd8b6..0c54e38 100644 --- a/sources/internal/zhizaoying-v1.yaml +++ b/sources/internal/zhizaoying-v1.yaml @@ -8,6 +8,8 @@ sources: license_note: "自制课件，仅内部参考，禁止再分发/商用" fetch_method: manual-export status: archived + subjects: [it] + stages: [primary, junior] items: - id: zhizaoying-v1-all title: 智造营V1 整包归档 diff --git a/sources/source.schema.json b/sources/source.schema.json index 0ba94c9..6ea67db 100644 --- a/sources/source.schema.json +++ b/sources/source.schema.json @@ -3,47 +3,230 @@ "title": "edu-content source registry", "description": "机器可读的 L1 资源清单。一个 YAML 文件 = 一个域，内含多条 source。source 是四层血缘链的根 id。", "type": "object", - "required": ["sources"], + "required": [ + "sources" + ], "additionalProperties": false, "properties": { "sources": { "type": "array", - "items": { "$ref": "#/definitions/source" } + "items": { + "$ref": "#/definitions/source" + } } }, "definitions": { "source": { "type": "object", - "required": ["id", "title", "publisher", "type", "license", "status"], + "required": [ + "id", + "title", + "publisher", + "type", + "license", + "status", + "subjects" + ], "additionalProperties": false, "properties": { - "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$", "description": "全局唯一 kebab-case，血缘链根 id" }, - "title": { "type": "string" }, - "publisher": { "type": "string" }, - "type": { "type": "string", "enum": ["dataset", "website", "book", "internal", "tool"] }, - "url": { "type": ["string", "null"], "description": "internal 可为 null" }, - "license": { "type": "string", "enum": ["official-free", "cc-by", "cc-by-sa", "cc-nc-education", "restricted", "internal-only", "llm-generated", "unknown"] }, - "license_note": { "type": "string" }, - "fetch_method": { "type": "string", "enum": ["curl", "git-clone", "manual-download", "manual-export", "generated"], "description": "unknown 允许暂存" }, - "status": { "type": "string", "enum": ["planned", "fetching", "fetched", "failed", "archived"] }, - "fetched_at": { "type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$|^$" }, - "items": { "type": "array", "items": { "$ref": "#/definitions/item" } } + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "全局唯一 kebab-case，血缘链根 id" + }, + "title": { + "type": "string" + }, + "publisher": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "dataset", + "website", + "book", + "internal", + "tool" + ] + }, + "url": { + "type": [ + "string", + "null" + ], + "description": "internal 可为 null" + }, + "license": { + "type": "string", + "enum": [ + "official-free", + "mit", + "cc-by", + "cc-by-sa", + "cc-nc-education", + "restricted", + "internal-only", + "llm-generated", + "unknown" + ] + }, + "license_note": { + "type": "string" + }, + "fetch_method": { + "type": "string", + "enum": [ + "curl", + "git-clone", + "manual-download", + "manual-export", + "generated" + ], + "description": "unknown 允许暂存" + }, + "status": { + "type": "string", + "enum": [ + "planned", + "fetching", + "fetched", + "failed", + "archived" + ] + }, + "fetched_at": { + "type": [ + "string", + "null" + ], + "pattern": "^\\d{4}-\\d{2}-\\d{2}$|^$" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/item" + } + }, + "subjects": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "chinese", + "math", + "english", + "science", + "morality", + "physics", + "chemistry", + "biology", + "history", + "geography", + "it", + "ket", + "pet", + "fce", + "amc8", + "cspj" + ] + }, + "description": "该来源覆盖的学科/考试域，取值与 structured/ 目录名一致；用于回答『某学科有哪些来源』" + }, + "stages": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "primary", + "junior" + ] + }, + "description": "覆盖学段：primary=小学，junior=初中" + }, + "url_template": { + "type": [ + "string", + "null" + ], + "description": "fetch_method=curl 时的下载模板，{item_id} 占位；item.url 优先级更高。例：https://host/sims/{item_id}/{item_id}_zh_CN.html" + } } }, "item": { "type": "object", - "required": ["id", "title"], + "required": [ + "id", + "title" + ], "additionalProperties": false, "properties": { - "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$", "description": "source 内唯一" }, - "title": { "type": "string" }, - "oss_key": { "type": ["string", "null"], "description": "fetched 后必填，形如 edu-raw/exams/ket/vocab/cambridge-official/a2-key-vocab.pdf" }, - "sha256": { "type": ["string", "null"], "pattern": "^[a-f0-9]{64}$|^$" }, - "size": { "type": ["integer", "null"], "minimum": 0 }, - "media_type": { "type": "string" }, - "fetch_date": { "type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$|^$" }, - "status": { "type": "string", "enum": ["planned", "fetching", "fetched", "failed", "archived"] }, - "note": { "type": "string" } + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$", + "description": "source 内唯一" + }, + "title": { + "type": "string" + }, + "oss_key": { + "type": [ + "string", + "null" + ], + "description": "fetched 后必填，形如 edu-raw/exams/ket/vocab/cambridge-official/a2-key-vocab.pdf" + }, + "sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^[a-f0-9]{64}$|^$" + }, + "size": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "media_type": { + "type": "string" + }, + "fetch_date": { + "type": [ + "string", + "null" + ], + "pattern": "^\\d{4}-\\d{2}-\\d{2}$|^$" + }, + "status": { + "type": "string", + "enum": [ + "planned", + "fetching", + "fetched", + "failed", + "archived" + ] + }, + "note": { + "type": "string" + }, + "url": { + "type": [ + "string", + "null" + ], + "description": "该条目的直接下载地址；缺省时由 source.url_template 渲染" + }, + "src_path": { + "type": [ + "string", + "null" + ], + "description": "fetch_method=git-clone 时，条目在仓库内的相对路径（打包为 oss_key 指向的归档）" + } } } } diff --git a/tools/fetch_source.py b/tools/fetch_source.py new file mode 100644 index 0000000..f89d9e9 --- /dev/null +++ b/tools/fetch_source.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""fetch_source.py — 按 sources/*.yaml 自动抓取 L2 原始数据到本地 staging。 + +只处理**可自动化**的 fetch_method： + - curl : 逐 item 下载单文件（item.url，或 source.url_template 渲染 {item_id}） + - git-clone : 浅克隆一次仓库，再按 item.src_path 打包成 .tar.gz + +manual-download / manual-export / generated 会被跳过（需人工介入）。 + +用法： + # 看有哪些可自动抓取（不下载） + python tools/fetch_source.py --list + + # 试跑：只打印将要下载的 URL / 目标路径 + python tools/fetch_source.py --source phet --dry-run + + # 实际下载到 .staging/，并回填 sha256/size/fetch_date/status=fetched + python tools/fetch_source.py --source phet --apply + python tools/fetch_source.py --source chinese-poetry --apply + python tools/fetch_source.py --subject physics --apply # 按学科批量 + python tools/fetch_source.py --all-auto --apply # 所有可自动化来源 + +下载产物落在 /.staging/{source_id}/...（已 gitignore），随后用 + python tools/oss_sync.py --raw-root .staging --apply +上传到 OSS 并对齐 oss_key。 + +注意：回填 YAML 走 yaml_patch（行级替换），**保留文件内注释**。 +""" +import os, sys, glob, hashlib, datetime, argparse, subprocess, shutil, tarfile, urllib.request + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from yaml_patch import patch_item_fields, patch_source_fields + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STAGING = os.path.join(REPO, ".staging") +CACHE = os.path.join(REPO, ".cache", "repos") +AUTO_METHODS = {"curl", "git-clone"} +UA = "edu-content-fetcher/1.0 (+https://git.aibeaver.cn/fengjunchen/edu-content)" + + +def sha256_file(p): + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def load_sources(): + import yaml + out = [] + for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)): + try: + doc = yaml.safe_load(open(path, encoding="utf-8")) + except Exception as e: + print(f"WARN: {path} 解析失败: {e}") + continue + for s in (doc or {}).get("sources", []) or []: + out.append((path, s)) + return out + + +def item_url(source, item): + if item.get("url"): + return item["url"] + tpl = source.get("url_template") + if tpl: + return tpl.replace("{item_id}", item["id"]) + return None + + +def fetch_curl(source, item, dest_dir, dry_run): + url = item_url(source, item) + if not url: + return None, "无 url / url_template，跳过" + fname = os.path.basename(item.get("oss_key") or "") or f"{item['id']}" + dest = os.path.join(dest_dir, fname) + if dry_run: + return dest, f"[dry-run] {url} -> {dest}" + os.makedirs(dest_dir, exist_ok=True) + req = urllib.request.Request(url, headers={"User-Agent": UA}) + try: + with urllib.request.urlopen(req, timeout=120) as r, open(dest, "wb") as f: + shutil.copyfileobj(r, f) + except Exception as e: + return None, f"下载失败: {type(e).__name__}: {e}" + return dest, f"ok {os.path.getsize(dest)}B" + + +def ensure_clone(source, dry_run): + """浅克隆到 .cache/repos/{id}，返回本地路径。""" + url = source.get("url") + if not url: + return None, "source.url 为空，无法 clone" + local = os.path.join(CACHE, source["id"]) + if os.path.isdir(os.path.join(local, ".git")): + return local, "复用已有克隆" + if dry_run: + return local, f"[dry-run] git clone --depth 1 {url} {local}" + os.makedirs(os.path.dirname(local), exist_ok=True) + r = subprocess.run(["git", "clone", "--depth", "1", url, local], + capture_output=True, text=True, timeout=1800) + if r.returncode != 0: + return None, f"clone 失败: {(r.stderr or '').strip()[:200]}" + return local, "克隆完成" + + +def fetch_git_item(source, item, repo_dir, dest_dir, dry_run): + """按 item.src_path 从克隆仓库取内容。 + + 目录 → 打包 .tar.gz；单文件 → 直接复制（除非 oss_key 显式要 .tar.gz）。 + 否则会产出「名叫 KET.json 其实是 tar.gz」的坑数据。 + """ + rel = item.get("src_path") + if not rel: + return None, "缺 src_path，跳过" + src = os.path.join(repo_dir, rel) + fname = os.path.basename(item.get("oss_key") or f"{item['id']}.tar.gz") + dest = os.path.join(dest_dir, fname) + want_tar = fname.endswith((".tar.gz", ".tgz")) + if dry_run: + action = "tar" if want_tar else "copy" + return dest, f"[dry-run] {action} {rel} -> {dest}" + if not os.path.exists(src): + return None, f"仓库内路径不存在: {rel}" + if os.path.isdir(src) and not want_tar: + return None, f"src_path 是目录但 oss_key 不是 .tar.gz（{fname}），请修正 oss_key" + os.makedirs(dest_dir, exist_ok=True) + if want_tar: + with tarfile.open(dest, "w:gz") as tf: + tf.add(src, arcname=rel) + else: + shutil.copy2(src, dest) + return dest, f"ok {os.path.getsize(dest)}B" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--source", help="按 source id 抓取") + ap.add_argument("--subject", help="按学科代码抓取（chinese/physics/science/...）") + ap.add_argument("--all-auto", action="store_true", help="所有 fetch_method 可自动化的来源") + ap.add_argument("--list", action="store_true", help="只列出可自动抓取的来源") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--apply", action="store_true") + ap.add_argument("--limit", type=int, default=0, help="每个 source 最多抓 N 个条目（调试用）") + args = ap.parse_args() + + all_sources = load_sources() + + if args.list: + print(f"{'source':22} {'method':12} {'items':>5} subjects") + print("-" * 70) + for path, s in all_sources: + mark = "" if s.get("fetch_method") in AUTO_METHODS else " (需人工)" + print(f"{s['id']:22} {str(s.get('fetch_method')):12} " + f"{len(s.get('items') or []):5} {','.join(s.get('subjects') or [])}{mark}") + return + + # 选择目标 + targets = [] + for path, s in all_sources: + if s.get("fetch_method") not in AUTO_METHODS: + continue + if args.source and s["id"] != args.source: + continue + if args.subject and args.subject not in (s.get("subjects") or []): + continue + if not (args.source or args.subject or args.all_auto): + continue + targets.append((path, s)) + + if not targets: + print("没有匹配的可自动抓取来源。用 --list 查看，或检查 --source/--subject 拼写。") + sys.exit(0 if (args.source or args.subject or args.all_auto) else 2) + + dry = args.dry_run or not args.apply + today = datetime.date.today().isoformat() + total_ok = total_fail = 0 + + for path, s in targets: + sid = s["id"] + method = s["fetch_method"] + items = s.get("items") or [] + if args.limit: + items = items[:args.limit] + print(f"\n=== {sid} ({method}) — {len(items)} 个条目 ===") + if not items: + print(" items 为空，无可抓取内容（该来源尚未登记具体条目）") + continue + + dest_dir = os.path.join(STAGING, sid) + repo_dir = None + if method == "git-clone": + repo_dir, msg = ensure_clone(s, dry) + print(f" clone: {msg}") + if repo_dir is None: + total_fail += len(items) + continue + + ok_here = 0 + for it in items: + if method == "curl": + dest, msg = fetch_curl(s, it, dest_dir, dry) + else: + dest, msg = fetch_git_item(s, it, repo_dir, dest_dir, dry) + print(f" {it['id']:34} {msg}") + if dest is None: + total_fail += 1 + continue + total_ok += 1 + ok_here += 1 + if not dry and os.path.isfile(dest): + patch_item_fields(path, sid, it["id"], { + "sha256": sha256_file(dest), + "size": os.path.getsize(dest), + "fetch_date": today, + "status": "fetched", + }) + if not dry and ok_here: + patch_source_fields(path, sid, {"status": "fetched", "fetched_at": today}) + + print(f"\n合计：成功 {total_ok} | 失败/跳过 {total_fail}") + if dry: + print("[dry-run] 未实际下载。加 --apply 执行。") + else: + print(f"产物在 {STAGING}/，已回填 sha256/size 到 sources YAML（注释保留）。") + print("下一步上传 OSS： python tools/oss_sync.py --raw-root .staging --apply") + + +if __name__ == "__main__": + main() diff --git a/tools/index_build.py b/tools/index_build.py index a2ab956..036a354 100644 --- a/tools/index_build.py +++ b/tools/index_build.py @@ -1,58 +1,185 @@ #!/usr/bin/env python3 """index_build.py — 生成 INDEX.json（机器可读库全貌）。 -汇总：资源域、各数据集记录数、license 分布、OSS 前缀、红线摘要。 +汇总：资源域、学科覆盖矩阵、抓取进度、各数据集记录数、license 分布、OSS 前缀、红线摘要。 注意：本文件由工具生成，勿手改；CI 在 lint 通过后自动重跑。 """ import os, glob, json, datetime, collections REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -def source_summary(): +# 与 sources/source.schema.json 的 subjects 枚举保持一致 +SUBJECTS = [ + "chinese", "math", "english", "science", "morality", + "physics", "chemistry", "biology", "history", "geography", "it", + "ket", "pet", "fce", "amc8", "cspj", +] +STAGES = ["primary", "junior"] +# 可全自动抓取的 fetch_method（fetch_source.py 支持） +AUTO_METHODS = {"curl", "git-clone", "api"} +PRODUCT_SAFE = {"official-free", "mit", "cc-by", "cc-by-sa", "llm-generated"} + + +def load_sources(): + """返回 [(domain, source_dict, yaml_rel_path)]""" import yaml - domains, n_sources, lic = {}, 0, collections.Counter() - for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True): + out = [] + for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)): rel = os.path.relpath(path, os.path.join(REPO, "sources")) - domain = os.path.dirname(rel) - try: doc = yaml.safe_load(open(path, encoding="utf-8")) - except Exception: continue + domain = os.path.dirname(rel) or "." + try: + doc = yaml.safe_load(open(path, encoding="utf-8")) + except Exception: + continue for s in (doc or {}).get("sources", []) or []: - n_sources += 1 - lic[s.get("license")] += 1 - domains.setdefault(domain, 0) - domains[domain] += 1 - return {"domains": domains, "n_sources": n_sources, "license_breakdown": dict(lic)} + out.append((domain, s, rel)) + return out + + +def source_summary(rows): + domains, lic = collections.Counter(), collections.Counter() + for domain, s, _ in rows: + domains[domain] += 1 + lic[s.get("license")] += 1 + return { + "domains": dict(domains), + "n_sources": len(rows), + "license_breakdown": dict(lic), + } + + +def fetch_progress(rows): + """条目级抓取进度：planned / fetched / uploaded。""" + st = collections.Counter() + n_items = 0 + auto, manual = 0, 0 + for _, s, _ in rows: + if s.get("fetch_method") in AUTO_METHODS: + auto += 1 + else: + manual += 1 + for it in s.get("items", []) or []: + n_items += 1 + st[it.get("status") or "planned"] += 1 + return { + "n_items": n_items, + "by_status": dict(st), + "auto_fetchable_sources": auto, + "manual_sources": manual, + } + + +def subject_coverage(rows): + """学科 → 资源覆盖矩阵。每个学科给出源清单、学段、可自动抓取数、条目进度。""" + cov = {} + for subj in SUBJECTS: + cov[subj] = { + "n_sources": 0, + "stages": [], + "auto_fetchable": 0, + "n_items": 0, + "n_fetched": 0, + "product_safe_sources": 0, + "sources": [], + } + for domain, s, rel in rows: + subs = s.get("subjects") or [] + auto = s.get("fetch_method") in AUTO_METHODS + items = s.get("items", []) or [] + n_fetched = sum(1 for it in items if it.get("status") in ("fetched", "uploaded")) + safe = s.get("license") in PRODUCT_SAFE + entry = { + "id": s.get("id"), + "title": s.get("title"), + "license": s.get("license"), + "fetch_method": s.get("fetch_method"), + "auto_fetchable": auto, + "product_safe": safe, + "stages": s.get("stages") or [], + "n_items": len(items), + "n_fetched": n_fetched, + "yaml": rel, + } + for subj in subs: + if subj not in cov: + cov[subj] = {"n_sources": 0, "stages": [], "auto_fetchable": 0, + "n_items": 0, "n_fetched": 0, "product_safe_sources": 0, "sources": []} + c = cov[subj] + c["n_sources"] += 1 + c["auto_fetchable"] += 1 if auto else 0 + c["n_items"] += len(items) + c["n_fetched"] += n_fetched + c["product_safe_sources"] += 1 if safe else 0 + c["sources"].append(entry) + for st in s.get("stages") or []: + if st not in c["stages"]: + c["stages"].append(st) + for c in cov.values(): + c["stages"] = [x for x in STAGES if x in c["stages"]] + return cov + + +def gaps(cov): + """缺口清单：完全无源 / 有源但一条都没抓下来 / 有源但无 product_safe。""" + no_source = sorted([k for k, v in cov.items() if v["n_sources"] == 0]) + no_data = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["n_fetched"] == 0]) + no_safe = sorted([k for k, v in cov.items() if v["n_sources"] > 0 and v["product_safe_sources"] == 0]) + return { + "subjects_without_source": no_source, + "subjects_without_fetched_data": no_data, + "subjects_without_product_safe_source": no_safe, + } + def structured_summary(): datasets, lic = {}, collections.Counter() - for path in glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True): + for path in sorted(glob.glob(os.path.join(REPO, "structured", "**", "*.jsonl"), recursive=True)): rel = os.path.relpath(path, os.path.join(REPO, "structured")) cnt = 0 for line in open(path, encoding="utf-8"): line = line.strip() - if not line: continue + if not line: + continue cnt += 1 - try: r = json.loads(line) - except Exception: continue + try: + r = json.loads(line) + except Exception: + continue l = r.get("license") - if l: lic[l] += 1 + if l: + lic[l] += 1 datasets[rel] = cnt return {"datasets": datasets, "n_records": sum(datasets.values()), "license_breakdown": dict(lic)} + def main(): + rows = load_sources() + cov = subject_coverage(rows) idx = { "generated_at": datetime.date.today().isoformat(), "repo": "edu-content", "model": "L1 source registry (git) + L2 raw (OSS edu-raw/) + L3 structured (git)", "oss": {"bucket": "edu-content-hy", "region": "oss-cn-heyuan", "prefixes": ["edu-raw/", "edu-derived/"]}, "redline": "restricted / internal-only / unknown 数据永不可进 derived，永不可对外产品使用", - "sources": source_summary(), + "product_safe_licenses": sorted(PRODUCT_SAFE), + "sources": source_summary(rows), + "fetch_progress": fetch_progress(rows), + "subject_coverage": cov, + "gaps": gaps(cov), "structured": structured_summary(), "entrypoints": {"human": "README.md", "machine": "INDEX.json", "license": "LICENSES.md", "conventions": "docs/conventions.md"}, } out = os.path.join(REPO, "INDEX.json") json.dump(idx, open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=2) - print(f"已生成 {out}：{idx['sources']['n_sources']} source，{idx['structured']['n_records']} 结构化记录") + fp = idx["fetch_progress"] + g = idx["gaps"] + print(f"已生成 {out}") + print(f" source {idx['sources']['n_sources']} 条 | 条目 {fp['n_items']} 个 | 状态 {fp['by_status']}") + print(f" 可自动抓取源 {fp['auto_fetchable_sources']} / 需人工 {fp['manual_sources']}") + print(f" 无源学科：{', '.join(g['subjects_without_source']) or '（无）'}") + print(f" 有源但零数据：{', '.join(g['subjects_without_fetched_data']) or '（无）'}") + print(f" 结构化记录 {idx['structured']['n_records']} 条") + if __name__ == "__main__": main() diff --git a/tools/lint_sources.py b/tools/lint_sources.py index 95450eb..ca7089b 100644 --- a/tools/lint_sources.py +++ b/tools/lint_sources.py @@ -2,7 +2,7 @@ """lint_sources.py — 校验 sources/ 资源清单合规。 检查项： -- 每个 source 必填字段、id kebab-case 唯一、license/type/status 枚举合法 +- 每个 source 必填字段、id kebab-case 唯一、license/type/status/subjects 枚举合法 - status=fetched 的 item 必须有 oss_key + 64位 sha256 - license=unknown 且 fetched_at 距今 > 30 天 → warning @@ -11,9 +11,12 @@ import sys, os, glob, re, datetime, hashlib REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"} +LICENSES = {"official-free","mit","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"} TYPES = {"dataset","website","book","internal","tool"} STATUSES = {"planned","fetching","fetched","failed","archived"} +SUBJECTS = {"chinese","math","english","science","morality","physics","chemistry", + "biology","history","geography","it","ket","pet","fce","amc8","cspj"} +STAGES = {"primary","junior"} ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") SHA_RE = re.compile(r"^[a-f0-9]{64}$") @@ -42,6 +45,13 @@ def main(): if s.get("type") not in TYPES: err(f"{sid}: type 非法 {s.get('type')}"); errors += 1 if s.get("license") not in LICENSES: err(f"{sid}: license 非法 {s.get('license')}"); errors += 1 if s.get("status") not in STATUSES: err(f"{sid}: status 非法 {s.get('status')}"); errors += 1 + subs = s.get("subjects") + if not subs: err(f"{sid}: 缺必填 subjects（覆盖学科，见 LICENSES.md 同级约定）"); errors += 1 + else: + for x in subs: + if x not in SUBJECTS: err(f"{sid}: subjects 含非法值 {x}"); errors += 1 + for x in (s.get("stages") or []): + if x not in STAGES: err(f"{sid}: stages 含非法值 {x}"); errors += 1 if s.get("license") == "unknown" and s.get("fetched_at"): try: d = datetime.date.fromisoformat(s["fetched_at"]) diff --git a/tools/lint_structured.py b/tools/lint_structured.py index 79258af..79b96ce 100644 --- a/tools/lint_structured.py +++ b/tools/lint_structured.py @@ -12,7 +12,7 @@ import sys, os, glob, json REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -LICENSES = {"official-free","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"} +LICENSES = {"official-free","mit","cc-by","cc-by-sa","cc-nc-education","restricted","internal-only","llm-generated","unknown"} def load_source_ids(): import yaml diff --git a/tools/oss_sync.py b/tools/oss_sync.py index 05ac3e0..c467961 100644 --- a/tools/oss_sync.py +++ b/tools/oss_sync.py @@ -16,6 +16,9 @@ """ import os, sys, glob, json, hashlib, datetime, argparse, yaml +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from yaml_patch import patch_item_fields, patch_source_fields + REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SHA_RE = __import__("re").compile(r"^[a-f0-9]{64}$") @@ -87,19 +90,16 @@ def main(): for path, sid, iid, local, oss_key, sha, size in pending: b.put_object_from_file(oss_key, local) print(f" uploaded {oss_key}") - # 回填 - doc = yaml.safe_load(open(path, encoding="utf-8")) - for s in doc["sources"]: - if s.get("id") == sid: - for it in s.get("items", []): - if it.get("id") == iid: - it["oss_key"] = oss_key - it["sha256"] = sha - it["size"] = size - it["fetch_date"] = today - it["status"] = "fetched" - yaml.safe_dump(doc, open(path, "w", encoding="utf-8"), allow_unicode=True, sort_keys=False) - print("已回填 sha256/size 到 sources YAML。请 git commit。") + # 回填（行级替换，保留 YAML 注释；勿用 safe_dump 整file重写） + patch_item_fields(path, sid, iid, { + "oss_key": oss_key, + "sha256": sha, + "size": size, + "fetch_date": today, + "status": "fetched", + }) + patch_source_fields(path, sid, {"status": "fetched", "fetched_at": today}) + print("已回填 oss_key/sha256/size 到 sources YAML（注释保留）。请 git commit。") if __name__ == "__main__": main() diff --git a/tools/stage_local.py b/tools/stage_local.py new file mode 100644 index 0000000..f2129ca --- /dev/null +++ b/tools/stage_local.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""stage_local.py — 把「已在本机的原始文件」归位到 .staging 并回填 L1 元数据。 + +适用场景：fetch_method 为 manual-download / manual-export 的来源（官方网站需人工点下载、 +需登录、或历史上已经下载过），文件已存在本地磁盘，但 sources/*.yaml 的 item 还挂在 +status: planned、sha256/size 为空。 + +设计原则：**本机绝对路径只在命令行出现，绝不写进仓库**（换机器就失效，属脏数据）。 +仓库里只保留 oss_key / sha256 / size / status —— 与机器无关的事实。 + +用法： + # 单个文件 + python tools/stage_local.py --source cambridge-official --item a2-key-vocab \ + --file "/abs/path/a2-key-vocabulary-list.pdf" + + # 批量：从 TSV 读（sourceitemabs_path），文件本身放仓库外或 .staging 里 + python tools/stage_local.py --manifest /tmp/stage.tsv --apply + + # 默认 dry-run，加 --apply 才真正复制 + 回填 +""" +import os, sys, glob, shutil, hashlib, datetime, argparse + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from yaml_patch import patch_item_fields, patch_source_fields + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STAGING = os.path.join(REPO, ".staging") + + +def sha256_file(p): + h = hashlib.sha256() + with open(p, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def load_index(): + """返回 {(source_id, item_id): (yaml_path, source_dict, item_dict)}""" + import yaml + idx = {} + for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)): + try: + doc = yaml.safe_load(open(path, encoding="utf-8")) + except Exception: + continue + for s in (doc or {}).get("sources", []) or []: + for it in s.get("items", []) or []: + idx[(s["id"], it["id"])] = (path, s, it) + return idx + + +def stage_one(idx, sid, iid, src_file, apply_): + key = (sid, iid) + if key not in idx: + return False, f"sources 里找不到 {sid}/{iid}（先在 YAML 登记 item）" + if not os.path.isfile(src_file): + return False, f"本地文件不存在: {src_file}" + path, s, it = idx[key] + oss_key = it.get("oss_key") + if not oss_key: + return False, f"{sid}/{iid} 缺 oss_key，无法确定归位文件名" + fname = os.path.basename(oss_key.rstrip("/")) + if not fname or oss_key.endswith("/"): + return False, f"{sid}/{iid} 的 oss_key 是目录（{oss_key}），需改成具体文件名" + dest_dir = os.path.join(STAGING, sid) + dest = os.path.join(dest_dir, fname) + size = os.path.getsize(src_file) + if not apply_: + return True, f"[dry-run] {src_file} -> {os.path.relpath(dest, REPO)} ({size}B)" + os.makedirs(dest_dir, exist_ok=True) + if os.path.abspath(src_file) != os.path.abspath(dest): + shutil.copy2(src_file, dest) + sha = sha256_file(dest) + patch_item_fields(path, sid, iid, { + "sha256": sha, + "size": os.path.getsize(dest), + "fetch_date": datetime.date.today().isoformat(), + "status": "fetched", + }) + patch_source_fields(path, sid, {"status": "fetched", + "fetched_at": datetime.date.today().isoformat()}) + return True, f"ok {size}B sha256={sha[:12]}..." + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--source") + ap.add_argument("--item") + ap.add_argument("--file") + ap.add_argument("--manifest", help="TSV: sourceitemabs_path，# 开头为注释") + ap.add_argument("--apply", action="store_true") + args = ap.parse_args() + + jobs = [] + if args.manifest: + for ln in open(args.manifest, encoding="utf-8-sig"): # utf-8-sig 吃掉 BOM + ln = ln.lstrip("\ufeff").strip() + if not ln or ln.startswith("#"): + continue + parts = ln.split("\t") + if len(parts) < 3: + print(f"WARN 跳过格式错误行: {ln[:80]}") + continue + jobs.append((parts[0].strip(), parts[1].strip(), parts[2].strip())) + if args.source and args.item and args.file: + jobs.append((args.source, args.item, args.file)) + if not jobs: + ap.error("需要 --source/--item/--file 或 --manifest") + + idx = load_index() + ok = fail = 0 + for sid, iid, f in jobs: + good, msg = stage_one(idx, sid, iid, f, args.apply) + print(f" {sid}/{iid:28} {msg}") + ok += 1 if good else 0 + fail += 0 if good else 1 + print(f"\n合计：成功 {ok} | 失败 {fail}") + if not args.apply: + print("[dry-run] 未复制。加 --apply 执行。") + else: + print("已回填 sha256/size/status 到 sources YAML（注释保留）。") + + +if __name__ == "__main__": + main() diff --git a/tools/yaml_patch.py b/tools/yaml_patch.py new file mode 100644 index 0000000..fab8924 --- /dev/null +++ b/tools/yaml_patch.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""yaml_patch.py — 对 sources/*.yaml 做**行级**字段回填，保留注释与原有格式。 + +为什么需要它： +`yaml.safe_load` + `yaml.safe_dump` 会把整个文件重写，**注释全部丢失**。 +sources/*.yaml 里的注释承载了关键信息（下载模板、许可解释、未纳入原因、运维提示）， +必须保住。因此所有自动回填（fetch_source.py / oss_sync.py）统一走本模块的行级替换。 + +约定的缩进层级（与 sources/*.yaml 一致）： + sources: + - id: # 2 空格 + "- " + status: planned # 4 空格 + items: + - id: # 6 空格 + "- " + oss_key: ... # 8 空格 +""" +import re + + +def _yaml_scalar(v): + """把 Python 值渲染成 YAML 标量。""" + if v is None: + return "null" + if isinstance(v, bool): + return "true" if v else "false" + if isinstance(v, (int, float)): + return str(v) + s = str(v) + # 需要引号的情况：空串、含特殊起始字符、含冒号空格、纯数字样式 + if s == "" or s[0] in "&*?|-<>=!%@`{[\"'" or ": " in s or s.endswith(":"): + return '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"' + return s + + +def patch_item_fields(path, source_id, item_id, fields): + """回填某 source 下某 item 的字段。返回 True 表示有改动。 + + fields: dict，如 {"sha256": "ab..", "size": 123, "status": "fetched"} + 已存在的键 → 原地替换；不存在的键 → 追加到该 item 块末尾。 + """ + with open(path, encoding="utf-8") as f: + lines = f.read().split("\n") + + src_re = re.compile(r"^ - id: (\S+)\s*$") + item_re = re.compile(r"^ - id: (\S+)\s*$") + + # 1) 定位 source 块范围 + s_start = s_end = None + for i, ln in enumerate(lines): + m = src_re.match(ln) + if m: + if m.group(1) == source_id and s_start is None: + s_start = i + elif s_start is not None and s_end is None: + s_end = i + break + if s_start is None: + return False + if s_end is None: + s_end = len(lines) + + # 2) 在 source 块内定位 item 块范围 + i_start = i_end = None + for i in range(s_start, s_end): + m = item_re.match(lines[i]) + if m: + if m.group(1) == item_id and i_start is None: + i_start = i + elif i_start is not None and i_end is None: + i_end = i + break + if i_start is None: + return False + if i_end is None: + # item 块延伸到 source 块末尾；剔除尾部空行与注释行 + j = s_end - 1 + while j > i_start and (lines[j].strip() == "" or lines[j].lstrip().startswith("#")): + j -= 1 + i_end = j + 1 + + # 3) 逐字段替换或追加 + changed = False + for key, val in fields.items(): + new_line = f" {key}: {_yaml_scalar(val)}" + hit = None + for i in range(i_start, i_end): + if re.match(rf"^ {re.escape(key)}:", lines[i]): + hit = i + break + if hit is not None: + if lines[hit] != new_line: + lines[hit] = new_line + changed = True + else: + lines.insert(i_end, new_line) + i_end += 1 + changed = True + + if changed: + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + return changed + + +def patch_source_fields(path, source_id, fields): + """回填 source 级字段（如 status / fetched_at）。返回 True 表示有改动。""" + with open(path, encoding="utf-8") as f: + lines = f.read().split("\n") + + src_re = re.compile(r"^ - id: (\S+)\s*$") + s_start = s_end = None + for i, ln in enumerate(lines): + m = src_re.match(ln) + if m: + if m.group(1) == source_id and s_start is None: + s_start = i + elif s_start is not None and s_end is None: + s_end = i + break + if s_start is None: + return False + if s_end is None: + s_end = len(lines) + + # items: 之前才是 source 级字段区 + limit = s_end + for i in range(s_start, s_end): + if re.match(r"^ items:", lines[i]): + limit = i + break + + changed = False + for key, val in fields.items(): + new_line = f" {key}: {_yaml_scalar(val)}" + hit = None + for i in range(s_start, limit): + if re.match(rf"^ {re.escape(key)}:", lines[i]): + hit = i + break + if hit is not None: + if lines[hit] != new_line: + lines[hit] = new_line + changed = True + else: + lines.insert(limit, new_line) + limit += 1 + changed = True + + if changed: + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + return changed