Drjunchenfeng
a006c8c504
一些检查失败了
ci / lint (push) Failing after 52s
登记 chinese-poetry / ck12 / phet，并让 source schema、lint 与 INDEX 能按学科和抓取进度消费。 Co-authored-by: Cursor <cursoragent@cursor.com>
127 行
4.8 KiB
Python
127 行
4.8 KiB
Python
#!/usr/bin/env python3
|
||
"""stage_local.py — 把「已在本机的原始文件」归位到 .staging 并回填 L1 元数据。
|
||
|
||
适用场景：fetch_method 为 manual-download / manual-export 的来源（官方网站需人工点下载、
|
||
需登录、或历史上已经下载过），文件已存在本地磁盘，但 sources/*.yaml 的 item 还挂在
|
||
status: planned、sha256/size 为空。
|
||
|
||
设计原则：**本机绝对路径只在命令行出现，绝不写进仓库**（换机器就失效，属脏数据）。
|
||
仓库里只保留 oss_key / sha256 / size / status —— 与机器无关的事实。
|
||
|
||
用法：
|
||
# 单个文件
|
||
python tools/stage_local.py --source cambridge-official --item a2-key-vocab \
|
||
--file "/abs/path/a2-key-vocabulary-list.pdf"
|
||
|
||
# 批量：从 TSV 读（source<TAB>item<TAB>abs_path），文件本身放仓库外或 .staging 里
|
||
python tools/stage_local.py --manifest /tmp/stage.tsv --apply
|
||
|
||
# 默认 dry-run，加 --apply 才真正复制 + 回填
|
||
"""
|
||
import os, sys, glob, shutil, hashlib, datetime, argparse
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from yaml_patch import patch_item_fields, patch_source_fields
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
STAGING = os.path.join(REPO, ".staging")
|
||
|
||
|
||
def sha256_file(p):
|
||
h = hashlib.sha256()
|
||
with open(p, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def load_index():
|
||
"""返回 {(source_id, item_id): (yaml_path, source_dict, item_dict)}"""
|
||
import yaml
|
||
idx = {}
|
||
for path in sorted(glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True)):
|
||
try:
|
||
doc = yaml.safe_load(open(path, encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
for s in (doc or {}).get("sources", []) or []:
|
||
for it in s.get("items", []) or []:
|
||
idx[(s["id"], it["id"])] = (path, s, it)
|
||
return idx
|
||
|
||
|
||
def stage_one(idx, sid, iid, src_file, apply_):
|
||
key = (sid, iid)
|
||
if key not in idx:
|
||
return False, f"sources 里找不到 {sid}/{iid}（先在 YAML 登记 item）"
|
||
if not os.path.isfile(src_file):
|
||
return False, f"本地文件不存在: {src_file}"
|
||
path, s, it = idx[key]
|
||
oss_key = it.get("oss_key")
|
||
if not oss_key:
|
||
return False, f"{sid}/{iid} 缺 oss_key，无法确定归位文件名"
|
||
fname = os.path.basename(oss_key.rstrip("/"))
|
||
if not fname or oss_key.endswith("/"):
|
||
return False, f"{sid}/{iid} 的 oss_key 是目录（{oss_key}），需改成具体文件名"
|
||
dest_dir = os.path.join(STAGING, sid)
|
||
dest = os.path.join(dest_dir, fname)
|
||
size = os.path.getsize(src_file)
|
||
if not apply_:
|
||
return True, f"[dry-run] {src_file} -> {os.path.relpath(dest, REPO)} ({size}B)"
|
||
os.makedirs(dest_dir, exist_ok=True)
|
||
if os.path.abspath(src_file) != os.path.abspath(dest):
|
||
shutil.copy2(src_file, dest)
|
||
sha = sha256_file(dest)
|
||
patch_item_fields(path, sid, iid, {
|
||
"sha256": sha,
|
||
"size": os.path.getsize(dest),
|
||
"fetch_date": datetime.date.today().isoformat(),
|
||
"status": "fetched",
|
||
})
|
||
patch_source_fields(path, sid, {"status": "fetched",
|
||
"fetched_at": datetime.date.today().isoformat()})
|
||
return True, f"ok {size}B sha256={sha[:12]}..."
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--source")
|
||
ap.add_argument("--item")
|
||
ap.add_argument("--file")
|
||
ap.add_argument("--manifest", help="TSV: source<TAB>item<TAB>abs_path，# 开头为注释")
|
||
ap.add_argument("--apply", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
jobs = []
|
||
if args.manifest:
|
||
for ln in open(args.manifest, encoding="utf-8-sig"): # utf-8-sig 吃掉 BOM
|
||
ln = ln.lstrip("\ufeff").strip()
|
||
if not ln or ln.startswith("#"):
|
||
continue
|
||
parts = ln.split("\t")
|
||
if len(parts) < 3:
|
||
print(f"WARN 跳过格式错误行: {ln[:80]}")
|
||
continue
|
||
jobs.append((parts[0].strip(), parts[1].strip(), parts[2].strip()))
|
||
if args.source and args.item and args.file:
|
||
jobs.append((args.source, args.item, args.file))
|
||
if not jobs:
|
||
ap.error("需要 --source/--item/--file 或 --manifest")
|
||
|
||
idx = load_index()
|
||
ok = fail = 0
|
||
for sid, iid, f in jobs:
|
||
good, msg = stage_one(idx, sid, iid, f, args.apply)
|
||
print(f" {sid}/{iid:28} {msg}")
|
||
ok += 1 if good else 0
|
||
fail += 0 if good else 1
|
||
print(f"\n合计：成功 {ok} | 失败 {fail}")
|
||
if not args.apply:
|
||
print("[dry-run] 未复制。加 --apply 执行。")
|
||
else:
|
||
print("已回填 sha256/size/status 到 sources YAML（注释保留）。")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|