edu-content/tools/oss_sync.py

#!/usr/bin/env python3
"""oss_sync.py — 上传 L2 原始文件到 OSS 并回填 sha256/size/fetch_date。

用法：
  python tools/oss_sync.py --raw-root <DIR> --dry-run      # 仅报告将要上传的文件
  EDU_OSS_KEY_ID=... EDU_OSS_KEY_SECRET=... \\
  python tools/oss_sync.py --raw-root <DIR> --apply        # 实际上传 + 回填 YAML

约定：
- 本地 DIR 结构：<raw-root>/{source-id}/{filename...}，与 OSS key 的
  edu-raw/{k9|exams|internal}/{学科或考试}/{类目}/{source-id}/{filename} 对应。
- 上传后自动把 oss_key / sha256 / size / fetch_date 写回对应 source YAML 的 item。

凭据：从环境变量 EDU_OSS_KEY_ID / EDU_OSS_KEY_SECRET / EDU_OSS_BUCKET / EDU_OSS_ENDPOINT 读取，
      绝不写死、不进 git。缺省 bucket=edu-content-hy，endpoint=https://oss-cn-heyuan.aliyuncs.com。
"""
import os, sys, glob, json, hashlib, datetime, argparse, yaml

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from yaml_patch import patch_item_fields, patch_source_fields

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SHA_RE = __import__("re").compile(r"^[a-f0-9]{64}$")

def sha256_file(p):
    h = hashlib.sha256()
    with open(p, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()

def load_sources():
    import yaml
    docs = []
    for path in glob.glob(os.path.join(REPO, "sources", "**", "*.yaml"), recursive=True):
        try: doc = yaml.safe_load(open(path, encoding="utf-8"))
        except Exception: continue
        docs.append((path, doc))
    return docs

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--raw-root", required=True)
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--apply", action="store_true")
    args = ap.parse_args()
    if not os.path.isdir(args.raw_root):
        print("raw-root 不存在:", args.raw_root); sys.exit(2)

    docs = load_sources()
    pending = []
    for path, doc in docs:
        for s in (doc or {}).get("sources", []) or []:
            sid = s.get("id")
            for it in s.get("items", []) or []:
                # 本地文件：raw-root/{sid}/{filename}
                local = os.path.join(args.raw_root, sid, os.path.basename(it.get("oss_key") or it.get("title") or it.get("id")))
                if os.path.isfile(local):
                    sha = sha256_file(local)
                    oss_key = it.get("oss_key") or f"edu-raw/exams/_staging/{sid}/{os.path.basename(local)}"
                    pending.append((path, sid, it.get("id"), local, oss_key, sha, os.path.getsize(local)))

    if not pending:
        print("无可上传文件（raw-root 下未匹配到 source item 对应的本地文件）。")
        sys.exit(0)

    print(f"待上传 {len(pending)} 个文件：")
    for p, sid, iid, local, ok, sha, size in pending:
        print(f"  {sid}/{iid}: {local} -> {ok} ({size}B, sha256={sha[:12]}...)")

    if args.dry_run or not args.apply:
        print("\n[dry-run] 未实际上传。加 --apply 执行。")
        sys.exit(0)

    # 实际上传
    try:
        import oss2
    except Exception as e:
        print("oss2 不可用，无法上传:", e); sys.exit(3)
    key_id = os.environ.get("EDU_OSS_KEY_ID")
    key_secret = os.environ.get("EDU_OSS_KEY_SECRET")
    bucket = os.environ.get("EDU_OSS_BUCKET", "edu-content-hy")
    endpoint = os.environ.get("EDU_OSS_ENDPOINT", "https://oss-cn-heyuan.aliyuncs.com")
    if not (key_id and key_secret):
        print("缺少 EDU_OSS_KEY_ID / EDU_OSS_KEY_SECRET 环境变量"); sys.exit(3)
    auth = oss2.Auth(key_id, key_secret)
    b = oss2.Bucket(auth, endpoint, bucket)
    today = datetime.date.today().isoformat()
    # 上传 + 回填 YAML
    for path, sid, iid, local, oss_key, sha, size in pending:
        b.put_object_from_file(oss_key, local)
        print(f"  uploaded {oss_key}")
        # 回填（行级替换，保留 YAML 注释；勿用 safe_dump 整file重写）
        patch_item_fields(path, sid, iid, {
            "oss_key": oss_key,
            "sha256": sha,
            "size": size,
            "fetch_date": today,
            "status": "fetched",
        })
        patch_source_fields(path, sid, {"status": "fetched", "fetched_at": today})
    print("已回填 oss_key/sha256/size 到 sources YAML（注释保留）。请 git commit。")

if __name__ == "__main__":
    main()