真实语料 11 个 / 10,697,067 B(STEP 6、DXF 3、DWG 1、3MF 1),来源 ViewDesign-deepseek/corpus(与 kimi、Assembly 仓交叉校验),按 sha256 前 12 位命名;3 个文件做同长度 PII 脱敏(几何零改动),逐资产来源/哈希/许可判定见 corpus/real_manifest.json 与 real_provenance.md。 manifest 工具扩展:gen_manifest.py 扫描 corpus/real 并做登记表双向复核(未登记文件/哈希不符 exit 1),verify_manifest.py --strict 覆盖 tests/data + corpus/real;实测 gen --check 与 verify --strict 均 exit 0(63 合成 + 11 真实)。 driver 增加真实语料识别(real_corpus_present/real_step_files),修复 Get-StepFileCount 扩展名过滤 bug(首跑误记 11 STEP → 6 STEP)。 真实语料实测(run-20261119-225558):B1 import 6/6、clean 0/12=0%、perturbed 0/24=0%(381→372s);B2 3/19(与合成一致,无真实脏模型);B5 单线程 3 轮 bit-identical 通过、2/4/8 线程 digest 不一致且 8 线程拓扑不同(282v/434e),8 线程加速比 0.94x;B6 4/4 bounded_pass(4×30s≈180s)。KPI 表与门限建议见 docs/planning/benchmarks-kcp-wave1.md §7.1 与 docs/planning/wave2/corpus-notes.md。
248 lines
11 KiB
Python
248 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""KCP-1.0 Wave 1 / W4:校验 corpus/manifest.json 与磁盘现状一致(防清单撒谎)。
|
||
|
||
检查项
|
||
------
|
||
1. manifest 中每个 present 资产:文件存在、大小一致、sha256 一致。
|
||
2. manifest 的 missing 段:标记 missing 的目录确实不存在(若意外存在则报错,
|
||
避免“清单说缺失、实际已入库”的维护漂移)。
|
||
3. tests/data 下出现但 manifest 未登记的文件(--strict 时视为错误)。
|
||
4. 真实语料(real_corpus 段)逐资产校验存在/大小/sha256,且 corpus/real/ 下
|
||
未登记文件在 --strict 时视为错误;real_corpus 与 corpus/real_manifest.json
|
||
登记表必须一致(防止只改一处)。
|
||
5. 旧清单格式/缺字段(--check 时提示重新生成)。
|
||
|
||
用法
|
||
----
|
||
python corpus/verify_manifest.py # 默认仓库根 = 脚本上级目录
|
||
python corpus/verify_manifest.py --json # 机器可读输出
|
||
python corpus/verify_manifest.py --strict # 未登记文件也视为失败
|
||
python corpus/verify_manifest.py --skip-hash # 大仓库/快速模式
|
||
退出码:0 = 通过;1 = 校验失败。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
REQUIRED_ASSET_FIELDS = [
|
||
"path",
|
||
"format",
|
||
"category",
|
||
"synthetic",
|
||
"present",
|
||
"size_bytes",
|
||
"sha256",
|
||
"source",
|
||
"license",
|
||
]
|
||
REAL_ROOT_REL = "corpus/real"
|
||
REAL_REGISTRY_REL = "corpus/real_manifest.json"
|
||
|
||
|
||
def sha256_of(path: Path) -> str:
|
||
h = hashlib.sha256()
|
||
with path.open("rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def verify(repo_root: Path, manifest_path: Path, skip_hash: bool, strict: bool) -> dict:
|
||
report: dict = {
|
||
"manifest": str(manifest_path),
|
||
"errors": [],
|
||
"warnings": [],
|
||
"checked_assets": 0,
|
||
"checked_bytes": 0,
|
||
"checked_real_assets": 0,
|
||
}
|
||
if not manifest_path.is_file():
|
||
report["errors"].append(f"manifest not found: {manifest_path}")
|
||
return report
|
||
try:
|
||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError as e:
|
||
report["errors"].append(f"manifest is not valid JSON: {e}")
|
||
return report
|
||
|
||
for key in ("schema_version", "counts", "assets"):
|
||
if key not in manifest:
|
||
report["errors"].append(f"manifest missing top-level field: {key}")
|
||
assets = manifest.get("assets")
|
||
if not isinstance(assets, list):
|
||
report["errors"].append("manifest.assets is not a list")
|
||
return report
|
||
|
||
listed: set[str] = set()
|
||
for asset in assets:
|
||
for field in REQUIRED_ASSET_FIELDS:
|
||
if field not in asset:
|
||
report["errors"].append(f"asset missing field '{field}': {asset.get('path')}")
|
||
rel = asset.get("path", "")
|
||
listed.add(rel)
|
||
path = repo_root / rel
|
||
if not path.is_file():
|
||
report["errors"].append(f"asset listed but missing on disk: {rel}")
|
||
continue
|
||
report["checked_assets"] += 1
|
||
size = path.stat().st_size
|
||
report["checked_bytes"] += size
|
||
if size != asset.get("size_bytes"):
|
||
report["errors"].append(
|
||
f"size mismatch: {rel} manifest={asset.get('size_bytes')} disk={size}"
|
||
)
|
||
if not skip_hash and asset.get("sha256"):
|
||
digest = sha256_of(path)
|
||
if digest != asset.get("sha256"):
|
||
report["errors"].append(f"sha256 mismatch: {rel}")
|
||
|
||
# missing 段反向校验
|
||
for entry in manifest.get("missing", []):
|
||
rel = entry.get("asset", "")
|
||
exists = (repo_root / rel).exists() if rel else False
|
||
if entry.get("status") == "missing" and exists:
|
||
report["errors"].append(
|
||
f"manifest claims missing but path exists: {rel} "
|
||
f"(if real corpus was added, update manifest real_corpus section)"
|
||
)
|
||
if entry.get("status") == "present_unexpected":
|
||
report["warnings"].append(f"path exists but listed as unexpected: {rel}")
|
||
|
||
# 真实语料(real_corpus 段)校验:逐资产 + counts + 登记表一致性 + 未登记文件
|
||
real = manifest.get("real_corpus")
|
||
real_listed: set[str] = set()
|
||
if isinstance(real, dict) and real.get("status") == "present":
|
||
real_assets = real.get("assets")
|
||
if not isinstance(real_assets, list):
|
||
report["errors"].append("real_corpus.assets is not a list")
|
||
real_assets = []
|
||
for asset in real_assets:
|
||
rel = asset.get("path", "")
|
||
if not rel:
|
||
report["errors"].append("real_corpus asset missing path")
|
||
continue
|
||
real_listed.add(rel)
|
||
path = repo_root / rel
|
||
if not path.is_file():
|
||
report["errors"].append(f"real asset listed but missing on disk: {rel}")
|
||
continue
|
||
report["checked_real_assets"] = report.get("checked_real_assets", 0) + 1
|
||
size = path.stat().st_size
|
||
if size != asset.get("size_bytes"):
|
||
report["errors"].append(
|
||
f"real size mismatch: {rel} manifest={asset.get('size_bytes')} disk={size}"
|
||
)
|
||
if not skip_hash and asset.get("sha256"):
|
||
if sha256_of(path) != asset.get("sha256"):
|
||
report["errors"].append(f"real sha256 mismatch: {rel}")
|
||
rc_counts = real.get("counts", {})
|
||
if isinstance(rc_counts, dict):
|
||
if rc_counts.get("assets") != len(real_assets):
|
||
report["errors"].append(
|
||
f"real_corpus.counts.assets={rc_counts.get('assets')} != assets[]={len(real_assets)}"
|
||
)
|
||
real_bytes = sum(a.get("size_bytes", 0) for a in real_assets)
|
||
if rc_counts.get("bytes") != real_bytes:
|
||
report["errors"].append(
|
||
f"real_corpus.counts.bytes={rc_counts.get('bytes')} != sum={real_bytes}"
|
||
)
|
||
# 与来源登记表 corpus/real_manifest.json 双向一致
|
||
registry_path = repo_root / REAL_REGISTRY_REL
|
||
if not registry_path.is_file():
|
||
report["errors"].append(
|
||
f"real_corpus present but registry missing: {REAL_REGISTRY_REL}"
|
||
)
|
||
else:
|
||
try:
|
||
registry = json.loads(registry_path.read_text(encoding="utf-8"))
|
||
reg_assets = {
|
||
a.get("path"): a for a in registry.get("assets", []) if isinstance(a, dict)
|
||
}
|
||
man_assets = {a.get("path"): a for a in real_assets if isinstance(a, dict)}
|
||
for rel in sorted(set(reg_assets) - set(man_assets)):
|
||
report["errors"].append(f"registry asset not in manifest.real_corpus: {rel}")
|
||
for rel in sorted(set(man_assets) - set(reg_assets)):
|
||
report["errors"].append(f"manifest.real_corpus asset not in registry: {rel}")
|
||
for rel in sorted(set(reg_assets) & set(man_assets)):
|
||
if reg_assets[rel].get("sha256") != man_assets[rel].get("sha256"):
|
||
report["errors"].append(f"registry/manifest sha256 differ: {rel}")
|
||
if reg_assets[rel].get("size_bytes") != man_assets[rel].get("size_bytes"):
|
||
report["errors"].append(f"registry/manifest size differ: {rel}")
|
||
except json.JSONDecodeError as e:
|
||
report["errors"].append(f"registry is not valid JSON: {e}")
|
||
# corpus/real 下磁盘有而清单未登记(--strict 视为错误)
|
||
real_root = repo_root / REAL_ROOT_REL
|
||
if real_root.is_dir():
|
||
for path in real_root.rglob("*"):
|
||
if not path.is_file():
|
||
continue
|
||
rel = path.relative_to(repo_root).as_posix()
|
||
if rel not in real_listed:
|
||
msg = f"real file on disk but not in manifest: {rel}"
|
||
(report["errors"] if strict else report["warnings"]).append(msg)
|
||
elif isinstance(real, dict) and real.get("status") == "missing":
|
||
if (repo_root / REAL_ROOT_REL).exists():
|
||
report["errors"].append(
|
||
f"manifest says {REAL_ROOT_REL} missing but path exists"
|
||
" (update manifest real_corpus section)"
|
||
)
|
||
|
||
# 未登记文件检查(tests/data 递归)
|
||
data_root = repo_root / manifest.get("repo_data_root", "tests/data")
|
||
if data_root.is_dir():
|
||
for path in data_root.rglob("*"):
|
||
if not path.is_file():
|
||
continue
|
||
rel = path.relative_to(repo_root).as_posix()
|
||
if rel not in listed:
|
||
msg = f"file on disk but not in manifest: {rel}"
|
||
(report["errors"] if strict else report["warnings"]).append(msg)
|
||
|
||
# 计数一致性(如果 manifest 自报 counts)
|
||
counts = manifest.get("counts", {})
|
||
if isinstance(counts, dict) and assets:
|
||
if counts.get("assets_present") != len(assets):
|
||
report["errors"].append(
|
||
f"counts.assets_present={counts.get('assets_present')} != assets[]={len(assets)}"
|
||
)
|
||
return report
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--repo-root", default=None, help="仓库根目录(默认脚本上级目录)")
|
||
ap.add_argument("--manifest", default=None, help="manifest.json 路径")
|
||
ap.add_argument("--skip-hash", action="store_true", help="跳过 sha256 计算")
|
||
ap.add_argument("--strict", action="store_true", help="未登记文件视为错误")
|
||
ap.add_argument("--json", action="store_true", help="输出 JSON 报告")
|
||
args = ap.parse_args(argv)
|
||
|
||
script_dir = Path(__file__).resolve().parent
|
||
repo_root = Path(args.repo_root).resolve() if args.repo_root else script_dir.parent
|
||
manifest_path = Path(args.manifest) if args.manifest else repo_root / "corpus" / "manifest.json"
|
||
|
||
report = verify(repo_root, manifest_path, args.skip_hash, args.strict)
|
||
ok = not report["errors"]
|
||
if args.json:
|
||
print(json.dumps({"ok": ok, **report}, ensure_ascii=False, indent=2))
|
||
else:
|
||
print(
|
||
f"[verify_manifest] {'OK' if ok else 'FAIL'} "
|
||
f"assets={report['checked_assets']} bytes={report['checked_bytes']} "
|
||
f"real_assets={report['checked_real_assets']}"
|
||
)
|
||
for e in report["errors"]:
|
||
print(f" ERROR: {e}")
|
||
for w in report["warnings"]:
|
||
print(f" WARN: {w}")
|
||
return 0 if ok else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|