Files
hm 2936b4a888 feat(corpus): W12 首批真实语料导入与 B1/B2/B5/B6 真实 KPI 基线
真实语料 11 个 / 10,697,067 B(STEP 6、DXF 3、DWG 1、3MF 1),来源 ViewDesign-deepseek/corpus(与 kimi、Assembly 仓交叉校验),按 sha256 前 12 位命名;3 个文件做同长度 PII 脱敏(几何零改动),逐资产来源/哈希/许可判定见 corpus/real_manifest.json 与 real_provenance.md。

manifest 工具扩展:gen_manifest.py 扫描 corpus/real 并做登记表双向复核(未登记文件/哈希不符 exit 1),verify_manifest.py --strict 覆盖 tests/data + corpus/real;实测 gen --check 与 verify --strict 均 exit 0(63 合成 + 11 真实)。

driver 增加真实语料识别(real_corpus_present/real_step_files),修复 Get-StepFileCount 扩展名过滤 bug(首跑误记 11 STEP → 6 STEP)。

真实语料实测(run-20261119-225558):B1 import 6/6、clean 0/12=0%、perturbed 0/24=0%(381→372s);B2 3/19(与合成一致,无真实脏模型);B5 单线程 3 轮 bit-identical 通过、2/4/8 线程 digest 不一致且 8 线程拓扑不同(282v/434e),8 线程加速比 0.94x;B6 4/4 bounded_pass(4×30s≈180s)。KPI 表与门限建议见 docs/planning/benchmarks-kcp-wave1.md §7.1 与 docs/planning/wave2/corpus-notes.md。
2026-11-19 23:06:19 +08:00

565 lines
25 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""KCP-1.0 Wave 1 / W4:生成 corpus/manifest.json 与 corpus/manifest.csv。
诚实性要求
----------
1. 只登记**仓库内真实存在**的文件(逐个 stat + sha256),不虚构“真实语料”。
2. 真实语料(corpus/real/)的来源/许可元数据以 corpus/real_manifest.json 为
权威登记表:生成时逐个复核 size/sha256,登记表与磁盘不一致、磁盘有而未
登记、登记有而磁盘缺失,均直接失败(不静默通过)。corpus/realfile 等缺失
目录仍显式写入 "missing" 段并标注 verified_missing。
3. 每个资产标注 synthetic / source / license / license_review_required /
redistributable,便于审计与法务复核。
用法
----
python corpus/gen_manifest.py # 默认 repo root = 脚本上级目录
python corpus/gen_manifest.py --repo-root D:/3_Source/VDE_cp_corpus
python corpus/gen_manifest.py --check # 只校验不写文件(等价 verify_manifest.py)
"""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import os
import sys
from collections import Counter
from pathlib import Path
SCHEMA_VERSION = "1.0"
GENERATOR = "corpus/gen_manifest.py"
HARNESS = "KCP-1.0 Wave 1 / W4 corpus+benchmark"
DATA_ROOT_REL = "tests/data"
REAL_ROOT_REL = "corpus/real"
REAL_REGISTRY_REL = "corpus/real_manifest.json"
CSV_COLUMNS = [
"path",
"format",
"category",
"synthetic",
"present",
"size_bytes",
"sha256",
"source",
"license",
"license_review_required",
"redistributable",
]
# 格式识别(扩展名小写)
FORMAT_BY_EXT = {
".step": "STEP",
".stp": "STEP",
".iges": "IGES",
".igs": "IGES",
".stl": "STL",
".obj": "OBJ",
".3mf": "3MF",
".dxf": "DXF",
".dwg": "DWG",
".vde": "VDE",
".txt": "TXT",
".csv": "CSV",
}
SYNTHETIC_SOURCE = (
"仓库内合成夹具(tools/test_fixtures/ 生成器自产或手工构造;"
"不代表真实客户模型)"
)
def classify(rel: str) -> dict:
"""按仓库内已知资产给出 category/source/license/synthetic。"""
p = rel.replace("\\", "/")
name = os.path.basename(p)
low = name.lower()
# 1) 合成 STEP 夹具
if p.startswith("fixtures/step/"):
return dict(
category="synthetic_step_fixture",
synthetic=True,
source=SYNTHETIC_SOURCE,
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="B-Rep 导出器生成的合成 STEP(ap214/ap242/测试立方体)",
)
# 2) OCCT 官方回归模型
if p == "fixtures/real/step/occ__xr1-tu-plt.stp":
return dict(
category="real_reference_model",
synthetic=False,
source="OCCT (OpenCASCADE Technology) 官方回归/示例模型 xr1-tu-plt"
"(来源登记见 docs/reference/220、docs/reference/267 与 "
"tools/test_fixtures/README.md)",
license="OCCT LGPL-2.1 with OCCT exception(上游许可证;再分发需保留版权与许可声明)",
license_review_required=False,
notes="80x60 板 + 6 通孔、12 内环、198 面/1176 边;当前唯一真实几何参照模型",
)
# 3) QCad 示例图纸
if p == "fixtures/real/drawing/qcad__flange.dxf":
return dict(
category="real_reference_drawing",
synthetic=False,
source="QCad 官方示例图纸(flange)",
license="待法务复核:QCad 示例图纸随 QCad 发行,QCad CE 为 GPLv2;入库前需按 corpus/README.md §6 复核",
license_review_required=True,
notes="真实参照 DXF 图纸;许可证需上游确认后方可用于对外发行",
)
# 4) 合成 DXF(R12/R2000/R2010/R2018 版本变体)
if p.startswith("fixtures/real/drawing/synth_") and low.endswith(".dxf"):
return dict(
category="synthetic_drawing_fixture",
synthetic=True,
source=SYNTHETIC_SOURCE,
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="合成 DXF,按 DXF 版本变体(R12/R2000/R2010/R2018)覆盖解析路径",
)
# 5) 合成 DWG
if low.endswith(".dwg") and ("synth" in low):
return dict(
category="synthetic_drawing_fixture",
synthetic=True,
source=SYNTHETIC_SOURCE,
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="合成 DWG(AC1015/AC1032 版本变体),非真实客户图纸",
)
# 6) 合成 3MF
if low.endswith(".3mf"):
return dict(
category="synthetic_mesh_fixture",
synthetic=True,
source=SYNTHETIC_SOURCE,
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="合成 3MF 网格夹具(assembly/deflate/transform 变体)",
)
# 7) 手工脏模型夹具
if low.endswith(".vde"):
return dict(
category="handcrafted_defect_fixture",
synthetic=True,
source="仓库内手工构造的脏模型夹具(W0 阶段)",
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="开放体 + 5 悬挂边;用于 validate/heal/failure 语义测试,B2 复用",
)
# 8) 说明性文本
if low.endswith(".txt"):
return dict(
category="metadata_text",
synthetic=True,
source="仓库内说明文本(非几何数据)",
license="自产(仓库内),随仓库许可",
license_review_required=False,
notes="原生语料说明/清单文本,不含几何",
)
return dict(
category="unclassified",
synthetic=None,
source="未登记来源(需人工补登记)",
license="未知",
license_review_required=True,
notes="生成器未识别的资产,需在 corpus/gen_manifest.py 中补充分类规则",
)
def sha256_of(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def scan_assets(repo_root: Path) -> list[dict]:
data_root = repo_root / DATA_ROOT_REL
assets: list[dict] = []
if not data_root.is_dir():
return assets
for path in sorted(data_root.rglob("*")):
if not path.is_file():
continue
rel = path.relative_to(data_root).as_posix()
info = classify(rel)
ext = path.suffix.lower()
assets.append(
{
"path": f"{DATA_ROOT_REL}/{rel}",
"format": FORMAT_BY_EXT.get(ext, ext.lstrip(".").upper() or "UNKNOWN"),
"category": info["category"],
"synthetic": info["synthetic"],
"present": True,
"size_bytes": path.stat().st_size,
"sha256": sha256_of(path),
"source": info["source"],
"license": info["license"],
"license_review_required": info["license_review_required"],
"notes": info.get("notes", ""),
}
)
return assets
def load_real_registry(repo_root: Path) -> dict:
"""读取 corpus/real_manifest.json(真实语料来源/许可权威登记表)。"""
path = repo_root / REAL_REGISTRY_REL
if not path.is_file():
raise SystemExit(
f"[gen_manifest] 缺少真实语料来源登记文件 {REAL_REGISTRY_REL}:"
"corpus/real 存在时必须逐资产登记来源/许可"
)
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise SystemExit(f"[gen_manifest] {REAL_REGISTRY_REL} 不是合法 JSON: {exc}")
if not isinstance(data.get("assets"), list):
raise SystemExit(f"[gen_manifest] {REAL_REGISTRY_REL} 缺少 assets[]")
return data
def scan_real_assets(repo_root: Path, registry: dict) -> list[dict]:
"""扫描 corpus/real/,与登记表逐项复核(size/sha256/存在性双向)。"""
real_root = repo_root / REAL_ROOT_REL
registry_assets = {a.get("path", ""): a for a in registry.get("assets", [])}
on_disk = sorted(p for p in real_root.rglob("*") if p.is_file()) if real_root.is_dir() else []
if not on_disk and not registry_assets:
return []
assets: list[dict] = []
seen: set[str] = set()
for path in on_disk:
rel = path.relative_to(repo_root).as_posix()
entry = registry_assets.get(rel)
if entry is None:
raise SystemExit(
f"[gen_manifest] 未登记的真实语料: {rel}(先写入 {REAL_REGISTRY_REL} 再生成清单)"
)
seen.add(rel)
size = path.stat().st_size
digest = sha256_of(path)
if size != entry.get("size_bytes"):
raise SystemExit(
f"[gen_manifest] 真实语料大小不一致: {rel} registry={entry.get('size_bytes')} disk={size}"
)
if digest != entry.get("sha256"):
raise SystemExit(
f"[gen_manifest] 真实语料 sha256 不一致: {rel} registry={entry.get('sha256')} disk={digest}"
)
source = entry.get("source", "")
if not source:
raise SystemExit(f"[gen_manifest] 真实语料缺少 source(来源登记): {rel}")
if not entry.get("license"):
raise SystemExit(f"[gen_manifest] 真实语料缺少 license 判定: {rel}")
ext = path.suffix.lower()
asset = {
"path": rel,
"format": entry.get("format")
or FORMAT_BY_EXT.get(ext, ext.lstrip(".").upper() or "UNKNOWN"),
"category": entry.get("category", "real_model"),
"synthetic": False,
"present": True,
"size_bytes": size,
"sha256": digest,
"source": source,
"source_path": entry.get("source_path", ""),
"source_registration": entry.get("source_registration", ""),
"source_repos": entry.get("source_repos", []),
"sanitized": bool(entry.get("sanitized")),
"acquired_at": entry.get("acquired_at", registry.get("acquired_at", "")),
"license": entry["license"],
"license_basis": entry.get("license_basis", []),
"license_review_required": bool(entry.get("license_review_required")),
"redistributable": bool(entry.get("redistributable")),
"notes": entry.get("notes", ""),
}
if entry.get("original_sha256"):
asset["original_sha256"] = entry["original_sha256"]
assets.append(asset)
missing = sorted(set(registry_assets) - seen)
if missing:
raise SystemExit(
f"[gen_manifest] 登记表存在但磁盘缺失的真实语料: {', '.join(missing)}"
"(若已删除,请同步更新 registry)"
)
return assets
def build_real_corpus_spec(
repo_root: Path, real_assets: list[dict], registry: dict | None
) -> dict:
root_exists = (repo_root / REAL_ROOT_REL).is_dir()
if real_assets:
status = "present"
elif root_exists:
status = "present_empty"
else:
status = "missing"
redistributable = [a for a in real_assets if a["redistributable"]]
by_category = Counter(a["category"] for a in real_assets)
by_format = Counter(a["format"] for a in real_assets)
spec = {
"root": REAL_ROOT_REL,
"realfile_root": "corpus/realfile",
"status": status,
"purpose": "真实/内部授权 CAD 语料(STEP/IGES/STL/3MF/DXF/DWG),"
"用于 B1 STEP 布尔通过率、B2 healing 恢复曲线、KC-Q06 corpus gate 的真实语料部分",
"entry_registry": REAL_REGISTRY_REL if registry else "",
"updated_from_git": git_head(repo_root),
"acquired_at": (registry or {}).get("acquired_at", ""),
"layout": {
"step": "corpus/real/step/<vendor>__<part>__<sha256-12>.step",
"iges": "corpus/real/iges/<vendor>__<part>__<sha256-12>.igs",
"mesh": "corpus/real/mesh/<vendor>__<part>__<sha256-12>.stl|3mf|obj",
"drawing": "corpus/real/drawing/<vendor>__<part>__<sha256-12>.dxf|dwg",
"assembly": "corpus/real/assembly/<vendor>__<part>__<sha256-12>.step",
"realfile": "corpus/realfile/<vendor>__<file>__<sha256-12>.<ext>",
},
"naming": "<vendor>__<part>__<sha256前12位>.<ext>;vendor 为来源标识(非客户名),"
"part 为脱敏件号;禁止包含真实客户名称/个人信息",
"accepted_formats": ["step", "stp", "iges", "igs", "stl", "obj", "3mf", "dxf", "dwg"],
"counts": {
"assets": len(real_assets),
"bytes": sum(a["size_bytes"] for a in real_assets),
"redistributable_assets": len(redistributable),
"non_redistributable_assets": len(real_assets) - len(redistributable),
"sanitized_assets": sum(1 for a in real_assets if a["sanitized"]),
},
"by_category": dict(sorted(by_category.items())),
"by_format": dict(sorted(by_format.items())),
"proposed_target": {
"step_models": 20,
"dirty_or_open_models": 8,
"drawing_files": 40,
"note": "目标值待 W1/主线评审后固化;W12 首批已入库见 counts",
},
"source_policy": "来源必须可追溯(source/source_path 非空)+ 许可判定可审计"
"(license/license_basis);许可不明者不得进入 corpus/real",
"license_policy": [
"只接收许可判定清晰(允许项目内部测试)的语料;许可证不明者不得进入 corpus/real",
"禁止把无许可证的第三方模型伪装成合成夹具或真实语料",
"redistributable=false 的资产按 corpus/README.md §4.1 本应只进 corpus/realfile;"
"W12 首批按任务指示放 corpus/real(内部仓库、corpus/ 默认不入发行包),"
"该偏离已记录在 corpus/real_provenance.md §2,对外发布前必须迁移或重获许可",
],
"acquisition_flow": [
"1. 来源登记:corpus/real_manifest.json 填写 id/path/source/source_path/license/license_basis",
"2. 法务初筛:许可必须允许内部测试;不明确者不入库(登记排除清单)",
"3. 脱敏:删除标题栏客户名/图号/个人信息(同长度占位符替换,登记 original_sha256),"
"按 <vendor>__<part>__<sha256-12> 重命名",
"4. 校验:python corpus/gen_manifest.py && python corpus/verify_manifest.py --strict",
"5. 入库:manifest.real_corpus.status=present,PR 附来源与许可依据",
],
"validation_flow": [
"python corpus/verify_manifest.py --strict(存在性/size/sha256/未登记文件)",
"python corpus/gen_manifest.py --check(清单是否最新)",
"build/tests/benchmark/bench_kcp.exe --suite b1 --fixtures-root corpus/real(B1 真实语料)",
],
"license_policy_note": "本目录资产全部为项目内部资产,redistributable=false;"
"真实语料来自项目内部仓库本地 corpus/(gitignored),"
"逐资产来源/许可判定见 entry_registry 与 corpus/real_provenance.md",
"assets": real_assets,
}
return spec
def build_missing_section(repo_root: Path) -> list[dict]:
entries = [
{
"asset": "corpus/real",
"status": "missing",
"purpose": "真实/公开授权几何语料(B1/B2/真实语料回归)",
"expected_layout": "corpus/real-layout.txt",
"blocking_kpi": "B1 真实语料通过率、B2 真实脏模型恢复率、KC-Q06 corpus gate 真实语料部分",
},
{
"asset": "corpus/realfile",
"status": "missing",
"purpose": "不可再分发的真实原生文件语料(realfile 测试用,不入发行包)",
"expected_layout": "corpus/real-layout.txt",
"blocking_kpi": "RealFile 语料类测试(当前 79 条 skip 中的语料缺失项)",
},
]
for e in entries:
exists = (repo_root / e["asset"]).exists()
e["verified_missing"] = not exists
if exists:
if e["asset"] == REAL_ROOT_REL:
# 真实语料目录已入库:不再是 missing(内容由 real_corpus 段登记)。
e["status"] = "present"
e["note"] = "已入库真实语料;逐资产登记见 corpus/real_corpus.assets"
else:
e["status"] = "present_unexpected"
e["warning"] = (
"目录存在但清单仍标记 missing:若为占位目录请保持为空;"
"若已入库真实语料,请更新 manifest 的 real_corpus 段"
)
return entries
def git_head(repo_root: Path) -> str:
# 优先用 git 命令(兼容 worktree:此时 .git 是文件而非目录)。
try:
import subprocess
out = subprocess.run(
["git", "-C", str(repo_root), "rev-parse", "HEAD"],
capture_output=True,
text=True,
timeout=10,
)
if out.returncode == 0 and out.stdout.strip():
return out.stdout.strip()
except (OSError, ValueError):
pass
try:
head = (repo_root / ".git" / "HEAD").read_text(encoding="utf-8").strip()
if head.startswith("ref: "):
ref = head[len("ref: "):].strip()
ref_file = repo_root / ".git" / ref
if ref_file.is_file():
return ref_file.read_text(encoding="utf-8").strip()
packed = repo_root / ".git" / "packed-refs"
if packed.is_file():
for line in packed.read_text(encoding="utf-8").splitlines():
parts = line.split()
if len(parts) == 2 and parts[1] == ref:
return parts[0]
return head
except OSError:
return "unknown"
def build_manifest(repo_root: Path) -> dict:
assets = scan_assets(repo_root)
registry: dict | None = None
real_assets: list[dict] = []
if (repo_root / REAL_ROOT_REL).is_dir():
registry = load_real_registry(repo_root)
real_assets = scan_real_assets(repo_root, registry)
by_category: Counter = Counter(a["category"] for a in assets)
by_format: Counter = Counter(a["format"] for a in assets)
synthetic_count = sum(1 for a in assets if a["synthetic"] is True)
review_required = sum(1 for a in assets if a["license_review_required"])
real_review_required = sum(1 for a in real_assets if a["license_review_required"])
missing = build_missing_section(repo_root)
real_corpus = build_real_corpus_spec(repo_root, real_assets, registry)
return {
"schema_version": SCHEMA_VERSION,
"harness": HARNESS,
"generator": GENERATOR,
"repo_data_root": DATA_ROOT_REL,
"generated_from_git": git_head(repo_root),
"counts": {
"assets_present": len(assets),
"assets_missing": len([m for m in missing if m["status"] == "missing"]),
"synthetic_assets": synthetic_count,
"non_synthetic_assets": len(assets) - synthetic_count,
"license_review_required": review_required,
"total_bytes": sum(a["size_bytes"] for a in assets),
"real_assets": len(real_assets),
"real_bytes": sum(a["size_bytes"] for a in real_assets),
"real_license_review_required": real_review_required,
"total_assets_including_real": len(assets) + len(real_assets),
"total_bytes_including_real": sum(a["size_bytes"] for a in assets)
+ sum(a["size_bytes"] for a in real_assets),
},
"by_category": dict(sorted(by_category.items())),
"by_format": dict(sorted(by_format.items())),
"missing": missing,
"real_corpus": real_corpus,
"licensing_summary": {
"synthetic_fixtures": "自产,随仓库许可(无第三方再分发限制)",
"occt_regression_model": "OCCT LGPL-2.1 with OCCT exception;再分发保留声明",
"qcad_example": "许可待法务复核(license_review_required=true)",
"real_corpus_assets": "项目内部资产(自研/内部导出),redistributable=false,"
"仅内部测试;逐资产判定见 real_corpus.assets[].license",
"unknown": "禁止入库;见 real_corpus.license_policy",
},
"assets": assets,
}
def write_csv(manifest: dict, csv_path: Path) -> None:
rows = list(manifest["assets"]) + list(manifest["real_corpus"]["assets"])
with csv_path.open("w", encoding="utf-8", newline="") as f:
w = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
w.writeheader()
for a in rows:
row = {k: a.get(k, "") for k in CSV_COLUMNS}
row["synthetic"] = "true" if row["synthetic"] is True else (
"false" if row["synthetic"] is False else ""
)
row["present"] = "true" if row["present"] else "false"
row["license_review_required"] = "true" if row["license_review_required"] else "false"
row["redistributable"] = "" if a.get("redistributable") is None else (
"true" if a.get("redistributable") else "false"
)
w.writerow(row)
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--repo-root", default=None, help="仓库根目录(默认脚本上级目录)")
ap.add_argument("--output", default=None, help="manifest.json 输出路径")
ap.add_argument("--csv", default=None, help="manifest.csv 输出路径")
ap.add_argument("--check", action="store_true", help="只校验:不写文件,比对现有清单")
args = ap.parse_args(argv)
script_dir = Path(__file__).resolve().parent
repo_root = Path(args.repo_root).resolve() if args.repo_root else script_dir.parent
out_json = Path(args.output) if args.output else repo_root / "corpus" / "manifest.json"
out_csv = Path(args.csv) if args.csv else repo_root / "corpus" / "manifest.csv"
manifest = build_manifest(repo_root)
new_json = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
if args.check:
if not out_json.is_file():
print(f"[gen_manifest] MISSING {out_json}", file=sys.stderr)
return 1
old_json = out_json.read_text(encoding="utf-8")
# generated_from_git 记录“生成时的 HEAD”,提交后旧 HEAD 自然成为父提交,
# 因此 --check 只比对语料内容,不把 git HEAD 变动算作 stale(否则每次提交后必失败)。
try:
old_cmp = json.loads(old_json)
new_cmp = json.loads(new_json)
except json.JSONDecodeError as exc:
print(f"[gen_manifest] STALE {out_json}: 无法解析旧清单: {exc}", file=sys.stderr)
return 1
old_cmp.pop("generated_from_git", None)
new_cmp.pop("generated_from_git", None)
if json.dumps(old_cmp, ensure_ascii=False, sort_keys=True) == json.dumps(
new_cmp, ensure_ascii=False, sort_keys=True
):
print(
f"[gen_manifest] OK {out_json} ({manifest['counts']['assets_present']} assets; "
f"generated_from_git 忽略,informational)"
)
return 0
print(f"[gen_manifest] STALE {out_json}: 请重新生成并 review diff", file=sys.stderr)
return 1
out_json.parent.mkdir(parents=True, exist_ok=True)
out_json.write_text(new_json, encoding="utf-8")
write_csv(manifest, out_csv)
c = manifest["counts"]
print(
f"[gen_manifest] wrote {out_json} + {out_csv}\n"
f" assets={c['assets_present']} synthetic={c['synthetic_assets']} "
f"non_synthetic={c['non_synthetic_assets']} bytes={c['total_bytes']} "
f"missing={c['assets_missing']} license_review_required={c['license_review_required']}\n"
f" real_assets={c['real_assets']} real_bytes={c['real_bytes']} "
f"real_license_review_required={c['real_license_review_required']} "
f"real_status={manifest['real_corpus']['status']}"
)
return 0
if __name__ == "__main__":
sys.exit(main())