真实语料 11 个 / 10,697,067 B(STEP 6、DXF 3、DWG 1、3MF 1),来源 ViewDesign-deepseek/corpus(与 kimi、Assembly 仓交叉校验),按 sha256 前 12 位命名;3 个文件做同长度 PII 脱敏(几何零改动),逐资产来源/哈希/许可判定见 corpus/real_manifest.json 与 real_provenance.md。 manifest 工具扩展:gen_manifest.py 扫描 corpus/real 并做登记表双向复核(未登记文件/哈希不符 exit 1),verify_manifest.py --strict 覆盖 tests/data + corpus/real;实测 gen --check 与 verify --strict 均 exit 0(63 合成 + 11 真实)。 driver 增加真实语料识别(real_corpus_present/real_step_files),修复 Get-StepFileCount 扩展名过滤 bug(首跑误记 11 STEP → 6 STEP)。 真实语料实测(run-20261119-225558):B1 import 6/6、clean 0/12=0%、perturbed 0/24=0%(381→372s);B2 3/19(与合成一致,无真实脏模型);B5 单线程 3 轮 bit-identical 通过、2/4/8 线程 digest 不一致且 8 线程拓扑不同(282v/434e),8 线程加速比 0.94x;B6 4/4 bounded_pass(4×30s≈180s)。KPI 表与门限建议见 docs/planning/benchmarks-kcp-wave1.md §7.1 与 docs/planning/wave2/corpus-notes.md。
565 lines
25 KiB
Python
565 lines
25 KiB
Python
#!/usr/bin/env python3
|
||
"""KCP-1.0 Wave 1 / W4:生成 corpus/manifest.json 与 corpus/manifest.csv。
|
||
|
||
诚实性要求
|
||
----------
|
||
1. 只登记**仓库内真实存在**的文件(逐个 stat + sha256),不虚构“真实语料”。
|
||
2. 真实语料(corpus/real/)的来源/许可元数据以 corpus/real_manifest.json 为
|
||
权威登记表:生成时逐个复核 size/sha256,登记表与磁盘不一致、磁盘有而未
|
||
登记、登记有而磁盘缺失,均直接失败(不静默通过)。corpus/realfile 等缺失
|
||
目录仍显式写入 "missing" 段并标注 verified_missing。
|
||
3. 每个资产标注 synthetic / source / license / license_review_required /
|
||
redistributable,便于审计与法务复核。
|
||
|
||
用法
|
||
----
|
||
python corpus/gen_manifest.py # 默认 repo root = 脚本上级目录
|
||
python corpus/gen_manifest.py --repo-root D:/3_Source/VDE_cp_corpus
|
||
python corpus/gen_manifest.py --check # 只校验不写文件(等价 verify_manifest.py)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import sys
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
SCHEMA_VERSION = "1.0"
|
||
GENERATOR = "corpus/gen_manifest.py"
|
||
HARNESS = "KCP-1.0 Wave 1 / W4 corpus+benchmark"
|
||
DATA_ROOT_REL = "tests/data"
|
||
REAL_ROOT_REL = "corpus/real"
|
||
REAL_REGISTRY_REL = "corpus/real_manifest.json"
|
||
CSV_COLUMNS = [
|
||
"path",
|
||
"format",
|
||
"category",
|
||
"synthetic",
|
||
"present",
|
||
"size_bytes",
|
||
"sha256",
|
||
"source",
|
||
"license",
|
||
"license_review_required",
|
||
"redistributable",
|
||
]
|
||
|
||
# 格式识别(扩展名小写)
|
||
FORMAT_BY_EXT = {
|
||
".step": "STEP",
|
||
".stp": "STEP",
|
||
".iges": "IGES",
|
||
".igs": "IGES",
|
||
".stl": "STL",
|
||
".obj": "OBJ",
|
||
".3mf": "3MF",
|
||
".dxf": "DXF",
|
||
".dwg": "DWG",
|
||
".vde": "VDE",
|
||
".txt": "TXT",
|
||
".csv": "CSV",
|
||
}
|
||
|
||
SYNTHETIC_SOURCE = (
|
||
"仓库内合成夹具(tools/test_fixtures/ 生成器自产或手工构造;"
|
||
"不代表真实客户模型)"
|
||
)
|
||
|
||
|
||
def classify(rel: str) -> dict:
|
||
"""按仓库内已知资产给出 category/source/license/synthetic。"""
|
||
p = rel.replace("\\", "/")
|
||
name = os.path.basename(p)
|
||
low = name.lower()
|
||
|
||
# 1) 合成 STEP 夹具
|
||
if p.startswith("fixtures/step/"):
|
||
return dict(
|
||
category="synthetic_step_fixture",
|
||
synthetic=True,
|
||
source=SYNTHETIC_SOURCE,
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="B-Rep 导出器生成的合成 STEP(ap214/ap242/测试立方体)",
|
||
)
|
||
# 2) OCCT 官方回归模型
|
||
if p == "fixtures/real/step/occ__xr1-tu-plt.stp":
|
||
return dict(
|
||
category="real_reference_model",
|
||
synthetic=False,
|
||
source="OCCT (OpenCASCADE Technology) 官方回归/示例模型 xr1-tu-plt"
|
||
"(来源登记见 docs/reference/220、docs/reference/267 与 "
|
||
"tools/test_fixtures/README.md)",
|
||
license="OCCT LGPL-2.1 with OCCT exception(上游许可证;再分发需保留版权与许可声明)",
|
||
license_review_required=False,
|
||
notes="80x60 板 + 6 通孔、12 内环、198 面/1176 边;当前唯一真实几何参照模型",
|
||
)
|
||
# 3) QCad 示例图纸
|
||
if p == "fixtures/real/drawing/qcad__flange.dxf":
|
||
return dict(
|
||
category="real_reference_drawing",
|
||
synthetic=False,
|
||
source="QCad 官方示例图纸(flange)",
|
||
license="待法务复核:QCad 示例图纸随 QCad 发行,QCad CE 为 GPLv2;入库前需按 corpus/README.md §6 复核",
|
||
license_review_required=True,
|
||
notes="真实参照 DXF 图纸;许可证需上游确认后方可用于对外发行",
|
||
)
|
||
# 4) 合成 DXF(R12/R2000/R2010/R2018 版本变体)
|
||
if p.startswith("fixtures/real/drawing/synth_") and low.endswith(".dxf"):
|
||
return dict(
|
||
category="synthetic_drawing_fixture",
|
||
synthetic=True,
|
||
source=SYNTHETIC_SOURCE,
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="合成 DXF,按 DXF 版本变体(R12/R2000/R2010/R2018)覆盖解析路径",
|
||
)
|
||
# 5) 合成 DWG
|
||
if low.endswith(".dwg") and ("synth" in low):
|
||
return dict(
|
||
category="synthetic_drawing_fixture",
|
||
synthetic=True,
|
||
source=SYNTHETIC_SOURCE,
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="合成 DWG(AC1015/AC1032 版本变体),非真实客户图纸",
|
||
)
|
||
# 6) 合成 3MF
|
||
if low.endswith(".3mf"):
|
||
return dict(
|
||
category="synthetic_mesh_fixture",
|
||
synthetic=True,
|
||
source=SYNTHETIC_SOURCE,
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="合成 3MF 网格夹具(assembly/deflate/transform 变体)",
|
||
)
|
||
# 7) 手工脏模型夹具
|
||
if low.endswith(".vde"):
|
||
return dict(
|
||
category="handcrafted_defect_fixture",
|
||
synthetic=True,
|
||
source="仓库内手工构造的脏模型夹具(W0 阶段)",
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="开放体 + 5 悬挂边;用于 validate/heal/failure 语义测试,B2 复用",
|
||
)
|
||
# 8) 说明性文本
|
||
if low.endswith(".txt"):
|
||
return dict(
|
||
category="metadata_text",
|
||
synthetic=True,
|
||
source="仓库内说明文本(非几何数据)",
|
||
license="自产(仓库内),随仓库许可",
|
||
license_review_required=False,
|
||
notes="原生语料说明/清单文本,不含几何",
|
||
)
|
||
return dict(
|
||
category="unclassified",
|
||
synthetic=None,
|
||
source="未登记来源(需人工补登记)",
|
||
license="未知",
|
||
license_review_required=True,
|
||
notes="生成器未识别的资产,需在 corpus/gen_manifest.py 中补充分类规则",
|
||
)
|
||
|
||
|
||
def sha256_of(path: Path) -> str:
|
||
h = hashlib.sha256()
|
||
with path.open("rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def scan_assets(repo_root: Path) -> list[dict]:
|
||
data_root = repo_root / DATA_ROOT_REL
|
||
assets: list[dict] = []
|
||
if not data_root.is_dir():
|
||
return assets
|
||
for path in sorted(data_root.rglob("*")):
|
||
if not path.is_file():
|
||
continue
|
||
rel = path.relative_to(data_root).as_posix()
|
||
info = classify(rel)
|
||
ext = path.suffix.lower()
|
||
assets.append(
|
||
{
|
||
"path": f"{DATA_ROOT_REL}/{rel}",
|
||
"format": FORMAT_BY_EXT.get(ext, ext.lstrip(".").upper() or "UNKNOWN"),
|
||
"category": info["category"],
|
||
"synthetic": info["synthetic"],
|
||
"present": True,
|
||
"size_bytes": path.stat().st_size,
|
||
"sha256": sha256_of(path),
|
||
"source": info["source"],
|
||
"license": info["license"],
|
||
"license_review_required": info["license_review_required"],
|
||
"notes": info.get("notes", ""),
|
||
}
|
||
)
|
||
return assets
|
||
|
||
|
||
def load_real_registry(repo_root: Path) -> dict:
|
||
"""读取 corpus/real_manifest.json(真实语料来源/许可权威登记表)。"""
|
||
path = repo_root / REAL_REGISTRY_REL
|
||
if not path.is_file():
|
||
raise SystemExit(
|
||
f"[gen_manifest] 缺少真实语料来源登记文件 {REAL_REGISTRY_REL}:"
|
||
"corpus/real 存在时必须逐资产登记来源/许可"
|
||
)
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError as exc:
|
||
raise SystemExit(f"[gen_manifest] {REAL_REGISTRY_REL} 不是合法 JSON: {exc}")
|
||
if not isinstance(data.get("assets"), list):
|
||
raise SystemExit(f"[gen_manifest] {REAL_REGISTRY_REL} 缺少 assets[]")
|
||
return data
|
||
|
||
|
||
def scan_real_assets(repo_root: Path, registry: dict) -> list[dict]:
|
||
"""扫描 corpus/real/,与登记表逐项复核(size/sha256/存在性双向)。"""
|
||
real_root = repo_root / REAL_ROOT_REL
|
||
registry_assets = {a.get("path", ""): a for a in registry.get("assets", [])}
|
||
on_disk = sorted(p for p in real_root.rglob("*") if p.is_file()) if real_root.is_dir() else []
|
||
if not on_disk and not registry_assets:
|
||
return []
|
||
assets: list[dict] = []
|
||
seen: set[str] = set()
|
||
for path in on_disk:
|
||
rel = path.relative_to(repo_root).as_posix()
|
||
entry = registry_assets.get(rel)
|
||
if entry is None:
|
||
raise SystemExit(
|
||
f"[gen_manifest] 未登记的真实语料: {rel}(先写入 {REAL_REGISTRY_REL} 再生成清单)"
|
||
)
|
||
seen.add(rel)
|
||
size = path.stat().st_size
|
||
digest = sha256_of(path)
|
||
if size != entry.get("size_bytes"):
|
||
raise SystemExit(
|
||
f"[gen_manifest] 真实语料大小不一致: {rel} registry={entry.get('size_bytes')} disk={size}"
|
||
)
|
||
if digest != entry.get("sha256"):
|
||
raise SystemExit(
|
||
f"[gen_manifest] 真实语料 sha256 不一致: {rel} registry={entry.get('sha256')} disk={digest}"
|
||
)
|
||
source = entry.get("source", "")
|
||
if not source:
|
||
raise SystemExit(f"[gen_manifest] 真实语料缺少 source(来源登记): {rel}")
|
||
if not entry.get("license"):
|
||
raise SystemExit(f"[gen_manifest] 真实语料缺少 license 判定: {rel}")
|
||
ext = path.suffix.lower()
|
||
asset = {
|
||
"path": rel,
|
||
"format": entry.get("format")
|
||
or FORMAT_BY_EXT.get(ext, ext.lstrip(".").upper() or "UNKNOWN"),
|
||
"category": entry.get("category", "real_model"),
|
||
"synthetic": False,
|
||
"present": True,
|
||
"size_bytes": size,
|
||
"sha256": digest,
|
||
"source": source,
|
||
"source_path": entry.get("source_path", ""),
|
||
"source_registration": entry.get("source_registration", ""),
|
||
"source_repos": entry.get("source_repos", []),
|
||
"sanitized": bool(entry.get("sanitized")),
|
||
"acquired_at": entry.get("acquired_at", registry.get("acquired_at", "")),
|
||
"license": entry["license"],
|
||
"license_basis": entry.get("license_basis", []),
|
||
"license_review_required": bool(entry.get("license_review_required")),
|
||
"redistributable": bool(entry.get("redistributable")),
|
||
"notes": entry.get("notes", ""),
|
||
}
|
||
if entry.get("original_sha256"):
|
||
asset["original_sha256"] = entry["original_sha256"]
|
||
assets.append(asset)
|
||
missing = sorted(set(registry_assets) - seen)
|
||
if missing:
|
||
raise SystemExit(
|
||
f"[gen_manifest] 登记表存在但磁盘缺失的真实语料: {', '.join(missing)}"
|
||
"(若已删除,请同步更新 registry)"
|
||
)
|
||
return assets
|
||
|
||
|
||
def build_real_corpus_spec(
|
||
repo_root: Path, real_assets: list[dict], registry: dict | None
|
||
) -> dict:
|
||
root_exists = (repo_root / REAL_ROOT_REL).is_dir()
|
||
if real_assets:
|
||
status = "present"
|
||
elif root_exists:
|
||
status = "present_empty"
|
||
else:
|
||
status = "missing"
|
||
redistributable = [a for a in real_assets if a["redistributable"]]
|
||
by_category = Counter(a["category"] for a in real_assets)
|
||
by_format = Counter(a["format"] for a in real_assets)
|
||
spec = {
|
||
"root": REAL_ROOT_REL,
|
||
"realfile_root": "corpus/realfile",
|
||
"status": status,
|
||
"purpose": "真实/内部授权 CAD 语料(STEP/IGES/STL/3MF/DXF/DWG),"
|
||
"用于 B1 STEP 布尔通过率、B2 healing 恢复曲线、KC-Q06 corpus gate 的真实语料部分",
|
||
"entry_registry": REAL_REGISTRY_REL if registry else "",
|
||
"updated_from_git": git_head(repo_root),
|
||
"acquired_at": (registry or {}).get("acquired_at", ""),
|
||
"layout": {
|
||
"step": "corpus/real/step/<vendor>__<part>__<sha256-12>.step",
|
||
"iges": "corpus/real/iges/<vendor>__<part>__<sha256-12>.igs",
|
||
"mesh": "corpus/real/mesh/<vendor>__<part>__<sha256-12>.stl|3mf|obj",
|
||
"drawing": "corpus/real/drawing/<vendor>__<part>__<sha256-12>.dxf|dwg",
|
||
"assembly": "corpus/real/assembly/<vendor>__<part>__<sha256-12>.step",
|
||
"realfile": "corpus/realfile/<vendor>__<file>__<sha256-12>.<ext>",
|
||
},
|
||
"naming": "<vendor>__<part>__<sha256前12位>.<ext>;vendor 为来源标识(非客户名),"
|
||
"part 为脱敏件号;禁止包含真实客户名称/个人信息",
|
||
"accepted_formats": ["step", "stp", "iges", "igs", "stl", "obj", "3mf", "dxf", "dwg"],
|
||
"counts": {
|
||
"assets": len(real_assets),
|
||
"bytes": sum(a["size_bytes"] for a in real_assets),
|
||
"redistributable_assets": len(redistributable),
|
||
"non_redistributable_assets": len(real_assets) - len(redistributable),
|
||
"sanitized_assets": sum(1 for a in real_assets if a["sanitized"]),
|
||
},
|
||
"by_category": dict(sorted(by_category.items())),
|
||
"by_format": dict(sorted(by_format.items())),
|
||
"proposed_target": {
|
||
"step_models": 20,
|
||
"dirty_or_open_models": 8,
|
||
"drawing_files": 40,
|
||
"note": "目标值待 W1/主线评审后固化;W12 首批已入库见 counts",
|
||
},
|
||
"source_policy": "来源必须可追溯(source/source_path 非空)+ 许可判定可审计"
|
||
"(license/license_basis);许可不明者不得进入 corpus/real",
|
||
"license_policy": [
|
||
"只接收许可判定清晰(允许项目内部测试)的语料;许可证不明者不得进入 corpus/real",
|
||
"禁止把无许可证的第三方模型伪装成合成夹具或真实语料",
|
||
"redistributable=false 的资产按 corpus/README.md §4.1 本应只进 corpus/realfile;"
|
||
"W12 首批按任务指示放 corpus/real(内部仓库、corpus/ 默认不入发行包),"
|
||
"该偏离已记录在 corpus/real_provenance.md §2,对外发布前必须迁移或重获许可",
|
||
],
|
||
"acquisition_flow": [
|
||
"1. 来源登记:corpus/real_manifest.json 填写 id/path/source/source_path/license/license_basis",
|
||
"2. 法务初筛:许可必须允许内部测试;不明确者不入库(登记排除清单)",
|
||
"3. 脱敏:删除标题栏客户名/图号/个人信息(同长度占位符替换,登记 original_sha256),"
|
||
"按 <vendor>__<part>__<sha256-12> 重命名",
|
||
"4. 校验:python corpus/gen_manifest.py && python corpus/verify_manifest.py --strict",
|
||
"5. 入库:manifest.real_corpus.status=present,PR 附来源与许可依据",
|
||
],
|
||
"validation_flow": [
|
||
"python corpus/verify_manifest.py --strict(存在性/size/sha256/未登记文件)",
|
||
"python corpus/gen_manifest.py --check(清单是否最新)",
|
||
"build/tests/benchmark/bench_kcp.exe --suite b1 --fixtures-root corpus/real(B1 真实语料)",
|
||
],
|
||
"license_policy_note": "本目录资产全部为项目内部资产,redistributable=false;"
|
||
"真实语料来自项目内部仓库本地 corpus/(gitignored),"
|
||
"逐资产来源/许可判定见 entry_registry 与 corpus/real_provenance.md",
|
||
"assets": real_assets,
|
||
}
|
||
return spec
|
||
|
||
|
||
def build_missing_section(repo_root: Path) -> list[dict]:
|
||
entries = [
|
||
{
|
||
"asset": "corpus/real",
|
||
"status": "missing",
|
||
"purpose": "真实/公开授权几何语料(B1/B2/真实语料回归)",
|
||
"expected_layout": "corpus/real-layout.txt",
|
||
"blocking_kpi": "B1 真实语料通过率、B2 真实脏模型恢复率、KC-Q06 corpus gate 真实语料部分",
|
||
},
|
||
{
|
||
"asset": "corpus/realfile",
|
||
"status": "missing",
|
||
"purpose": "不可再分发的真实原生文件语料(realfile 测试用,不入发行包)",
|
||
"expected_layout": "corpus/real-layout.txt",
|
||
"blocking_kpi": "RealFile 语料类测试(当前 79 条 skip 中的语料缺失项)",
|
||
},
|
||
]
|
||
for e in entries:
|
||
exists = (repo_root / e["asset"]).exists()
|
||
e["verified_missing"] = not exists
|
||
if exists:
|
||
if e["asset"] == REAL_ROOT_REL:
|
||
# 真实语料目录已入库:不再是 missing(内容由 real_corpus 段登记)。
|
||
e["status"] = "present"
|
||
e["note"] = "已入库真实语料;逐资产登记见 corpus/real_corpus.assets"
|
||
else:
|
||
e["status"] = "present_unexpected"
|
||
e["warning"] = (
|
||
"目录存在但清单仍标记 missing:若为占位目录请保持为空;"
|
||
"若已入库真实语料,请更新 manifest 的 real_corpus 段"
|
||
)
|
||
return entries
|
||
|
||
|
||
def git_head(repo_root: Path) -> str:
|
||
# 优先用 git 命令(兼容 worktree:此时 .git 是文件而非目录)。
|
||
try:
|
||
import subprocess
|
||
|
||
out = subprocess.run(
|
||
["git", "-C", str(repo_root), "rev-parse", "HEAD"],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
)
|
||
if out.returncode == 0 and out.stdout.strip():
|
||
return out.stdout.strip()
|
||
except (OSError, ValueError):
|
||
pass
|
||
try:
|
||
head = (repo_root / ".git" / "HEAD").read_text(encoding="utf-8").strip()
|
||
if head.startswith("ref: "):
|
||
ref = head[len("ref: "):].strip()
|
||
ref_file = repo_root / ".git" / ref
|
||
if ref_file.is_file():
|
||
return ref_file.read_text(encoding="utf-8").strip()
|
||
packed = repo_root / ".git" / "packed-refs"
|
||
if packed.is_file():
|
||
for line in packed.read_text(encoding="utf-8").splitlines():
|
||
parts = line.split()
|
||
if len(parts) == 2 and parts[1] == ref:
|
||
return parts[0]
|
||
return head
|
||
except OSError:
|
||
return "unknown"
|
||
|
||
|
||
def build_manifest(repo_root: Path) -> dict:
|
||
assets = scan_assets(repo_root)
|
||
registry: dict | None = None
|
||
real_assets: list[dict] = []
|
||
if (repo_root / REAL_ROOT_REL).is_dir():
|
||
registry = load_real_registry(repo_root)
|
||
real_assets = scan_real_assets(repo_root, registry)
|
||
by_category: Counter = Counter(a["category"] for a in assets)
|
||
by_format: Counter = Counter(a["format"] for a in assets)
|
||
synthetic_count = sum(1 for a in assets if a["synthetic"] is True)
|
||
review_required = sum(1 for a in assets if a["license_review_required"])
|
||
real_review_required = sum(1 for a in real_assets if a["license_review_required"])
|
||
missing = build_missing_section(repo_root)
|
||
real_corpus = build_real_corpus_spec(repo_root, real_assets, registry)
|
||
return {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"harness": HARNESS,
|
||
"generator": GENERATOR,
|
||
"repo_data_root": DATA_ROOT_REL,
|
||
"generated_from_git": git_head(repo_root),
|
||
"counts": {
|
||
"assets_present": len(assets),
|
||
"assets_missing": len([m for m in missing if m["status"] == "missing"]),
|
||
"synthetic_assets": synthetic_count,
|
||
"non_synthetic_assets": len(assets) - synthetic_count,
|
||
"license_review_required": review_required,
|
||
"total_bytes": sum(a["size_bytes"] for a in assets),
|
||
"real_assets": len(real_assets),
|
||
"real_bytes": sum(a["size_bytes"] for a in real_assets),
|
||
"real_license_review_required": real_review_required,
|
||
"total_assets_including_real": len(assets) + len(real_assets),
|
||
"total_bytes_including_real": sum(a["size_bytes"] for a in assets)
|
||
+ sum(a["size_bytes"] for a in real_assets),
|
||
},
|
||
"by_category": dict(sorted(by_category.items())),
|
||
"by_format": dict(sorted(by_format.items())),
|
||
"missing": missing,
|
||
"real_corpus": real_corpus,
|
||
"licensing_summary": {
|
||
"synthetic_fixtures": "自产,随仓库许可(无第三方再分发限制)",
|
||
"occt_regression_model": "OCCT LGPL-2.1 with OCCT exception;再分发保留声明",
|
||
"qcad_example": "许可待法务复核(license_review_required=true)",
|
||
"real_corpus_assets": "项目内部资产(自研/内部导出),redistributable=false,"
|
||
"仅内部测试;逐资产判定见 real_corpus.assets[].license",
|
||
"unknown": "禁止入库;见 real_corpus.license_policy",
|
||
},
|
||
"assets": assets,
|
||
}
|
||
|
||
|
||
def write_csv(manifest: dict, csv_path: Path) -> None:
|
||
rows = list(manifest["assets"]) + list(manifest["real_corpus"]["assets"])
|
||
with csv_path.open("w", encoding="utf-8", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=CSV_COLUMNS)
|
||
w.writeheader()
|
||
for a in rows:
|
||
row = {k: a.get(k, "") for k in CSV_COLUMNS}
|
||
row["synthetic"] = "true" if row["synthetic"] is True else (
|
||
"false" if row["synthetic"] is False else ""
|
||
)
|
||
row["present"] = "true" if row["present"] else "false"
|
||
row["license_review_required"] = "true" if row["license_review_required"] else "false"
|
||
row["redistributable"] = "" if a.get("redistributable") is None else (
|
||
"true" if a.get("redistributable") else "false"
|
||
)
|
||
w.writerow(row)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--repo-root", default=None, help="仓库根目录(默认脚本上级目录)")
|
||
ap.add_argument("--output", default=None, help="manifest.json 输出路径")
|
||
ap.add_argument("--csv", default=None, help="manifest.csv 输出路径")
|
||
ap.add_argument("--check", action="store_true", help="只校验:不写文件,比对现有清单")
|
||
args = ap.parse_args(argv)
|
||
|
||
script_dir = Path(__file__).resolve().parent
|
||
repo_root = Path(args.repo_root).resolve() if args.repo_root else script_dir.parent
|
||
out_json = Path(args.output) if args.output else repo_root / "corpus" / "manifest.json"
|
||
out_csv = Path(args.csv) if args.csv else repo_root / "corpus" / "manifest.csv"
|
||
|
||
manifest = build_manifest(repo_root)
|
||
new_json = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
|
||
|
||
if args.check:
|
||
if not out_json.is_file():
|
||
print(f"[gen_manifest] MISSING {out_json}", file=sys.stderr)
|
||
return 1
|
||
old_json = out_json.read_text(encoding="utf-8")
|
||
# generated_from_git 记录“生成时的 HEAD”,提交后旧 HEAD 自然成为父提交,
|
||
# 因此 --check 只比对语料内容,不把 git HEAD 变动算作 stale(否则每次提交后必失败)。
|
||
try:
|
||
old_cmp = json.loads(old_json)
|
||
new_cmp = json.loads(new_json)
|
||
except json.JSONDecodeError as exc:
|
||
print(f"[gen_manifest] STALE {out_json}: 无法解析旧清单: {exc}", file=sys.stderr)
|
||
return 1
|
||
old_cmp.pop("generated_from_git", None)
|
||
new_cmp.pop("generated_from_git", None)
|
||
if json.dumps(old_cmp, ensure_ascii=False, sort_keys=True) == json.dumps(
|
||
new_cmp, ensure_ascii=False, sort_keys=True
|
||
):
|
||
print(
|
||
f"[gen_manifest] OK {out_json} ({manifest['counts']['assets_present']} assets; "
|
||
f"generated_from_git 忽略,informational)"
|
||
)
|
||
return 0
|
||
print(f"[gen_manifest] STALE {out_json}: 请重新生成并 review diff", file=sys.stderr)
|
||
return 1
|
||
|
||
out_json.parent.mkdir(parents=True, exist_ok=True)
|
||
out_json.write_text(new_json, encoding="utf-8")
|
||
write_csv(manifest, out_csv)
|
||
c = manifest["counts"]
|
||
print(
|
||
f"[gen_manifest] wrote {out_json} + {out_csv}\n"
|
||
f" assets={c['assets_present']} synthetic={c['synthetic_assets']} "
|
||
f"non_synthetic={c['non_synthetic_assets']} bytes={c['total_bytes']} "
|
||
f"missing={c['assets_missing']} license_review_required={c['license_review_required']}\n"
|
||
f" real_assets={c['real_assets']} real_bytes={c['real_bytes']} "
|
||
f"real_license_review_required={c['real_license_review_required']} "
|
||
f"real_status={manifest['real_corpus']['status']}"
|
||
)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|