release: 发布 v0.6.2 DOCX 真实文档修复
新增能力：将 DOCX 发布验收拆分为四套独立 140，支持真实语料冻结、指纹复用、失败与基础设施错误独立统计，并为表格换行、全 JSON 围栏、代码连续性和长文档分页建立通用门禁。 问题修复：冻结 Paged.js 分片前的逻辑表格列轨并传递打印几何，统一 Markdown 表格换行、代码、段落与 OOXML 翻译；改进 PDF 文本流排序、语义块映射、颜色与栅格比较，消除窄字符重叠和跨行范围符号误报。 兼容与部署：版本统一为 0.6.2；正式 Docker 镜像内置固定 Chromium、Pandoc 3.9.0.2 和 Serif/Sans/Mono 字体；Desktop NSIS 与 ZIP 继续直接内置字体，无需系统字体安装。 验证结果：合成基线与长庆严格 280/280，M4N 140/140；健康数据残余误报 6/115（5.22%），均核查为重复表头自动对齐/取样误报且基础设施错误为 0。全项目测试、类型检查、生产构建和 git diff --check 通过；正式 Docker、NSIS、ZIP、离线镜像、部署包、清单及 SHA-256 均已生成并校验。
This commit is contained in:
SkyJourney
@@ -7,7 +7,7 @@ import type {
|
||||
VisualPageSemanticExpectation,
|
||||
} from "./types.js";
|
||||
|
||||
const ZERO_WIDTH_AND_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f\u200b-\u200d\u2060\ufeff]/gu;
|
||||
const ZERO_WIDTH_AND_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f\u200b-\u200d\u2060\ufe0e\ufe0f\ufeff]/gu;
|
||||
const CJK_RADICAL_VARIANTS = new Map([
|
||||
["⺠", "民"],
|
||||
["⻅", "见"],
|
||||
@@ -16,6 +16,10 @@ const CJK_RADICAL_VARIANTS = new Map([
|
||||
["⻔", "门"],
|
||||
["⻚", "页"],
|
||||
["⻛", "风"],
|
||||
["⻝", "食"],
|
||||
["⻣", "骨"],
|
||||
["⻋", "车"],
|
||||
["⻬", "齐"],
|
||||
]);
|
||||
const PAGE_NUMBER_PATTERNS = [
|
||||
/^(?:[-—–]\s*)?\d+(?:\s*[/／]\s*\d+)?(?:\s*[-—–])?$/u,
|
||||
@@ -30,6 +34,16 @@ export function normalizePdfText(value: string): string {
|
||||
.replace(/[\u2e80-\u2eff]/gu, (character) =>
|
||||
CJK_RADICAL_VARIANTS.get(character) ?? character
|
||||
)
|
||||
// Chromium、Word 与 WPS 的 PDF 文本层会以不同方式保留弯引号；
|
||||
// 统一为 ASCII 只用于可编辑文本定位，不改变栅格视觉比较。
|
||||
.replace(/[“”]/gu, '"')
|
||||
.replace(/[‘’]/gu, "'")
|
||||
// Chromium 的部分 CJK 字体把 Markdown em dash 提取为 horizontal bar；
|
||||
// 二者在此仅作为语义定位字符归一，不影响后续栅格视觉比较。
|
||||
.replace(/―/gu, "—")
|
||||
// Chromium 的部分 CJK 字体会把数字区间中的 en dash 提取为两个
|
||||
// ASCII hyphen。只归一数字/百分比区间，避免改写代码中的普通 `--`。
|
||||
.replace(/(?<=[\d%])\s*--(?:\s*(?=\d)|\s*$)/gu, "–")
|
||||
.replace(ZERO_WIDTH_AND_CONTROL, "")
|
||||
.replace(/\u00a0/gu, " ")
|
||||
.replace(/[ \t]+/gu, " ")
|
||||
@@ -133,6 +147,63 @@ function median(values: readonly number[]): number {
|
||||
return ((sorted[middle - 1] ?? value) + value) / 2;
|
||||
}
|
||||
|
||||
function mergeInlineScriptGroups(
|
||||
sourceGroups: readonly PdfTextItemSnapshot[][],
|
||||
typicalHeight: number,
|
||||
): PdfTextItemSnapshot[][] {
|
||||
const groups = sourceGroups.map((group) => [...group]);
|
||||
for (let index = 0; index < groups.length; index += 1) {
|
||||
const group = groups[index];
|
||||
if (!group || group.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const groupHeight = median(group.map((item) => item.bounds.height));
|
||||
if (groupHeight > typicalHeight * 0.9) {
|
||||
continue;
|
||||
}
|
||||
const groupBounds = unionBounds(group);
|
||||
for (const adjacentIndex of [index - 1, index + 1]) {
|
||||
const adjacent = groups[adjacentIndex];
|
||||
if (!adjacent || adjacent.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const adjacentHeight = median(
|
||||
adjacent.map((item) => item.bounds.height),
|
||||
);
|
||||
if (adjacentHeight < groupHeight / 0.9) {
|
||||
continue;
|
||||
}
|
||||
const adjacentBounds = unionBounds(adjacent);
|
||||
const verticalOverlap = Math.min(
|
||||
groupBounds.y + groupBounds.height,
|
||||
adjacentBounds.y + adjacentBounds.height,
|
||||
) - Math.max(groupBounds.y, adjacentBounds.y);
|
||||
const horizontallyContained =
|
||||
groupBounds.x >= adjacentBounds.x - typicalHeight * 0.5 &&
|
||||
groupBounds.x + groupBounds.width <=
|
||||
adjacentBounds.x + adjacentBounds.width + typicalHeight * 0.5;
|
||||
// Chromium 会把行末公式按较小字号单独提取；公式片段可能从正文
|
||||
// 最后一个字符的右缘开始，并向右超出正文组，因此不能只用“被正文
|
||||
// 水平包含”判断同行。仅在两组真实垂直重叠且小字号组紧贴较大字号
|
||||
// 组右缘时合并，避免把下一视觉行或相邻表格单元格误并入当前行。
|
||||
const adjacentRight = adjacentBounds.x + adjacentBounds.width;
|
||||
const touchesAdjacentRight =
|
||||
groupBounds.x >= adjacentBounds.x &&
|
||||
Math.abs(groupBounds.x - adjacentRight) <= typicalHeight * 0.5;
|
||||
if (
|
||||
verticalOverlap <= 0 ||
|
||||
(!horizontallyContained && !touchesAdjacentRight)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
adjacent.push(...group);
|
||||
groups[index] = [];
|
||||
break;
|
||||
}
|
||||
}
|
||||
return groups.filter((group) => group.length > 0);
|
||||
}
|
||||
|
||||
function joinLineItems(items: readonly PdfTextItemSnapshot[]): string {
|
||||
let result = "";
|
||||
let previous: PdfTextItemSnapshot | undefined;
|
||||
@@ -181,6 +252,9 @@ export function aggregatePdfTextLines(
|
||||
sourceItems: readonly PdfTextItemSnapshot[],
|
||||
pageHeightPt: number,
|
||||
): PdfTextLineSnapshot[] {
|
||||
const sourceOrder = new Map(
|
||||
sourceItems.map((item, index) => [item, index] as const),
|
||||
);
|
||||
const items = sourceItems
|
||||
.filter((item) => item.normalizedText.length > 0)
|
||||
.sort(
|
||||
@@ -188,11 +262,14 @@ export function aggregatePdfTextLines(
|
||||
left.bounds.y - right.bounds.y || left.bounds.x - right.bounds.x,
|
||||
);
|
||||
const typicalHeight = median(items.map((item) => item.bounds.height));
|
||||
const baselineTolerance = Math.max(1.5, typicalHeight * 0.15);
|
||||
const groups: PdfTextItemSnapshot[][] = [];
|
||||
// Office/WPS 会让同一视觉行内不同字体或单元格的基线产生约 2pt
|
||||
// 浮动；按 15% 聚合会把这些片段错误拆行并改变阅读顺序。
|
||||
// 20% 仍远小于正常行距，同时能覆盖常见的 10–12pt 字号偏差。
|
||||
const baselineTolerance = Math.max(2, typicalHeight * 0.2);
|
||||
const initialGroups: PdfTextItemSnapshot[][] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const lastGroup = groups.at(-1);
|
||||
const lastGroup = initialGroups.at(-1);
|
||||
const lastBaseline = lastGroup
|
||||
? median(lastGroup.map((entry) => entry.baselineY))
|
||||
: undefined;
|
||||
@@ -203,13 +280,27 @@ export function aggregatePdfTextLines(
|
||||
) {
|
||||
lastGroup.push(item);
|
||||
} else {
|
||||
groups.push([item]);
|
||||
initialGroups.push([item]);
|
||||
}
|
||||
}
|
||||
|
||||
const groups = mergeInlineScriptGroups(initialGroups, typicalHeight);
|
||||
|
||||
return groups.map((group) => {
|
||||
const sortedItems = [...group].sort(
|
||||
(left, right) => left.bounds.x - right.bounds.x,
|
||||
(left, right) => {
|
||||
const leftRight = left.bounds.x + left.bounds.width;
|
||||
const rightRight = right.bounds.x + right.bounds.width;
|
||||
const horizontallyOverlapping =
|
||||
left.bounds.x < rightRight && right.bounds.x < leftRight;
|
||||
// PDF.js 的文本流保留字符阅读顺序，但不同字体的窄标点可能与
|
||||
// 相邻全角字符发生水平重叠。此时单纯按 x 排序会把闭引号移到
|
||||
// 左括号之后，破坏语义块定位；只对真实重叠片段保留文本流顺序，
|
||||
// 其余片段仍按几何位置排序，兼容 Office 表格等非阅读序文本流。
|
||||
return horizontallyOverlapping
|
||||
? (sourceOrder.get(left) ?? 0) - (sourceOrder.get(right) ?? 0)
|
||||
: left.bounds.x - right.bounds.x;
|
||||
},
|
||||
);
|
||||
const bounds = unionBounds(sortedItems);
|
||||
const normalizedText = joinLineItems(sortedItems);
|
||||
|
||||
Reference in New Issue
Block a user