From 1eee1c0c6acab6ea681194a5f5fb497b3ba63df3 Mon Sep 17 00:00:00 2001 From: null Date: Wed, 9 Sep 2026 02:41:46 +0800 Subject: [PATCH] feat: reuse component IDs and preserve XML line breaks --- README.md | 49 +++ package.json | 2 +- src/constants.ts | 1 + src/convert.ts | 13 +- src/id-reuse.ts | 358 ++++++++++++++++++++ src/identity.ts | 26 ++ src/ids.ts | 69 +++- src/index.ts | 1 + src/jsonFormat.ts | 158 ++++++--- src/model.ts | 18 +- src/xmlFormat.ts | 65 +++- tests/fixtures/laevat-basic-record.json | 4 + tests/id-reuse.test.ts | 412 ++++++++++++++++++++++++ tests/newlines.test.ts | 106 ++++++ 14 files changed, 1209 insertions(+), 73 deletions(-) create mode 100644 src/id-reuse.ts create mode 100644 src/identity.ts create mode 100644 tests/fixtures/laevat-basic-record.json create mode 100644 tests/id-reuse.test.ts create mode 100644 tests/newlines.test.ts diff --git a/README.md b/README.md index b862b57..6eeb2b5 100644 --- a/README.md +++ b/README.md @@ -157,3 +157,52 @@ interface WikiJsonToXmlBatchResult { ## 许可证 MIT + +### Reuse component IDs when editing + +Starting with 0.6.0, JSON renderers accept an optional `referenceJson`: + +```ts +import { submitJsonToXml, xmlToSubmitJson } from '@eihrteam/xml' + +const xml = submitJsonToXml(originalSubmitJson).text +const result = xmlToSubmitJson(editedXml || xml, { + referenceJson: originalSubmitJson, +}) +``` + +The reference can be a JSON string or object: a submit envelope (`{ item, +commitMsg }`), a Wiki response (`{ data: { item } }`), or a bare item. Its `itemId` +must match the item being rendered. Invalid references throw conversion errors. +The option is also available on `renderSubmitJson`, `renderWikiJson`, +`documentToWikiJsonObject`, `xmlToWikiJson`, `convert` with a Wiki JSON target, +and the direct JSON-to-JSON conversion helpers. `RenderJsonOptions` describes +shared rendering options; `RenderWikiJsonOptions` adds Wiki envelope options. + +Matching preserves widget, tab, document, block, list item, table row/column, +audio and video element identities. Exact content is matched first, followed by +unique local titles/resources, then compatible positions between matched +siblings. Unique unchanged content can move between parents; repeated content +is matched in order without reusing one identity for multiple copies. Table +cells follow the matched row and column identities. In ambiguous cases, new +IDs are generated. Resource and business IDs retain their existing semantics. + +XML stays unchanged and carries no hidden identity attributes. Empty blocks +removed by existing normalization do not consume other blocks' identities. +This is not a lossless JSON round trip: formatting and existing normalization +of non-ID fields are unchanged. Without `referenceJson`, IDs are generated as +before. Editors should cache successful conversions and use their latest +successful JSON as the next reference so newly inserted components keep their +IDs between diff, format conversion and submission. Failed conversions must +not replace that reference. + +### Explicit line breaks + +Text content containing LF, CRLF or CR is rendered with one closed `

` +per newline, including consecutive and leading/trailing breaks. The parser +accepts both `

` and `
` and restores inline `\n` characters, keeping +the containing paragraph and its component identity intact. Formatting wrappers, +links, pronunciations, table content, image descriptions and audio profiles +preserve these explicit breaks. Existing XML layout newlines continue to separate +paragraphs. Literal text such as `
` is escaped normally; a `br` element with +content is rejected. diff --git a/package.json b/package.json index 24210a8..be6b527 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@eihrteam/xml", - "version": "0.5.0", + "version": "0.6.0", "description": "Convert SKLand Endfield Wiki public item JSON and XML locally.", "license": "MIT", "type": "module", diff --git a/src/constants.ts b/src/constants.ts index e66f06d..c032d8d 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -17,6 +17,7 @@ export const INLINE_TAGS: ReadonlySet = new Set([ 'pron', 'a', 'entry', + 'br', ]) export const BASE_BLOCK_TAGS: ReadonlySet = new Set([ 'h1', diff --git a/src/convert.ts b/src/convert.ts index a888a73..e2c9e3d 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -5,6 +5,7 @@ import { parseSubmitJson, renderSubmitJson, type RenderWikiJsonOptions, + type RenderJsonOptions, } from './jsonFormat.js' import { documentFromXmlText, documentToXmlText } from './xmlFormat.js' import type { @@ -125,17 +126,17 @@ export function submitJsonToXml(json: string | object): ConversionResult { return { text: documentToXmlText(document), warnings } } -export function xmlToSubmitJson(xml: string): ConversionResult { +export function xmlToSubmitJson(xml: string, options: RenderJsonOptions = {}): ConversionResult { const [document, warnings] = documentFromXmlText(xml) - return { text: renderSubmitJson(document), warnings } + return { text: renderSubmitJson(document, options), warnings } } -export function wikiJsonToSubmitJson(json: string | object): ConversionResult { +export function wikiJsonToSubmitJson(json: string | object, options: RenderJsonOptions = {}): ConversionResult { const [document, warnings] = documentFromJsonText(json) - return { text: renderSubmitJson(document), warnings } + return { text: renderSubmitJson(document, options), warnings } } -export function submitJsonToWikiJson(json: string | object): ConversionResult { +export function submitJsonToWikiJson(json: string | object, options: RenderWikiJsonOptions = {}): ConversionResult { const [document, warnings] = parseSubmitJson(json) - return { text: documentToJsonText(document), warnings } + return { text: documentToJsonText(document, options), warnings } } diff --git a/src/id-reuse.ts b/src/id-reuse.ts new file mode 100644 index 0000000..818174b --- /dev/null +++ b/src/id-reuse.ts @@ -0,0 +1,358 @@ +import { IdFactory } from './ids.js' +import { identity, withIdentity, type Identity } from './identity.js' +import { normalizeBlocks, type Block, type DocumentModel, type ImageIntro } from './model.js' + +interface Node { + object: object + type: string + keys: string[] + children: Record + signature: string + movable: boolean +} + +function canonicalValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalValue) + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => [key, canonicalValue(entry)]), + ) + } + return value +} + +interface Tree { + root: Node + tables: Array<{ block: Block; rows: Node[]; columns: Node[]; cells: Node[] }> +} + +function tree( + document: DocumentModel, + introBlocks: (intro: ImageIntro) => Block[], + source: boolean, + signatures: Map, +): Tree { + // Intern structural signatures instead of embedding escaped subtree strings. + // The index is shared by the two trees and released after this conversion. + function node( + object: object, + type: string, + value: unknown, + children: Record = {}, + keys: string[] = [], + movable = true, + ): Node { + const key = JSON.stringify([ + type, + canonicalValue(value), + Object.entries(children).map(([role, nodes]) => [role, nodes.map((entry) => entry.signature)]), + ]) + let signature = signatures.get(key) + if (signature === undefined) { + signature = String(signatures.size) + signatures.set(key, signature) + } + return { object, type, children, keys: keys.filter(Boolean), movable, signature } + } + const tables: Tree['tables'] = [] + const blocks = (items: Block[]): Node[] => { + const normalized = normalizeBlocks(items) + items.splice(0, items.length, ...normalized) + return items.map(blockNode) + } + const documentNode = (items: Block[]): Node => node(items, 'document', null, { blocks: blocks(items) }, [], false) + function blockNode(block: Block): Node { + switch (block.blockType) { + case 'paragraph': + return node(block, 'paragraph', [block.kind, block.align, block.inlines]) + case 'quote': + return node(block, 'quote', null, { blocks: blocks(block.children) }) + case 'list': + return node(block, 'list', block.ordered, { + items: block.items.map((item) => node(item, 'listItem', null, { blocks: blocks(item.blocks) })), + }) + case 'complexTable': { + const ids = identity(block) + const cells = block.cells.map((cell) => + node(cell, 'cell', [cell.rowSpan, cell.colSpan], { blocks: blocks(cell.blocks) }, [], false), + ) + const rows = Array.from({ length: block.rowCount }, (_, index) => { + const object = withIdentity({}, ids.rowIds?.[index] ? { id: ids.rowIds[index] } : {}) + return node( + object, + 'row', + cells.filter((_, i) => block.cells[i]!.rowIndex === index).map((cell) => cell.signature), + {}, + [], + false, + ) + }) + const columns = Array.from({ length: block.columnCount }, (_, index) => { + const object = withIdentity({}, ids.columnIds?.[index] ? { id: ids.columnIds[index] } : {}) + return node( + object, + 'column', + [ + block.columnWidths[index], + cells.filter((_, i) => block.cells[i]!.columnIndex === index).map((cell) => cell.signature), + ], + {}, + [], + false, + ) + }) + tables.push({ block, rows, columns, cells }) + return node(block, 'complexTable', block.headerMode, { rows, columns, cells }) + } + case 'image': + return node(block, 'image', block, {}, [`resource:${block.imageId}`, `url:${block.url}`]) + case 'externalVideo': + return node(block, 'externalVideo', block, {}, [`video:${block.videoKind}:${block.videoId}`]) + case 'horizontalLine': + return node(block, 'horizontalLine', block.kind) + } + } + const audioNodes = (items: DocumentModel['chapterGroups'][number]['chapters'][number]['audios']): Node[] => + items.map((audio) => + node(audio, 'audio', audio, {}, [ + audio.title && `title:${audio.title}`, + audio.resourceUrl && `url:${audio.resourceUrl}`, + ]), + ) + const root = node( + document, + 'root', + null, + { + description: [documentNode(document.description)], + groups: document.chapterGroups.map((group) => + node( + group, + 'group', + group.title, + { + chapters: group.chapters.map((chapter) => + node( + chapter, + `chapter:${chapter.chapterType}`, + [chapter.title, chapter.size, chapter.tableRows], + { + content: [documentNode(chapter.content)], + audios: audioNodes(chapter.audios), + tabs: (chapter.audioTabs ?? chapter.tabs).map((tab) => { + if ('audios' in tab) + return node( + tab, + 'audioTab', + [tab.title ?? '', tab.icon ?? ''], + { audios: audioNodes(tab.audios) }, + [tab.title ? `title:${tab.title}` : ''], + ) + const intro = tab.intro + const description = intro + ? source + ? (identity(intro).introBlocks ?? introBlocks(intro)) + : introBlocks(intro) + : null + return node( + tab, + 'tab', + [tab.title ?? '', tab.icon ?? '', intro && [intro.name, intro.introType, intro.imageUrl]], + { + content: [documentNode(tab.content)], + intro: description ? [documentNode(description)] : [], + }, + [tab.title ? `title:${tab.title}` : ''], + ) + }), + }, + [chapter.title && `title:${chapter.title}`], + ), + ), + }, + [group.title && `title:${group.title}`], + ), + ), + }, + [], + false, + ) + return { root, tables } +} + +function flatten(root: Node): Node[] { + return [root, ...Object.values(root.children).flatMap((children) => children.flatMap(flatten))] +} + +function buckets(nodes: Node[], key: (node: Node) => string): Map { + const result = new Map() + for (const entry of nodes) { + const value = key(entry) + const list = result.get(value) ?? [] + list.push(entry) + result.set(value, list) + } + return result +} + +/** Match identities before writing any JSON references. No string-wide ID replacement. */ +export function prepareIdReuse( + document: DocumentModel, + reference: DocumentModel, + introBlocks: (intro: ImageIntro) => Block[], + reserved: Set = new Set(), +): IdFactory { + // Work with normalized model nodes so discarded empty blocks cannot consume IDs. + const signatures = new Map() + const next = tree(document, introBlocks, false, signatures) + const old = tree(reference, introBlocks, true, signatures) + const matches = new Map() + const used = new Set() + const nextNodes = flatten(next.root) + const oldNodes = flatten(old.root) + const oldGlobal = buckets( + oldNodes.filter((n) => n.movable), + (n) => n.signature, + ) + const nextGlobal = buckets( + nextNodes.filter((n) => n.movable), + (n) => n.signature, + ) + + function pair(current: Node, previous: Node): boolean { + if (matches.has(current) || used.has(previous) || current.type !== previous.type) return false + matches.set(current, previous) + used.add(previous) + return true + } + + function matchSiblings(current: Node[], previous: Node[]): void { + const available = () => current.filter((n) => !matches.has(n)) + const remaining = () => previous.filter((n) => !used.has(n)) + // Unique exact content first; then unique title/resource keys within this parent. + function unique(key: (n: Node) => string): void { + const a = buckets(available(), key) + const b = buckets(remaining(), key) + for (const [value, entries] of a) { + const candidates = b.get(value) + if (entries.length === 1 && candidates?.length === 1) pair(entries[0]!, candidates[0]!) + } + } + unique((n) => n.signature) + const keys = new Set(available().flatMap((n) => n.keys)) + for (const key of keys) { + const a = available().filter((n) => n.keys.includes(key)) + const b = remaining().filter((n) => n.keys.includes(key)) + if (a.length === 1 && b.length === 1) pair(a[0]!, b[0]!) + } + // Global matches are restricted to unique, unchanged semantic subtrees. + for (const entry of available()) { + const candidates = oldGlobal.get(entry.signature) + if (entry.movable && nextGlobal.get(entry.signature)?.length === 1 && candidates?.length === 1) + pair(entry, candidates[0]!) + } + // Ordered duplicate content (LCS) avoids assigning one identity to copies. + const a = available() + const b = remaining() + const lengths = Array.from({ length: a.length + 1 }, () => new Uint32Array(b.length + 1)) + for (let i = a.length - 1; i >= 0; i--) { + for (let j = b.length - 1; j >= 0; j--) { + lengths[i]![j] = + a[i]!.signature === b[j]!.signature + ? lengths[i + 1]![j + 1]! + 1 + : Math.max(lengths[i + 1]![j]!, lengths[i]![j + 1]!) + } + } + let i = 0 + let j = 0 + while (i < a.length && j < b.length) { + if (a[i]!.signature === b[j]!.signature) { + pair(a[i++]!, b[j++]!) + } else if (lengths[i + 1]![j]! > lengths[i]![j + 1]!) i++ + else j++ + } + // Use monotonic matched siblings as anchors. Only equal, compatible gaps align. + const anchors: Array<[number, number]> = [[-1, -1]] + let lastOldIndex = -1 + current.forEach((entry, index) => { + const match = matches.get(entry) + const oldIndex = match ? previous.indexOf(match) : -1 + if (oldIndex > lastOldIndex) { + anchors.push([index, oldIndex]) + lastOldIndex = oldIndex + } + }) + anchors.push([current.length, previous.length]) + for (let index = 1; index < anchors.length; index++) { + const [startA, startB] = anchors[index - 1]! + const [endA, endB] = anchors[index]! + const gapA = current.slice(startA + 1, endA).filter((n) => !matches.has(n)) + const gapB = previous.slice(startB + 1, endB).filter((n) => !used.has(n)) + if (gapA.length === gapB.length && gapA.every((n, k) => n.type === gapB[k]!.type)) { + gapA.forEach((n, k) => pair(n, gapB[k]!)) + } + } + } + + pair(next.root, old.root) + const visited = new Set() + function descend(current: Node): void { + if (visited.has(current)) return + visited.add(current) + const previous = matches.get(current) + for (const [role, children] of Object.entries(current.children)) { + // Table cells are aligned by matched row AND column, below, never by flat index. + if (current.type === 'complexTable' && role === 'cells') continue + matchSiblings(children, previous?.children[role] ?? []) + children.forEach(descend) + } + if (current.type === 'complexTable') { + const currentTable = next.tables.find((table) => table.block === current.object)! + const oldTable = old.tables.find((table) => table.block === previous?.object) + if (oldTable && currentTable.block.blockType === 'complexTable' && oldTable.block.blockType === 'complexTable') { + const previousBlock = oldTable.block + currentTable.cells.forEach((cellNode, index) => { + const cell = currentTable.block.blockType === 'complexTable' ? currentTable.block.cells[index]! : null + if (!cell) return + const row = oldTable.rows.indexOf(matches.get(currentTable.rows[cell.rowIndex]!)!) + const col = oldTable.columns.indexOf(matches.get(currentTable.columns[cell.columnIndex]!)!) + const oldIndex = previousBlock.cells.findIndex((c) => c.rowIndex === row && c.columnIndex === col) + if (oldIndex >= 0) pair(cellNode, oldTable.cells[oldIndex]!) + }) + } + currentTable.cells.forEach(descend) + } + } + descend(next.root) + + for (const entry of oldNodes) { + const ids = identity(entry.object) + for (const value of [ids.id, ids.documentId, ids.elementId, ...(ids.rowIds ?? []), ...(ids.columnIds ?? [])]) { + if (value) reserved.add(value) + } + } + // Business video IDs are also block-map keys; fresh IDs must not collide with them. + for (const entry of nextNodes) { + if (entry.type === 'externalVideo') reserved.add((entry.object as { videoId: string }).videoId) + } + for (const [entry, previous] of matches) { + const ids = identity(previous.object) + // Intro caches and table vectors refer to source nodes; assign those separately. + const selected: Identity = {} + if (ids.id) selected.id = ids.id + if (ids.documentId) selected.documentId = ids.documentId + if (ids.elementId) selected.elementId = ids.elementId + withIdentity(entry.object, selected) + } + const factory = new IdFactory(reserved, true) + for (const table of next.tables) { + withIdentity(table.block, { + ...identity(table.block), + rowIds: table.rows.map((row) => identity(row.object).id ?? factory.itemId()), + columnIds: table.columns.map((col) => identity(col.object).id ?? factory.itemId()), + }) + } + return factory +} diff --git a/src/identity.ts b/src/identity.ts new file mode 100644 index 0000000..4815953 --- /dev/null +++ b/src/identity.ts @@ -0,0 +1,26 @@ +/** Private identity sidecar: never serialized into XML or public model fields. */ +export interface Identity { + id?: string + documentId?: string + elementId?: string + rowIds?: string[] + columnIds?: string[] + introBlocks?: import('./model.js').Block[] +} + +const identities = new WeakMap() + +export function identity(value: object): Identity { + return identities.get(value) ?? {} +} + +export function withIdentity(value: T, ids: Identity): T { + identities.set(value, ids) + return value +} + +export function copyIdentity(source: object, target: T): T { + const ids = identities.get(source) + if (ids) identities.set(target, ids) + return target +} diff --git a/src/ids.ts b/src/ids.ts index 82301c9..8563239 100644 --- a/src/ids.ts +++ b/src/ids.ts @@ -1,14 +1,69 @@ +import { identity, withIdentity } from './identity.js' +import { XmlWikiConversionError } from './model.js' + export class IdFactory { private readonly alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' - private token(length: number) { + private readonly reserved: Set + private readonly preserve: boolean + + constructor(reserved: Set = new Set(), preserve = false) { + this.reserved = reserved + this.preserve = preserve + } + + get preservesIds(): boolean { + return this.preserve + } + + private readonly claimed = new Map>() + private documentScope: object = {} + + beginDocument(blocks: object): void { + this.documentScope = blocks + } + + reuse( + value: object, + key: 'id' | 'documentId' | 'elementId', + create: () => string, + scope: object | string = this.documentScope, + ): string { + const candidate = this.preserve ? identity(value)[key] : undefined + if (!candidate) return create() + const used = this.claimed.get(scope) ?? new Set() + if (used.has(candidate)) return create() + used.add(candidate) + this.claimed.set(scope, used) + return candidate + } + + tableId(value: object, key: 'rowIds' | 'columnIds', index: number): string { + const ids = this.preserve ? identity(value)[key] : undefined + const id = ids?.[index] + return this.reuse(withIdentity({}, id ? { id } : {}), 'id', () => this.itemId(), ids ?? value) + } + + fixedBlockId(id: string): string { + const used = this.claimed.get(this.documentScope) ?? new Set() + if (used.has(id)) throw new XmlWikiConversionError(`Duplicate fixed video block ID '${id}'.`) + used.add(id) + this.claimed.set(this.documentScope, used) + return id + } + + private token(length: number, prefix = ''): string { const bytes = new Uint8Array(length) - crypto.getRandomValues(bytes) - let value = '' - for (let i = 0; i < length; i += 1) { - value += this.alphabet[bytes[i]! % this.alphabet.length] + for (;;) { + crypto.getRandomValues(bytes) + let value = prefix + for (let i = 0; i < length; i += 1) { + value += this.alphabet[bytes[i]! % this.alphabet.length] + } + if (this.reserved.has(value)) continue + this.reserved.add(value) + return value } - return value } widgetId(): string { @@ -24,7 +79,7 @@ export class IdFactory { } tabId(): string { - return `tab_${this.token(12)}` + return this.token(12, 'tab_') } audioId(): string { diff --git a/src/index.ts b/src/index.ts index a2dec50..fa55fb7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -54,6 +54,7 @@ export { parseSubmitJson, renderSubmitJson, type RenderWikiJsonOptions, + type RenderJsonOptions, } from './jsonFormat.js' export { documentFromXmlText as parseXmlWithWarnings, diff --git a/src/jsonFormat.ts b/src/jsonFormat.ts index 5f93179..ef27314 100644 --- a/src/jsonFormat.ts +++ b/src/jsonFormat.ts @@ -1,3 +1,5 @@ +import { identity, withIdentity } from './identity.js' +import { prepareIdReuse } from './id-reuse.js' import { DEFAULT_JSON_TEXT_COLOR, JSON_TO_XML_COLOR, XML_TO_JSON_COLOR } from './colors.js' import { JSON_ENTRY_TO_XML, @@ -39,7 +41,11 @@ import { validateComplexTableBlock, } from './model.js' -export interface RenderWikiJsonOptions { +export interface RenderJsonOptions { + referenceJson?: string | object +} + +export interface RenderWikiJsonOptions extends RenderJsonOptions { wrapInfoRoot?: boolean envelope?: Record } @@ -73,11 +79,11 @@ function audioItemsFromList(raw: unknown[], location: string): AudioItem[] { return raw.map((entry, index) => { const entryLocation = `${location}[${index}]` const entryMap = ensureMapping(entry, entryLocation) - return { + return withIdentity({ title: requireString(entryMap, `${entryLocation}.title`, 'title'), profile: requireString(entryMap, `${entryLocation}.profile`, 'profile'), resourceUrl: requireString(entryMap, `${entryLocation}.resourceUrl`, 'resourceUrl'), - } + }, typeof entryMap.id === 'string' ? { id: entryMap.id } : {}) }) } @@ -263,20 +269,49 @@ export function parseSubmitJson(source: string | object): [DocumentModel, string return [buildDocumentFromItem(parsed.item, {}, commitMsg), []] } +function reservedJsonIds(value: unknown, ids = new Set()): Set { + if (Array.isArray(value)) { + for (const entry of value) reservedJsonIds(entry, ids) + } else if (isRecord(value)) { + for (const [key, entry] of Object.entries(value)) { + if (['id', 'parentId', 'tabId', 'elementId'].includes(key) && typeof entry === 'string') ids.add(entry) + if (['blockIds', 'childIds', 'itemIds', 'rowIds', 'columnIds'].includes(key) && Array.isArray(entry)) { + for (const id of entry) if (typeof id === 'string') ids.add(id) + } + if (['widgetCommonMap', 'documentMap', 'blockMap', 'itemMap', 'tabDataMap', 'rowMap', 'columnMap', 'cellMap'].includes(key) && isRecord(entry)) { + for (const id of Object.keys(entry)) ids.add(id) + } + reservedJsonIds(entry, ids) + } + } + return ids +} + function buildWikiJsonObject( document: DocumentModel, options: RenderWikiJsonOptions, target: JsonRenderTarget, ) { - const factory = new IdFactory() - const widgetCommonMap: Record = {} - const documentMap: Record = {} + let factory = new IdFactory() + if (options.referenceJson !== undefined) { + const parsed = ensureMapping(parseJsonInput(options.referenceJson), 'referenceJson') + const reference = isRecord(parsed.item) + ? parseSubmitJson(parsed)[0] + : documentFromJsonText(parsed)[0] + if (reference.itemId !== document.itemId) { + throw new EndfieldWikitextConversionError('referenceJson itemId does not match the rendered itemId.') + } + document = structuredClone(document) + factory = prepareIdReuse(document, reference, (intro) => introDocumentBlocks(intro, true), reservedJsonIds(parsed)) + } + const widgetCommonMap: Record = Object.create(null) + const documentMap: Record = Object.create(null) const chapterGroups: Array> = [] for (const group of document.chapterGroups) { const widgets: Array> = [] for (const chapter of group.chapters) { - const widgetId = factory.widgetId() + const widgetId = factory.reuse(chapter, 'id', () => factory.widgetId(), 'widgets') widgets.push({ id: widgetId, title: chapter.title, size: chapter.size }) widgetCommonMap[widgetId] = chapterToJson(chapter, documentMap, factory, target) } @@ -353,8 +388,8 @@ export function documentToJsonText( return `${JSON.stringify(payload, null, 4)}\n` } -export function renderSubmitJson(document: DocumentModel): string { - const item = buildWikiJsonObject(document, {}, 'submit') +export function renderSubmitJson(document: DocumentModel, options: RenderJsonOptions = {}): string { + const item = buildWikiJsonObject(document, options, 'submit') const payload = { commitMsg: document.commitMsg ?? '', item, @@ -363,6 +398,21 @@ export function renderSubmitJson(document: DocumentModel): string { } function chapterFromJson( + widget: Record, + widgetCommonMap: Record, + documentMap: Record, +): Chapter { + const chapter = readChapterFromJson(widget, widgetCommonMap, documentMap) + const id = widget.id as string + const common = widgetCommonMap[id] as Record + const tabs = (common.tabList ?? []) as Array<{ tabId: string }> + ;(chapter.audioTabs ?? chapter.tabs).forEach((tab, index) => { + withIdentity(tab, { id: tabs[index]!.tabId }) + }) + return withIdentity(chapter, { id }) +} + +function readChapterFromJson( widget: Record, widgetCommonMap: Record, documentMap: Record @@ -487,7 +537,12 @@ function chapterFromJson( } if (descriptionId) { const descriptionDoc = ensureMapping(documentMap[descriptionId], `documentMap[${descriptionId}]`) - intro.description = blocksToInlines(blocksFromDocument(descriptionDoc)) + const introBlocks = withIdentity(blocksFromDocument(descriptionDoc), { + id: typeof descriptionDoc.id === 'string' ? descriptionDoc.id : 'document-id', + documentId: descriptionId, + }) + intro.description = blocksToInlines(introBlocks) + withIdentity(intro, { introBlocks }) } } @@ -537,20 +592,23 @@ function chapterContentFromRef( throw new EndfieldWikitextConversionError(`Expected string at '${chapterTitle}.${refName}.content'.`) } const document = ensureMapping(documentMap[documentId], `documentMap[${documentId}]`) - return blocksFromDocument(document) + return withIdentity(blocksFromDocument(document), { + id: typeof document.id === 'string' ? document.id : 'document-id', + documentId, + }) } function blocksFromDocument(document: Record) { const blockIds = requireList(document, 'document.blockIds', 'blockIds') const blockMap = requireMapping(document, 'document.blockMap', 'blockMap') - return normalizeBlocks( + return withIdentity(normalizeBlocks( blockIds.map((blockId) => { if (typeof blockId !== 'string') { throw new EndfieldWikitextConversionError('Expected block ids to be strings.') } return blockFromJson(blockId, blockMap, { allowTables: true }) }) - ) + ), { id: typeof document.id === 'string' ? document.id : 'document-id' }) } function blocksFromChildIds( @@ -569,6 +627,22 @@ function blocksFromChildIds( } function blockFromJson( + blockId: string, + blockMap: Record, + options: { allowTables: boolean }, +): Block { + const block = readBlockFromJson(blockId, blockMap, options) + const raw = blockMap[blockId] as Record + const table = raw.table as { rowIds: string[]; columnIds: string[] } | undefined + const video = raw.externalVideo as { elementId?: string } | undefined + return withIdentity(block, { + id: blockId, + ...(table ? { rowIds: [...table.rowIds], columnIds: [...table.columnIds] } : {}), + ...(video?.elementId ? { elementId: video.elementId } : {}), + }) +} + +function readBlockFromJson( blockId: string, blockMap: Record, options: { allowTables: boolean } @@ -609,9 +683,9 @@ function blockFromJson( } const item = ensureMapping(itemMap[itemId], `itemMap[${itemId}]`) const childIds = requireList(item, `item[${itemId}].childIds`, 'childIds') - items.push({ + items.push(withIdentity({ blocks: blocksFromChildIds(childIds, blockMap, options), - }) + }, { id: itemId })) } return { @@ -843,10 +917,10 @@ function chapterToJson( if (audioTabs.length) { const tabList: Array> = [] - const tabDataMap: Record = {} + const tabDataMap: Record = Object.create(null) for (const tab of audioTabs) { - const tabId = factory.tabId() + const tabId = factory.reuse(tab, 'id', () => factory.tabId(), chapter) tabList.push({ tabId, title: tab.title ?? '', @@ -887,10 +961,10 @@ function chapterToJson( if (chapter.tabs.length) { const tabList: Array> = [] - const tabDataMap: Record = {} + const tabDataMap: Record = Object.create(null) for (const tab of chapter.tabs) { - const tabId = factory.tabId() + const tabId = factory.reuse(tab, 'id', () => factory.tabId(), chapter) const tabDescriptor: Record = { tabId, title: tab.title ?? '', @@ -908,9 +982,7 @@ function chapterToJson( // Split the intro description inlines on `\n` so each line becomes // its own paragraph block — the wiki renders one block per line, // so a `\n` inside a single text run would be collapsed to a space. - const introBlocks = tab.intro.description.length - ? splitInlinesByNewline(tab.intro.description).map((line) => paragraph(line)) - : [] + const introBlocks = introDocumentBlocks(tab.intro, factory.preservesIds) tabPayload.intro = { name: tab.intro.name, @@ -959,7 +1031,7 @@ function audioItemsToJson( title: entry.title, profile: entry.profile, resourceUrl: entry.resourceUrl, - ...(target === 'submit' ? { id: factory.audioId() } : {}), + ...(target === 'submit' ? { id: factory.reuse(entry, 'id', () => factory.audioId(), audios) } : {}), })) } @@ -968,22 +1040,24 @@ function storeDocument( documentMap: Record, factory: IdFactory ): string { - const documentId = factory.widgetId() + const documentId = factory.reuse(blocks, 'documentId', () => factory.widgetId(), 'documents') documentMap[documentId] = buildDocumentPayload(blocks, factory) return documentId } function buildDocumentPayload(blocks: Block[], factory: IdFactory): Record { const blockIds: string[] = [] - const blockMap: Record = {} + const blockMap: Record = Object.create(null) + factory.beginDocument(blocks) + const rootId = factory.reuse(blocks, 'id', () => 'document-id') for (const block of normalizeBlocks(blocks)) { - const blockId = appendBlock(block, blockMap, 'document-id', factory) + const blockId = appendBlock(block, blockMap, rootId, factory) blockIds.push(blockId) } return { - id: 'document-id', + id: rootId, blockIds, blockMap, authorMap: {}, @@ -998,7 +1072,7 @@ function appendBlock( factory: IdFactory ): string { if (isExternalVideo(block)) { - const blockId = block.videoId + const blockId = factory.preservesIds ? factory.fixedBlockId(block.videoId) : block.videoId blockMap[blockId] = { kind: 'externalVideo', id: blockId, @@ -1006,7 +1080,7 @@ function appendBlock( externalVideo: { id: blockId, kind: block.videoKind, - elementId: factory.elementId(), + elementId: factory.reuse(block, 'elementId', () => factory.elementId(), 'elements'), type: 'external-video', children: [{ text: '' }], }, @@ -1014,7 +1088,7 @@ function appendBlock( return blockId } - const blockId = factory.blockId() + const blockId = factory.reuse(block, 'id', () => factory.blockId()) if (isParagraph(block)) { blockMap[blockId] = { @@ -1047,10 +1121,10 @@ function appendBlock( if (isList(block)) { const itemIds: string[] = [] - const itemMap: Record = {} + const itemMap: Record = Object.create(null) for (const item of block.items) { - const itemId = factory.itemId() + const itemId = factory.reuse(item, 'id', () => factory.itemId(), block) itemIds.push(itemId) const childIds = normalizeBlocks(item.blocks).map((child) => appendBlock(child, blockMap, itemId, factory) @@ -1110,15 +1184,15 @@ function appendBlock( if (isComplexTable(block)) { validateComplexTableBlock(block) - const rowIds = Array.from({ length: block.rowCount }, () => factory.itemId()) - const columnIds = Array.from({ length: block.columnCount }, () => factory.itemId()) + const rowIds = Array.from({ length: block.rowCount }, (_, index) => factory.tableId(block, 'rowIds', index)) + const columnIds = Array.from({ length: block.columnCount }, (_, index) => factory.tableId(block, 'columnIds', index)) - const rowMap: Record = {} + const rowMap: Record = Object.create(null) for (const rowId of rowIds) { rowMap[rowId] = { id: rowId } } - const columnMap: Record = {} + const columnMap: Record = Object.create(null) for (let i = 0; i < columnIds.length; i += 1) { const columnId = columnIds[i]! columnMap[columnId] = { @@ -1127,7 +1201,7 @@ function appendBlock( } } - const cellMap: Record = {} + const cellMap: Record = Object.create(null) for (const cell of [...block.cells].sort((a, b) => { if (a.rowIndex !== b.rowIndex) { return a.rowIndex - b.rowIndex @@ -1175,6 +1249,16 @@ function appendBlock( throw new EndfieldWikitextConversionError(`Unsupported block type '${(block as Block).blockType}'.`) } +function introDocumentBlocks(intro: ImageIntro, useCached = false): Block[] { + const previous = identity(intro).introBlocks + if (useCached && previous) return previous + const blocks = intro.description.length + ? splitInlinesByNewline(intro.description).map((line) => paragraph(line)) + : [] + withIdentity(intro, { ...identity(intro), introBlocks: blocks }) + return blocks +} + /** * Split a sequence of inlines on `\n` boundaries inside text runs, producing * one Inline[] per line. Used for imgIntro descriptions where each line must diff --git a/src/model.ts b/src/model.ts index 9ce4ce2..befa546 100644 --- a/src/model.ts +++ b/src/model.ts @@ -1,3 +1,5 @@ +import { copyIdentity } from './identity.js' + export class XmlWikiConversionError extends Error { constructor(message: string) { super(message) @@ -305,17 +307,17 @@ export function normalizeBlocks(blocks: Block[]): Block[] { if (nextParagraph.kind !== 'body' && !nextParagraph.inlines.length) { continue } - normalized.push(nextParagraph) + normalized.push(copyIdentity(block, nextParagraph)) continue } if (isQuote(block)) { const children = normalizeBlocks(block.children) if (children.length) { - normalized.push({ + normalized.push(copyIdentity(block, { blockType: 'quote', children, - }) + })) } continue } @@ -325,15 +327,15 @@ export function normalizeBlocks(blocks: Block[]): Block[] { for (const item of block.items) { const itemBlocks = normalizeBlocks(item.blocks) if (itemBlocks.length) { - items.push({ blocks: itemBlocks }) + items.push(copyIdentity(item, { blocks: itemBlocks })) } } if (items.length) { - normalized.push({ + normalized.push(copyIdentity(block, { blockType: 'list', ordered: block.ordered, items, - }) + })) } continue } @@ -363,7 +365,7 @@ export function normalizeBlocks(blocks: Block[]): Block[] { columnCount: block.columnCount, } validateComplexTableBlock(nextTable) - normalized.push(nextTable) + normalized.push(copyIdentity(block, nextTable)) continue } @@ -387,7 +389,7 @@ export function normalizeBlocks(blocks: Block[]): Block[] { collapsed.push(block) } - return collapsed + return copyIdentity(blocks, collapsed) } export function blocksToPlainText(blocks: Block[]): string { diff --git a/src/xmlFormat.ts b/src/xmlFormat.ts index e078fa3..42b0026 100644 --- a/src/xmlFormat.ts +++ b/src/xmlFormat.ts @@ -262,7 +262,7 @@ function chapterFromXml(chapter: Element): Chapter { } rows.push({ - cells: cells.map((cell) => [cell.getAttribute('label') || '', normalizedText(directTextContent(cell))]), + cells: cells.map((cell) => [cell.getAttribute('label') || '', contentTextWithBreaks(cell)]), }) } @@ -355,7 +355,7 @@ function chapterFromXml(chapter: Element): Chapter { function audioItemsFromXml(audiosContainer: Element): AudioItem[] { return childrenByTag(audiosContainer, 'audio').map((audio) => ({ title: audio.getAttribute('name') || '', - profile: normalizedMultilineText(directTextContent(audio)), + profile: contentTextWithBreaks(audio, true), resourceUrl: audio.getAttribute('src') || '', })) } @@ -607,7 +607,7 @@ function parseBlockElement( size: normalizedText(directTextContent(requireChild(element, 'size'))), imageId: explicitId ? normalizedText(directTextContent(explicitId)) : inferredParts![0], imageFormat: explicitFormat ? normalizedText(directTextContent(explicitFormat)) : inferredParts![1], - description: normalizedText(directTextContent(requireChild(element, 'description')), ''), + description: contentTextWithBreaks(requireChild(element, 'description')), } } @@ -914,6 +914,11 @@ function parseInlineElement( const color = state?.color ?? null const preserveNewlines = state?.preserveNewlines ?? false + if (element.tagName === 'br') { + assertEmptyLineBreak(element) + return [textRun('\n', { bold, italic, underline, strike, color })] + } + if (element.tagName === 'b') { return parseInlineContainer(element, { bold: true, italic, underline, strike, color, preserveNewlines }) } @@ -952,7 +957,7 @@ function parseInlineElement( return [ { inlineType: 'pronunciation', - content: normalizedText(element.textContent), + content: contentTextWithBreaks(element), }, ] } @@ -962,7 +967,7 @@ function parseInlineElement( { inlineType: 'link', href: element.getAttribute('href') || '', - text: normalizedText(element.textContent), + text: contentTextWithBreaks(element), }, ] } @@ -1187,10 +1192,7 @@ function renderAudioList(audios: AudioItem[], indent: number) { for (const audio of audios) { const attrs = `name=${quoteAttr(audio.title)} src=${quoteAttr(audio.resourceUrl)}` lines.push(`${pad} `) } @@ -1208,7 +1210,7 @@ function renderSimpleTable(rows: TableRow[], indent: number) { } lines.push(`${pad} `) for (const [label, value] of row.cells) { - lines.push(`${pad} ${escapeXmlText(value)}`) + lines.push(`${pad} ${renderTextWithBreaks(value)}`) } lines.push(`${pad} `) } @@ -1302,7 +1304,7 @@ function renderBlock(block: Block, indent: number): string[] { `${pad} ${escapeXmlText(block.height)}`, `${pad} ${escapeXmlText(block.size)}`, `${pad} ${escapeXmlText(block.url)}`, - `${pad} ${escapeXmlText(block.description)}`, + `${pad} ${renderTextWithBreaks(block.description)}`, `${pad}`, ] } @@ -1373,13 +1375,48 @@ function renderComplexTable(block: ComplexTableBlock, indent: number) { return lines } +/** XML layout newlines remain paragraph separators; explicit breaks stay inline. */ +function renderTextWithBreaks(value: string): string { + return escapeXmlText(value).replace(/\r\n|\r|\n/g, '

') +} + +function assertEmptyLineBreak(element: Element): void { + if (Array.from(element.childNodes).some((child) => + child.nodeType === ELEMENT_NODE || Boolean(child.nodeValue?.trim()) + )) { + throw new EndfieldWikitextConversionError('
must not contain content.') + } +} + +/** Normalize layout whitespace without collapsing explicit br elements. */ +function contentTextWithBreaks(element: Element, multiline = false): string { + const parts = [''] + function visit(parent: Element): void { + for (const child of Array.from(parent.childNodes)) { + if (child.nodeType === TEXT_NODE || child.nodeType === CDATA_SECTION_NODE) { + parts[parts.length - 1] += child.nodeValue ?? '' + } else if (child.nodeType === ELEMENT_NODE) { + const nested = child as Element + if (nested.tagName === 'br') { + assertEmptyLineBreak(nested) + parts.push('') + } else { + visit(nested) + } + } + } + } + visit(element) + return parts.map((part) => multiline ? normalizedMultilineText(part) : normalizedText(part)).join('\n') +} + function renderInlines(inlines: Inline[]) { return inlines.map((inline) => renderInline(inline)).join('') } function renderInline(inline: Inline) { if (isTextRun(inline)) { - let value = escapeXmlText(inline.text) + let value = renderTextWithBreaks(inline.text) if (inline.color !== null) { value = `${value}` } @@ -1399,11 +1436,11 @@ function renderInline(inline: Inline) { } if (inline.inlineType === 'pronunciation') { - return `${escapeXmlText(inline.content)}` + return `${renderTextWithBreaks(inline.content)}` } if (inline.inlineType === 'link') { - return `${escapeXmlText(inline.text)}` + return `${renderTextWithBreaks(inline.text)}` } if (inline.inlineType === 'entry') { diff --git a/tests/fixtures/laevat-basic-record.json b/tests/fixtures/laevat-basic-record.json new file mode 100644 index 0000000..e2eedb4 --- /dev/null +++ b/tests/fixtures/laevat-basic-record.json @@ -0,0 +1,4 @@ +{ + "title": "基础档案", + "content": "【代号】莱万汀\n【性别】女\n【身份认证】罗德岛\n【生日】和管理员的一样\n【种族】萨卡兹\n【矿石病感染情况】\n参照医学检测报告，确认为非感染者。\n\n【综合体检测试】\n生理强度：优良\n作战技巧：标准\n战术规划：标准\n源石技艺适应性：优良\n\n注：她随身携带的环，需由武库工程师定期检查维护。——终末地安全专家 卡拉德" +} diff --git a/tests/id-reuse.test.ts b/tests/id-reuse.test.ts new file mode 100644 index 0000000..5fe32cc --- /dev/null +++ b/tests/id-reuse.test.ts @@ -0,0 +1,412 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, test, vi } from 'vitest' +import { + parseSubmitJson, + parseXml, + renderSubmitJson, + renderXml, + submitJsonToXml, + xmlToSubmitJson, + xmlToWikiJson, + type Block, + type Chapter, + type ComplexTableBlock, +} from '../src/index.js' +import { paragraph, textRun } from '../src/model.js' + +const sample = readFileSync(new URL('./fixtures/sample.xml', import.meta.url), 'utf8') +const p = (text: string): Block => paragraph([textRun(text)]) +const chapter = (title: string, content: Block[] = []): Chapter => ({ + title, + size: 'large', + chapterType: 'common', + content, + tabs: [], + audios: [], + tableRows: [], +}) +function source(chapters: Chapter[]) { + const model = parseXml(sample) + model.description = [] + model.chapterGroups = [{ title: 'Group', chapters }] + return JSON.parse(renderSubmitJson(model)) +} +function edit(reference: any, change: (model: ReturnType) => void) { + const model = parseSubmitJson(reference)[0] + change(model) + return JSON.parse(xmlToSubmitJson(renderXml(model), { referenceJson: reference }).text) +} +function widget(item: any, index = 0) { + const descriptor = item.document.chapterGroup[0].widgets[index] + return { descriptor, data: item.document.widgetCommonMap[descriptor.id] } +} +function doc(item: any, index = 0) { + return item.document.documentMap[widget(item, index).data.tabDataMap.default.content] +} +function assertReferences(item: any) { + for (const group of item.document.chapterGroup) { + for (const entry of group.widgets) expect(item.document.widgetCommonMap[entry.id]).toBeDefined() + } + for (const data of Object.values(item.document.widgetCommonMap) as any[]) { + for (const tab of data.tabList) expect(data.tabDataMap[tab.tabId]).toBeDefined() + for (const tab of Object.values(data.tabDataMap) as any[]) { + if (tab.content) expect(item.document.documentMap[tab.content]).toBeDefined() + if (tab.intro?.description) expect(item.document.documentMap[tab.intro.description]).toBeDefined() + } + } + for (const document of [item.brief.description, ...Object.values(item.document.documentMap)] as any[]) { + if (!document) continue + const parents = new Set([document.id, ...Object.keys(document.blockMap)]) + const refs: Array<[string, string[]]> = [[document.id, document.blockIds]] + for (const [id, block] of Object.entries(document.blockMap) as Array<[string, any]>) { + expect(block.id).toBe(id) + if (block.quote) refs.push([id, block.quote.childIds]) + if (block.list) { + expect(block.list.id).toBe(id) + for (const itemId of block.list.itemIds) { + expect(block.list.itemMap[itemId].id).toBe(itemId) + parents.add(itemId) + refs.push([itemId, block.list.itemMap[itemId].childIds]) + } + } + if (block.table) { + expect(block.table.id).toBe(id) + for (const row of block.table.rowIds) expect(block.table.rowMap[row].id).toBe(row) + for (const col of block.table.columnIds) expect(block.table.columnMap[col].id).toBe(col) + for (const [key, cell] of Object.entries(block.table.cellMap) as Array<[string, any]>) { + expect(cell.id).toBe(key) + expect( + block.table.rowIds.some((row: string) => + block.table.columnIds.some((col: string) => key === `${row}_${col}`), + ), + ).toBe(true) + parents.add(key) + refs.push([key, cell.childIds]) + } + } + } + for (const [parent, ids] of refs) { + expect(new Set(ids).size).toBe(ids.length) + for (const id of ids) expect(document.blockMap[id]?.parentId).toBe(parent) + } + for (const block of Object.values(document.blockMap) as any[]) expect(parents.has(block.parentId)).toBe(true) + } +} + +describe('reference JSON identities', () => { + test('preserves all IDs across the rich fixture including tables, nested lists, audio and intros', () => { + const original = JSON.parse(renderSubmitJson(parseXml(sample))) + const output = xmlToSubmitJson(submitJsonToXml(original).text, { referenceJson: original }) + expect(JSON.parse(output.text)).toEqual(original) + assertReferences(original.item) + }) + + test('accepts all reference envelopes and rejects invalid or different items', () => { + const original = source([chapter('One', [p('hello')])]) + const xml = submitJsonToXml(original).text + for (const referenceJson of [ + original, + original.item, + { data: { item: original.item } }, + JSON.stringify(original), + ]) { + expect(JSON.parse(xmlToSubmitJson(xml, { referenceJson }).text)).toEqual(original) + expect(JSON.parse(xmlToWikiJson(xml, { referenceJson }).text).document).toEqual(original.item.document) + } + expect(() => xmlToSubmitJson(xml, { referenceJson: '{' })).toThrow() + expect(() => xmlToSubmitJson(xml, { referenceJson: {} })).toThrow() + expect(() => xmlToSubmitJson(xml, { referenceJson: { ...original.item, itemId: 'other' } })).toThrow(/itemId/) + expect(() => xmlToSubmitJson(xml, { referenceJson: { ...original.item, document: {} } })).toThrow() + }) + + test('retains IDs for text and title edits and a nonstandard document root', () => { + const original = source([chapter('One', [p('hello'), p('world')])]) + const before = doc(original.item) + before.id = 'custom-root' + for (const block of Object.values(before.blockMap) as any[]) block.parentId = 'custom-root' + const output = edit(original, (model) => { + model.chapterGroups[0]!.chapters[0]!.title = 'Renamed' + model.chapterGroups[0]!.chapters[0]!.content[0] = p('edited') + }) + expect(widget(output.item).descriptor.id).toBe(widget(original.item).descriptor.id) + expect(doc(output.item).blockIds).toEqual(before.blockIds) + expect(doc(output.item).id).toBe('custom-root') + assertReferences(output.item) + }) + + test.each(['start', 'middle', 'delete', 'reorder'] as const)('preserves unaffected blocks on %s', (operation) => { + const original = source([chapter('One', [p('A'), p('B'), p('C')])]) + const ids = doc(original.item).blockIds + const output = edit(original, (model) => { + const list = model.chapterGroups[0]!.chapters[0]!.content + if (operation === 'start') list.unshift(p('new')) + if (operation === 'middle') list.splice(1, 0, p('new')) + if (operation === 'delete') list.splice(1, 1) + if (operation === 'reorder') list.reverse() + }) + const result = doc(output.item).blockIds + if (operation === 'start') expect(result.slice(1)).toEqual(ids) + if (operation === 'middle') expect([result[0], ...result.slice(2)]).toEqual(ids) + if (operation === 'delete') expect(result).toEqual([ids[0], ids[2]]) + if (operation === 'reorder') expect(result).toEqual([...ids].reverse()) + assertReferences(output.item) + }) + + test('matches unique moves across containers and does not reuse an ID twice for copied content', () => { + const original = source([chapter('One', [p('move me'), p('stay')]), chapter('Two', [p('second')])]) + const movedId = doc(original.item).blockIds[0] + const moved = edit(original, (model) => { + const [one, two] = model.chapterGroups[0]!.chapters + two!.content.push(one!.content.shift()!) + }) + expect(doc(moved.item, 1).blockIds[1]).toBe(movedId) + const copied = edit(original, (model) => { + const list = model.chapterGroups[0]!.chapters[0]!.content + list.push(structuredClone(list[0]!)) + }) + expect(new Set(doc(copied.item).blockIds).size).toBe(3) + expect(doc(copied.item).blockIds[0]).toBe(movedId) + assertReferences(moved.item) + assertReferences(copied.item) + }) + + test('handles duplicate titles, duplicate paragraphs, type replacement and clipped empty paragraphs', () => { + const original = source([chapter('Same', [p('duplicate'), p('duplicate')]), chapter('Same', [p('distinct')])]) + const first = doc(original.item) + const copy = structuredClone(first.blockMap[first.blockIds[0]]) + copy.id = 'trimmed-empty' + copy.text.inlineElements = [] + first.blockMap[copy.id] = copy + first.blockIds.unshift(copy.id) + const output = edit(original, (model) => { + model.chapterGroups[0]!.chapters.reverse() + }) + expect(doc(output.item, 1).blockIds).toEqual(first.blockIds.slice(1)) + const replaced = edit(original, (model) => { + model.chapterGroups[0]!.chapters[0]!.content[0] = { blockType: 'horizontalLine', kind: 'single' } + }) + expect(doc(replaced.item).blockIds[0]).not.toBe(first.blockIds[1]) + assertReferences(replaced.item) + }) + + function table(): ComplexTableBlock { + return { + blockType: 'complexTable', + headerMode: 'none', + rowCount: 2, + columnCount: 2, + columnWidths: [100, 200], + cells: [0, 1].flatMap((rowIndex) => + [0, 1].map((columnIndex) => ({ + rowIndex, + columnIndex, + rowSpan: 1, + colSpan: 1, + blocks: [p(`${rowIndex}:${columnIndex}`)], + })), + ), + } + } + test.each(['row', 'column'] as const)('retains table identities when inserting a %s', (axis) => { + const original = source([chapter('Table', [table()])]) + const before = Object.values(doc(original.item).blockMap).find((b: any) => b.table) as any + const output = edit(original, (model) => { + const value = model.chapterGroups[0]!.chapters[0]!.content[0] as ComplexTableBlock + if (axis === 'row') { + value.rowCount++ + value.cells.forEach((cell) => cell.rowIndex++) + for (let columnIndex = 0; columnIndex < 2; columnIndex++) + value.cells.push({ rowIndex: 0, columnIndex, rowSpan: 1, colSpan: 1, blocks: [p(`new ${columnIndex}`)] }) + } else { + value.columnCount++ + value.columnWidths.unshift(50) + value.cells.forEach((cell) => cell.columnIndex++) + for (let rowIndex = 0; rowIndex < 2; rowIndex++) + value.cells.push({ rowIndex, columnIndex: 0, rowSpan: 1, colSpan: 1, blocks: [p(`new ${rowIndex}`)] }) + } + }) + const after = doc(output.item).blockMap[before.id].table + if (axis === 'row') { + expect(after.rowIds.slice(1)).toEqual(before.table.rowIds) + expect(after.columnIds).toEqual(before.table.columnIds) + } else { + expect(after.columnIds.slice(1)).toEqual(before.table.columnIds) + expect(after.rowIds).toEqual(before.table.rowIds) + } + for (const key of Object.keys(before.table.cellMap)) expect(after.cellMap[key]).toEqual(before.table.cellMap[key]) + assertReferences(output.item) + }) + + test('preserves audio identities when all editable fields change and video element identity', () => { + const audio: Chapter = { + ...chapter('Audio'), + chapterType: 'audio', + audioTabs: [ + { + title: 'Chinese', + icon: null, + audios: [{ title: 'Old', profile: 'Old text', resourceUrl: 'https://old.test/a.wav' }], + }, + { + title: 'English', + icon: null, + audios: [{ title: 'Keep', profile: 'Keep text', resourceUrl: 'https://old.test/b.wav' }], + }, + ], + } + const original = source([ + audio, + chapter('Video', [{ blockType: 'externalVideo', videoKind: 'bilibili', videoId: 'BV12345' }]), + ]) + const output = edit(original, (model) => { + model.chapterGroups[0]!.chapters[0]!.audioTabs![0]!.audios[0] = { + title: 'New', + profile: 'New text', + resourceUrl: 'https://new.test/a.wav', + } + }) + const before = widget(original.item).data + const after = widget(output.item).data + expect(after.tabList).toEqual(before.tabList) + for (const tab of before.tabList) + expect(after.tabDataMap[tab.tabId].audioList[0].id).toBe(before.tabDataMap[tab.tabId].audioList[0].id) + expect(doc(output.item, 1)).toEqual(doc(original.item, 1)) + assertReferences(output.item) + }) + test('reserves even discarded IDs and retries random collisions', () => { + const original = source([chapter('One', [p('stay')])]) + const data = doc(original.item) + const discarded = 'aaaaaaaaaaaa' + data.blockMap[discarded] = { + ...structuredClone(data.blockMap[data.blockIds[0]]), + id: discarded, + text: { kind: 'body', inlineElements: [] }, + } + data.blockIds.unshift(discarded) + const random = vi.spyOn(crypto, 'getRandomValues').mockImplementationOnce((array) => { + ;(array as Uint8Array).fill(0) + return array + }) + try { + const output = edit(original, (model) => { + model.chapterGroups[0]!.chapters[0]!.content.push(p('new')) + }) + expect(doc(output.item).blockIds[1]).not.toBe(discarded) + expect(random.mock.calls.length).toBeGreaterThanOrEqual(2) + assertReferences(output.item) + } finally { + random.mockRestore() + } + }) + + test('preserves unusual map keys without changing object prototypes', () => { + const original = source([chapter('One', [p('keep')])]) + const oldWidget = widget(original.item) + const oldId = oldWidget.descriptor.id + original.item.document.chapterGroup[0].widgets[0].id = '__proto__' + Object.defineProperty(original.item.document.widgetCommonMap, '__proto__', { + value: oldWidget.data, + enumerable: true, + }) + delete original.item.document.widgetCommonMap[oldId] + const output = edit(original, () => {}) + expect(Object.hasOwn(output.item.document.widgetCommonMap, '__proto__')).toBe(true) + assertReferences(output.item) + }) + + test('supports deeply nested identity trees without expanding escaped subtree signatures', () => { + let block = p('leaf') + for (let index = 0; index < 30; index++) block = { blockType: 'quote', children: [block] } + const original = source([chapter('Nested', [block])]) + expect(edit(original, () => {})).toEqual(original) + }) + + test.each(['row', 'column'] as const)('preserves retained table identities after deleting a %s', (axis) => { + const original = source([chapter('Table', [table()])]) + const before = Object.values(doc(original.item).blockMap).find((block: any) => block.table) as any + const output = edit(original, (model) => { + const value = model.chapterGroups[0]!.chapters[0]!.content[0] as ComplexTableBlock + if (axis === 'row') { + value.rowCount-- + value.cells = value.cells.filter((cell) => cell.rowIndex === 1) + value.cells.forEach((cell) => cell.rowIndex--) + } else { + value.columnCount-- + value.columnWidths.shift() + value.cells = value.cells.filter((cell) => cell.columnIndex === 1) + value.cells.forEach((cell) => cell.columnIndex--) + } + }) + const after = doc(output.item).blockMap[before.id].table + expect(after.rowIds).toEqual(axis === 'row' ? before.table.rowIds.slice(1) : before.table.rowIds) + expect(after.columnIds).toEqual(axis === 'column' ? before.table.columnIds.slice(1) : before.table.columnIds) + assertReferences(output.item) + }) + + test('keeps caller-owned reference and model values unchanged and honors changed intro text', () => { + const original = JSON.parse(renderSubmitJson(parseXml(sample))) + const model = parseSubmitJson(original)[0] + const intro = model.chapterGroups + .flatMap((group) => group.chapters) + .flatMap((entry) => entry.tabs) + .find((tab) => tab.intro)?.intro + expect(intro).toBeTruthy() + intro!.description = [textRun('Edited intro')] + const snapshot = structuredClone(model) + const originalSnapshot = structuredClone(original) + const output = JSON.parse(renderSubmitJson(model, { referenceJson: original })) + expect(model).toEqual(snapshot) + expect(original).toEqual(originalSnapshot) + expect(submitJsonToXml(output).text).toContain('Edited intro') + // The option-free renderer must not use parser-side intro caches either. + expect(submitJsonToXml(renderSubmitJson(model)).text).toContain('Edited intro') + }) + test('keeps local list-item and row/column namespaces independent', () => { + const list = (text: string): Block => ({ blockType: 'list', ordered: false, items: [{ blocks: [p(text)] }] }) + const original = source([chapter('Lists and tables', [list('one'), list('two'), table()])]) + const data = doc(original.item) + for (const block of Object.values(data.blockMap) as any[]) { + if (!block.list) continue + const oldId = block.list.itemIds[0] + const item = block.list.itemMap[oldId] + item.id = 'shared-local-id' + for (const child of item.childIds) data.blockMap[child].parentId = item.id + block.list.itemIds = [item.id] + block.list.itemMap = { [item.id]: item } + } + const value = (Object.values(data.blockMap).find((block: any) => block.table) as any).table + const oldColumn = value.columnIds[0] + const newColumn = value.rowIds[0] + value.columnIds[0] = newColumn + value.columnMap[newColumn] = { ...value.columnMap[oldColumn], id: newColumn } + delete value.columnMap[oldColumn] + for (const row of value.rowIds) { + const previous = `${row}_${oldColumn}` + const id = `${row}_${newColumn}` + const cell = value.cellMap[previous] + cell.id = id + for (const child of cell.childIds) data.blockMap[child].parentId = id + value.cellMap[id] = cell + delete value.cellMap[previous] + } + const output = edit(original, () => {}) + expect(output).toEqual(original) + assertReferences(output.item) + }) + + test('retains table axes and remaining cell identities when merging cells', () => { + const original = source([chapter('Table', [table()])]) + const before = Object.values(doc(original.item).blockMap).find((block: any) => block.table) as any + const output = edit(original, (model) => { + const value = model.chapterGroups[0]!.chapters[0]!.content[0] as ComplexTableBlock + value.cells[0]!.colSpan = 2 + value.cells.splice(1, 1) + }) + const after = doc(output.item).blockMap[before.id].table + expect(after.rowIds).toEqual(before.table.rowIds) + expect(after.columnIds).toEqual(before.table.columnIds) + const unchanged = `${before.table.rowIds[1]}_${before.table.columnIds[1]}` + expect(after.cellMap[unchanged]).toEqual(before.table.cellMap[unchanged]) + assertReferences(output.item) + }) +}) diff --git a/tests/newlines.test.ts b/tests/newlines.test.ts new file mode 100644 index 0000000..c39dd5b --- /dev/null +++ b/tests/newlines.test.ts @@ -0,0 +1,106 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, test } from 'vitest' +import { + parseXml, + renderXml, + renderSubmitJson, + submitJsonToXml, + xmlToSubmitJson, + type Block, + type Inline, +} from '../src/index.js' +import { paragraph, textRun } from '../src/model.js' + +const sample = readFileSync(new URL('./fixtures/sample.xml', import.meta.url), 'utf8') +const record = JSON.parse(readFileSync(new URL('./fixtures/laevat-basic-record.json', import.meta.url), 'utf8')) as { + title: string + content: string +} +function document(blocks: Block[]) { + const model = parseXml(sample) + model.description = [] + model.chapterGroups = [ + { + title: '资料', + chapters: [ + { + title: '档案', + size: 'middle', + chapterType: 'common', + content: blocks, + tabs: [], + audios: [], + tableRows: [], + }, + ], + }, + ] + return model +} +const content = (xml: string) => parseXml(xml).chapterGroups[0]!.chapters[0]!.content + +describe('explicit inline line breaks', () => { + test.each(['甲\n乙', '甲\r\n乙\r丙', '\n甲\n\n乙\n', '<标签> & 值\n下一行'])( + 'round-trips %j as closed br elements in one block', + (text) => { + const model = document([paragraph([textRun(text)])]) + const xml = renderXml(model) + expect(xml.match(/
<\/br>/g)?.length).toBe(text.match(/\r\n|\r|\n/g)?.length) + expect(xml).not.toContain('
') + expect(content(xml)).toEqual([paragraph([textRun(text.replace(/\r\n|\r/g, '\n'))])]) + }, + ) + + test('preserves formatting around consecutive breaks and accepts self-closing input', () => { + const inlines: Inline[] = [textRun('甲\n\n乙', { bold: true, italic: true, color: 'f_red' })] + const xml = renderXml(document([paragraph(inlines, 'heading1')])) + expect(xml).toContain('甲



乙') + expect(content(xml.replaceAll('

', '
'))).toEqual([paragraph(inlines, 'heading1')]) + }) + + test('supports breaks in links, pronunciations, lists, quotes and table cells', () => { + const inlines: Inline[] = [ + { inlineType: 'link', href: 'https://example.test', text: '甲\n乙' }, + { inlineType: 'pronunciation', content: '丙\n丁' }, + ] + const blocks: Block[] = [ + { blockType: 'quote', children: [paragraph(inlines)] }, + { blockType: 'list', ordered: false, items: [{ blocks: [paragraph([textRun('一\n二')])] }] }, + { + blockType: 'complexTable', + headerMode: 'none', + columnWidths: [100], + rowCount: 1, + columnCount: 1, + cells: [{ rowIndex: 0, columnIndex: 0, rowSpan: 1, colSpan: 1, blocks: [paragraph([textRun('上\n下')])] }], + }, + ] + expect(content(renderXml(document(blocks)))).toEqual(blocks) + }) + + test('retains legacy paragraph separators, literal escaped markup and rejects nonempty br', () => { + const model = document([paragraph([textRun('甲')]), paragraph([textRun('乙')])]) + const xml = renderXml(model) + expect(content(xml)).toHaveLength(2) + expect(renderXml(document([paragraph([textRun('

')])]))).toContain('<br></br>') + expect(() => parseXml(xml.replace('甲', '甲
lost text
'))).toThrow(/must not contain content/) + }) + + test('preserves audio profile newlines, including empty lines', () => { + const model = document([]) + const chapter = model.chapterGroups[0]!.chapters[0]! + chapter.chapterType = 'audio' + chapter.audios = [{ title: 'Audio', profile: '\n甲\n\n乙\n', resourceUrl: 'https://example.test/a.wav' }] + const xml = renderXml(model) + expect(xml).toContain('

甲



乙

') + expect(parseXml(xml).chapterGroups[0]!.chapters[0]!.audios).toEqual(chapter.audios) + }) + + test('keeps Laevat archive text and its block identity through JSON/XML/JSON', () => { + const original = renderSubmitJson(document([paragraph([textRun(record.content)])])) + const xml = submitJsonToXml(original).text + expect(xml).toContain('【代号】莱万汀

【性别】女') + expect(xml).toContain('非感染者。



【综合体检测试】') + expect(JSON.parse(xmlToSubmitJson(xml, { referenceJson: original }).text)).toEqual(JSON.parse(original)) + }) +})