From 13ab43300693e19d49a26ac09e90baecdf2b24d8 Mon Sep 17 00:00:00 2001 From: Mindaugas Rukas <3825716+mindaugasrukas@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:13:55 -0700 Subject: [PATCH 1/2] Escape lone surrogates in symbol documentation --- snapshots/input/syntax/src/surrogates.ts | 6 +++++ snapshots/output/syntax/src/surrogates.ts | 29 +++++++++++++++++++++++ src/FileIndexer.test.ts | 21 ++++++++++++++++ src/FileIndexer.ts | 11 ++++++++- 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 snapshots/input/syntax/src/surrogates.ts create mode 100644 snapshots/output/syntax/src/surrogates.ts create mode 100644 src/FileIndexer.test.ts diff --git a/snapshots/input/syntax/src/surrogates.ts b/snapshots/input/syntax/src/surrogates.ts new file mode 100644 index 00000000..69cfca69 --- /dev/null +++ b/snapshots/input/syntax/src/surrogates.ts @@ -0,0 +1,6 @@ +// format-options: showDocs + +export const high = '\ud800' +export const low = '\udfff' +export const pair = '\ud83d\ude00' +export const mixed = '\ud800\ud83d\ude00\udfff' diff --git a/snapshots/output/syntax/src/surrogates.ts b/snapshots/output/syntax/src/surrogates.ts new file mode 100644 index 00000000..4274bd10 --- /dev/null +++ b/snapshots/output/syntax/src/surrogates.ts @@ -0,0 +1,29 @@ +// language TypeScript +// < definition syntax 1.0.0 src/`surrogates.ts`/ +//documentation ```ts +// > module "surrogates.ts" +// > ``` + +// format-options: showDocs + +export const high = '\ud800' +// ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/high. +// documentation ```ts +// > var high: "\ud800" +// > ``` +export const low = '\udfff' +// ^^^ definition syntax 1.0.0 src/`surrogates.ts`/low. +// documentation ```ts +// > var low: "\udfff" +// > ``` +export const pair = '\ud83d\ude00' +// ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/pair. +// documentation ```ts +// > var pair: "😀" +// > ``` +export const mixed = '\ud800\ud83d\ude00\udfff' +// ^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/mixed. +// documentation ```ts +// > var mixed: "\ud800😀\udfff" +// > ``` + diff --git a/src/FileIndexer.test.ts b/src/FileIndexer.test.ts new file mode 100644 index 00000000..59bb4877 --- /dev/null +++ b/src/FileIndexer.test.ts @@ -0,0 +1,21 @@ +import { test } from 'uvu' +import * as assert from 'uvu/assert' + +import { escapeLoneSurrogates } from './FileIndexer' + +test('documentation preserves valid Unicode and existing escapes', () => { + const text = 'ASCII café 中文 😀 \\ud800' + assert.is(escapeLoneSurrogates(text), text) +}) + +test('documentation escapes unpaired UTF-16 code units', () => { + assert.is(escapeLoneSurrogates('\ud800'), '\\ud800') + assert.is(escapeLoneSurrogates('\udfff'), '\\udfff') + assert.is(escapeLoneSurrogates('a\ud83cx\udfffb'), 'a\\ud83cx\\udfffb') + assert.is( + escapeLoneSurrogates('\ud800\ud800\udc00\udc00'), + '\\ud800\ud800\udc00\\udc00' + ) +}) + +test.run() diff --git a/src/FileIndexer.ts b/src/FileIndexer.ts index bcae42ee..e7231efc 100644 --- a/src/FileIndexer.ts +++ b/src/FileIndexer.ts @@ -21,6 +21,15 @@ import * as scip from './scip' import { ScipSymbol } from './ScipSymbol' import * as ts_inline from './TypeScriptInternal' +// TypeScript signatures can contain lone UTF-16 surrogates from string literals. +// Protobuf strings require valid UTF-8; keep these code units visible as escapes. +export function escapeLoneSurrogates(value: string): string { + return value.replace( + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? '\\u' + surrogate.charCodeAt(0).toString(16).padStart(4, '0') + ) +} + export class FileIndexer { private localCounter = new Counter() private propertyCounters: Map = new Map() @@ -355,7 +364,7 @@ export class FileIndexer { this.document.symbols.push( new scip.scip.SymbolInformation({ symbol: symbol.value, - documentation, + documentation: documentation.map(escapeLoneSurrogates), relationships: this.relationships(declaration, symbol), kind: symbolKind(declaration, sym), }) From d4c0c043704c911a78e616252b98823feaf9bc97 Mon Sep 17 00:00:00 2001 From: Mindaugas Rukas <3825716+mindaugasrukas@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:55:36 -0700 Subject: [PATCH 2/2] Normalize lone surrogates in documentation and symbols --- snapshots/input/syntax/src/surrogates.ts | 16 ++++ snapshots/output/syntax/src/surrogates.ts | 68 ++++++++++++++++- src/FileIndexer.test.ts | 93 +++++++++++++++++++---- src/FileIndexer.ts | 15 +--- src/ScipSymbol.ts | 5 +- tsconfig.json | 2 +- 6 files changed, 170 insertions(+), 29 deletions(-) diff --git a/snapshots/input/syntax/src/surrogates.ts b/snapshots/input/syntax/src/surrogates.ts index 69cfca69..76a7420e 100644 --- a/snapshots/input/syntax/src/surrogates.ts +++ b/snapshots/input/syntax/src/surrogates.ts @@ -4,3 +4,19 @@ export const high = '\ud800' export const low = '\udfff' export const pair = '\ud83d\ude00' export const mixed = '\ud800\ud83d\ude00\udfff' + +export const adjacent = '\ud800\ud800\udc00\udc00' +export const unicode = 'ASCII café 中文 😀' +export const escaped = '\\ud800' + +export function C() {} +const prototype = (C.prototype = { + '\ud800': 1, + 'low\udfff': 2, + '\ud83d\ude00': 3, + '\\ud800': 4, +}) +prototype['\ud800'] +prototype['low\udfff'] +prototype['\ud83d\ude00'] +prototype['\\ud800'] diff --git a/snapshots/output/syntax/src/surrogates.ts b/snapshots/output/syntax/src/surrogates.ts index 4274bd10..dbd79488 100644 --- a/snapshots/output/syntax/src/surrogates.ts +++ b/snapshots/output/syntax/src/surrogates.ts @@ -9,12 +9,12 @@ export const high = '\ud800' // ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/high. // documentation ```ts -// > var high: "\ud800" +// > var high: "�" // > ``` export const low = '\udfff' // ^^^ definition syntax 1.0.0 src/`surrogates.ts`/low. // documentation ```ts -// > var low: "\udfff" +// > var low: "�" // > ``` export const pair = '\ud83d\ude00' // ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/pair. @@ -24,6 +24,68 @@ export const pair = '\ud83d\ude00' export const mixed = '\ud800\ud83d\ude00\udfff' // ^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/mixed. // documentation ```ts -// > var mixed: "\ud800😀\udfff" +// > var mixed: "�😀�" // > ``` +export const adjacent = '\ud800\ud800\udc00\udc00' +// ^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/adjacent. +// documentation ```ts +// > var adjacent: "�𐀀�" +// > ``` +export const unicode = 'ASCII café 中文 😀' +// ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/unicode. +// documentation ```ts +// > var unicode: "ASCII café 中文 😀" +// > ``` +export const escaped = '\\ud800' +// ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/escaped. +// documentation ```ts +// > var escaped: "\\ud800" +// > ``` + +export function C() {} +// ^ definition syntax 1.0.0 src/`surrogates.ts`/C(). +// documentation ```ts +// > function C(): void +// > ``` +const prototype = (C.prototype = { +// ^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/prototype. +// documentation ```ts +// > var prototype: { '\uD800': number; 'low\... +// > ``` +// ^ reference syntax 1.0.0 src/`surrogates.ts`/C(). +// ^^^^^^^^^ reference typescript 6.0.3 lib/`lib.es5.d.ts`/Function#prototype. + '\ud800': 1, +//^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`�`. +//documentation ```ts +// > (property) '\ud800': number +// > ``` + 'low\udfff': 2, +//^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`low�`. +//documentation ```ts +// > (property) 'low\udfff': number +// > ``` + '\ud83d\ude00': 3, +//^^^^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`😀`. +//documentation ```ts +// > (property) '\ud83d\ude00': number +// > ``` + '\\ud800': 4, +//^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`. +//documentation ```ts +// > (property) '\\ud800': number +// > ``` +}) +prototype['\ud800'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`�`. +prototype['low\udfff'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`low�`. +prototype['\ud83d\ude00'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`😀`. +prototype['\\ud800'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`. + diff --git a/src/FileIndexer.test.ts b/src/FileIndexer.test.ts index b37fb497..9408b202 100644 --- a/src/FileIndexer.test.ts +++ b/src/FileIndexer.test.ts @@ -4,7 +4,7 @@ import * as ts from 'typescript' import { test } from 'uvu' import * as assert from 'uvu/assert' -import { escapeLoneSurrogates, FileIndexer } from './FileIndexer' +import { FileIndexer } from './FileIndexer' import { Input } from './Input' import { Packages } from './Packages' import { scip } from './scip' @@ -62,19 +62,86 @@ test('local prototype owners stay out of the global symbol table', () => { ) }) -test('documentation preserves valid Unicode and existing escapes', () => { - const text = 'ASCII café 中文 😀 \\ud800' - assert.is(escapeLoneSurrogates(text), text) -}) +test('lone surrogates in documentation and symbols survive serialization', () => { + const cwd = path.resolve('snapshots/input/syntax') + const fileName = path.join(cwd, 'src/surrogates.ts') + const program = ts.createProgram([fileName], { + noEmit: true, + target: ts.ScriptTarget.ES2022, + }) + const sourceFile = program.getSourceFile(fileName)! + sourceFile.moduleName = 'module-\ud800' + const document = new scip.Document() + new FileIndexer( + program.getTypeChecker(), + { + cwd, + projectRoot: cwd, + projectDisplayName: 'syntax', + output: '', + inferTsconfig: false, + progressBar: false, + yarnWorkspaces: false, + yarnBerryWorkspaces: false, + pnpmWorkspaces: false, + globalCaches: true, + indexedProjects: new Set(), + writeIndex: () => {}, + }, + Input.fromFile(fileName), + document, + new Map(), + new Map(), + new Packages(cwd), + sourceFile + ).index() -test('documentation escapes unpaired UTF-16 code units', () => { - assert.is(escapeLoneSurrogates('\ud800'), '\\ud800') - assert.is(escapeLoneSurrogates('\udfff'), '\\udfff') - assert.is(escapeLoneSurrogates('a\ud83cx\udfffb'), 'a\\ud83cx\\udfffb') - assert.is( - escapeLoneSurrogates('\ud800\ud800\udc00\udc00'), - '\\ud800\ud800\udc00\\udc00' - ) + const index = new scip.Index({ documents: [document] }) + const decoded = scip.Index.deserializeBinary(index.serializeBinary()) + .documents[0] + assert.equal(decoded.toObject(), document.toObject()) + assert.equal(decoded.symbols[0].documentation, [ + '```ts\nmodule "module-�"\n```', + ]) + + const signatures = [ + 'var high: "�"', + 'var low: "�"', + 'var pair: "😀"', + 'var mixed: "�😀�"', + 'var adjacent: "�𐀀�"', + 'var unicode: "ASCII café 中文 😀"', + 'var escaped: "\\\\ud800"', + ] + for (const signature of signatures) { + assert.ok( + decoded.symbols.some(info => + info.documentation.includes('```ts\n' + signature + '\n```') + ), + `missing signature: ${signature}` + ) + } + + for (const name of ['�', 'low�', '😀', '\\ud800']) { + const suffix = '/C().`' + name + '`.' + const info = decoded.symbols.find(info => info.symbol.endsWith(suffix)) + assert.ok(info, `missing property symbol: ${suffix}`) + const occurrences = decoded.occurrences.filter( + occurrence => occurrence.symbol === info.symbol + ) + assert.ok( + occurrences.some( + occurrence => + (occurrence.symbol_roles & scip.SymbolRole.Definition) !== 0 + ) + ) + assert.ok( + occurrences.some( + occurrence => + (occurrence.symbol_roles & scip.SymbolRole.Definition) === 0 + ) + ) + } }) test.run() diff --git a/src/FileIndexer.ts b/src/FileIndexer.ts index 79beca1b..cca1dac3 100644 --- a/src/FileIndexer.ts +++ b/src/FileIndexer.ts @@ -21,15 +21,6 @@ import * as scip from './scip' import { ScipSymbol } from './ScipSymbol' import * as ts_inline from './TypeScriptInternal' -// TypeScript signatures can contain lone UTF-16 surrogates from string literals. -// Protobuf strings require valid UTF-8; keep these code units visible as escapes. -export function escapeLoneSurrogates(value: string): string { - return value.replace( - /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? '\\u' + surrogate.charCodeAt(0).toString(16).padStart(4, '0') - ) -} - export class FileIndexer { private localCounter = new Counter() private propertyCounters: Map = new Map() @@ -90,7 +81,9 @@ export class FileIndexer { this.document.symbols.push( new scip.scip.SymbolInformation({ symbol: symbol.value, - documentation: ['```ts\nmodule "' + moduleName + '"\n```'], + documentation: [ + ('```ts\nmodule "' + moduleName + '"\n```').toWellFormed(), + ], kind: scip.scip.SymbolInformation.Kind.File, }) ) @@ -387,7 +380,7 @@ export class FileIndexer { this.document.symbols.push( new scip.scip.SymbolInformation({ symbol: symbol.value, - documentation: documentation.map(escapeLoneSurrogates), + documentation: documentation.map(text => text.toWellFormed()), relationships: this.relationships(declaration, symbol), kind: symbolKind(declaration, sym), }) diff --git a/src/ScipSymbol.ts b/src/ScipSymbol.ts index 78531c5b..9d8d75a0 100644 --- a/src/ScipSymbol.ts +++ b/src/ScipSymbol.ts @@ -2,7 +2,10 @@ import { descriptorString } from './Descriptor' import * as scip from './scip' export class ScipSymbol { - private constructor(public readonly value: string) {} + private constructor(public readonly value: string) { + // Every symbol reference must use the same protobuf-safe identifier. + this.value = value.toWellFormed() + } public isEmpty(): boolean { return this.value === '' diff --git a/tsconfig.json b/tsconfig.json index 382d3c14..ff413be7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2022", "module": "Node16", "moduleResolution": "Node16", - "lib": ["es2022"], + "lib": ["es2022", "es2024.string"], "rootDir": ".", "outDir": "dist", "resolveJsonModule": true,