diff --git a/snapshots/input/syntax/src/surrogates.ts b/snapshots/input/syntax/src/surrogates.ts new file mode 100644 index 00000000..76a7420e --- /dev/null +++ b/snapshots/input/syntax/src/surrogates.ts @@ -0,0 +1,22 @@ +// format-options: showDocs + +export const high = '\ud800' +export const low = '\udfff' +export const pair = '\ud83d\ude00' +export const mixed = '\ud800\ud83d\ude00\udfff' + +export const adjacent = '\ud800\ud800\udc00\udc00' +export const unicode = 'ASCII café 中文 😀' +export const escaped = '\\ud800' + +export function C() {} +const prototype = (C.prototype = { + '\ud800': 1, + 'low\udfff': 2, + '\ud83d\ude00': 3, + '\\ud800': 4, +}) +prototype['\ud800'] +prototype['low\udfff'] +prototype['\ud83d\ude00'] +prototype['\\ud800'] diff --git a/snapshots/output/syntax/src/surrogates.ts b/snapshots/output/syntax/src/surrogates.ts new file mode 100644 index 00000000..dbd79488 --- /dev/null +++ b/snapshots/output/syntax/src/surrogates.ts @@ -0,0 +1,91 @@ +// language TypeScript +// < definition syntax 1.0.0 src/`surrogates.ts`/ +//documentation ```ts +// > module "surrogates.ts" +// > ``` + +// format-options: showDocs + +export const high = '\ud800' +// ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/high. +// documentation ```ts +// > var high: "�" +// > ``` +export const low = '\udfff' +// ^^^ definition syntax 1.0.0 src/`surrogates.ts`/low. +// documentation ```ts +// > var low: "�" +// > ``` +export const pair = '\ud83d\ude00' +// ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/pair. +// documentation ```ts +// > var pair: "😀" +// > ``` +export const mixed = '\ud800\ud83d\ude00\udfff' +// ^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/mixed. +// documentation ```ts +// > var mixed: "�😀�" +// > ``` + +export const adjacent = '\ud800\ud800\udc00\udc00' +// ^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/adjacent. +// documentation ```ts +// > var adjacent: "�𐀀�" +// > ``` +export const unicode = 'ASCII café 中文 😀' +// ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/unicode. +// documentation ```ts +// > var unicode: "ASCII café 中文 😀" +// > ``` +export const escaped = '\\ud800' +// ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/escaped. +// documentation ```ts +// > var escaped: "\\ud800" +// > ``` + +export function C() {} +// ^ definition syntax 1.0.0 src/`surrogates.ts`/C(). +// documentation ```ts +// > function C(): void +// > ``` +const prototype = (C.prototype = { +// ^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/prototype. +// documentation ```ts +// > var prototype: { '\uD800': number; 'low\... +// > ``` +// ^ reference syntax 1.0.0 src/`surrogates.ts`/C(). +// ^^^^^^^^^ reference typescript 6.0.3 lib/`lib.es5.d.ts`/Function#prototype. + '\ud800': 1, +//^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`�`. +//documentation ```ts +// > (property) '\ud800': number +// > ``` + 'low\udfff': 2, +//^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`low�`. +//documentation ```ts +// > (property) 'low\udfff': number +// > ``` + '\ud83d\ude00': 3, +//^^^^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`😀`. +//documentation ```ts +// > (property) '\ud83d\ude00': number +// > ``` + '\\ud800': 4, +//^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`. +//documentation ```ts +// > (property) '\\ud800': number +// > ``` +}) +prototype['\ud800'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`�`. +prototype['low\udfff'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`low�`. +prototype['\ud83d\ude00'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`😀`. +prototype['\\ud800'] +//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype. +// ^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`. + diff --git a/src/FileIndexer.test.ts b/src/FileIndexer.test.ts index 1e44e609..9408b202 100644 --- a/src/FileIndexer.test.ts +++ b/src/FileIndexer.test.ts @@ -62,4 +62,86 @@ test('local prototype owners stay out of the global symbol table', () => { ) }) +test('lone surrogates in documentation and symbols survive serialization', () => { + const cwd = path.resolve('snapshots/input/syntax') + const fileName = path.join(cwd, 'src/surrogates.ts') + const program = ts.createProgram([fileName], { + noEmit: true, + target: ts.ScriptTarget.ES2022, + }) + const sourceFile = program.getSourceFile(fileName)! + sourceFile.moduleName = 'module-\ud800' + const document = new scip.Document() + new FileIndexer( + program.getTypeChecker(), + { + cwd, + projectRoot: cwd, + projectDisplayName: 'syntax', + output: '', + inferTsconfig: false, + progressBar: false, + yarnWorkspaces: false, + yarnBerryWorkspaces: false, + pnpmWorkspaces: false, + globalCaches: true, + indexedProjects: new Set(), + writeIndex: () => {}, + }, + Input.fromFile(fileName), + document, + new Map(), + new Map(), + new Packages(cwd), + sourceFile + ).index() + + const index = new scip.Index({ documents: [document] }) + const decoded = scip.Index.deserializeBinary(index.serializeBinary()) + .documents[0] + assert.equal(decoded.toObject(), document.toObject()) + assert.equal(decoded.symbols[0].documentation, [ + '```ts\nmodule "module-�"\n```', + ]) + + const signatures = [ + 'var high: "�"', + 'var low: "�"', + 'var pair: "😀"', + 'var mixed: "�😀�"', + 'var adjacent: "�𐀀�"', + 'var unicode: "ASCII café 中文 😀"', + 'var escaped: "\\\\ud800"', + ] + for (const signature of signatures) { + assert.ok( + decoded.symbols.some(info => + info.documentation.includes('```ts\n' + signature + '\n```') + ), + `missing signature: ${signature}` + ) + } + + for (const name of ['�', 'low�', '😀', '\\ud800']) { + const suffix = '/C().`' + name + '`.' + const info = decoded.symbols.find(info => info.symbol.endsWith(suffix)) + assert.ok(info, `missing property symbol: ${suffix}`) + const occurrences = decoded.occurrences.filter( + occurrence => occurrence.symbol === info.symbol + ) + assert.ok( + occurrences.some( + occurrence => + (occurrence.symbol_roles & scip.SymbolRole.Definition) !== 0 + ) + ) + assert.ok( + occurrences.some( + occurrence => + (occurrence.symbol_roles & scip.SymbolRole.Definition) === 0 + ) + ) + } +}) + test.run() diff --git a/src/FileIndexer.ts b/src/FileIndexer.ts index a763e58e..cca1dac3 100644 --- a/src/FileIndexer.ts +++ b/src/FileIndexer.ts @@ -81,7 +81,9 @@ export class FileIndexer { this.document.symbols.push( new scip.scip.SymbolInformation({ symbol: symbol.value, - documentation: ['```ts\nmodule "' + moduleName + '"\n```'], + documentation: [ + ('```ts\nmodule "' + moduleName + '"\n```').toWellFormed(), + ], kind: scip.scip.SymbolInformation.Kind.File, }) ) @@ -378,7 +380,7 @@ export class FileIndexer { this.document.symbols.push( new scip.scip.SymbolInformation({ symbol: symbol.value, - documentation, + documentation: documentation.map(text => text.toWellFormed()), relationships: this.relationships(declaration, symbol), kind: symbolKind(declaration, sym), }) diff --git a/src/ScipSymbol.ts b/src/ScipSymbol.ts index 78531c5b..9d8d75a0 100644 --- a/src/ScipSymbol.ts +++ b/src/ScipSymbol.ts @@ -2,7 +2,10 @@ import { descriptorString } from './Descriptor' import * as scip from './scip' export class ScipSymbol { - private constructor(public readonly value: string) {} + private constructor(public readonly value: string) { + // Every symbol reference must use the same protobuf-safe identifier. + this.value = value.toWellFormed() + } public isEmpty(): boolean { return this.value === '' diff --git a/tsconfig.json b/tsconfig.json index 382d3c14..ff413be7 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2022", "module": "Node16", "moduleResolution": "Node16", - "lib": ["es2022"], + "lib": ["es2022", "es2024.string"], "rootDir": ".", "outDir": "dist", "resolveJsonModule": true,