Normalize lone surrogates in documentation and symbols

snapshots/input/syntax/src/surrogates.ts

@@ -0,0 +1,22 @@
+// format-options: showDocs
+
+export const high = '\ud800'
+export const low = '\udfff'
+export const pair = '\ud83d\ude00'
+export const mixed = '\ud800\ud83d\ude00\udfff'
+
+export const adjacent = '\ud800\ud800\udc00\udc00'
+export const unicode = 'ASCII café 中文 😀'
+export const escaped = '\\ud800'
+
+export function C() {}
+const prototype = (C.prototype = {
+  '\ud800': 1,
+  'low\udfff': 2,
+  '\ud83d\ude00': 3,
+  '\\ud800': 4,
+})
+prototype['\ud800']
+prototype['low\udfff']
+prototype['\ud83d\ude00']
+prototype['\\ud800']

snapshots/output/syntax/src/surrogates.ts

@@ -0,0 +1,91 @@
+// language TypeScript
+// < definition syntax 1.0.0 src/`surrogates.ts`/
+//documentation ```ts
+//            > module "surrogates.ts"
+//            > ```
+
+// format-options: showDocs
+
+export const high = '\ud800'
+//           ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/high.
+//           documentation ```ts
+//                       > var high: "�"
+//                       > ```
+export const low = '\udfff'
+//           ^^^ definition syntax 1.0.0 src/`surrogates.ts`/low.
+//           documentation ```ts
+//                       > var low: "�"
+//                       > ```
+export const pair = '\ud83d\ude00'
+//           ^^^^ definition syntax 1.0.0 src/`surrogates.ts`/pair.
+//           documentation ```ts
+//                       > var pair: "😀"
+//                       > ```
+export const mixed = '\ud800\ud83d\ude00\udfff'
+//           ^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/mixed.
+//           documentation ```ts
+//                       > var mixed: "�😀�"
+//                       > ```
+
+export const adjacent = '\ud800\ud800\udc00\udc00'
+//           ^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/adjacent.
+//           documentation ```ts
+//                       > var adjacent: "�𐀀�"
+//                       > ```
+export const unicode = 'ASCII café 中文 😀'
+//           ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/unicode.
+//           documentation ```ts
+//                       > var unicode: "ASCII café 中文 😀"
+//                       > ```
+export const escaped = '\\ud800'
+//           ^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/escaped.
+//           documentation ```ts
+//                       > var escaped: "\\ud800"
+//                       > ```
+
+export function C() {}
+//              ^ definition syntax 1.0.0 src/`surrogates.ts`/C().
+//              documentation ```ts
+//                          > function C(): void
+//                          > ```
+const prototype = (C.prototype = {
+//    ^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/prototype.
+//    documentation ```ts
+//                > var prototype: { '\uD800': number; 'low\...
+//                > ```
+//                 ^ reference syntax 1.0.0 src/`surrogates.ts`/C().
+//                   ^^^^^^^^^ reference typescript 6.0.3 lib/`lib.es5.d.ts`/Function#prototype.
+  '\ud800': 1,
+//^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`�`.
+//documentation ```ts
+//            > (property) '\ud800': number
+//            > ```
+  'low\udfff': 2,
+//^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`low�`.
+//documentation ```ts
+//            > (property) 'low\udfff': number
+//            > ```
+  '\ud83d\ude00': 3,
+//^^^^^^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`😀`.
+//documentation ```ts
+//            > (property) '\ud83d\ude00': number
+//            > ```
+  '\\ud800': 4,
+//^^^^^^^^^ definition syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`.
+//documentation ```ts
+//            > (property) '\\ud800': number
+//            > ```
+})
+prototype['\ud800']
+//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype.
+//        ^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`�`.
+prototype['low\udfff']
+//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype.
+//        ^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`low�`.
+prototype['\ud83d\ude00']
+//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype.
+//        ^^^^^^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`😀`.
+prototype['\\ud800']
+//^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/prototype.
+//        ^^^^^^^^^ reference syntax 1.0.0 src/`surrogates.ts`/C().`\ud800`.
+

src/FileIndexer.test.ts

@@ -62,4 +62,86 @@ test('local prototype owners stay out of the global symbol table', () => {
   )
 })
 
+test('lone surrogates in documentation and symbols survive serialization', () => {
+  const cwd = path.resolve('snapshots/input/syntax')
+  const fileName = path.join(cwd, 'src/surrogates.ts')
+  const program = ts.createProgram([fileName], {
+    noEmit: true,
+    target: ts.ScriptTarget.ES2022,
+  })
+  const sourceFile = program.getSourceFile(fileName)!
+  sourceFile.moduleName = 'module-\ud800'
+  const document = new scip.Document()
+  new FileIndexer(
+    program.getTypeChecker(),
+    {
+      cwd,
+      projectRoot: cwd,
+      projectDisplayName: 'syntax',
+      output: '',
+      inferTsconfig: false,
+      progressBar: false,
+      yarnWorkspaces: false,
+      yarnBerryWorkspaces: false,
+      pnpmWorkspaces: false,
+      globalCaches: true,
+      indexedProjects: new Set(),
+      writeIndex: () => {},
+    },
+    Input.fromFile(fileName),
+    document,
+    new Map(),
+    new Map(),
+    new Packages(cwd),
+    sourceFile
+  ).index()
+
+  const index = new scip.Index({ documents: [document] })
+  const decoded = scip.Index.deserializeBinary(index.serializeBinary())
+    .documents[0]
+  assert.equal(decoded.toObject(), document.toObject())
+  assert.equal(decoded.symbols[0].documentation, [
+    '```ts\nmodule "module-�"\n```',
+  ])
+
+  const signatures = [
+    'var high: "�"',
+    'var low: "�"',
+    'var pair: "😀"',
+    'var mixed: "�😀�"',
+    'var adjacent: "�𐀀�"',
+    'var unicode: "ASCII café 中文 😀"',
+    'var escaped: "\\\\ud800"',
+  ]
+  for (const signature of signatures) {
+    assert.ok(
+      decoded.symbols.some(info =>
+        info.documentation.includes('```ts\n' + signature + '\n```')
+      ),
+      `missing signature: ${signature}`
+    )
+  }
+
+  for (const name of ['�', 'low�', '😀', '\\ud800']) {
+    const suffix = '/C().`' + name + '`.'
+    const info = decoded.symbols.find(info => info.symbol.endsWith(suffix))
+    assert.ok(info, `missing property symbol: ${suffix}`)
+    const occurrences = decoded.occurrences.filter(
+      occurrence => occurrence.symbol === info.symbol
+    )
+    assert.ok(
+      occurrences.some(
+        occurrence =>
+          (occurrence.symbol_roles & scip.SymbolRole.Definition) !== 0
+      )
+    )
+    assert.ok(
+      occurrences.some(
+        occurrence =>
+          (occurrence.symbol_roles & scip.SymbolRole.Definition) === 0
+      )
+    )
+  }
+})
+
 test.run()

src/FileIndexer.ts

@@ -81,7 +81,9 @@ export class FileIndexer {
     this.document.symbols.push(
       new scip.scip.SymbolInformation({
         symbol: symbol.value,
-        documentation: ['```ts\nmodule "' + moduleName + '"\n```'],
+        documentation: [
+          ('```ts\nmodule "' + moduleName + '"\n```').toWellFormed(),
+        ],
         kind: scip.scip.SymbolInformation.Kind.File,
       })
     )
@@ -378,7 +380,7 @@ export class FileIndexer {
     this.document.symbols.push(
       new scip.scip.SymbolInformation({
         symbol: symbol.value,
-        documentation,
+        documentation: documentation.map(text => text.toWellFormed()),
         relationships: this.relationships(declaration, symbol),
         kind: symbolKind(declaration, sym),
       })

src/ScipSymbol.ts

@@ -2,7 +2,10 @@ import { descriptorString } from './Descriptor'
 import * as scip from './scip'
 
 export class ScipSymbol {
-  private constructor(public readonly value: string) {}
+  private constructor(public readonly value: string) {
+    // Every symbol reference must use the same protobuf-safe identifier.
+    this.value = value.toWellFormed()
+  }
 
   public isEmpty(): boolean {
     return this.value === ''

tsconfig.json

@@ -3,7 +3,7 @@
     "target": "es2022",
     "module": "Node16",
     "moduleResolution": "Node16",
-    "lib": ["es2022"],
+    "lib": ["es2022", "es2024.string"],
     "rootDir": ".",
     "outDir": "dist",
     "resolveJsonModule": true,