diff --git a/src/web-ui/src/app/scenes/session/fileDropPreviewKind.ts b/src/web-ui/src/app/scenes/session/fileDropPreviewKind.ts index fc73d3a1a5..1d92ba6620 100644 --- a/src/web-ui/src/app/scenes/session/fileDropPreviewKind.ts +++ b/src/web-ui/src/app/scenes/session/fileDropPreviewKind.ts @@ -7,7 +7,7 @@ const extensions: Record = Object.fromEntries( spreadsheet: 'xls xlsx xlsm csv tsv ods numbers', presentation: 'ppt pptx odp key', document: 'doc docx odt rtf txt md markdown pages', - code: 'js jsx ts tsx py rs go java c h cpp hpp cs rb php swift kt vue svelte html css scss json yaml yml toml xml sql sh ps1 ipynb', + code: 'js jsx ts tsx py rs go java c h cpp hpp cs rb php swift kt vue svelte html css scss json yaml yml toml xml sql sh ps1 ipynb cj', archive: 'zip rar 7z tar gz bz2 xz tgz', audio: 'mp3 wav flac aac m4a ogg opus', video: 'mp4 mov mkv avi webm m4v wmv', diff --git a/src/web-ui/src/infrastructure/appearance/appearanceDomainTokens.ts b/src/web-ui/src/infrastructure/appearance/appearanceDomainTokens.ts index 4932766fd9..c934ca63b9 100644 --- a/src/web-ui/src/infrastructure/appearance/appearanceDomainTokens.ts +++ b/src/web-ui/src/infrastructure/appearance/appearanceDomainTokens.ts @@ -66,6 +66,7 @@ export const APPEARANCE_LANGUAGE_TOKENS = { javascriptReact: languageToken('cyan'), python: languageToken('blue'), rust: languageToken('orange'), + cangjie: languageToken('red'), go: languageToken('cyan'), java: languageToken('orange'), kotlin: languageToken('purple'), diff --git a/src/web-ui/src/infrastructure/appearance/builtins/appearancePresetOutput.test.ts b/src/web-ui/src/infrastructure/appearance/builtins/appearancePresetOutput.test.ts index 643bb8b4a1..5764d96378 100644 --- a/src/web-ui/src/infrastructure/appearance/builtins/appearancePresetOutput.test.ts +++ b/src/web-ui/src/infrastructure/appearance/builtins/appearancePresetOutput.test.ts @@ -465,7 +465,7 @@ describe('builtin appearance preset output', () => { "type": "dark", }, { - "hash": "44d681ba137355f1db61559e01e190cb805b25f1e62fc396e7ebe834a429cf3a", + "hash": "c1fe8c49368e8ff96debaeba117ce40b22b2f9c48e2e5d86f372b8fd032380fe", "id": "openbitfun-dark", "type": "dark", }, diff --git a/src/web-ui/src/infrastructure/appearance/builtins/dark.ts b/src/web-ui/src/infrastructure/appearance/builtins/dark.ts index bdb2a961b2..054e8292f3 100644 --- a/src/web-ui/src/infrastructure/appearance/builtins/dark.ts +++ b/src/web-ui/src/infrastructure/appearance/builtins/dark.ts @@ -174,7 +174,12 @@ export const openBitFunDarkPalette: AppearancePalette = { monaco: { base: 'vs-dark', inherit: true, - rules: [], + // `vs-dark` has no rule for the `function` token, so function names fall back + // to the default foreground. Declared function names and call sites use that + // token (Cangjie, and any other language grammar that emits it). + rules: [ + { token: 'function', foreground: 'DCDCAA' }, + ], colors: { background: DARK_BACKGROUND_PRIMARY, foreground: DARK_TEXT_PRIMARY, diff --git a/src/web-ui/src/infrastructure/language-detection/core/LanguageRegistry.ts b/src/web-ui/src/infrastructure/language-detection/core/LanguageRegistry.ts index cbef9c62a0..ad22dda3ca 100644 --- a/src/web-ui/src/infrastructure/language-detection/core/LanguageRegistry.ts +++ b/src/web-ui/src/infrastructure/language-detection/core/LanguageRegistry.ts @@ -98,6 +98,21 @@ const BUILTIN_LANGUAGES: Language[] = [ lineCommentPrefix: '//', blockComment: { start: '/*', end: '*/' }, }, + { + id: 'cangjie', + name: 'Cangjie', + category: 'programming', + extensions: ['cj'], + monacoId: 'cangjie', + prismId: 'cangjie', + textmateScope: 'source.cj', + iconType: 'code', + color: APPEARANCE_LANGUAGE_TOKENS.cangjie, + aliases: ['cj'], + supportsComments: true, + lineCommentPrefix: '//', + blockComment: { start: '/*', end: '*/' }, + }, { id: 'go', name: 'Go', diff --git a/src/web-ui/src/infrastructure/language-detection/utils/helpers.test.ts b/src/web-ui/src/infrastructure/language-detection/utils/helpers.test.ts index e4ad27bf02..e2d9d4d3c7 100644 --- a/src/web-ui/src/infrastructure/language-detection/utils/helpers.test.ts +++ b/src/web-ui/src/infrastructure/language-detection/utils/helpers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { getEditorType } from './helpers'; +import { detectLanguage, getEditorType, getMonacoLanguage } from './helpers'; describe('getEditorType', () => { it('routes PDF files to the PDF viewer case-insensitively', () => { @@ -12,3 +12,11 @@ describe('getEditorType', () => { expect(getEditorType('legacy.HTM')).toBe('html-preview'); }); }); + +describe('Cangjie language detection', () => { + it('maps .cj sources to the Cangjie editor language', () => { + expect(getMonacoLanguage('main.cj')).toBe('cangjie'); + expect(detectLanguage('src/app.cj').language.id).toBe('cangjie'); + expect(getEditorType('main.cj')).toBe('code-editor'); + }); +}); diff --git a/src/web-ui/src/infrastructure/markdown/AsyncPrismSyntaxHighlighter.test.tsx b/src/web-ui/src/infrastructure/markdown/AsyncPrismSyntaxHighlighter.test.tsx index 0063d203fd..e0b47de983 100644 --- a/src/web-ui/src/infrastructure/markdown/AsyncPrismSyntaxHighlighter.test.tsx +++ b/src/web-ui/src/infrastructure/markdown/AsyncPrismSyntaxHighlighter.test.tsx @@ -3,14 +3,10 @@ import React, { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getLoadedPrismSyntaxHighlighter, loadPrismSyntaxHighlighter } from '@/shared/utils/syntaxHighlighterLoader'; import { AsyncPrismSyntaxHighlighter } from './AsyncPrismSyntaxHighlighter'; -vi.mock('@/shared/utils/syntaxHighlighterLoader', () => ({ - getLoadedPrismSyntaxHighlighter: () => null, - loadPrismSyntaxHighlighter: () => new Promise(() => {}), -})); - vi.mock('@/shared/utils/startupTaskScheduling', () => ({ scheduleAfterStartupPaint: () => () => {}, })); @@ -49,4 +45,22 @@ describe('AsyncPrismSyntaxHighlighter appearance contract', () => { expect(container.querySelector('pre[data-openbitfun-part="codePre"]')).not.toBeNull(); expect(container.querySelector('code[data-openbitfun-part="codeContent"]')).not.toBeNull(); }); + + it('highlights Cangjie when another consumer has already loaded Prism', async () => { + const highlighter = await loadPrismSyntaxHighlighter(); + expect(getLoadedPrismSyntaxHighlighter()).toBe(highlighter); + + await act(async () => { + root.render( + + {'func add(a: Int64): Int64 { return a + 1 }'} + , + ); + }); + + expect(container.querySelector('.code-block-fallback')).toBeNull(); + expect(container.querySelector('code .token.keyword')?.textContent).toBe('func'); + expect(container.querySelector('code .token.function')?.textContent).toBe('add'); + expect(container.querySelector('code .token.builtin')?.textContent).toBe('Int64'); + }); }); diff --git a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx index 7d426b9b96..445b63e1ea 100644 --- a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx +++ b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx @@ -229,6 +229,7 @@ const EDITOR_OPENABLE_EXTENSIONS = new Set([ 'gitignore', 'gitattributes', 'editorconfig', 'log', 'dockerfile', 'makefile', 'mk', 'gradle', 'properties', 'plist', 'tex', 'mermaid', 'svg', + 'cj', ]); const EDITOR_OPENABLE_BASENAMES = new Set([ 'dockerfile', @@ -792,6 +793,8 @@ function formatCodeLanguageLabel(lang: string): string { makefile: 'Makefile', toml: 'TOML', xml: 'XML', + cangjie: 'Cangjie', + cj: 'Cangjie', rust: 'Rust', typescript: 'TypeScript', javascript: 'JavaScript', diff --git a/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.test.ts b/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.test.ts new file mode 100644 index 0000000000..68f02f9e92 --- /dev/null +++ b/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from 'vitest'; +import Prism from 'prismjs'; +import SyntaxHighlighter from 'react-syntax-highlighter/dist/esm/prism-async-light'; +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { cangjiePrismGrammar } from '@/tools/editor/languages/cangjie.language'; +import { ensureCangjiePrismLanguage } from './cangjiePrismLanguage'; + +const grammar = cangjiePrismGrammar as unknown as Prism.Grammar; + +describe('cangjie Prism grammar', () => { + it('tokenizes declarations for chat code fences', () => { + const html = Prism.highlight('func add(a: Int64): Int64 {\n return a + 1\n}', grammar, 'cangjie'); + + expect(html).toContain('token keyword'); + expect(html).toContain('token function'); + expect(html).toContain('token builtin'); + expect(html).toContain('token punctuation'); + }); + + it('keeps comments, strings and annotations whole', () => { + const html = Prism.highlight('// note\nlet s = "a ${b} c"\n@Derive(ToString) struct P {}', grammar, 'cangjie'); + + expect(html).toContain('token comment'); + expect(html).toContain('token string'); + expect(html).toContain('token annotation function'); + expect(html).toContain('token class-name'); + }); +}); + +describe('ensureCangjiePrismLanguage', () => { + it('registers once and highlights with the real async highlighter', async () => { + const highlighter = SyntaxHighlighter as typeof SyntaxHighlighter & { + preload: () => Promise; + }; + await highlighter.preload(); + const registerLanguage = vi.spyOn(highlighter, 'registerLanguage'); + + try { + ensureCangjiePrismLanguage(highlighter); + ensureCangjiePrismLanguage(highlighter); + + expect(registerLanguage).toHaveBeenCalledTimes(1); + expect(registerLanguage).toHaveBeenCalledWith('cangjie', expect.any(Function)); + const html = renderToStaticMarkup(React.createElement(highlighter, { + language: 'cangjie', + useInlineStyles: false, + children: 'func add(a: Int64): Int64 { return a + 1 }', + })); + expect(html).toContain('class="token keyword"'); + expect(html).toContain('class="token function"'); + expect(html).toContain('class="token builtin"'); + } finally { + registerLanguage.mockRestore(); + } + }); +}); diff --git a/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.ts b/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.ts new file mode 100644 index 0000000000..9a77ec6903 --- /dev/null +++ b/src/web-ui/src/infrastructure/markdown/cangjiePrismLanguage.ts @@ -0,0 +1,48 @@ +/** + * Registers the Cangjie grammar on the lazily loaded Prism highlighter. + * + * Markdown code fences are rendered by `react-syntax-highlighter`'s + * `prism-async-light`, which only knows the languages listed in its own + * refractor loader map. Cangjie is not one of them, and unknown languages + * degrade to plain text, so the grammar built for the editor is registered on + * the loaded instance instead of adding a dependency on refractor. + */ + +import type * as Prism from 'prismjs'; +import { createLogger } from '@/shared/utils/logger'; +import { cangjiePrismGrammar } from '@/tools/editor/languages/cangjie.language'; + +const log = createLogger('cangjiePrismLanguage'); + +interface AsyncHighlighterRegistrar { + registerLanguage?: (name: string, grammar: typeof registerCangjiePrismGrammar) => void; +} + +const PRISM_LANGUAGE_ID = 'cangjie'; +let registered = false; + +// Refractor accepts language modules, rather than Prism grammar objects. +function registerCangjiePrismGrammar(prism: typeof Prism): void { + prism.languages[PRISM_LANGUAGE_ID] = cangjiePrismGrammar; +} +registerCangjiePrismGrammar.displayName = PRISM_LANGUAGE_ID; + +/** Idempotently register the Cangjie Prism grammar on a loaded highlighter. */ +export function ensureCangjiePrismLanguage(highlighter: unknown): void { + if (registered) { + return; + } + + const registerLanguage = (highlighter as AsyncHighlighterRegistrar | null | undefined)?.registerLanguage; + if (typeof registerLanguage !== 'function') { + log.warn('Prism highlighter does not expose registerLanguage; Cangjie fences stay plain'); + return; + } + + try { + registerLanguage.call(highlighter, PRISM_LANGUAGE_ID, registerCangjiePrismGrammar); + registered = true; + } catch (error) { + log.warn('Failed to register Cangjie Prism grammar', error); + } +} diff --git a/src/web-ui/src/shared/context-system/core/types/CodeSnippetContextImpl.tsx b/src/web-ui/src/shared/context-system/core/types/CodeSnippetContextImpl.tsx index 3744a1771a..1b76349b88 100644 --- a/src/web-ui/src/shared/context-system/core/types/CodeSnippetContextImpl.tsx +++ b/src/web-ui/src/shared/context-system/core/types/CodeSnippetContextImpl.tsx @@ -170,6 +170,7 @@ export function getLanguageDisplayName(language?: string): string { 'cpp': 'C++', 'c': 'C', 'csharp': 'C#', + 'cangjie': 'Cangjie', 'html': 'HTML', 'css': 'CSS', 'scss': 'SCSS', diff --git a/src/web-ui/src/shared/utils/syntaxHighlighterLoader.test.ts b/src/web-ui/src/shared/utils/syntaxHighlighterLoader.test.ts index 5596cad98a..4c04bb0e78 100644 --- a/src/web-ui/src/shared/utils/syntaxHighlighterLoader.test.ts +++ b/src/web-ui/src/shared/utils/syntaxHighlighterLoader.test.ts @@ -5,7 +5,8 @@ it('keeps the fallback available until the AST engine is ready and shares concur let finishEngine!: () => void; const engineReady = new Promise(resolve => { finishEngine = resolve; }); const preload = vi.fn(() => engineReady); - const component = Object.assign(() => null, { preload }); + const registerLanguage = vi.fn(); + const component = Object.assign(() => null, { preload, registerLanguage }); vi.doMock('react-syntax-highlighter/dist/esm/prism-async-light', () => ({ default: component })); try { const loader = await import('./syntaxHighlighterLoader'); @@ -14,9 +15,11 @@ it('keeps the fallback available until the AST engine is ready and shares concur expect(second).toBe(first); await vi.waitFor(() => expect(preload).toHaveBeenCalledTimes(1)); expect(loader.getLoadedPrismSyntaxHighlighter()).toBeNull(); + expect(registerLanguage).not.toHaveBeenCalled(); finishEngine(); expect(await first).toBe(component); expect(loader.getLoadedPrismSyntaxHighlighter()).toBe(component); + expect(registerLanguage).toHaveBeenCalledExactlyOnceWith('cangjie', expect.any(Function)); } finally { vi.doUnmock('react-syntax-highlighter/dist/esm/prism-async-light'); vi.resetModules(); diff --git a/src/web-ui/src/shared/utils/syntaxHighlighterLoader.ts b/src/web-ui/src/shared/utils/syntaxHighlighterLoader.ts index 832ec450e5..d396be451c 100644 --- a/src/web-ui/src/shared/utils/syntaxHighlighterLoader.ts +++ b/src/web-ui/src/shared/utils/syntaxHighlighterLoader.ts @@ -1,4 +1,5 @@ import type React from 'react'; +import { ensureCangjiePrismLanguage } from '@/infrastructure/markdown/cangjiePrismLanguage'; import { createModuleLoader } from './moduleLoader'; type SyntaxHighlighterComponent = React.ComponentType; @@ -21,6 +22,9 @@ export const loadPrismSyntaxHighlighter = createModuleLoader(() => preload: () => Promise; }; await component.preload(); + // All consumers, including CodePreview and cached Markdown renders, must + // receive the custom grammar before the component becomes available. + ensureCangjiePrismLanguage(component); prismSyntaxHighlighterComponent = component; return prismSyntaxHighlighterComponent; }, diff --git a/src/web-ui/src/tools/editor/components/EditorStatusBar.tsx b/src/web-ui/src/tools/editor/components/EditorStatusBar.tsx index 45b3352032..12145eacdb 100644 --- a/src/web-ui/src/tools/editor/components/EditorStatusBar.tsx +++ b/src/web-ui/src/tools/editor/components/EditorStatusBar.tsx @@ -69,6 +69,7 @@ const getLanguageDisplayName = (language: string): string => { 'dockerfile': 'Dockerfile', 'plaintext': 'Plain Text', 'toml': 'TOML', + 'cangjie': 'Cangjie', 'ini': 'INI', 'vue': 'Vue', 'svelte': 'Svelte', diff --git a/src/web-ui/src/tools/editor/components/StatusBarPopovers/StatusBarPopovers.tsx b/src/web-ui/src/tools/editor/components/StatusBarPopovers/StatusBarPopovers.tsx index 737a06be07..f19158b12b 100644 --- a/src/web-ui/src/tools/editor/components/StatusBarPopovers/StatusBarPopovers.tsx +++ b/src/web-ui/src/tools/editor/components/StatusBarPopovers/StatusBarPopovers.tsx @@ -273,6 +273,7 @@ const getLanguageDisplayName = (id: string, aliases?: string[]): string => { bash: 'Bash', plaintext: 'Plain Text', toml: 'TOML', + cangjie: 'Cangjie', vue: 'Vue', svelte: 'Svelte', graphql: 'GraphQL', diff --git a/src/web-ui/src/tools/editor/languages/cangjie.language.test.ts b/src/web-ui/src/tools/editor/languages/cangjie.language.test.ts new file mode 100644 index 0000000000..ea33deb5bb --- /dev/null +++ b/src/web-ui/src/tools/editor/languages/cangjie.language.test.ts @@ -0,0 +1,123 @@ +// @vitest-environment jsdom +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import type * as Monaco from 'monaco-editor'; +import { CANGJIE_LANGUAGE_ID, registerCangjieLanguage } from './cangjie.language'; + +let monaco: typeof Monaco; + +// Loading the real Monaco module graph can exceed 10 seconds on a cold Windows run. +beforeAll(async () => { + window.matchMedia ??= vi.fn().mockReturnValue({ matches: false, addEventListener() {}, removeEventListener() {} }); + monaco = await import('monaco-editor/esm/vs/editor/editor.api'); + registerCangjieLanguage(monaco); +}, 60_000); + +function tokenize(text: string): Monaco.editor.Token[][] { + return monaco.editor.tokenize(text, CANGJIE_LANGUAGE_ID); +} + +function plainType(type: string): string { + return type.replace(/\.cangjie$/, ''); +} + +function typesOnLine(text: string, lineIndex: number): Array<{ offset: number; type: string }> { + return tokenize(text)[lineIndex].map((token) => ({ offset: token.offset, type: plainType(token.type) })); +} + +function escapeForRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Token type covering the first standalone occurrence of `word` on `lineIndex`. + * Whole-word matching matters because single-letter parameter names also appear + * inside neighbouring identifiers. + */ +function typeOfWord(text: string, word: string, lineIndex = 0): string { + const line = text.split('\n')[lineIndex]; + const matcher = new RegExp(`(?:^|[^\\w$])(${escapeForRegExp(word)})(?![\\w$])`); + const match = matcher.exec(line); + expect(match, `"${word}" must appear on line ${lineIndex}`).not.toBeNull(); + const column = match!.index + match![0].indexOf(match![1]); + const covering = [...typesOnLine(text, lineIndex)].reverse().find((token) => token.offset <= column); + return covering?.type ?? ''; +} + +describe('cangjie Monarch tokenizer', () => { + it('registers the language with the .cj extension', () => { + const language = monaco.languages.getLanguages().find((entry) => entry.id === CANGJIE_LANGUAGE_ID); + expect(language?.extensions).toContain('.cj'); + }); + + it('highlights declarations, parameters and builtin types', () => { + const line = 'func add(a: Int64, b: Int64): Int64 { return a + b }'; + expect(typeOfWord(line, 'func')).toBe('keyword'); + expect(typeOfWord(line, 'add')).toBe('function'); + expect(typeOfWord(line, 'a')).toBe('variable'); + expect(typeOfWord(line, 'Int64')).toBe('type.builtin'); + expect(typeOfWord(line, 'return')).toBe('keyword'); + }); + + it('separates declared types, variables and call sites', () => { + const struct = 'public struct Point { let x: Float64 }'; + expect(typeOfWord(struct, 'struct')).toBe('keyword'); + expect(typeOfWord(struct, 'Point')).toBe('type.identifier'); + expect(typeOfWord(struct, 'x')).toBe('variable'); + + const call = 'let total = calculate(1, 2)'; + expect(typeOfWord(call, 'total')).toBe('variable'); + expect(typeOfWord(call, 'calculate')).toBe('function'); + }); + + it('highlights macro invocations', () => { + const line = '@Derive(ToString)'; + expect(typesOnLine(line, 0)[0]?.type).toBe('keyword'); + expect(typeOfWord(line, 'Derive')).toBe('function'); + }); + + it('highlights every numeric literal form', () => { + const line = 'let n = 0x1Fu8 + 0b1010 + 0o717 + 2.5e3f32 + 42'; + expect(typeOfWord(line, '0x1Fu8')).toBe('number.hex'); + expect(typeOfWord(line, '0b1010')).toBe('number.binary'); + expect(typeOfWord(line, '0o717')).toBe('number.octal'); + expect(typeOfWord(line, '2.5e3f32')).toBe('number.float'); + expect(typeOfWord(line, '42')).toBe('number'); + }); + + it('tokenizes strings, escapes and interpolation', () => { + const line = 'let s = "hi ${name} \\n tail"'; + expect(typeOfWord(line, 'hi')).toBe('string'); + expect(typeOfWord(line, '\\n')).toBe('string.escape'); + // Interpolated expressions are code again, not string content. + expect(typeOfWord(line, 'name')).not.toBe('string'); + }); + + it('keeps raw strings opaque and stops them at the closing hashes', () => { + const text = 'let s = #"a "quoted" b"#\nlet t = 1'; + expect(typeOfWord(text, '"quoted"')).toBe('string'); + expect(typeOfWord(text, 'let', 1)).toBe('keyword'); + }); + + it('supports multiline strings and nested block comments', () => { + const multiline = 'let s = """line one\nline two"""\nlet t = 2'; + expect(typeOfWord(multiline, 'line two', 1)).toBe('string'); + expect(typeOfWord(multiline, 'let', 2)).toBe('keyword'); + + const nested = '/* outer /* inner */ still comment */\nlet t = 3'; + expect(typeOfWord(nested, 'still')).toBe('comment'); + expect(typeOfWord(nested, 'let', 1)).toBe('keyword'); + }); + + it('keeps range operators and member access distinct', () => { + const rangeLine = 'for (i in 0..10) { }'; + expect(typesOnLine(rangeLine, 0).find((token) => token.offset === rangeLine.indexOf('..'))?.type).toBe('operator'); + + const memberLine = 'let n = list.size'; + expect(typesOnLine(memberLine, 0).find((token) => token.offset === memberLine.indexOf('.'))?.type).toBe('delimiter'); + }); + + it('does not leak an unterminated single-line string into the next line', () => { + expect(typesOnLine('let s = "open\nlet t = 4', 1)[0]?.type).toBe('keyword'); + expect(typesOnLine("let c = '\nlet t = 5", 1)[0]?.type).toBe('keyword'); + }); +}); diff --git a/src/web-ui/src/tools/editor/languages/cangjie.language.ts b/src/web-ui/src/tools/editor/languages/cangjie.language.ts new file mode 100644 index 0000000000..84f311cb36 --- /dev/null +++ b/src/web-ui/src/tools/editor/languages/cangjie.language.ts @@ -0,0 +1,298 @@ +/** + * Cangjie language definition for Monaco Editor. + * + * Derived from the TextMate grammar shipped with the CodeArts Agent Cangjie + * extension (`huaweicloud.cangjie`, language id `Cangjie`, scope `source.cj`). + * Monaco's standalone build has no Oniguruma runtime, so TextMate grammars + * cannot be loaded directly; this module rewrites the same lexical rules as a + * Monarch tokenizer and exposes a Prism/refractor grammar for markdown code + * fences, both built from the same keyword tables. + * + * Scope mapping: + * - `keyword.control.Cangjie` -> `keyword` + * - `support.class.Cangjie` -> `type.identifier` (declared / PascalCase types) + * - builtin type keywords -> `type.builtin` (deviation: the reference + * grammar classes `Int64`/`Bool`/... as keywords; a type token keeps the + * builtin types visually distinct from control flow) + * - `support.function.Cangjie` -> `function` (declaration names and call sites) + * - `support.variable.Cangjie` -> `variable` (declared names, parameters) + * - `constant.numeric.*` -> `number` / `number.hex` / `number.float` + * - `string.*` -> `string` (plus `${...}` interpolation) + * - `comment.*` -> `comment` (nested block comments) + * - everything else -> default foreground (the reference grammar + * paints every identifier as `support.variable`; leaving free identifiers + * uncolored keeps whole files readable) + * + * @module languages/cangjie + */ + +import type * as Monaco from 'monaco-editor'; +import { createLogger } from '@/shared/utils/logger'; + +const log = createLogger('cangjie.language'); + +export const CANGJIE_LANGUAGE_ID = 'cangjie'; + +/** Declaration / control keywords (`keyword.control.Cangjie` in the reference grammar). */ +export const CANGJIE_KEYWORDS = [ + 'package', 'import', + 'class', 'interface', 'struct', 'enum', 'extend', 'func', 'let', 'var', 'const', 'type', + 'init', 'this', 'super', + 'if', 'else', 'case', 'try', 'catch', 'finally', 'for', 'do', 'while', + 'throw', 'return', 'continue', 'break', 'is', 'as', 'in', 'match', 'where', + 'spawn', 'synchronized', 'macro', 'quote', 'static', + 'public', 'internal', 'private', 'protected', 'override', 'abstract', 'open', + 'redef', 'sealed', 'external', 'foreign', 'inout', 'mut', 'unsafe', 'operator', 'prop', + 'main', 'This', 'Nothing', 'true', 'false', + // Experimental / research-feature keywords present in the reference grammar. + 'VArray', 'differentiable', 'grad', 'vjp', 'valWithGrad', 'adjoint', 'adjointOf', + 'when', 'except', 'include', 'primal', 'stage', +] as const; + +/** Builtin value and collection types. */ +export const CANGJIE_BUILTIN_TYPES = [ + 'Int8', 'Int16', 'Int32', 'Int64', 'IntNative', + 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'UIntNative', + 'Float16', 'Float32', 'Float64', + 'Rune', 'Bool', 'Unit', 'String', 'Option', 'Byte', 'Array', 'Range', +] as const; + +const ID_START = 'A-Za-z_\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF'; +const ID_PART = `${ID_START}0-9`; +const IDENT = `[${ID_START}][${ID_PART}]*`; +const ESCAPED_IDENT = '`[^`\\r\\n]+`'; +const IDENT_OR_ESCAPED = `(?:${ESCAPED_IDENT}|${IDENT})`; + +/** + * Longest-first alternation so `UIntNative` wins over `UInt8`, `adjointOf` over + * `adjoint`, and similar prefix pairs. + */ +function wordRegex(words: readonly string[]): RegExp { + const alternation = [...words] + .sort((left, right) => right.length - left.length) + .join('|'); + return new RegExp(`\\b(?:${alternation})\\b`); +} + +const KEYWORD_REGEX = wordRegex(CANGJIE_KEYWORDS); +const BUILTIN_TYPE_REGEX = wordRegex(CANGJIE_BUILTIN_TYPES); + +const INTEGER_SUFFIX = '(?:u8|u16|u32|u64|i8|i16|i32|i64)?'; +const FLOAT_SUFFIX = '(?:f16|f32|f64)?'; + +/** Cangjie language configuration for Monaco (comments, pairs, word pattern). */ +export const cangjieLanguageConfig: Monaco.languages.LanguageConfiguration = { + comments: { + lineComment: '//', + blockComment: ['/*', '*/'], + }, + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'], + ], + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"', notIn: ['string', 'comment'] }, + { open: "'", close: "'", notIn: ['string', 'comment'] }, + ], + surroundingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + { open: "'", close: "'" }, + ], + wordPattern: /(-?\d*\.\d\w*)|([^\s~$!@#%^&*()\-=+{}[\]\\|;:'",.<>/?]+)/, +}; + +/** Cangjie Monarch tokenizer definition. */ +export const cangjieLanguageTokens: Monaco.languages.IMonarchLanguage = { + defaultToken: '', + ignoreCase: false, + brackets: [ + { open: '{', close: '}', token: 'delimiter.curly' }, + { open: '[', close: ']', token: 'delimiter.square' }, + { open: '(', close: ')', token: 'delimiter.parenthesis' }, + ], + tokenizer: { + root: [ + [/[ \t\r\n]+/, ''], + + // Comments: Cangjie allows nested block comments. + [/\/\/.*$/, 'comment'], + [/\/\*/, { token: 'comment', next: '@blockComment' }], + + // Macro invocation: `@Name` / `@std.collection.Name`. + [new RegExp(`(@)(${IDENT}(?:\\.${IDENT})*)`), ['keyword', 'function']], + + // Declarations, so the declared name gets its own token. + [new RegExp(`\\b(func)(\\s+)(${IDENT_OR_ESCAPED})`), ['keyword', '', 'function']], + [new RegExp(`\\b(macro)(\\s+)(${IDENT_OR_ESCAPED})`), ['keyword', '', 'function']], + [new RegExp(`\\b(class|interface|struct|enum|extend|type)(\\s+)(${IDENT_OR_ESCAPED})`), ['keyword', '', 'type.identifier']], + [new RegExp(`\\b(let|var|const|prop)(\\s+)(${IDENT_OR_ESCAPED})`), ['keyword', '', 'variable']], + + // Parameter and field names of the form `name: Type` after `(` or `,`. + [new RegExp(`([,(])(\\s*)(${IDENT_OR_ESCAPED})(\\s*)(:)`), ['delimiter', '', 'variable', '', 'operator']], + + [BUILTIN_TYPE_REGEX, 'type.builtin'], + [KEYWORD_REGEX, 'keyword'], + + // Numbers. + [new RegExp(`0[xX][0-9a-fA-F_]*\\.[0-9a-fA-F_]*[pP][+-]?\\d[\\d_]*${FLOAT_SUFFIX}`), 'number.float'], + [new RegExp(`0[xX][0-9a-fA-F][0-9a-fA-F_]*${INTEGER_SUFFIX}`), 'number.hex'], + [new RegExp(`0[bB][01][01_]*${INTEGER_SUFFIX}`), 'number.binary'], + [new RegExp(`0[oO][0-7][0-7_]*${INTEGER_SUFFIX}`), 'number.octal'], + [new RegExp(`\\.\\d[\\d_]*(?:[eE][+-]?\\d[\\d_]*)?${FLOAT_SUFFIX}`), 'number.float'], + [new RegExp(`\\d[\\d_]*\\.\\d[\\d_]*(?:[eE][+-]?\\d[\\d_]*)?${FLOAT_SUFFIX}`), 'number.float'], + [new RegExp(`\\d[\\d_]*[eE][+-]?\\d[\\d_]*${FLOAT_SUFFIX}`), 'number.float'], + [new RegExp(`\\d[\\d_]*${INTEGER_SUFFIX}`), 'number'], + + // Raw multiline strings: #"..."#, ##'...'##. Raw strings take no escapes. + [/#+"/, { token: 'string.quote', next: '@rawHashDouble' }], + [/#+'/, { token: 'string.quote', next: '@rawHashSingle' }], + // Raw and prefixed single-line strings. + [/r"[^"\r\n]*"/, 'string'], + [/r'[^'\r\n]*'/, 'string'], + [/b"(?:[^"\\\r\n]|\\.)*"/, 'string'], + [/b'(?:[^'\\\r\n]|\\.)*'/, 'string'], + [/J"(?:[^"\\\r\n]|\\.)*"/, 'string'], + // Multiline strings. + [/"""/, { token: 'string.quote', next: '@tripleDouble' }], + [/'''/, { token: 'string.quote', next: '@tripleSingle' }], + // Single-line strings. The lookahead keeps a quote that ends the line from + // opening a state that would leak into the next line. + [/"(?=.)/, { token: 'string.quote', next: '@singleDouble' }], + [/'(?=.)/, { token: 'string.quote', next: '@singleSingle' }], + [/["']/, 'string.invalid'], + + [/[{}[\]()]/, '@brackets'], + [/[,;]/, 'delimiter'], + [/(?:=>|->|==|!=|<=|>=|&&|\|\||\.\.=|\?\?|\.\.|<<|>>|\*\*|[+\-*/%=<>!&|^~?:])/, 'operator'], + [/\./, 'delimiter'], + + // Declared types, call sites, then plain identifiers. + [/[A-Z][A-Za-z0-9_]*/, 'type.identifier'], + [new RegExp(`${IDENT}(?=\\s*\\()`), 'function'], + [new RegExp(ESCAPED_IDENT), 'variable'], + [new RegExp(IDENT), ''], + [/./, ''], + ], + blockComment: [ + [/\*\//, { token: 'comment', next: '@pop' }], + [/\/\*/, { token: 'comment', next: '@push' }], + [/[^/*]+/, 'comment'], + [/[/*]/, 'comment'], + ], + singleDouble: [ + [/\$\{/, { token: 'delimiter.curly', next: '@interpolation' }], + [/\\./, 'string.escape'], + [/[^"\\$]+$/, { token: 'string', next: '@pop' }], + [/[^"\\$]+/, 'string'], + [/"/, { token: 'string.quote', next: '@pop' }], + [/\$/, 'string'], + [/\\/, { token: 'string.invalid', next: '@pop' }], + ], + singleSingle: [ + [/\$\{/, { token: 'delimiter.curly', next: '@interpolation' }], + [/\\./, 'string.escape'], + [/[^'\\$]+$/, { token: 'string', next: '@pop' }], + [/[^'\\$]+/, 'string'], + [/'/, { token: 'string.quote', next: '@pop' }], + [/\$/, 'string'], + [/\\/, { token: 'string.invalid', next: '@pop' }], + ], + tripleDouble: [ + [/\$\{/, { token: 'delimiter.curly', next: '@interpolation' }], + [/\\./, 'string.escape'], + [/"""/, { token: 'string.quote', next: '@pop' }], + [/[^"\\$]+/, 'string'], + [/["$]/, 'string'], + [/\\/, 'string.escape.invalid'], + ], + tripleSingle: [ + [/\$\{/, { token: 'delimiter.curly', next: '@interpolation' }], + [/\\./, 'string.escape'], + [/'''/, { token: 'string.quote', next: '@pop' }], + [/[^'\\$]+/, 'string'], + [/'|\$/, 'string'], + [/\\/, 'string.escape.invalid'], + ], + rawHashDouble: [ + [/"#+/, { token: 'string.quote', next: '@pop' }], + [/[^"]+/, 'string'], + [/"/, 'string'], + ], + rawHashSingle: [ + [/'#+/, { token: 'string.quote', next: '@pop' }], + [/[^']+/, 'string'], + [/'/, 'string'], + ], + interpolation: [ + [/\}/, { token: 'delimiter.curly', next: '@pop' }], + [/\{/, { token: 'delimiter.curly', next: '@push' }], + { include: '@root' }, + ], + }, +}; + +/** + * Prism/refractor grammar for markdown code fences, so `cangjie` fenced blocks + * in chat highlight with the same vocabulary as the editor. + */ +export const cangjiePrismGrammar = { + comment: [ + { pattern: /\/\*[\s\S]*?\*\//, greedy: true }, + { pattern: /\/\/[^\r\n]*/, greedy: true }, + ], + string: [ + { pattern: /#+["'][\s\S]*?["']#+/, greedy: true }, + { pattern: /(?:r|b|J)?"""[\s\S]*?"""/, greedy: true }, + { pattern: /(?:r|b|J)?'''[\s\S]*?'''/, greedy: true }, + { pattern: /r"[^"\r\n]*"|r'[^'\r\n]*'/ }, + { pattern: /b"(?:[^"\\\r\n]|\\.)*"|b'(?:[^'\\\r\n]|\\.)*'/ }, + { pattern: /J"(?:[^"\\\r\n]|\\.)*"/ }, + { pattern: /"(?:[^"\\\r\n]|\\.)*"/ }, + { pattern: /'(?:[^'\\\r\n]|\\.)*'/ }, + ], + annotation: { + pattern: /@[A-Za-z_][A-Za-z0-9_.]*/, + alias: 'function', + }, + keyword: KEYWORD_REGEX, + builtin: BUILTIN_TYPE_REGEX, + 'class-name': /\b[A-Z][A-Za-z0-9_]*\b/, + function: new RegExp(`${IDENT}(?=\\s*\\()`), + number: [ + { pattern: new RegExp(`0[xX][0-9a-fA-F][0-9a-fA-F_]*${INTEGER_SUFFIX}`), alias: 'constant' }, + { pattern: new RegExp(`0[bB][01][01_]*${INTEGER_SUFFIX}`), alias: 'constant' }, + { pattern: new RegExp(`0[oO][0-7][0-7_]*${INTEGER_SUFFIX}`), alias: 'constant' }, + { pattern: new RegExp(`\\d[\\d_]*(?:\\.\\d[\\d_]*)?(?:[eE][+-]?\\d[\\d_]*)?${FLOAT_SUFFIX}`) }, + ], + operator: /(?:=>|->|==|!=|<=|>=|&&|\|\||\.\.=|\?\?|\.\.|<<|>>|\*\*|[+\-*/%=<>!&|^~?:])/, + punctuation: /[{}[\];(),.]/, +}; + +/** Register the Cangjie language, configuration and tokenizer with Monaco. */ +export function registerCangjieLanguage(monaco: typeof Monaco): void { + try { + const exists = monaco.languages.getLanguages().some((language) => language.id === CANGJIE_LANGUAGE_ID); + + if (!exists) { + monaco.languages.register({ + id: CANGJIE_LANGUAGE_ID, + extensions: ['.cj', '.cj.macrocall'], + aliases: ['Cangjie', 'cangjie', 'cj'], + mimetypes: ['text/x-cangjie'], + }); + } + + monaco.languages.setLanguageConfiguration(CANGJIE_LANGUAGE_ID, cangjieLanguageConfig); + monaco.languages.setMonarchTokensProvider(CANGJIE_LANGUAGE_ID, cangjieLanguageTokens); + } catch (error) { + log.error('Failed to register Cangjie language', error); + throw error; + } +} diff --git a/src/web-ui/src/tools/editor/services/MonacoInitManager.ts b/src/web-ui/src/tools/editor/services/MonacoInitManager.ts index 5a0e8f546d..4078e5d5d9 100644 --- a/src/web-ui/src/tools/editor/services/MonacoInitManager.ts +++ b/src/web-ui/src/tools/editor/services/MonacoInitManager.ts @@ -2,8 +2,8 @@ * Monaco initialization manager (singleton). * * Handles Monaco library initialization (once), TypeScript/JavaScript language - * configuration, custom language registration (TOML), and EditorOpener registration - * for cross-file navigation. + * configuration, custom language registration (TOML, Mermaid, Cangjie), and + * EditorOpener registration for cross-file navigation. * * Visual settings are attached by MonacoAppearanceAdapter. */ @@ -12,6 +12,7 @@ import { loader } from '@monaco-editor/react'; import type * as Monaco from 'monaco-editor'; import { registerMermaidLanguage } from '../languages/mermaid.language'; import { registerTomlLanguage } from '../languages/toml.language'; +import { registerCangjieLanguage } from '../languages/cangjie.language'; import { getMonacoPath, getMonacoWorkerPath, logMonacoResourceCheck } from '../utils/monacoPathHelper'; import { setMonacoRuntime } from './monacoRuntime'; import { createLogger } from '@/shared/utils/logger'; @@ -193,8 +194,10 @@ class MonacoInitManager { try { registerTomlLanguage(monaco); registerMermaidLanguage(monaco); + registerCangjieLanguage(monaco); log.debug('TOML language registered'); log.debug('Mermaid language registered'); + log.debug('Cangjie language registered'); } catch (error) { log.error('Failed to register custom Monaco languages', error); }