From 3a700c621e679d5a9adf04117e6435f8f19d1793 Mon Sep 17 00:00:00 2001 From: Jeremy Levartovsky <140487034+JayOfTheKeyboard@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:25:07 +1000 Subject: [PATCH] fix(context): keep generics and JSX inside code when removing MDX tags cleanMdxContent removed every -shaped token, including inside fenced blocks and inline code, so createTRPCClient() was indexed as createTRPCClient(), List as List, and vanished. Every such code line was lost in the four doc sets measured: NestJS 289, tRPC 298, Kysely 44, Drizzle 88. All are kept now. The tag regex now matches code first and keeps it whole: fences that open and close on a line of their own, closed by the same fence (an unclosed fence runs to the end, as in CommonMark), and inline code spans. createSection no longer runs cleanMdxContent again. Markdown is cleaned before it is split, and the second pass stripped generics from the later parts of a split code block. It also stripped them from AsciiDoc and rST sections only when they were long enough to split (JUnit lost 7 of 16 such lines, e.g. tasks.withType()). --- .changeset/keep-code-generics.md | 5 ++ packages/context/src/build.test.ts | 93 ++++++++++++++++++++++++++++++ packages/context/src/build.ts | 30 +++++++--- 3 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 .changeset/keep-code-generics.md diff --git a/.changeset/keep-code-generics.md b/.changeset/keep-code-generics.md new file mode 100644 index 0000000..d883da5 --- /dev/null +++ b/.changeset/keep-code-generics.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Keep generic types and JSX inside code examples. The MDX tag cleanup also ran inside fenced blocks and inline code, so `createTRPCClient()` was indexed as `createTRPCClient()`, `List` as `List`, and `` vanished. Every such code line was damaged in the four doc sets measured: 289 in the NestJS docs, 298 in tRPC, 44 in Kysely and 88 in Drizzle. MDX component tags such as `` outside code are still removed. diff --git a/packages/context/src/build.test.ts b/packages/context/src/build.test.ts index 2e8cb9f..3054820 100644 --- a/packages/context/src/build.test.ts +++ b/packages/context/src/build.test.ts @@ -169,6 +169,87 @@ Regular content. expect(result.sections[0].content).toContain("Regular content"); }); + it("keeps generics and JSX inside code while removing MDX component tags", () => { + const source = `## Section + + +Call \`createClient()\` to get a typed \`Promise\`. + + +\`\`\`tsx +const client = createClient({}); +root.render(); +\`\`\` + +~~~java +List names = new ArrayList(); +~~~ +`; + + const { content } = parseMarkdown(source, "docs/client.mdx").sections[0]; + + expect(content).not.toContain(""); + expect(content).toContain("`createClient()`"); + expect(content).toContain("`Promise`"); + expect(content).toContain("const client = createClient({});"); + expect(content).toContain("root.render();"); + expect(content).toContain("List names = new ArrayList();"); + }); + + it("pairs code fences by line and length", () => { + const source = `## Section + +\`\`\`js +const fence = "\`\`\`"; +render(); +\`\`\` + +Wrap code in \`\`\` fences. + +App router content. + +\`\`\`\`md +\`\`\`ts +\`\`\` +const client = createClient(); +\`\`\`\` + +\`\`\`ts +const rest = createClient(); +`; + + const { content } = parseMarkdown(source, "docs/fences.mdx").sections[0]; + + expect(content).not.toContain(""); + expect(content).toContain("App router content."); + expect(content).toContain("render();"); + expect(content).toContain("const client = createClient();"); + expect(content).toContain("const rest = createClient();"); + }); + + it("keeps generics in a code block split across section parts", () => { + const filler = `const value = compute("${"x".repeat(300)}");`; + const code = Array.from({ length: 12 }, () => filler).join("\n\n"); + const source = `## Setup\n\n\`\`\`ts\n${code}\n\nconst trpc = createTRPCClient({});\n\`\`\`\n`; + + const { sections } = parseMarkdown(source, "docs/setup.md"); + + expect(sections.length).toBeGreaterThan(1); + expect(sections.at(-1)?.content).toContain( + "createTRPCClient({});", + ); + }); + + it("keeps generics inside code converted from HTML", () => { + const { content } = parseHtml( + '

Section

Returns Optional<User>.

Map<String, List<User>> byTeam;\nList<User> users;
', + "users.html", + ).sections[0]; + + expect(content).toContain("`Optional`"); + expect(content).toContain("List users;"); + }); + it("splits large sections at paragraph boundaries", () => { // Create content that exceeds MAX_CHUNK_TOKENS (800) const largeParagraph = "This is a paragraph. ".repeat(50); // ~1000 chars = ~250 tokens @@ -258,6 +339,18 @@ Use the library by importing it into your project. expect(result.sections[2].sectionTitle).toBe("Usage"); }); + it("keeps generics in a section long enough to split", () => { + const block = `[source,java]\n----\nList names = load("${"x".repeat(300)}");\n----`; + const source = `= Guide\n\n== Usage\n\n${Array(12).fill(block).join("\n\n")}\n`; + + const { sections } = parseAsciidoc(source, "docs/guide.adoc"); + + expect(sections.length).toBeGreaterThan(1); + for (const section of sections) { + expect(section.content).toContain("List names"); + } + }); + it("extracts attributes as frontmatter", () => { const source = `:doctitle: My API Reference :description: Complete API reference for the library diff --git a/packages/context/src/build.ts b/packages/context/src/build.ts index 96fceee..8d18dc0 100644 --- a/packages/context/src/build.ts +++ b/packages/context/src/build.ts @@ -186,10 +186,23 @@ function astToMarkdown(nodes: Content[], source: string): string { return ""; } +/** + * React-style tags like or , unless they sit inside a fenced + * block or an inline code span. There the same shape is a generic type + * (`List`, `createTRPCClient`) or a JSX example (``), + * and removing it breaks the code. Code is matched first so it is skipped whole. + * A fence opens and closes only on a line of its own, so a ``` inside a code line + * or a sentence does not pair up with the next block; an unclosed fence runs to + * the end, as it does in CommonMark. + */ +const MDX_TAG_OUTSIDE_CODE = + /^[ \t]*(`{3,}|~{3,})[^\n]*(?:[\s\S]*?\n[ \t]*\1[`~]*[ \t]*$|[\s\S]*)|`[^`\n]+`|(<\/?[A-Z][a-zA-Z]*\s*\/?>)/gm; + /** Remove MDX-specific tags from content. */ function cleanMdxContent(content: string): string { - // Remove React-style tags like , , etc. - let cleaned = content.replace(/<\/?[A-Z][a-zA-Z]*\s*\/?>/g, ""); + let cleaned = content.replace(MDX_TAG_OUTSIDE_CODE, (match, _fence, tag) => + tag ? "" : match, + ); // Remove empty lines created by tag removal cleaned = cleaned.replace(/\n{3,}/g, "\n\n"); return cleaned.trim(); @@ -203,9 +216,12 @@ function createSection( content: string, partNum: number, ): DocSection | null { - const cleanedContent = cleanMdxContent(content); - const tokens = estimateTokens(cleanedContent); - if (!cleanedContent || tokens < MIN_CHUNK_TOKENS) { + // Not cleanMdxContent: Markdown is cleaned whole before it is split, and a second + // pass here would strip generics from a code block split across parts, whose + // opening fence is in an earlier part. + const trimmed = content.trim(); + const tokens = estimateTokens(trimmed); + if (!trimmed || tokens < MIN_CHUNK_TOKENS) { return null; } return { @@ -213,9 +229,9 @@ function createSection( docTitle, sectionTitle: partNum > 1 ? `${sectionTitle} (part ${partNum})` : sectionTitle, - content: cleanedContent, + content: trimmed, tokens, - hasCode: hasCodeBlock(cleanedContent), + hasCode: hasCodeBlock(trimmed), }; }