diff --git a/.changeset/keep-code-generics.md b/.changeset/keep-code-generics.md new file mode 100644 index 0000000..d883da5 --- /dev/null +++ b/.changeset/keep-code-generics.md @@ -0,0 +1,5 @@ +--- +"@neuledge/context": patch +--- + +Keep generic types and JSX inside code examples. The MDX tag cleanup also ran inside fenced blocks and inline code, so `createTRPCClient()` was indexed as `createTRPCClient()`, `List` as `List`, and `` vanished. Every such code line was damaged in the four doc sets measured: 289 in the NestJS docs, 298 in tRPC, 44 in Kysely and 88 in Drizzle. MDX component tags such as `` outside code are still removed. diff --git a/packages/context/src/build.test.ts b/packages/context/src/build.test.ts index 2e8cb9f..3054820 100644 --- a/packages/context/src/build.test.ts +++ b/packages/context/src/build.test.ts @@ -169,6 +169,87 @@ Regular content. expect(result.sections[0].content).toContain("Regular content"); }); + it("keeps generics and JSX inside code while removing MDX component tags", () => { + const source = `## Section + + +Call \`createClient()\` to get a typed \`Promise\`. + + +\`\`\`tsx +const client = createClient({}); +root.render(); +\`\`\` + +~~~java +List names = new ArrayList(); +~~~ +`; + + const { content } = parseMarkdown(source, "docs/client.mdx").sections[0]; + + expect(content).not.toContain(""); + expect(content).toContain("`createClient()`"); + expect(content).toContain("`Promise`"); + expect(content).toContain("const client = createClient({});"); + expect(content).toContain("root.render();"); + expect(content).toContain("List names = new ArrayList();"); + }); + + it("pairs code fences by line and length", () => { + const source = `## Section + +\`\`\`js +const fence = "\`\`\`"; +render(); +\`\`\` + +Wrap code in \`\`\` fences. + +App router content. + +\`\`\`\`md +\`\`\`ts +\`\`\` +const client = createClient(); +\`\`\`\` + +\`\`\`ts +const rest = createClient(); +`; + + const { content } = parseMarkdown(source, "docs/fences.mdx").sections[0]; + + expect(content).not.toContain(""); + expect(content).toContain("App router content."); + expect(content).toContain("render();"); + expect(content).toContain("const client = createClient();"); + expect(content).toContain("const rest = createClient();"); + }); + + it("keeps generics in a code block split across section parts", () => { + const filler = `const value = compute("${"x".repeat(300)}");`; + const code = Array.from({ length: 12 }, () => filler).join("\n\n"); + const source = `## Setup\n\n\`\`\`ts\n${code}\n\nconst trpc = createTRPCClient({});\n\`\`\`\n`; + + const { sections } = parseMarkdown(source, "docs/setup.md"); + + expect(sections.length).toBeGreaterThan(1); + expect(sections.at(-1)?.content).toContain( + "createTRPCClient({});", + ); + }); + + it("keeps generics inside code converted from HTML", () => { + const { content } = parseHtml( + '

Section

Returns Optional<User>.

Map<String, List<User>> byTeam;\nList<User> users;
', + "users.html", + ).sections[0]; + + expect(content).toContain("`Optional`"); + expect(content).toContain("List users;"); + }); + it("splits large sections at paragraph boundaries", () => { // Create content that exceeds MAX_CHUNK_TOKENS (800) const largeParagraph = "This is a paragraph. ".repeat(50); // ~1000 chars = ~250 tokens @@ -258,6 +339,18 @@ Use the library by importing it into your project. expect(result.sections[2].sectionTitle).toBe("Usage"); }); + it("keeps generics in a section long enough to split", () => { + const block = `[source,java]\n----\nList names = load("${"x".repeat(300)}");\n----`; + const source = `= Guide\n\n== Usage\n\n${Array(12).fill(block).join("\n\n")}\n`; + + const { sections } = parseAsciidoc(source, "docs/guide.adoc"); + + expect(sections.length).toBeGreaterThan(1); + for (const section of sections) { + expect(section.content).toContain("List names"); + } + }); + it("extracts attributes as frontmatter", () => { const source = `:doctitle: My API Reference :description: Complete API reference for the library diff --git a/packages/context/src/build.ts b/packages/context/src/build.ts index 96fceee..8d18dc0 100644 --- a/packages/context/src/build.ts +++ b/packages/context/src/build.ts @@ -186,10 +186,23 @@ function astToMarkdown(nodes: Content[], source: string): string { return ""; } +/** + * React-style tags like or , unless they sit inside a fenced + * block or an inline code span. There the same shape is a generic type + * (`List`, `createTRPCClient`) or a JSX example (``), + * and removing it breaks the code. Code is matched first so it is skipped whole. + * A fence opens and closes only on a line of its own, so a ``` inside a code line + * or a sentence does not pair up with the next block; an unclosed fence runs to + * the end, as it does in CommonMark. + */ +const MDX_TAG_OUTSIDE_CODE = + /^[ \t]*(`{3,}|~{3,})[^\n]*(?:[\s\S]*?\n[ \t]*\1[`~]*[ \t]*$|[\s\S]*)|`[^`\n]+`|(<\/?[A-Z][a-zA-Z]*\s*\/?>)/gm; + /** Remove MDX-specific tags from content. */ function cleanMdxContent(content: string): string { - // Remove React-style tags like , , etc. - let cleaned = content.replace(/<\/?[A-Z][a-zA-Z]*\s*\/?>/g, ""); + let cleaned = content.replace(MDX_TAG_OUTSIDE_CODE, (match, _fence, tag) => + tag ? "" : match, + ); // Remove empty lines created by tag removal cleaned = cleaned.replace(/\n{3,}/g, "\n\n"); return cleaned.trim(); @@ -203,9 +216,12 @@ function createSection( content: string, partNum: number, ): DocSection | null { - const cleanedContent = cleanMdxContent(content); - const tokens = estimateTokens(cleanedContent); - if (!cleanedContent || tokens < MIN_CHUNK_TOKENS) { + // Not cleanMdxContent: Markdown is cleaned whole before it is split, and a second + // pass here would strip generics from a code block split across parts, whose + // opening fence is in an earlier part. + const trimmed = content.trim(); + const tokens = estimateTokens(trimmed); + if (!trimmed || tokens < MIN_CHUNK_TOKENS) { return null; } return { @@ -213,9 +229,9 @@ function createSection( docTitle, sectionTitle: partNum > 1 ? `${sectionTitle} (part ${partNum})` : sectionTitle, - content: cleanedContent, + content: trimmed, tokens, - hasCode: hasCodeBlock(cleanedContent), + hasCode: hasCodeBlock(trimmed), }; }