fix(context): keep generics and JSX inside code when removing MDX tags

.changeset/keep-code-generics.md

@@ -0,0 +1,5 @@
+---
+"@neuledge/context": patch
+---
+
+Keep generic types and JSX inside code examples. The MDX tag cleanup also ran inside fenced blocks and inline code, so `createTRPCClient<AppRouter>()` was indexed as `createTRPCClient()`, `List<String>` as `List`, and `<App />` vanished. Every such code line was damaged in the four doc sets measured: 289 in the NestJS docs, 298 in tRPC, 44 in Kysely and 88 in Drizzle. MDX component tags such as `<AppOnly>` outside code are still removed.

packages/context/src/build.test.ts

@@ -169,6 +169,87 @@ Regular content.
     expect(result.sections[0].content).toContain("Regular content");
   });
 
+  it("keeps generics and JSX inside code while removing MDX component tags", () => {
+    const source = `## Section
+
+<AppOnly>
+Call \`createClient<AppRouter>()\` to get a typed \`Promise<Response>\`.
+</AppOnly>
+
+\`\`\`tsx
+const client = createClient<AppRouter>({});
+root.render(<App />);
+\`\`\`
+
+~~~java
+List<String> names = new ArrayList<String>();
+~~~
+`;
+
+    const { content } = parseMarkdown(source, "docs/client.mdx").sections[0];
+
+    expect(content).not.toContain("<AppOnly>");
+    expect(content).toContain("`createClient<AppRouter>()`");
+    expect(content).toContain("`Promise<Response>`");
+    expect(content).toContain("const client = createClient<AppRouter>({});");
+    expect(content).toContain("root.render(<App />);");
+    expect(content).toContain("List<String> names = new ArrayList<String>();");
+  });
+
+  it("pairs code fences by line and length", () => {
+    const source = `## Section
+
+\`\`\`js
+const fence = "\`\`\`";
+render(<App />);
+\`\`\`
+
+Wrap code in \`\`\` fences.
+
+<AppOnly>App router content.</AppOnly>
+
+\`\`\`\`md
+\`\`\`ts
+\`\`\`
+const client = createClient<AppRouter>();
+\`\`\`\`
+
+\`\`\`ts
+const rest = createClient<AppRouter>();
+`;
+
+    const { content } = parseMarkdown(source, "docs/fences.mdx").sections[0];
+
+    expect(content).not.toContain("<AppOnly>");
+    expect(content).toContain("App router content.");
+    expect(content).toContain("render(<App />);");
+    expect(content).toContain("const client = createClient<AppRouter>();");
+    expect(content).toContain("const rest = createClient<AppRouter>();");
+  });
+
+  it("keeps generics in a code block split across section parts", () => {
+    const filler = `const value = compute("${"x".repeat(300)}");`;
+    const code = Array.from({ length: 12 }, () => filler).join("\n\n");
+    const source = `## Setup\n\n\`\`\`ts\n${code}\n\nconst trpc = createTRPCClient<AppRouter>({});\n\`\`\`\n`;
+
+    const { sections } = parseMarkdown(source, "docs/setup.md");
+
+    expect(sections.length).toBeGreaterThan(1);
+    expect(sections.at(-1)?.content).toContain(
+      "createTRPCClient<AppRouter>({});",
+    );
+  });
+
+  it("keeps generics inside code converted from HTML", () => {
+    const { content } = parseHtml(
+      '<h2>Section</h2><p>Returns <code>Optional&lt;User&gt;</code>.</p><pre><code class="language-java">Map&lt;String, List&lt;User&gt;&gt; byTeam;\nList&lt;User&gt; users;</code></pre>',
+      "users.html",
+    ).sections[0];
+
+    expect(content).toContain("`Optional<User>`");
+    expect(content).toContain("List<User> users;");
+  });
+
   it("splits large sections at paragraph boundaries", () => {
     // Create content that exceeds MAX_CHUNK_TOKENS (800)
     const largeParagraph = "This is a paragraph. ".repeat(50); // ~1000 chars = ~250 tokens
@@ -258,6 +339,18 @@ Use the library by importing it into your project.
     expect(result.sections[2].sectionTitle).toBe("Usage");
   });
 
+  it("keeps generics in a section long enough to split", () => {
+    const block = `[source,java]\n----\nList<String> names = load("${"x".repeat(300)}");\n----`;
+    const source = `= Guide\n\n== Usage\n\n${Array(12).fill(block).join("\n\n")}\n`;
+
+    const { sections } = parseAsciidoc(source, "docs/guide.adoc");
+
+    expect(sections.length).toBeGreaterThan(1);
+    for (const section of sections) {
+      expect(section.content).toContain("List<String> names");
+    }
+  });
+
   it("extracts attributes as frontmatter", () => {
     const source = `:doctitle: My API Reference
 :description: Complete API reference for the library

packages/context/src/build.ts

@@ -186,10 +186,23 @@ function astToMarkdown(nodes: Content[], source: string): string {
   return "";
 }
 
+/**
+ * React-style tags like <AppOnly> or <PagesOnly>, unless they sit inside a fenced
+ * block or an inline code span. There the same shape is a generic type
+ * (`List<String>`, `createTRPCClient<AppRouter>`) or a JSX example (`<App />`),
+ * and removing it breaks the code. Code is matched first so it is skipped whole.
+ * A fence opens and closes only on a line of its own, so a ``` inside a code line
+ * or a sentence does not pair up with the next block; an unclosed fence runs to
+ * the end, as it does in CommonMark.
+ */
+const MDX_TAG_OUTSIDE_CODE =
+  /^[ \t]*(`{3,}|~{3,})[^\n]*(?:[\s\S]*?\n[ \t]*\1[`~]*[ \t]*$|[\s\S]*)|`[^`\n]+`|(<\/?[A-Z][a-zA-Z]*\s*\/?>)/gm;
+
 /** Remove MDX-specific tags from content. */
 function cleanMdxContent(content: string): string {
-  // Remove React-style tags like <AppOnly>, <PagesOnly>, etc.
-  let cleaned = content.replace(/<\/?[A-Z][a-zA-Z]*\s*\/?>/g, "");
+  let cleaned = content.replace(MDX_TAG_OUTSIDE_CODE, (match, _fence, tag) =>
+    tag ? "" : match,
+  );
   // Remove empty lines created by tag removal
   cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
   return cleaned.trim();
@@ -203,19 +216,22 @@ function createSection(
   content: string,
   partNum: number,
 ): DocSection | null {
-  const cleanedContent = cleanMdxContent(content);
-  const tokens = estimateTokens(cleanedContent);
-  if (!cleanedContent || tokens < MIN_CHUNK_TOKENS) {
+  // Not cleanMdxContent: Markdown is cleaned whole before it is split, and a second
+  // pass here would strip generics from a code block split across parts, whose
+  // opening fence is in an earlier part.
+  const trimmed = content.trim();
+  const tokens = estimateTokens(trimmed);
+  if (!trimmed || tokens < MIN_CHUNK_TOKENS) {
     return null;
   }
   return {
     docPath,
     docTitle,
     sectionTitle:
       partNum > 1 ? `${sectionTitle} (part ${partNum})` : sectionTitle,
-    content: cleanedContent,
+    content: trimmed,
     tokens,
-    hasCode: hasCodeBlock(cleanedContent),
+    hasCode: hasCodeBlock(trimmed),
   };
 }