feat(desktop): find design-token sources in a picked folder

apps/desktop/src/main/design-system-tokens.test.ts

@@ -0,0 +1,177 @@
+import { mkdir, mkdtemp, rm, stat, symlink, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { DEFAULT_TOKEN_SCAN_LIMITS, scanTokenSources } from './design-system-tokens';
+
+const tempDirs: string[] = [];
+
+async function makeRepo(files: Record<string, string>): Promise<string> {
+  const root = await mkdtemp(path.join(os.tmpdir(), 'codesign-token-scan-'));
+  tempDirs.push(root);
+  for (const [relativePath, content] of Object.entries(files)) {
+    const fullPath = path.join(root, relativePath);
+    await mkdir(path.dirname(fullPath), { recursive: true });
+    await writeFile(fullPath, content);
+  }
+  return root;
+}
+
+afterEach(async () => {
+  await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
+});
+
+describe('scanTokenSources', () => {
+  it('extracts tokens from a Tailwind config, :root CSS, @theme CSS and DTCG JSON', async () => {
+    const root = await makeRepo({
+      'tailwind.config.js':
+        "module.exports = { theme: { extend: { colors: { brand: '#ff0066' } } } };",
+      'src/styles/tokens.css': ':root { --color-ink: #112233; }',
+      'src/app.css': '@import "tailwindcss";\n@theme {\n  --radius-md: 0.5rem;\n}\n',
+      'design/brand.tokens.json': '{"color":{"paper":{"$value":"#fafafa","$type":"color"}}}',
+      'src/index.ts': 'export {};',
+      'README.md': '# repo',
+    });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan.truncated).toBe(false);
+    expect(scan.skipped).toEqual([]);
+    expect(
+      scan.sources.map((source) => [
+        source.kind,
+        source.path,
+        source.tokens.map((token) => `${token.name}=${token.value}`),
+      ]),
+    ).toEqual([
+      ['dtcg-json', 'design/brand.tokens.json', ['color.paper=#fafafa']],
+      ['css', 'src/app.css', ['radius-md=0.5rem']],
+      ['css', 'src/styles/tokens.css', ['color-ink=#112233']],
+      ['tailwind-config', 'tailwind.config.js', ['colors.brand=#ff0066']],
+    ]);
+  });
+
+  it('reads both @theme and :root blocks from one CSS file', async () => {
+    const root = await makeRepo({
+      'app.css': '@theme {\n  --color-brand: #ff0066;\n}\n:root {\n  --radius: 0.5rem;\n}\n',
+    });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan.sources).toHaveLength(1);
+    expect(scan.sources[0]?.tokens.map((token) => token.name)).toEqual(['color-brand', 'radius']);
+  });
+
+  it('skips node_modules and the other ignored directories', async () => {
+    const root = await makeRepo({
+      'node_modules/pkg/tokens.css': ':root { --color-a: #111111; }',
+      'dist/tokens.css': ':root { --color-b: #222222; }',
+      '.git/tokens.css': ':root { --color-c: #333333; }',
+      'src/tokens.css': ':root { --color-d: #444444; }',
+    });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan.sources.map((source) => source.path)).toEqual(['src/tokens.css']);
+  });
+
+  it('enters directories down to maxDepth and no further', async () => {
+    const root = await makeRepo({
+      'a/b/tokens.css': ':root { --color-a: #111111; }',
+      'a/b/c/tokens.css': ':root { --color-b: #222222; }',
+    });
+
+    const scan = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxDepth: 2 });
+
+    expect(scan.sources.map((source) => source.path)).toEqual(['a/b/tokens.css']);
+  });
+
+  it('stops at maxFiles and reports that it did', async () => {
+    const root = await makeRepo({
+      'a1.txt': '',
+      'a2.txt': '',
+      'a3.txt': '',
+      'tokens.css': ':root { --color-a: #111111; }',
+    });
+
+    const capped = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFiles: 3 });
+    const roomy = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFiles: 4 });
+
+    expect(capped).toEqual({ sources: [], skipped: [], truncated: true });
+    expect(roomy.truncated).toBe(false);
+    expect(roomy.sources.map((source) => source.path)).toEqual(['tokens.css']);
+  });
+
+  it('reports files above maxFileBytes without reading them', async () => {
+    const root = await makeRepo({
+      'big.css': `/* ${'x'.repeat(100)} */\n:root { --color-a: #111111; }`,
+      'small.css': ':root { --color-b: #222222; }',
+    });
+
+    const scan = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFileBytes: 60 });
+
+    expect(scan.skipped).toEqual([{ path: 'big.css', reason: 'too-large' }]);
+    expect(scan.sources.map((source) => source.path)).toEqual(['small.css']);
+  });
+
+  it('reports a DTCG file that is not valid JSON', async () => {
+    const root = await makeRepo({ 'tokens.json': '{ not json' });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan.sources).toEqual([]);
+    expect(scan.skipped).toEqual([{ path: 'tokens.json', reason: 'invalid-json' }]);
+  });
+
+  it('ignores valid JSON that is not shaped like DTCG tokens', async () => {
+    const root = await makeRepo({
+      'tokens.json': '[1, 2, 3]',
+      'a/tokens.json': 'null',
+      'b/tokens.json': '{"name": "pkg", "version": "1.0.0", "files": ["dist"]}',
+    });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan).toEqual({ sources: [], skipped: [], truncated: false });
+  });
+
+  it('ignores a Tailwind config and CSS that are not well formed', async () => {
+    const root = await makeRepo({
+      'tailwind.config.js': "module.exports = { theme: { colors: { brand: '#fff', ",
+      'broken.css': ':root { --color-a: #fff; @theme {',
+    });
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan).toEqual({ sources: [], skipped: [], truncated: false });
+  });
+
+  it('parses a Tailwind config as text and never runs it', async () => {
+    const root = await makeRepo({});
+    const marker = path.join(root, 'ran.txt');
+    await writeFile(
+      path.join(root, 'tailwind.config.js'),
+      `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'ran');\n` +
+        "module.exports = { theme: { colors: { brand: '#ff0066' } } };\n",
+    );
+
+    const scan = await scanTokenSources(root);
+
+    expect(scan.sources[0]?.tokens.map((token) => token.value)).toEqual(['#ff0066']);
+    await expect(stat(marker)).rejects.toMatchObject({ code: 'ENOENT' });
+  });
+
+  it.skipIf(process.platform === 'win32')(
+    'does not follow symlinks out of the folder',
+    async () => {
+      const outside = await makeRepo({ 'tokens.css': ':root { --color-a: #111111; }' });
+      const root = await makeRepo({});
+      await symlink(outside, path.join(root, 'linked'));
+      await symlink(path.join(outside, 'tokens.css'), path.join(root, 'tokens.css'));
+
+      const scan = await scanTokenSources(root);
+
+      expect(scan.sources).toEqual([]);
+    },
+  );
+});

apps/desktop/src/main/design-system-tokens.ts

@@ -0,0 +1,141 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
+import { extractFromCssVars, extractFromTailwindConfig, importDtcgJson } from '@open-codesign/core';
+import type { DesignToken } from '@open-codesign/shared';
+import { IGNORED_DIRS } from './design-system';
+
+export interface TokenScanLimits {
+  /** Directories below the picked folder that are entered. */
+  maxDepth: number;
+  /** Files looked at before the walk stops. */
+  maxFiles: number;
+  /** Largest source file that is read. */
+  maxFileBytes: number;
+}
+
+export const DEFAULT_TOKEN_SCAN_LIMITS: TokenScanLimits = {
+  maxDepth: 6,
+  maxFiles: 10_000,
+  maxFileBytes: 256_000,
+};
+
+export type TokenSourceKind = 'tailwind-config' | 'css' | 'dtcg-json';
+
+export interface TokenSource {
+  kind: TokenSourceKind;
+  /** Path relative to the picked folder, with `/` separators. */
+  path: string;
+  tokens: DesignToken[];
+}
+
+export interface SkippedTokenSource {
+  path: string;
+  reason: 'too-large' | 'invalid-json';
+}
+
+export interface TokenScan {
+  sources: TokenSource[];
+  skipped: SkippedTokenSource[];
+  /** True when the walk stopped at `maxFiles`, so later files were not looked at. */
+  truncated: boolean;
+}
+
+const TAILWIND_CONFIG = /^tailwind\.config\.(?:js|cjs|mjs|ts)$/;
+const DTCG_FILE = /(?:^|\.)tokens\.json$/;
+const CSS_ROOT_BLOCK = /:root|\[data-theme/;
+const CSS_THEME_BLOCK = /@theme\b/;
+
+function sourceKind(fileName: string): TokenSourceKind | null {
+  if (TAILWIND_CONFIG.test(fileName)) return 'tailwind-config';
+  if (DTCG_FILE.test(fileName)) return 'dtcg-json';
+  if (fileName.endsWith('.css')) return 'css';
+  return null;
+}
+
+interface Candidate {
+  file: string;
+  kind: TokenSourceKind;
+}
+
+interface Walk {
+  candidates: Candidate[];
+  visited: number;
+  truncated: boolean;
+}
+
+// Directory entries from readdir are not followed through symlinks: a link is
+// neither isDirectory() nor isFile(), so the scan stays inside the picked folder.
+async function walk(
+  dir: string,
+  depth: number,
+  limits: TokenScanLimits,
+  state: Walk,
+): Promise<void> {
+  const entries = await readdir(dir, { withFileTypes: true, encoding: 'utf8' });
+  entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+  for (const entry of entries) {
+    if (state.truncated) return;
+    if (entry.isDirectory()) {
+      if (depth < limits.maxDepth && !IGNORED_DIRS.has(entry.name)) {
+        await walk(join(dir, entry.name), depth + 1, limits, state);
+      }
+    } else if (entry.isFile()) {
+      if (state.visited >= limits.maxFiles) {
+        state.truncated = true;
+        return;
+      }
+      state.visited += 1;
+      const kind = sourceKind(entry.name);
+      if (kind !== null) state.candidates.push({ file: join(dir, entry.name), kind });
+    }
+  }
+}
+
+async function readTokens(
+  file: string,
+  kind: TokenSourceKind,
+): Promise<DesignToken[] | 'invalid-json'> {
+  if (kind === 'tailwind-config') return extractFromTailwindConfig(file);
+  if (kind === 'dtcg-json') {
+    // importDtcgJson accepts any parsed value, so only malformed JSON marks a bad file.
+    // Read errors propagate instead of being reported as a skipped file.
+    try {
+      return importDtcgJson(JSON.parse(await readFile(file, 'utf-8')));
+    } catch (error) {
+      if (error instanceof SyntaxError) return 'invalid-json';
+      throw error;
+    }
+  }
+  const css = await readFile(file, 'utf-8');
+  const tokens: DesignToken[] = [];
+  if (CSS_THEME_BLOCK.test(css)) tokens.push(...(await extractFromTailwindConfig(file)));
+  if (CSS_ROOT_BLOCK.test(css)) tokens.push(...(await extractFromCssVars(file)));
+  return tokens;
+}
+
+/**
+ * Finds the files under `rootPath` that carry design tokens (a Tailwind config,
+ * CSS with `:root`, `[data-theme]` or `@theme` blocks, DTCG JSON) and extracts
+ * them. Files are read as text and never executed.
+ */
+export async function scanTokenSources(
+  rootPath: string,
+  limits: TokenScanLimits = DEFAULT_TOKEN_SCAN_LIMITS,
+): Promise<TokenScan> {
+  const state: Walk = { candidates: [], visited: 0, truncated: false };
+  await walk(rootPath, 0, limits, state);
+
+  const sources: TokenSource[] = [];
+  const skipped: SkippedTokenSource[] = [];
+  for (const { file, kind } of state.candidates) {
+    const path = relative(rootPath, file).split(sep).join('/');
+    if ((await stat(file)).size > limits.maxFileBytes) {
+      skipped.push({ path, reason: 'too-large' });
+      continue;
+    }
+    const tokens = await readTokens(file, kind);
+    if (tokens === 'invalid-json') skipped.push({ path, reason: 'invalid-json' });
+    else if (tokens.length > 0) sources.push({ kind, path, tokens });
+  }
+  return { sources, skipped, truncated: state.truncated };
+}

apps/desktop/src/main/design-system.ts

@@ -6,7 +6,7 @@ import {
   type StoredDesignSystem,
 } from '@open-codesign/shared';
 
-const IGNORED_DIRS = new Set([
+export const IGNORED_DIRS = new Set([
   '.git',
   '.idea',
   '.next',

packages/core/src/index.ts

@@ -36,6 +36,11 @@ export {
   ModelRegistry,
   SessionManager,
 } from './agent-session.js';
+export {
+  extractFromCssVars,
+  extractFromTailwindConfig,
+  importDtcgJson,
+} from './brand/index.js';
 export { DEFAULT_CONTEXT_WINDOW, resolveContextWindow } from './context-window.js';
 export {
   type BuildDesignContextPackInput,