diff --git a/apps/desktop/src/main/design-system-tokens.test.ts b/apps/desktop/src/main/design-system-tokens.test.ts new file mode 100644 index 00000000..881bd6eb --- /dev/null +++ b/apps/desktop/src/main/design-system-tokens.test.ts @@ -0,0 +1,177 @@ +import { mkdir, mkdtemp, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { DEFAULT_TOKEN_SCAN_LIMITS, scanTokenSources } from './design-system-tokens'; + +const tempDirs: string[] = []; + +async function makeRepo(files: Record): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), 'codesign-token-scan-')); + tempDirs.push(root); + for (const [relativePath, content] of Object.entries(files)) { + const fullPath = path.join(root, relativePath); + await mkdir(path.dirname(fullPath), { recursive: true }); + await writeFile(fullPath, content); + } + return root; +} + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +describe('scanTokenSources', () => { + it('extracts tokens from a Tailwind config, :root CSS, @theme CSS and DTCG JSON', async () => { + const root = await makeRepo({ + 'tailwind.config.js': + "module.exports = { theme: { extend: { colors: { brand: '#ff0066' } } } };", + 'src/styles/tokens.css': ':root { --color-ink: #112233; }', + 'src/app.css': '@import "tailwindcss";\n@theme {\n --radius-md: 0.5rem;\n}\n', + 'design/brand.tokens.json': '{"color":{"paper":{"$value":"#fafafa","$type":"color"}}}', + 'src/index.ts': 'export {};', + 'README.md': '# repo', + }); + + const scan = await scanTokenSources(root); + + expect(scan.truncated).toBe(false); + expect(scan.skipped).toEqual([]); + expect( + scan.sources.map((source) => [ + source.kind, + source.path, + source.tokens.map((token) => `${token.name}=${token.value}`), + ]), + ).toEqual([ + ['dtcg-json', 'design/brand.tokens.json', ['color.paper=#fafafa']], + ['css', 'src/app.css', ['radius-md=0.5rem']], + ['css', 'src/styles/tokens.css', ['color-ink=#112233']], + ['tailwind-config', 'tailwind.config.js', ['colors.brand=#ff0066']], + ]); + }); + + it('reads both @theme and :root blocks from one CSS file', async () => { + const root = await makeRepo({ + 'app.css': '@theme {\n --color-brand: #ff0066;\n}\n:root {\n --radius: 0.5rem;\n}\n', + }); + + const scan = await scanTokenSources(root); + + expect(scan.sources).toHaveLength(1); + expect(scan.sources[0]?.tokens.map((token) => token.name)).toEqual(['color-brand', 'radius']); + }); + + it('skips node_modules and the other ignored directories', async () => { + const root = await makeRepo({ + 'node_modules/pkg/tokens.css': ':root { --color-a: #111111; }', + 'dist/tokens.css': ':root { --color-b: #222222; }', + '.git/tokens.css': ':root { --color-c: #333333; }', + 'src/tokens.css': ':root { --color-d: #444444; }', + }); + + const scan = await scanTokenSources(root); + + expect(scan.sources.map((source) => source.path)).toEqual(['src/tokens.css']); + }); + + it('enters directories down to maxDepth and no further', async () => { + const root = await makeRepo({ + 'a/b/tokens.css': ':root { --color-a: #111111; }', + 'a/b/c/tokens.css': ':root { --color-b: #222222; }', + }); + + const scan = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxDepth: 2 }); + + expect(scan.sources.map((source) => source.path)).toEqual(['a/b/tokens.css']); + }); + + it('stops at maxFiles and reports that it did', async () => { + const root = await makeRepo({ + 'a1.txt': '', + 'a2.txt': '', + 'a3.txt': '', + 'tokens.css': ':root { --color-a: #111111; }', + }); + + const capped = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFiles: 3 }); + const roomy = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFiles: 4 }); + + expect(capped).toEqual({ sources: [], skipped: [], truncated: true }); + expect(roomy.truncated).toBe(false); + expect(roomy.sources.map((source) => source.path)).toEqual(['tokens.css']); + }); + + it('reports files above maxFileBytes without reading them', async () => { + const root = await makeRepo({ + 'big.css': `/* ${'x'.repeat(100)} */\n:root { --color-a: #111111; }`, + 'small.css': ':root { --color-b: #222222; }', + }); + + const scan = await scanTokenSources(root, { ...DEFAULT_TOKEN_SCAN_LIMITS, maxFileBytes: 60 }); + + expect(scan.skipped).toEqual([{ path: 'big.css', reason: 'too-large' }]); + expect(scan.sources.map((source) => source.path)).toEqual(['small.css']); + }); + + it('reports a DTCG file that is not valid JSON', async () => { + const root = await makeRepo({ 'tokens.json': '{ not json' }); + + const scan = await scanTokenSources(root); + + expect(scan.sources).toEqual([]); + expect(scan.skipped).toEqual([{ path: 'tokens.json', reason: 'invalid-json' }]); + }); + + it('ignores valid JSON that is not shaped like DTCG tokens', async () => { + const root = await makeRepo({ + 'tokens.json': '[1, 2, 3]', + 'a/tokens.json': 'null', + 'b/tokens.json': '{"name": "pkg", "version": "1.0.0", "files": ["dist"]}', + }); + + const scan = await scanTokenSources(root); + + expect(scan).toEqual({ sources: [], skipped: [], truncated: false }); + }); + + it('ignores a Tailwind config and CSS that are not well formed', async () => { + const root = await makeRepo({ + 'tailwind.config.js': "module.exports = { theme: { colors: { brand: '#fff', ", + 'broken.css': ':root { --color-a: #fff; @theme {', + }); + + const scan = await scanTokenSources(root); + + expect(scan).toEqual({ sources: [], skipped: [], truncated: false }); + }); + + it('parses a Tailwind config as text and never runs it', async () => { + const root = await makeRepo({}); + const marker = path.join(root, 'ran.txt'); + await writeFile( + path.join(root, 'tailwind.config.js'), + `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'ran');\n` + + "module.exports = { theme: { colors: { brand: '#ff0066' } } };\n", + ); + + const scan = await scanTokenSources(root); + + expect(scan.sources[0]?.tokens.map((token) => token.value)).toEqual(['#ff0066']); + await expect(stat(marker)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it.skipIf(process.platform === 'win32')( + 'does not follow symlinks out of the folder', + async () => { + const outside = await makeRepo({ 'tokens.css': ':root { --color-a: #111111; }' }); + const root = await makeRepo({}); + await symlink(outside, path.join(root, 'linked')); + await symlink(path.join(outside, 'tokens.css'), path.join(root, 'tokens.css')); + + const scan = await scanTokenSources(root); + + expect(scan.sources).toEqual([]); + }, + ); +}); diff --git a/apps/desktop/src/main/design-system-tokens.ts b/apps/desktop/src/main/design-system-tokens.ts new file mode 100644 index 00000000..9d374383 --- /dev/null +++ b/apps/desktop/src/main/design-system-tokens.ts @@ -0,0 +1,141 @@ +import { readdir, readFile, stat } from 'node:fs/promises'; +import { join, relative, sep } from 'node:path'; +import { extractFromCssVars, extractFromTailwindConfig, importDtcgJson } from '@open-codesign/core'; +import type { DesignToken } from '@open-codesign/shared'; +import { IGNORED_DIRS } from './design-system'; + +export interface TokenScanLimits { + /** Directories below the picked folder that are entered. */ + maxDepth: number; + /** Files looked at before the walk stops. */ + maxFiles: number; + /** Largest source file that is read. */ + maxFileBytes: number; +} + +export const DEFAULT_TOKEN_SCAN_LIMITS: TokenScanLimits = { + maxDepth: 6, + maxFiles: 10_000, + maxFileBytes: 256_000, +}; + +export type TokenSourceKind = 'tailwind-config' | 'css' | 'dtcg-json'; + +export interface TokenSource { + kind: TokenSourceKind; + /** Path relative to the picked folder, with `/` separators. */ + path: string; + tokens: DesignToken[]; +} + +export interface SkippedTokenSource { + path: string; + reason: 'too-large' | 'invalid-json'; +} + +export interface TokenScan { + sources: TokenSource[]; + skipped: SkippedTokenSource[]; + /** True when the walk stopped at `maxFiles`, so later files were not looked at. */ + truncated: boolean; +} + +const TAILWIND_CONFIG = /^tailwind\.config\.(?:js|cjs|mjs|ts)$/; +const DTCG_FILE = /(?:^|\.)tokens\.json$/; +const CSS_ROOT_BLOCK = /:root|\[data-theme/; +const CSS_THEME_BLOCK = /@theme\b/; + +function sourceKind(fileName: string): TokenSourceKind | null { + if (TAILWIND_CONFIG.test(fileName)) return 'tailwind-config'; + if (DTCG_FILE.test(fileName)) return 'dtcg-json'; + if (fileName.endsWith('.css')) return 'css'; + return null; +} + +interface Candidate { + file: string; + kind: TokenSourceKind; +} + +interface Walk { + candidates: Candidate[]; + visited: number; + truncated: boolean; +} + +// Directory entries from readdir are not followed through symlinks: a link is +// neither isDirectory() nor isFile(), so the scan stays inside the picked folder. +async function walk( + dir: string, + depth: number, + limits: TokenScanLimits, + state: Walk, +): Promise { + const entries = await readdir(dir, { withFileTypes: true, encoding: 'utf8' }); + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const entry of entries) { + if (state.truncated) return; + if (entry.isDirectory()) { + if (depth < limits.maxDepth && !IGNORED_DIRS.has(entry.name)) { + await walk(join(dir, entry.name), depth + 1, limits, state); + } + } else if (entry.isFile()) { + if (state.visited >= limits.maxFiles) { + state.truncated = true; + return; + } + state.visited += 1; + const kind = sourceKind(entry.name); + if (kind !== null) state.candidates.push({ file: join(dir, entry.name), kind }); + } + } +} + +async function readTokens( + file: string, + kind: TokenSourceKind, +): Promise { + if (kind === 'tailwind-config') return extractFromTailwindConfig(file); + if (kind === 'dtcg-json') { + // importDtcgJson accepts any parsed value, so only malformed JSON marks a bad file. + // Read errors propagate instead of being reported as a skipped file. + try { + return importDtcgJson(JSON.parse(await readFile(file, 'utf-8'))); + } catch (error) { + if (error instanceof SyntaxError) return 'invalid-json'; + throw error; + } + } + const css = await readFile(file, 'utf-8'); + const tokens: DesignToken[] = []; + if (CSS_THEME_BLOCK.test(css)) tokens.push(...(await extractFromTailwindConfig(file))); + if (CSS_ROOT_BLOCK.test(css)) tokens.push(...(await extractFromCssVars(file))); + return tokens; +} + +/** + * Finds the files under `rootPath` that carry design tokens (a Tailwind config, + * CSS with `:root`, `[data-theme]` or `@theme` blocks, DTCG JSON) and extracts + * them. Files are read as text and never executed. + */ +export async function scanTokenSources( + rootPath: string, + limits: TokenScanLimits = DEFAULT_TOKEN_SCAN_LIMITS, +): Promise { + const state: Walk = { candidates: [], visited: 0, truncated: false }; + await walk(rootPath, 0, limits, state); + + const sources: TokenSource[] = []; + const skipped: SkippedTokenSource[] = []; + for (const { file, kind } of state.candidates) { + const path = relative(rootPath, file).split(sep).join('/'); + if ((await stat(file)).size > limits.maxFileBytes) { + skipped.push({ path, reason: 'too-large' }); + continue; + } + const tokens = await readTokens(file, kind); + if (tokens === 'invalid-json') skipped.push({ path, reason: 'invalid-json' }); + else if (tokens.length > 0) sources.push({ kind, path, tokens }); + } + return { sources, skipped, truncated: state.truncated }; +} diff --git a/apps/desktop/src/main/design-system.ts b/apps/desktop/src/main/design-system.ts index df5d4627..fbcff63a 100644 --- a/apps/desktop/src/main/design-system.ts +++ b/apps/desktop/src/main/design-system.ts @@ -6,7 +6,7 @@ import { type StoredDesignSystem, } from '@open-codesign/shared'; -const IGNORED_DIRS = new Set([ +export const IGNORED_DIRS = new Set([ '.git', '.idea', '.next', diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2a575258..3eeeee86 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,6 +36,11 @@ export { ModelRegistry, SessionManager, } from './agent-session.js'; +export { + extractFromCssVars, + extractFromTailwindConfig, + importDtcgJson, +} from './brand/index.js'; export { DEFAULT_CONTEXT_WINDOW, resolveContextWindow } from './context-window.js'; export { type BuildDesignContextPackInput,