diff --git a/docs/docs/en/cli/fill.md b/docs/docs/en/cli/fill.md index fd7080ddaf..a8542edc95 100644 --- a/docs/docs/en/cli/fill.md +++ b/docs/docs/en/cli/fill.md @@ -1,6 +1,6 @@ --- createdAt: 2024-08-11 -updatedAt: 2026-09-27 +updatedAt: 2026-10-06 priority: 5 title: "intlayer fill: Translate Dictionaries with AI" description: "Fill missing translations, audit existing ones and translate your Intlayer dictionaries with AI from the CLI, locally or in CI." @@ -117,7 +117,9 @@ Affected dictionary keys for processing: app, comp-test, hello-world, lang-switc - **`--output-locales [outputLocales...]`**: Target locales to translate to. If not specified, all locales from your configuration will be used except the source locale. -- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content, `review` will fill missing content and review existing keys. +- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content and re-translate values whose source changed in git, `review` will fill missing content and review existing keys. + + > In `complete` mode, source values are compared with the last commit (`HEAD`), or with the base branch when `--git-diff` is set (with `--unpushed`, with the upstream branch). A translation is re-translated when its source value changed and the translation itself did not. No extra file is written. This applies to JSON, JS and TS content declaration files tracked by git. If that git ref can't be read (no git repository, no commit, or a shallow clone without the base branch), `fill` logs why and only fills missing translations. In CI, check out the full history (`fetch-depth: 0` with `actions/checkout`) to use `--git-diff`. **Git options:** diff --git a/packages/@intlayer/cli/src/fill/fill.ts b/packages/@intlayer/cli/src/fill/fill.ts index 3f4e4e7c77..b23f4c06c4 100644 --- a/packages/@intlayer/cli/src/fill/fill.ts +++ b/packages/@intlayer/cli/src/fill/fill.ts @@ -35,6 +35,11 @@ import { listTranslationsTasks, type TranslationTask, } from './listTranslationsTasks'; +import { + getSourceChangesRef, + loadPreviousDictionaries, + type PreviousDictionaries, +} from './sourceChanges'; import { translateDictionary } from './translateDictionary'; import { writeFill } from './writeFill'; @@ -177,14 +182,39 @@ export const fill = async (options?: FillOptions): Promise => { * * Create a list of per-locale dictionaries to translate * - * In 'complete' mode, filter only the missing locales to translate + * In 'complete' mode, filter only the locales with missing translations or + * with source values changed in git (compared to the last commit, or to the + * base branch with `--git-diff`) */ + let previousDictionaries: PreviousDictionaries = {}; + + if (mode === 'complete') { + const { ref, skipReason } = await getSourceChangesRef( + options?.gitOptions, + configuration.system.baseDir + ); + + if (ref) { + previousDictionaries = await loadPreviousDictionaries( + targetUnmergedDictionaries, + configuration, + ref + ); + } else { + appLogger( + `Source changes not checked: ${skipReason}. Only missing translations are filled.`, + { level: options?.gitOptions ? 'warn' : 'info' } + ); + } + } + const translationTasks: TranslationTask[] = listTranslationsTasks( targetUnmergedDictionaries.map((dictionary) => dictionary.localId!), outputLocales, mode, baseLocale, - configuration + configuration, + previousDictionaries ); // AI calls in flight at once (translateJSON + metadata audit) diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts new file mode 100644 index 0000000000..07778fa86d --- /dev/null +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts @@ -0,0 +1,122 @@ +import { getDictionaries } from '@intlayer/dictionaries-entry'; +import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { listMissingTranslationsWithConfig } from '../test'; +import { listTranslationsTasks } from './listTranslationsTasks'; +import type { PreviousDictionaries } from './sourceChanges'; + +vi.mock('@intlayer/dictionaries-entry', () => ({ getDictionaries: vi.fn() })); +vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ + getUnmergedDictionaries: vi.fn(), +})); +vi.mock('../test', () => ({ listMissingTranslationsWithConfig: vi.fn() })); +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({})); +vi.mock('@intlayer/config/logger', async (importOriginal) => ({ + ...(await importOriginal()), + getAppLogger: () => () => undefined, +})); + +const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; + +/** Fully translated dictionary, whose English title was edited to `Hello`. */ +const dictionary: Dictionary = { + key: 'home', + localId: dictionaryLocalId, + filePath: 'src/home.content.ts', + content: { + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: 'Salut', es: 'Hola' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue', es: 'Bienvenido' }, + }, + }, +}; + +const configuration = { + dictionary: { fill: true }, +} as IntlayerConfig; + +/** `dictionary` as it was in git, with the title and translations given. */ +const getPreviousDictionary = (title: Record): Dictionary => ({ + ...dictionary, + content: { + ...dictionary.content, + title: { nodeType: 'translation', translation: title }, + }, +}); + +const listTasks = (previousDictionaries?: PreviousDictionaries) => + listTranslationsTasks( + [dictionaryLocalId], + ['en', 'fr', 'es'], + 'complete', + 'en', + configuration, + previousDictionaries + ); + +describe('listTranslationsTasks', () => { + beforeEach(() => { + vi.mocked(getDictionaries).mockReturnValue({ home: dictionary }); + vi.mocked(getUnmergedDictionaries).mockReturnValue({ home: [dictionary] }); + vi.mocked(listMissingTranslationsWithConfig).mockReturnValue({ + missingTranslations: [], + } as unknown as ReturnType); + }); + + it('skips a fully translated dictionary whose source did not change', () => { + expect(listTasks()).toEqual([]); + expect( + listTasks({ + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hello', + fr: 'Salut', + es: 'Hola', + }), + }) + ).toEqual([]); + }); + + it('re-translates locales whose source value changed in git', () => { + const tasks = listTasks({ + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hi', + fr: 'Salut', + es: 'Hola', + }), + }); + + expect(tasks).toHaveLength(1); + expect(tasks[0]).toMatchObject({ + dictionaryKey: 'home', + sourceLocale: 'en', + targetLocales: ['fr', 'es'], + changedSourceContent: { + fr: { title: 'Hello' }, + es: { title: 'Hello' }, + }, + }); + }); + + it('keeps a translation edited in the same change as its source', () => { + const tasks = listTasks({ + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hi', + fr: 'Salut', + es: 'Holi', + }), + }); + + expect(tasks).toHaveLength(1); + expect(tasks[0]).toMatchObject({ + targetLocales: ['fr'], + changedSourceContent: { fr: { title: 'Hello' } }, + }); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts index d16a6a9ff6..4414489ca1 100644 --- a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts @@ -10,11 +10,15 @@ import { import { getFilterTranslationsOnlyDictionary } from '@intlayer/core/plugins'; import { getDictionaries } from '@intlayer/dictionaries-entry'; import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; -import { formatLocale } from '@intlayer/engine/utils'; +import { formatLocale, type Recursive } from '@intlayer/engine/utils'; import type { Locale } from '@intlayer/types/allLocales'; import type { IntlayerConfig } from '@intlayer/types/config'; import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; import { listMissingTranslationsWithConfig } from '../test'; +import { + getStaleSourceContent, + type PreviousDictionaries, +} from './sourceChanges'; export type TranslationTask = { dictionaryKey: string; @@ -23,6 +27,8 @@ export type TranslationTask = { targetLocales: Locale[]; dictionaryPreset: string; dictionaryFilePath: string; + /** Per target locale, source values changed since the git ref. */ + changedSourceContent: Partial>; }; export const listTranslationsTasks = ( @@ -30,7 +36,8 @@ export const listTranslationsTasks = ( outputLocales: Locale[], mode: 'complete' | 'review', baseLocale: Locale, - configuration: IntlayerConfig + configuration: IntlayerConfig, + previousDictionaries: PreviousDictionaries = {} ): TranslationTask[] => { const appLogger = getAppLogger(configuration); @@ -115,19 +122,48 @@ export const listTranslationsTasks = ( } /** - * In 'complete' mode, filter only the missing locales to translate + * Source values edited since the git ref, with a translation that was not + * updated. Only the base locale is a source of truth: other per-locale + * files are fill outputs. + */ + const changedSourceContent: Partial> = {}; + + if (sourceLocale === baseLocale) { + for (const locale of outputLocales) { + if (locale === sourceLocale) continue; + + const changedContent = getStaleSourceContent( + previousDictionaries[dictionaryLocalId], + targetUnmergedDictionary, + sourceLocale, + locale + ); + + if (changedContent !== undefined) { + changedSourceContent[locale] = changedContent; + } + } + } + + /** + * In 'complete' mode, filter only the locales with missing translations + * or with source values changed since the git ref * - * Skip the dictionary if there are no missing locales to translate + * Skip the dictionary if there are no locales to translate */ let outputLocalesList: Locale[] = outputLocales as Locale[]; if (mode === 'complete') { - outputLocalesList = - missingTranslations - .find( - (missingTranslation) => missingTranslation.key === dictionaryKey - ) - ?.locales.filter((locale) => outputLocales.includes(locale)) ?? []; + const missingLocales = + missingTranslations.find( + (missingTranslation) => missingTranslation.key === dictionaryKey + )?.locales ?? []; + + outputLocalesList = outputLocales.filter( + (locale) => + missingLocales.includes(locale) || + Object.hasOwn(changedSourceContent, locale) + ); } if (outputLocalesList.length === 0) { @@ -147,6 +183,7 @@ export const listTranslationsTasks = ( targetLocales: outputLocalesList, dictionaryPreset, dictionaryFilePath: targetUnmergedDictionary.filePath, + changedSourceContent, }); } diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.test.ts b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts new file mode 100644 index 0000000000..c6af8fe960 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts @@ -0,0 +1,197 @@ +import { getGitMergeBase, hasGitRef, readGitFile } from '@intlayer/engine/cli'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary } from '@intlayer/types/dictionary'; +import { describe, expect, it, vi } from 'vitest'; +import { + getChangedContent, + getSourceChangesRef, + getStaleSourceContent, + loadPreviousDictionaries, + omitChangedContent, +} from './sourceChanges'; + +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({ + getGitMergeBase: vi.fn(), + hasGitRef: vi.fn(), + readGitFile: vi.fn(), +})); + +const getDictionary = ( + title: Record, + subtitle: Record +): Dictionary => ({ + key: 'home', + content: { + title: { nodeType: 'translation', translation: title }, + subtitle: { nodeType: 'translation', translation: subtitle }, + }, +}); + +describe('getChangedContent', () => { + it('returns only the changed values', () => { + expect( + getChangedContent( + { a: 'Hi', b: 'Bye', c: { d: 'Same' } }, + { a: 'Hello', b: 'Bye', c: { d: 'Same' } } + ) + ).toEqual({ a: 'Hello' }); + }); + + it('returns undefined when nothing changed', () => { + expect(getChangedContent({ a: ['x', 'y'] }, { a: ['x', 'y'] })).toBe( + undefined + ); + }); + + it('compares arrays as a whole', () => { + expect(getChangedContent({ a: ['x', 'y'] }, { a: ['x', 'z'] })).toEqual({ + a: ['x', 'z'], + }); + }); +}); + +describe('omitChangedContent', () => { + it('removes changed values from the target content', () => { + expect( + omitChangedContent( + { a: 'Salut', b: 'Au revoir', c: { d: 'x', e: 'y' } }, + { a: 'Hello', c: { d: 'z' } } + ) + ).toEqual({ b: 'Au revoir', c: { e: 'y' } }); + }); +}); + +describe('getStaleSourceContent', () => { + const current = getDictionary( + { en: 'Hello', fr: 'Salut' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + it('returns source values changed since the previous version', () => { + const previous = getDictionary( + { en: 'Hi', fr: 'Salut' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + expect(getStaleSourceContent(previous, current, 'en', 'fr')).toEqual({ + title: 'Hello', + }); + }); + + it('keeps translations updated in the same change', () => { + const previous = getDictionary( + { en: 'Hi', fr: 'Coucou' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + expect(getStaleSourceContent(previous, current, 'en', 'fr')).toBe( + undefined + ); + }); + + it('returns undefined without a previous version', () => { + expect(getStaleSourceContent(undefined, current, 'en', 'fr')).toBe( + undefined + ); + }); + + it('compares per-locale files on their source only', () => { + const perLocale = (title: string): Dictionary => ({ + key: 'home', + locale: 'en', + content: { title }, + }); + + expect( + getStaleSourceContent(perLocale('Hi'), perLocale('Hello'), 'en', 'fr') + ).toEqual({ title: 'Hello' }); + }); +}); + +describe('getSourceChangesRef', () => { + it('compares to the last commit by default', async () => { + vi.mocked(hasGitRef).mockResolvedValue(true); + + expect(await getSourceChangesRef()).toEqual({ ref: 'HEAD' }); + expect(await getSourceChangesRef({ mode: ['uncommitted'] })).toEqual({ + ref: 'HEAD', + }); + }); + + it('compares to the upstream for unpushed changes', async () => { + vi.mocked(hasGitRef).mockResolvedValue(true); + + expect(await getSourceChangesRef({ mode: ['unpushed'] })).toEqual({ + ref: '@{push}', + }); + }); + + it('compares to the merge base with --git-diff', async () => { + vi.mocked(getGitMergeBase).mockResolvedValue('abc123'); + + expect( + await getSourceChangesRef({ mode: ['gitDiff'], baseRef: 'origin/main' }) + ).toEqual({ ref: 'abc123' }); + }); + + it('skips when git or the last commit is not available', async () => { + vi.mocked(hasGitRef).mockResolvedValue(false); + + const result = await getSourceChangesRef(undefined, '/project'); + + expect(hasGitRef).toHaveBeenCalledWith('HEAD', '/project'); + expect(result.ref).toBe(undefined); + expect(result.skipReason).toContain('git ref HEAD not found'); + }); + + it('skips with --git-diff when the merge base is missing', async () => { + // Shallow clone (e.g. actions/checkout with its default fetch-depth: 1) + vi.mocked(getGitMergeBase).mockResolvedValue(undefined); + + const result = await getSourceChangesRef({ + mode: ['gitDiff'], + baseRef: 'origin/main', + }); + + expect(result.ref).toBe(undefined); + expect(result.skipReason).toContain('fetch-depth: 0'); + }); +}); + +describe('loadPreviousDictionaries', () => { + const configuration = { + system: { baseDir: '/project' }, + } as IntlayerConfig; + + const getEntry = (location: string): Dictionary => ({ + key: 'home', + localId: `home::${location}::src/home.content.json`, + location, + filePath: 'src/home.content.json', + content: {}, + }); + + it('reads git only for local and hybrid dictionaries', async () => { + vi.mocked(readGitFile).mockResolvedValue(undefined); + + const previousDictionaries = await loadPreviousDictionaries( + [ + getEntry('local'), + getEntry('hybrid'), + getEntry('remote'), + getEntry('plugin'), + getEntry('sync-json::messages/{{locale}}.json'), + ], + configuration, + 'HEAD' + ); + + expect(previousDictionaries).toEqual({}); + expect(readGitFile).toHaveBeenCalledTimes(2); + expect(readGitFile).toHaveBeenCalledWith( + '/project/src/home.content.json', + 'HEAD' + ); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.ts b/packages/@intlayer/cli/src/fill/sourceChanges.ts new file mode 100644 index 0000000000..55c5d1d051 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/sourceChanges.ts @@ -0,0 +1,228 @@ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { getPerLocaleDictionary } from '@intlayer/core/plugins'; +import { + loadContentDeclaration, + processContentDeclaration, +} from '@intlayer/engine/build'; +import { + getGitMergeBase, + hasGitRef, + type ListGitFilesOptions, + readGitFile, +} from '@intlayer/engine/cli'; +import type { Recursive } from '@intlayer/engine/utils'; +import type { Locale } from '@intlayer/types/allLocales'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; + +/** Previous version of each content declaration, by dictionary local id. */ +export type PreviousDictionaries = Partial< + Record +>; + +export type SourceChangesRef = + | { ref: string; skipReason?: undefined } + | { ref?: undefined; skipReason: string }; + +/** + * Git ref the source values are compared against. The base branch with + * `--git-diff`, the upstream with `--unpushed`, the last commit otherwise. + * + * When the ref can't be read (no git, no commit, no upstream, or a shallow + * clone without the base branch), returns why instead, and `fill` only fills + * missing translations, as it did before source change detection. + */ +export const getSourceChangesRef = async ( + gitOptions?: ListGitFilesOptions, + cwd?: string +): Promise => { + if (gitOptions?.mode.includes('gitDiff')) { + const { baseRef = 'origin/main', currentRef = 'HEAD' } = gitOptions; + const mergeBase = await getGitMergeBase(baseRef, currentRef); + + if (mergeBase) return { ref: mergeBase }; + + return { + skipReason: `no merge base found between ${baseRef} and ${currentRef}. In CI, check out the full git history (fetch-depth: 0)`, + }; + } + + const ref = gitOptions?.mode.includes('unpushed') ? '@{push}' : 'HEAD'; + + if (await hasGitRef(ref, cwd)) return { ref }; + + return { + skipReason: `git ref ${ref} not found (no git repository, no commit or no upstream branch)`, + }; +}; + +/** + * Loads each local content declaration as it is at `ref` in git. + * Files that are unchanged, new, or not tracked by git are skipped. + */ +export const loadPreviousDictionaries = async ( + dictionaries: Dictionary[], + configuration: IntlayerConfig, + ref: string +): Promise => { + const previousDictionaries: PreviousDictionaries = {}; + + for (const dictionary of dictionaries) { + if (!dictionary.localId || !dictionary.filePath) continue; + if (!['local', 'hybrid'].includes(dictionary.location ?? 'local')) continue; + + const filePath = join(configuration.system.baseDir, dictionary.filePath); + const previousCode = await readGitFile(filePath, ref); + + if (previousCode === undefined) continue; + + const currentCode = await readFile(filePath, 'utf-8').catch( + () => undefined + ); + + if (previousCode === currentCode) continue; + + const loaded = await loadContentDeclaration( + filePath, + configuration, + undefined, + { logError: false, code: previousCode } + ); + + // A file can declare several dictionaries + const previousDeclaration = [loaded] + .flat() + .find((declaration) => declaration?.key === dictionary.key); + + if (!previousDeclaration) continue; + + const previousDictionary = await processContentDeclaration( + previousDeclaration, + configuration + ); + + if (previousDictionary) { + previousDictionaries[dictionary.localId] = previousDictionary; + } + } + + return previousDictionaries; +}; + +/** + * Content of a dictionary in one locale, in the same shape `fill` sends to + * translation (plain values, no translation nodes). + */ +export const getLocaleContent = ( + dictionary: Dictionary, + locale: Locale +): Recursive => getPerLocaleDictionary(dictionary, locale).content; + +const isPlainObject = ( + value: Recursive +): value is { [key: string]: Recursive } => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isEmptyObject = (value: Recursive): boolean => + isPlainObject(value) && Object.keys(value).length === 0; + +/** + * Returns the parts of `currentContent` that differ from `previousContent`. + * Arrays are compared as a whole. + * + * @example + * getChangedContent({ a: 'Hi', b: 'Bye' }, { a: 'Hello', b: 'Bye' }) + * // => { a: 'Hello' } + */ +export const getChangedContent = ( + previousContent: Recursive, + currentContent: Recursive +): Recursive | undefined => { + if (isPlainObject(previousContent) && isPlainObject(currentContent)) { + const changedContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(currentContent)) { + const changedValue = getChangedContent(previousContent[key], value); + + if (changedValue !== undefined) { + changedContent[key] = changedValue; + } + } + + return Object.keys(changedContent).length > 0 ? changedContent : undefined; + } + + return JSON.stringify(previousContent) === JSON.stringify(currentContent) + ? undefined + : currentContent; +}; + +/** + * Removes from `targetContent` every value present in `changedContent`. + * Changed arrays are removed as a whole. + * + * @example + * omitChangedContent({ a: 'Salut', b: 'Au revoir' }, { a: 'Hello' }) + * // => { b: 'Au revoir' } + */ +export const omitChangedContent = ( + targetContent: Recursive, + changedContent: Recursive | undefined +): Recursive | undefined => { + if (changedContent === undefined) return targetContent; + + if (!isPlainObject(targetContent) || !isPlainObject(changedContent)) { + return undefined; + } + + const remainingContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(targetContent)) { + const remainingValue = omitChangedContent(value, changedContent[key]); + + if (remainingValue !== undefined) { + remainingContent[key] = remainingValue; + } + } + + return remainingContent; +}; + +/** + * Source values edited since `previousDictionary` whose `targetLocale` + * translation was left as is, so it is now stale. In multilingual + * dictionaries, translations edited in the same change are kept. + */ +export const getStaleSourceContent = ( + previousDictionary: Dictionary | undefined, + currentDictionary: Dictionary, + sourceLocale: Locale, + targetLocale: Locale +): Recursive | undefined => { + if (!previousDictionary) return undefined; + + const changedSourceContent = getChangedContent( + getLocaleContent(previousDictionary, sourceLocale), + getLocaleContent(currentDictionary, sourceLocale) + ); + + if (changedSourceContent === undefined) return undefined; + + // Per-locale files hold no translation to compare with + if (typeof currentDictionary.locale === 'string') return changedSourceContent; + + const editedTargetContent = getChangedContent( + getLocaleContent(previousDictionary, targetLocale), + getLocaleContent(currentDictionary, targetLocale) + ); + + const staleContent = omitChangedContent( + changedSourceContent, + editedTargetContent + ); + + return staleContent === undefined || isEmptyObject(staleContent) + ? undefined + : staleContent; +}; diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.test.ts b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts new file mode 100644 index 0000000000..ab754c8226 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts @@ -0,0 +1,99 @@ +import type { AIConfig } from '@intlayer/ai'; +import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { AIClient } from '../utils/setupAI'; +import type { TranslationTask } from './listTranslationsTasks'; +import { translateDictionary } from './translateDictionary'; + +vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ + getUnmergedDictionaries: vi.fn(), +})); +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({})); +vi.mock('../utils/checkAccess', () => ({ + getAuthenticatedAPI: async () => ({}), +})); +vi.mock('@intlayer/config/logger', async (importOriginal) => ({ + ...(await importOriginal()), + getAppLogger: () => () => undefined, +})); + +const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; + +/** Fully translated dictionary, whose English title was edited to `Hello`. */ +const dictionary: Dictionary = { + key: 'home', + localId: dictionaryLocalId, + filePath: 'src/home.content.ts', + location: 'local', + title: 'Home', + description: 'Home page', + tags: [], + content: { + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: 'Salut' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue' }, + }, + }, +}; + +const configuration = { + internationalization: { defaultLocale: 'en', locales: ['en', 'fr'] }, + dictionary: { fill: true }, +} as unknown as IntlayerConfig; + +describe('translateDictionary', () => { + beforeEach(() => { + vi.mocked(getUnmergedDictionaries).mockReturnValue({ home: [dictionary] }); + }); + + it('re-translates only the source values changed since the last fill', async () => { + const translateJSON = vi.fn( + async ({ entryFileContent }: { entryFileContent: unknown }) => ({ + fileContent: Object.fromEntries( + Object.entries(entryFileContent as Record).map( + ([index, value]) => [index, `[fr] ${value}`] + ) + ), + }) + ); + + const task: TranslationTask = { + dictionaryKey: 'home', + dictionaryLocalId, + sourceLocale: 'en', + targetLocales: ['fr'], + dictionaryPreset: '', + dictionaryFilePath: 'src/home.content.ts', + changedSourceContent: { fr: { title: 'Hello' } }, + }; + + const result = await translateDictionary(task, configuration, { + mode: 'complete', + fillMetadata: false, + aiClient: { translateJSON } as unknown as AIClient, + aiConfig: {} as AIConfig, + }); + + expect(translateJSON).toHaveBeenCalledTimes(1); + expect(translateJSON.mock.calls[0]?.[0].entryFileContent).toEqual({ + 1: 'Hello', + }); + expect(result.dictionaryOutput?.content).toEqual({ + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: '[fr] Hello' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue' }, + }, + }); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.ts b/packages/@intlayer/cli/src/fill/translateDictionary.ts index dae4a43e94..8217f91e82 100644 --- a/packages/@intlayer/cli/src/fill/translateDictionary.ts +++ b/packages/@intlayer/cli/src/fill/translateDictionary.ts @@ -36,6 +36,7 @@ import { reinsertTranslatedContent, } from './extractTranslatableContent'; import type { TranslationTask } from './listTranslationsTasks'; +import { omitChangedContent } from './sourceChanges'; type TranslateDictionaryResult = TranslationTask & { dictionaryOutput: Dictionary | null; @@ -190,6 +191,9 @@ export const translateDictionary = async ( // Reset to base dictionary for each locale to ensure we filter from the original let dictionaryToProcess = structuredClone(baseUnmergedDictionary); + // Source values edited since the git ref + const changedSourceContent = task.changedSourceContent[targetLocale]; + let targetLocaleDictionary: Dictionary; if (typeof baseUnmergedDictionary.locale === 'string') { @@ -233,7 +237,9 @@ export const translateDictionary = async ( }; } else { // For multilingual dictionaries - if (mode === 'complete') { + // Skipped when source values changed, as it would drop them for + // already translated nodes. The filter below covers this case. + if (mode === 'complete' && changedSourceContent === undefined) { // Remove all nodes that don't have any content to translate dictionaryToProcess = getFilterMissingTranslationsDictionary( dictionaryToProcess, @@ -255,13 +261,19 @@ export const translateDictionary = async ( // Filter to only untranslated fields, preserving explicit null values as // default-locale fallback markers. Applied after both paths converge so // the same logic covers per-locale and multilingual dictionaries. + // Translations of changed source values are treated as missing. if (mode === 'complete') { + const upToDateTargetContent = omitChangedContent( + targetLocaleDictionary.content, + changedSourceContent + ); + dictionaryToProcess = { ...dictionaryToProcess, content: excludeObjectFormat( dictionaryToProcess.content, - targetLocaleDictionary.content + upToDateTargetContent ) ?? {}, }; } diff --git a/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts b/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts index 2421f0da72..0313901cd9 100644 --- a/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts +++ b/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts @@ -29,6 +29,12 @@ export type LoadExternalFileOptions = { * Values are restored (or deleted) after `runInNewContext` returns. */ preloadGlobals?: Record; + /** + * Content to load instead of reading `filePath` from disk (for example an + * older version of the file read from git). `filePath` is still used to + * pick the loader and resolve imports. + */ + code?: string; } & SandBoxContextOptions; /** @@ -146,12 +152,13 @@ export const loadExternalFile = async ( fileExtension === '.jsonc' ) { // Remove cache to force getting fresh content - const fileContent = await readFile(filePath, 'utf-8'); + const fileContent = + options?.code ?? (await readFile(filePath, 'utf-8')); return parseJSON5(fileContent); } // Rest is JS, MJS or TS - const code = await readFile(filePath, 'utf-8'); + const code = options?.code ?? (await readFile(filePath, 'utf-8')); const moduleResultString: string | undefined = await transpileTSToCJS( code, diff --git a/packages/@intlayer/engine/src/listGitFiles.test.ts b/packages/@intlayer/engine/src/listGitFiles.test.ts new file mode 100644 index 0000000000..56f8289d9d --- /dev/null +++ b/packages/@intlayer/engine/src/listGitFiles.test.ts @@ -0,0 +1,87 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { hasGitRef, readGitFile } from './listGitFiles'; + +const git = (cwd: string, ...args: string[]) => + execFileSync( + 'git', + [ + '-c', + 'user.name=test', + '-c', + 'user.email=test@example.com', + '-c', + 'commit.gpgsign=false', + ...args, + ], + { cwd, stdio: 'pipe' } + ); + +describe('git helpers used by fill', () => { + let testDir: string; + + beforeEach(async () => { + testDir = await mkdtemp(join(tmpdir(), 'list-git-files-')); + }); + + afterEach(async () => { + await rm(testDir, { recursive: true, force: true }); + }); + + it('returns nothing outside a git repository', async () => { + const filePath = join(testDir, 'home.content.json'); + await writeFile(filePath, '{"key":"home"}'); + + expect(await hasGitRef('HEAD', testDir)).toBe(false); + expect(await readGitFile(filePath)).toBe(undefined); + }); + + it('returns nothing in a repository without commits', async () => { + git(testDir, 'init', '-q'); + + expect(await hasGitRef('HEAD', testDir)).toBe(false); + }); + + it('reads the committed version of a file', async () => { + const filePath = join(testDir, 'home.content.json'); + git(testDir, 'init', '-q'); + await writeFile(filePath, '{"title":"Hi"}'); + git(testDir, 'add', '.'); + git(testDir, 'commit', '-q', '-m', 'init'); + await writeFile(filePath, '{"title":"Hello"}'); + + expect(await hasGitRef('HEAD', testDir)).toBe(true); + expect(await readGitFile(filePath)).toBe('{"title":"Hi"}'); + }); + + it('does not find the base branch in a shallow clone', async () => { + const originDir = join(testDir, 'origin'); + const cloneDir = join(testDir, 'clone'); + git(testDir, 'init', '-q', '-b', 'main', originDir); + await writeFile(join(originDir, 'home.content.json'), '{"title":"Hi"}'); + git(originDir, 'add', '.'); + git(originDir, 'commit', '-q', '-m', 'init'); + git(originDir, 'checkout', '-q', '-b', 'feature'); + await writeFile(join(originDir, 'home.content.json'), '{"title":"Hello"}'); + git(originDir, 'commit', '-q', '-am', 'edit'); + + // Same history as actions/checkout with its default fetch-depth: 1 + git( + testDir, + 'clone', + '-q', + '--depth', + '1', + '--branch', + 'feature', + `file://${originDir}`, + cloneDir + ); + + expect(await hasGitRef('HEAD', cloneDir)).toBe(true); + expect(await hasGitRef('origin/main', cloneDir)).toBe(false); + }); +}); diff --git a/packages/@intlayer/engine/src/listGitFiles.ts b/packages/@intlayer/engine/src/listGitFiles.ts index d6b79d5728..5c64b0a3bb 100644 --- a/packages/@intlayer/engine/src/listGitFiles.ts +++ b/packages/@intlayer/engine/src/listGitFiles.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import { log } from '@intlayer/config/built'; import { getAppLogger } from '@intlayer/config/logger'; import { simpleGit } from 'simple-git'; @@ -199,3 +199,61 @@ export const listGitLines = async ( // Return the list sorted for convenience return Array.from(changedLines).sort((a, b) => a - b); }; + +/** + * Returns the content of a file at a git ref, or `undefined` if the file does + * not exist at that ref or git is not available. + */ +export const readGitFile = async ( + filePath: string, + ref = 'HEAD' +): Promise => { + try { + const git = simpleGit(dirname(filePath)); + + return await git.show([`${ref}:./${basename(filePath)}`]); + } catch { + return undefined; + } +}; + +/** + * Returns `true` if `ref` points to a commit. Returns `false` if git is not + * installed, `cwd` is not in a git repository, or the ref is missing (no + * commit yet, no upstream branch, or a shallow clone without that history). + */ +export const hasGitRef = async ( + ref: string, + cwd?: string +): Promise => { + try { + const commit = await simpleGit(cwd).raw([ + 'rev-parse', + '--verify', + `${ref}^{commit}`, + ]); + + return commit.trim() !== ''; + } catch { + return false; + } +}; + +/** + * Returns the commit `currentRef` forked from `baseRef`, which is the side + * `git diff baseRef...currentRef` compares against. + */ +export const getGitMergeBase = async ( + baseRef = 'origin/main', + currentRef = 'HEAD' +): Promise => { + try { + const git = simpleGit(); + await git.fetch(baseRef).catch(() => undefined); + const mergeBase = await git.raw(['merge-base', baseRef, currentRef]); + + return mergeBase.trim() || undefined; + } catch { + return undefined; + } +}; diff --git a/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts b/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts index 2b35af0318..da5b62704d 100644 --- a/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts +++ b/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts @@ -51,6 +51,11 @@ export const ensureIntlayerBundle = async ( type LoadContentDeclarationOptions = { logError?: boolean; + /** + * Content to load instead of reading the file from disk. Only supported + * for JSON, JS and TS content declarations. + */ + code?: string; }; // Initialize a module-level cache @@ -103,11 +108,18 @@ export const loadContentDeclaration = async ( bundleFilePath?: string, options?: LoadContentDeclarationOptions ): Promise => { - if (extname(path) === '.md' || extname(path) === '.mdx') { + const isMarkdown = extname(path) === '.md' || extname(path) === '.mdx'; + const isYaml = extname(path) === '.yaml' || extname(path) === '.yml'; + + if (typeof options?.code === 'string' && (isMarkdown || isYaml)) { + return undefined; + } + + if (isMarkdown) { return loadMarkdownContentDeclaration(path, configuration); } - if (extname(path) === '.yaml' || extname(path) === '.yml') { + if (isYaml) { return loadYamlContentDeclaration(path); } @@ -122,6 +134,7 @@ export const loadContentDeclaration = async ( try { const dictionary = await loadExternalFile(path, { logError: options?.logError, + code: options?.code, projectRequire: build.require ?? getProjectRequire(), buildOptions: { packages: undefined, // It fixes the import of ESM packages in the content declaration