From 50f6c78b89475cff43b165bdb0f38b7fcf23e68d Mon Sep 17 00:00:00 2001 From: theluckystrike <51033404+theluckystrike@users.noreply.github.com> Date: Mon, 5 Oct 2026 06:33:07 +0000 Subject: [PATCH 1/3] fix(cli): re-translate keys whose source value changed in fill `intlayer fill` in complete mode only translated locales with missing values, so editing a source string left every existing translation stale. Record a hash of each source value a target locale was filled from in `intlayer.journal.json` at the project root, so the file can be committed and used in CI. On the next run, values whose hash changed are treated as missing for that locale and sent for translation again. Unchanged translations are kept. Locales filled before the journal existed are taken as up to date on the first run. Fixes #513 Co-Authored-By: Claude Opus 5.5 --- docs/docs/en/cli/fill.md | 6 +- packages/@intlayer/cli/src/fill/fill.ts | 47 +++- .../cli/src/fill/fillSourceSnapshot.test.ts | 257 ++++++++++++++++++ .../cli/src/fill/fillSourceSnapshot.ts | 204 ++++++++++++++ .../src/fill/listTranslationsTasks.test.ts | 96 +++++++ .../cli/src/fill/listTranslationsTasks.ts | 63 ++++- .../cli/src/fill/translateDictionary.test.ts | 98 +++++++ .../cli/src/fill/translateDictionary.ts | 16 +- 8 files changed, 771 insertions(+), 16 deletions(-) create mode 100644 packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts create mode 100644 packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts create mode 100644 packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts create mode 100644 packages/@intlayer/cli/src/fill/translateDictionary.test.ts diff --git a/docs/docs/en/cli/fill.md b/docs/docs/en/cli/fill.md index fd7080ddaf..bff9c8ecdf 100644 --- a/docs/docs/en/cli/fill.md +++ b/docs/docs/en/cli/fill.md @@ -1,6 +1,6 @@ --- createdAt: 2024-08-11 -updatedAt: 2026-09-27 +updatedAt: 2026-10-05 priority: 5 title: "intlayer fill: Translate Dictionaries with AI" description: "Fill missing translations, audit existing ones and translate your Intlayer dictionaries with AI from the CLI, locally or in CI." @@ -117,7 +117,9 @@ Affected dictionary keys for processing: app, comp-test, hello-world, lang-switc - **`--output-locales [outputLocales...]`**: Target locales to translate to. If not specified, all locales from your configuration will be used except the source locale. -- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content, `review` will fill missing content and review existing keys. +- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content and re-translate values whose source changed since the last `fill` run, `review` will fill missing content and review existing keys. + + > `complete` mode tracks source changes in an `intlayer.journal.json` file at the root of your project. Commit this file so the changes are also detected in CI and other clones. On the first run, existing translations are taken as up to date. **Git options:** diff --git a/packages/@intlayer/cli/src/fill/fill.ts b/packages/@intlayer/cli/src/fill/fill.ts index 3f4e4e7c77..4f9ef1db72 100644 --- a/packages/@intlayer/cli/src/fill/fill.ts +++ b/packages/@intlayer/cli/src/fill/fill.ts @@ -31,6 +31,12 @@ import { getTargetUnmergedDictionaries, } from '../getTargetDictionary'; import { setupAI } from '../utils/setupAI'; +import { + getSourceLocaleContent, + readFillSourceSnapshot, + setFillSourceSnapshot, + writeFillSourceSnapshot, +} from './fillSourceSnapshot'; import { listTranslationsTasks, type TranslationTask, @@ -177,14 +183,18 @@ export const fill = async (options?: FillOptions): Promise => { * * Create a list of per-locale dictionaries to translate * - * In 'complete' mode, filter only the missing locales to translate + * In 'complete' mode, filter only the locales with missing translations or + * with source values changed since the last fill */ + const fillSourceSnapshot = await readFillSourceSnapshot(configuration); + const translationTasks: TranslationTask[] = listTranslationsTasks( targetUnmergedDictionaries.map((dictionary) => dictionary.localId!), outputLocales, mode, baseLocale, - configuration + configuration, + fillSourceSnapshot ); // AI calls in flight at once (translateJSON + metadata audit) @@ -204,6 +214,7 @@ export const fill = async (options?: FillOptions): Promise => { const taskLimiter = getTaskLimiter(nbConcurrentTasks); let writtenCount = 0; + const writtenTasks: TranslationTask[] = []; const runners = translationTasks.map((task) => taskLimiter(async () => { @@ -284,9 +295,11 @@ export const fill = async (options?: FillOptions): Promise => { configuration ); writtenCount++; + writtenTasks.push(task); } else { await writeContentDeclaration(dictionaryOutput, configuration); writtenCount++; + writtenTasks.push(task); if (dictionaryOutput.filePath) { appLogger( @@ -301,6 +314,36 @@ export const fill = async (options?: FillOptions): Promise => { await Promise.all(runners); await (globalLimiter as any).onIdle(); + // Locales filled before the snapshot existed are taken as up to date + let nextFillSourceSnapshot = fillSourceSnapshot; + + for (const dictionary of targetUnmergedDictionaries) { + const sourceLocale = (dictionary.locale ?? baseLocale) as Locale; + + if (dictionary.filled || !dictionary.localId || sourceLocale !== baseLocale) + continue; + + nextFillSourceSnapshot = setFillSourceSnapshot( + nextFillSourceSnapshot, + dictionary.localId, + outputLocales.filter((locale) => locale !== sourceLocale), + getSourceLocaleContent(dictionary, sourceLocale), + false + ); + } + + for (const task of writtenTasks) { + nextFillSourceSnapshot = setFillSourceSnapshot( + nextFillSourceSnapshot, + task.dictionaryLocalId, + task.targetLocales, + task.sourceContent, + true + ); + } + + await writeFillSourceSnapshot(configuration, nextFillSourceSnapshot); + return { status: 'completed', dictionaryKeys: keysToProcess, diff --git a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts new file mode 100644 index 0000000000..913bb385f2 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts @@ -0,0 +1,257 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { excludeObjectFormat } from '@intlayer/engine/utils'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + getChangedSourceContent, + getSourceLocaleContent, + hashSourceContent, + omitChangedContent, + readFillSourceSnapshot, + setFillSourceSnapshot, + writeFillSourceSnapshot, +} from './fillSourceSnapshot'; + +const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; + +describe('hashSourceContent', () => { + it('hashes each value and keeps the object shape', () => { + const hashes = hashSourceContent({ + title: 'Hi', + nested: { label: 'Save' }, + items: ['a', 'b'], + }) as Record; + + expect(Object.keys(hashes)).toEqual(['title', 'nested', 'items']); + expect(typeof hashes.title).toBe('string'); + expect(hashes.title).not.toBe('Hi'); + expect(Object.keys(hashes.nested as object)).toEqual(['label']); + expect(typeof hashes.items).toBe('string'); + }); + + it('is stable for the same value', () => { + expect(hashSourceContent({ title: 'Hi' })).toEqual( + hashSourceContent({ title: 'Hi' }) + ); + expect(hashSourceContent({ title: 'Hi' })).not.toEqual( + hashSourceContent({ title: 'Hello' }) + ); + }); +}); + +describe('getChangedSourceContent', () => { + it('returns only the changed values', () => { + expect( + getChangedSourceContent( + hashSourceContent({ + title: 'Hi', + nested: { label: 'Save', hint: 'Click' }, + }), + { title: 'Hello', nested: { label: 'Save', hint: 'Tap' } } + ) + ).toEqual({ title: 'Hello', nested: { hint: 'Tap' } }); + }); + + it('returns undefined when nothing changed', () => { + expect( + getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { + title: 'Hi', + }) + ).toBeUndefined(); + }); + + it('ignores keys with no previous value', () => { + expect( + getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { + title: 'Hi', + added: 'New', + }) + ).toBeUndefined(); + expect(getChangedSourceContent(undefined, { title: 'Hi' })).toBeUndefined(); + }); + + it('returns a changed array as a whole', () => { + expect( + getChangedSourceContent(hashSourceContent({ items: ['a', 'b'] }), { + items: ['a', 'c'], + }) + ).toEqual({ items: ['a', 'c'] }); + }); + + it('returns a value whose shape changed', () => { + expect( + getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { + title: { short: 'Hi' }, + }) + ).toEqual({ title: { short: 'Hi' } }); + }); +}); + +describe('omitChangedContent', () => { + it('removes translations of changed values', () => { + expect( + omitChangedContent( + { title: 'Salut', nested: { label: 'Sauver', hint: 'Cliquer' } }, + { title: 'Hello', nested: { hint: 'Tap' } } + ) + ).toEqual({ nested: { label: 'Sauver' } }); + }); + + it('keeps the target as is when nothing changed', () => { + const targetContent = { title: 'Salut' }; + + expect(omitChangedContent(targetContent, undefined)).toBe(targetContent); + }); + + it('leaves changed and missing values to translate', () => { + const sourceContent = { title: 'Hello', subtitle: 'Welcome', cta: 'Go' }; + const targetContent = { title: 'Salut', subtitle: 'Bienvenue' }; + const changedContent = getChangedSourceContent( + hashSourceContent({ title: 'Hi', subtitle: 'Welcome' }), + sourceContent + ); + + expect( + excludeObjectFormat( + sourceContent, + omitChangedContent(targetContent, changedContent) + ) + ).toEqual({ title: 'Hello', cta: 'Go' }); + }); + + it('removes a changed array as a whole', () => { + expect( + omitChangedContent( + { items: ['a', 'b', 'c'], title: 'Salut' }, + { items: ['x'] } + ) + ).toEqual({ title: 'Salut' }); + }); +}); + +describe('setFillSourceSnapshot', () => { + const hiHashes = hashSourceContent({ title: 'Hi' }); + const helloHashes = hashSourceContent({ title: 'Hello' }); + + it('overwrites the recorded source of the given locales', () => { + const snapshot = setFillSourceSnapshot( + { [dictionaryLocalId]: { fr: hiHashes, es: hiHashes } }, + dictionaryLocalId, + ['fr'], + { title: 'Hello' }, + true + ); + + expect(snapshot[dictionaryLocalId]).toEqual({ + fr: helloHashes, + es: hiHashes, + }); + }); + + it('only fills unrecorded locales when not overwriting', () => { + const previousSnapshot = { + [dictionaryLocalId]: { fr: hiHashes }, + }; + + const snapshot = setFillSourceSnapshot( + previousSnapshot, + dictionaryLocalId, + ['fr', 'es'], + { title: 'Hello' }, + false + ); + + expect(snapshot[dictionaryLocalId]).toEqual({ + fr: hiHashes, + es: helloHashes, + }); + expect(previousSnapshot[dictionaryLocalId]).toEqual({ + fr: hiHashes, + }); + }); +}); + +describe('getSourceLocaleContent', () => { + it('returns the source locale values of a multilingual dictionary', () => { + const dictionary: Dictionary = { + key: 'home', + content: { + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: 'Bonjour' }, + }, + }, + }; + + expect(getSourceLocaleContent(dictionary, 'en')).toEqual({ + title: 'Hello', + }); + }); + + it('returns the content of a per-locale dictionary', () => { + const dictionary: Dictionary = { + key: 'home', + locale: 'en', + content: { title: 'Hello' }, + }; + + expect(getSourceLocaleContent(dictionary, 'en')).toEqual({ + title: 'Hello', + }); + }); +}); + +describe('readFillSourceSnapshot / writeFillSourceSnapshot', () => { + let baseDir: string; + let configuration: IntlayerConfig; + + beforeEach(async () => { + baseDir = await mkdtemp(join(tmpdir(), 'intlayer-fill-')); + configuration = { system: { baseDir } } as IntlayerConfig; + }); + + afterEach(async () => { + await rm(baseDir, { recursive: true, force: true }); + }); + + it('returns an empty snapshot when none was written', async () => { + expect(await readFillSourceSnapshot(configuration)).toEqual({}); + }); + + it('reads back the written snapshot', async () => { + const snapshot = { + [dictionaryLocalId]: { fr: hashSourceContent({ title: 'Hello' }) }, + }; + + await writeFillSourceSnapshot(configuration, snapshot); + + expect(await readFillSourceSnapshot(configuration)).toEqual(snapshot); + }); + + it('writes a sorted file at the project root', async () => { + const hashes = hashSourceContent({ title: 'Hello' }); + + await writeFillSourceSnapshot(configuration, { + 'b::local::src/b.content.ts': { fr: hashes, es: hashes }, + 'a::local::src/a.content.ts': { fr: hashes }, + }); + + const fileContent = await readFile( + join(baseDir, 'intlayer.journal.json'), + 'utf-8' + ); + const parsedContent = JSON.parse(fileContent); + + expect(Object.keys(parsedContent)).toEqual([ + 'a::local::src/a.content.ts', + 'b::local::src/b.content.ts', + ]); + expect(Object.keys(parsedContent['b::local::src/b.content.ts'])).toEqual([ + 'es', + 'fr', + ]); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts new file mode 100644 index 0000000000..33ebe9df3b --- /dev/null +++ b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts @@ -0,0 +1,204 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { getPerLocaleDictionary } from '@intlayer/core/plugins'; +import { + getPathHash, + type Recursive, + sortAlphabetically, +} from '@intlayer/engine/utils'; +import type { Locale } from '@intlayer/types/allLocales'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; + +/** + * Hash of each source value a target locale was last filled from, per + * dictionary. Lets `fill` detect source values edited after translation. + */ +export type FillSourceSnapshot = Partial< + Record>> +>; + +/** + * Stored at the project root, not in `.intlayer`, so it can be committed and + * shared with CI and other clones. + */ +const FILL_SOURCE_SNAPSHOT_FILE_NAME = 'intlayer.journal.json'; + +const getFillSourceSnapshotPath = (configuration: IntlayerConfig): string => + join(configuration.system.baseDir, FILL_SOURCE_SNAPSHOT_FILE_NAME); + +/** + * Reads the snapshot written by the previous `fill` run. + * Returns an empty snapshot if none exists or it cannot be parsed. + */ +export const readFillSourceSnapshot = async ( + configuration: IntlayerConfig +): Promise => { + try { + const fileContent = await readFile( + getFillSourceSnapshotPath(configuration), + 'utf-8' + ); + + return JSON.parse(fileContent) as FillSourceSnapshot; + } catch { + return {}; + } +}; + +/** + * Persists the snapshot for the next `fill` run. + * Dictionaries and locales are sorted to keep the file diff stable. + */ +export const writeFillSourceSnapshot = async ( + configuration: IntlayerConfig, + snapshot: FillSourceSnapshot +): Promise => { + const sortedSnapshot = Object.fromEntries( + Object.entries(snapshot) + .sort(([keyA], [keyB]) => sortAlphabetically(keyA, keyB)) + .map(([dictionaryLocalId, localeSnapshot]) => [ + dictionaryLocalId, + Object.fromEntries( + Object.entries(localeSnapshot ?? {}).sort(([localeA], [localeB]) => + sortAlphabetically(localeA, localeB) + ) + ), + ]) + ); + + await writeFile( + getFillSourceSnapshotPath(configuration), + `${JSON.stringify(sortedSnapshot, null, 2)}\n` + ); +}; + +/** + * Content of a dictionary in its source locale, in the same shape `fill` + * sends to translation (plain values, no translation nodes). + */ +export const getSourceLocaleContent = ( + dictionary: Dictionary, + sourceLocale: Locale +): Recursive => getPerLocaleDictionary(dictionary, sourceLocale).content; + +const isPlainObject = ( + value: Recursive +): value is { [key: string]: Recursive } => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** + * Replaces each value of `content` by a short hash, keeping the object shape. + * Arrays are hashed as a whole. + */ +export const hashSourceContent = (content: Recursive): Recursive => { + if (isPlainObject(content)) { + return Object.fromEntries( + Object.entries(content).map(([key, value]) => [ + key, + hashSourceContent(value), + ]) + ); + } + + return getPathHash(JSON.stringify(content) ?? ''); +}; + +/** + * Returns the parts of `currentContent` whose hash differs from + * `previousHashes` (built with `hashSourceContent`). + * + * Keys absent from `previousHashes` are ignored, as they are already handled + * as missing translations. Arrays are compared as a whole. + * + * @example + * getChangedSourceContent( + * hashSourceContent({ a: 'Hi', b: 'Bye' }), + * { a: 'Hello', b: 'Bye' } + * ) + * // => { a: 'Hello' } + */ +export const getChangedSourceContent = ( + previousHashes: Recursive, + currentContent: Recursive +): Recursive | undefined => { + if (previousHashes === undefined) return undefined; + + if (isPlainObject(previousHashes) && isPlainObject(currentContent)) { + const changedContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(currentContent)) { + const changedValue = getChangedSourceContent(previousHashes[key], value); + + if (changedValue !== undefined) { + changedContent[key] = changedValue; + } + } + + return Object.keys(changedContent).length > 0 ? changedContent : undefined; + } + + const isSameContent = + JSON.stringify(previousHashes) === + JSON.stringify(hashSourceContent(currentContent)); + + return isSameContent ? undefined : currentContent; +}; + +/** + * Removes from `targetContent` every value whose source changed, so it is + * treated as untranslated. Changed arrays are removed as a whole. + * + * @example + * omitChangedContent({ a: 'Salut', b: 'Au revoir' }, { a: 'Hello' }) + * // => { b: 'Au revoir' } + */ +export const omitChangedContent = ( + targetContent: Recursive, + changedContent: Recursive | undefined +): Recursive | undefined => { + if (changedContent === undefined) return targetContent; + + if (!isPlainObject(targetContent) || !isPlainObject(changedContent)) { + return undefined; + } + + const remainingContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(targetContent)) { + const remainingValue = omitChangedContent(value, changedContent[key]); + + if (remainingValue !== undefined) { + remainingContent[key] = remainingValue; + } + } + + return remainingContent; +}; + +/** + * Records the hashes of `sourceContent` as the source of `locales` for a + * dictionary. + * + * @param overwrite - When false, only locales with no recorded source are set + * (used to take a baseline of translations made before the snapshot existed). + * @returns A new snapshot; the input is not mutated. + */ +export const setFillSourceSnapshot = ( + snapshot: FillSourceSnapshot, + dictionaryLocalId: LocalDictionaryId, + locales: Locale[], + sourceContent: Recursive, + overwrite: boolean +): FillSourceSnapshot => { + const localeSnapshot = { ...snapshot[dictionaryLocalId] }; + const sourceHashes = hashSourceContent(sourceContent); + + for (const locale of locales) { + if (overwrite || !Object.hasOwn(localeSnapshot, locale)) { + localeSnapshot[locale] = sourceHashes; + } + } + + return { ...snapshot, [dictionaryLocalId]: localeSnapshot }; +}; diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts new file mode 100644 index 0000000000..332513de54 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts @@ -0,0 +1,96 @@ +import { getDictionaries } from '@intlayer/dictionaries-entry'; +import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { listMissingTranslationsWithConfig } from '../test'; +import { + type FillSourceSnapshot, + hashSourceContent, +} from './fillSourceSnapshot'; +import { listTranslationsTasks } from './listTranslationsTasks'; + +vi.mock('@intlayer/dictionaries-entry', () => ({ getDictionaries: vi.fn() })); +vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ + getUnmergedDictionaries: vi.fn(), +})); +vi.mock('../test', () => ({ listMissingTranslationsWithConfig: vi.fn() })); +vi.mock('@intlayer/config/logger', async (importOriginal) => ({ + ...(await importOriginal()), + getAppLogger: () => () => undefined, +})); + +const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; + +/** Fully translated dictionary, whose English title was edited to `Hello`. */ +const dictionary: Dictionary = { + key: 'home', + localId: dictionaryLocalId, + filePath: 'src/home.content.ts', + content: { + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: 'Salut', es: 'Hola' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue', es: 'Bienvenido' }, + }, + }, +}; + +const configuration = { + dictionary: { fill: true }, +} as IntlayerConfig; + +const listTasks = (fillSourceSnapshot?: FillSourceSnapshot) => + listTranslationsTasks( + [dictionaryLocalId], + ['en', 'fr', 'es'], + 'complete', + 'en', + configuration, + fillSourceSnapshot + ); + +describe('listTranslationsTasks', () => { + beforeEach(() => { + vi.mocked(getDictionaries).mockReturnValue({ home: dictionary }); + vi.mocked(getUnmergedDictionaries).mockReturnValue({ home: [dictionary] }); + vi.mocked(listMissingTranslationsWithConfig).mockReturnValue({ + missingTranslations: [], + } as unknown as ReturnType); + }); + + it('skips a fully translated dictionary whose source did not change', () => { + const sourceHashes = hashSourceContent({ + title: 'Hello', + subtitle: 'Welcome', + }); + + expect(listTasks()).toEqual([]); + expect( + listTasks({ + [dictionaryLocalId]: { fr: sourceHashes, es: sourceHashes }, + }) + ).toEqual([]); + }); + + it('re-translates locales filled from a since changed source value', () => { + const tasks = listTasks({ + [dictionaryLocalId]: { + fr: hashSourceContent({ title: 'Hi', subtitle: 'Welcome' }), + es: hashSourceContent({ title: 'Hello', subtitle: 'Welcome' }), + }, + }); + + expect(tasks).toHaveLength(1); + expect(tasks[0]).toMatchObject({ + dictionaryKey: 'home', + sourceLocale: 'en', + targetLocales: ['fr'], + sourceContent: { title: 'Hello', subtitle: 'Welcome' }, + changedSourceContent: { fr: { title: 'Hello' } }, + }); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts index d16a6a9ff6..ff5fc26185 100644 --- a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts @@ -10,11 +10,16 @@ import { import { getFilterTranslationsOnlyDictionary } from '@intlayer/core/plugins'; import { getDictionaries } from '@intlayer/dictionaries-entry'; import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; -import { formatLocale } from '@intlayer/engine/utils'; +import { formatLocale, type Recursive } from '@intlayer/engine/utils'; import type { Locale } from '@intlayer/types/allLocales'; import type { IntlayerConfig } from '@intlayer/types/config'; import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; import { listMissingTranslationsWithConfig } from '../test'; +import { + type FillSourceSnapshot, + getChangedSourceContent, + getSourceLocaleContent, +} from './fillSourceSnapshot'; export type TranslationTask = { dictionaryKey: string; @@ -23,6 +28,10 @@ export type TranslationTask = { targetLocales: Locale[]; dictionaryPreset: string; dictionaryFilePath: string; + /** Current source locale content, recorded once the task succeeds. */ + sourceContent: Recursive; + /** Per target locale, source values changed since that locale was filled. */ + changedSourceContent: Partial>; }; export const listTranslationsTasks = ( @@ -30,7 +39,8 @@ export const listTranslationsTasks = ( outputLocales: Locale[], mode: 'complete' | 'review', baseLocale: Locale, - configuration: IntlayerConfig + configuration: IntlayerConfig, + fillSourceSnapshot: FillSourceSnapshot = {} ): TranslationTask[] => { const appLogger = getAppLogger(configuration); @@ -114,20 +124,51 @@ export const listTranslationsTasks = ( continue; } + const sourceContent = getSourceLocaleContent( + targetUnmergedDictionary, + sourceLocale + ); + /** - * In 'complete' mode, filter only the missing locales to translate + * Source values edited since each locale was last filled. Only the base + * locale is a source of truth: other per-locale files are fill outputs. + */ + const changedSourceContent: Partial> = {}; + + if (sourceLocale === baseLocale) { + for (const locale of outputLocales) { + if (locale === sourceLocale) continue; + + const changedContent = getChangedSourceContent( + fillSourceSnapshot[dictionaryLocalId]?.[locale], + sourceContent + ); + + if (changedContent !== undefined) { + changedSourceContent[locale] = changedContent; + } + } + } + + /** + * In 'complete' mode, filter only the locales with missing translations + * or with source values changed since they were filled * - * Skip the dictionary if there are no missing locales to translate + * Skip the dictionary if there are no locales to translate */ let outputLocalesList: Locale[] = outputLocales as Locale[]; if (mode === 'complete') { - outputLocalesList = - missingTranslations - .find( - (missingTranslation) => missingTranslation.key === dictionaryKey - ) - ?.locales.filter((locale) => outputLocales.includes(locale)) ?? []; + const missingLocales = + missingTranslations.find( + (missingTranslation) => missingTranslation.key === dictionaryKey + )?.locales ?? []; + + outputLocalesList = outputLocales.filter( + (locale) => + missingLocales.includes(locale) || + Object.hasOwn(changedSourceContent, locale) + ); } if (outputLocalesList.length === 0) { @@ -147,6 +188,8 @@ export const listTranslationsTasks = ( targetLocales: outputLocalesList, dictionaryPreset, dictionaryFilePath: targetUnmergedDictionary.filePath, + sourceContent, + changedSourceContent, }); } diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.test.ts b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts new file mode 100644 index 0000000000..c38e45e689 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts @@ -0,0 +1,98 @@ +import type { AIConfig } from '@intlayer/ai'; +import { getUnmergedDictionaries } from '@intlayer/dictionaries-entry/unmerged'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { AIClient } from '../utils/setupAI'; +import type { TranslationTask } from './listTranslationsTasks'; +import { translateDictionary } from './translateDictionary'; + +vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ + getUnmergedDictionaries: vi.fn(), +})); +vi.mock('../utils/checkAccess', () => ({ + getAuthenticatedAPI: async () => ({}), +})); +vi.mock('@intlayer/config/logger', async (importOriginal) => ({ + ...(await importOriginal()), + getAppLogger: () => () => undefined, +})); + +const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; + +/** Fully translated dictionary, whose English title was edited to `Hello`. */ +const dictionary: Dictionary = { + key: 'home', + localId: dictionaryLocalId, + filePath: 'src/home.content.ts', + location: 'local', + title: 'Home', + description: 'Home page', + tags: [], + content: { + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: 'Salut' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue' }, + }, + }, +}; + +const configuration = { + internationalization: { defaultLocale: 'en', locales: ['en', 'fr'] }, + dictionary: { fill: true }, +} as unknown as IntlayerConfig; + +describe('translateDictionary', () => { + beforeEach(() => { + vi.mocked(getUnmergedDictionaries).mockReturnValue({ home: [dictionary] }); + }); + + it('re-translates only the source values changed since the last fill', async () => { + const translateJSON = vi.fn( + async ({ entryFileContent }: { entryFileContent: unknown }) => ({ + fileContent: Object.fromEntries( + Object.entries(entryFileContent as Record).map( + ([index, value]) => [index, `[fr] ${value}`] + ) + ), + }) + ); + + const task: TranslationTask = { + dictionaryKey: 'home', + dictionaryLocalId, + sourceLocale: 'en', + targetLocales: ['fr'], + dictionaryPreset: '', + dictionaryFilePath: 'src/home.content.ts', + sourceContent: { title: 'Hello', subtitle: 'Welcome' }, + changedSourceContent: { fr: { title: 'Hello' } }, + }; + + const result = await translateDictionary(task, configuration, { + mode: 'complete', + fillMetadata: false, + aiClient: { translateJSON } as unknown as AIClient, + aiConfig: {} as AIConfig, + }); + + expect(translateJSON).toHaveBeenCalledTimes(1); + expect(translateJSON.mock.calls[0]?.[0].entryFileContent).toEqual({ + 1: 'Hello', + }); + expect(result.dictionaryOutput?.content).toEqual({ + title: { + nodeType: 'translation', + translation: { en: 'Hello', fr: '[fr] Hello' }, + }, + subtitle: { + nodeType: 'translation', + translation: { en: 'Welcome', fr: 'Bienvenue' }, + }, + }); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.ts b/packages/@intlayer/cli/src/fill/translateDictionary.ts index dae4a43e94..5690d084c3 100644 --- a/packages/@intlayer/cli/src/fill/translateDictionary.ts +++ b/packages/@intlayer/cli/src/fill/translateDictionary.ts @@ -35,6 +35,7 @@ import { extractTranslatableContent, reinsertTranslatedContent, } from './extractTranslatableContent'; +import { omitChangedContent } from './fillSourceSnapshot'; import type { TranslationTask } from './listTranslationsTasks'; type TranslateDictionaryResult = TranslationTask & { @@ -190,6 +191,9 @@ export const translateDictionary = async ( // Reset to base dictionary for each locale to ensure we filter from the original let dictionaryToProcess = structuredClone(baseUnmergedDictionary); + // Source values edited since this locale was last filled + const changedSourceContent = task.changedSourceContent[targetLocale]; + let targetLocaleDictionary: Dictionary; if (typeof baseUnmergedDictionary.locale === 'string') { @@ -233,7 +237,9 @@ export const translateDictionary = async ( }; } else { // For multilingual dictionaries - if (mode === 'complete') { + // Skipped when source values changed, as it would drop them for + // already translated nodes. The filter below covers this case. + if (mode === 'complete' && changedSourceContent === undefined) { // Remove all nodes that don't have any content to translate dictionaryToProcess = getFilterMissingTranslationsDictionary( dictionaryToProcess, @@ -255,13 +261,19 @@ export const translateDictionary = async ( // Filter to only untranslated fields, preserving explicit null values as // default-locale fallback markers. Applied after both paths converge so // the same logic covers per-locale and multilingual dictionaries. + // Translations of changed source values are treated as missing. if (mode === 'complete') { + const upToDateTargetContent = omitChangedContent( + targetLocaleDictionary.content, + changedSourceContent + ); + dictionaryToProcess = { ...dictionaryToProcess, content: excludeObjectFormat( dictionaryToProcess.content, - targetLocaleDictionary.content + upToDateTargetContent ) ?? {}, }; } From 8edd33449e32557bda4d4569816234bff02f820f Mon Sep 17 00:00:00 2001 From: theluckystrike <51033404+theluckystrike@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:58:27 +0700 Subject: [PATCH 2/3] refactor(cli): detect changed source values in fill with git, drop the journal The journal file had to be committed after each fill and would conflict between team members. Fill now reads the previous version of each content declaration from git instead. It compares with HEAD by default, with the merge base of the base branch when --git-diff is set, and with the upstream when --unpushed is set. A source value that changed while its translation did not is sent for translation again. A translation edited in the same change is kept. Nothing extra is written to the project. loadContentDeclaration and loadExternalFile accept the file content as an option so the git version can be loaded with the same imports and aliases. --- docs/docs/en/cli/fill.md | 4 +- packages/@intlayer/cli/src/fill/fill.ts | 63 ++--- .../cli/src/fill/fillSourceSnapshot.test.ts | 257 ------------------ .../cli/src/fill/fillSourceSnapshot.ts | 204 -------------- .../src/fill/listTranslationsTasks.test.ts | 62 +++-- .../cli/src/fill/listTranslationsTasks.ts | 34 +-- .../cli/src/fill/sourceChanges.test.ts | 122 +++++++++ .../@intlayer/cli/src/fill/sourceChanges.ts | 207 ++++++++++++++ .../cli/src/fill/translateDictionary.test.ts | 3 +- .../cli/src/fill/translateDictionary.ts | 4 +- .../src/loadExternalFile/loadExternalFile.ts | 11 +- packages/@intlayer/engine/src/listGitFiles.ts | 38 ++- .../loadContentDeclaration.ts | 17 +- 13 files changed, 475 insertions(+), 551 deletions(-) delete mode 100644 packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts delete mode 100644 packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts create mode 100644 packages/@intlayer/cli/src/fill/sourceChanges.test.ts create mode 100644 packages/@intlayer/cli/src/fill/sourceChanges.ts diff --git a/docs/docs/en/cli/fill.md b/docs/docs/en/cli/fill.md index bff9c8ecdf..049b312295 100644 --- a/docs/docs/en/cli/fill.md +++ b/docs/docs/en/cli/fill.md @@ -117,9 +117,9 @@ Affected dictionary keys for processing: app, comp-test, hello-world, lang-switc - **`--output-locales [outputLocales...]`**: Target locales to translate to. If not specified, all locales from your configuration will be used except the source locale. -- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content and re-translate values whose source changed since the last `fill` run, `review` will fill missing content and review existing keys. +- **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content and re-translate values whose source changed in git, `review` will fill missing content and review existing keys. - > `complete` mode tracks source changes in an `intlayer.journal.json` file at the root of your project. Commit this file so the changes are also detected in CI and other clones. On the first run, existing translations are taken as up to date. + > In `complete` mode, source values are compared with the last commit (`HEAD`), or with the base branch when `--git-diff` is set (with `--unpushed`, with the upstream branch). A translation is re-translated when its source value changed and the translation itself did not. No extra file is written. This applies to JSON, JS and TS content declaration files tracked by git. **Git options:** diff --git a/packages/@intlayer/cli/src/fill/fill.ts b/packages/@intlayer/cli/src/fill/fill.ts index 4f9ef1db72..516bc5ee2d 100644 --- a/packages/@intlayer/cli/src/fill/fill.ts +++ b/packages/@intlayer/cli/src/fill/fill.ts @@ -31,16 +31,15 @@ import { getTargetUnmergedDictionaries, } from '../getTargetDictionary'; import { setupAI } from '../utils/setupAI'; -import { - getSourceLocaleContent, - readFillSourceSnapshot, - setFillSourceSnapshot, - writeFillSourceSnapshot, -} from './fillSourceSnapshot'; import { listTranslationsTasks, type TranslationTask, } from './listTranslationsTasks'; +import { + getSourceChangesRef, + loadPreviousDictionaries, + type PreviousDictionaries, +} from './sourceChanges'; import { translateDictionary } from './translateDictionary'; import { writeFill } from './writeFill'; @@ -184,9 +183,22 @@ export const fill = async (options?: FillOptions): Promise => { * Create a list of per-locale dictionaries to translate * * In 'complete' mode, filter only the locales with missing translations or - * with source values changed since the last fill + * with source values changed in git (compared to the last commit, or to the + * base branch with `--git-diff`) */ - const fillSourceSnapshot = await readFillSourceSnapshot(configuration); + let previousDictionaries: PreviousDictionaries = {}; + + if (mode === 'complete') { + const sourceChangesRef = await getSourceChangesRef(options?.gitOptions); + + if (sourceChangesRef) { + previousDictionaries = await loadPreviousDictionaries( + targetUnmergedDictionaries, + configuration, + sourceChangesRef + ); + } + } const translationTasks: TranslationTask[] = listTranslationsTasks( targetUnmergedDictionaries.map((dictionary) => dictionary.localId!), @@ -194,7 +206,7 @@ export const fill = async (options?: FillOptions): Promise => { mode, baseLocale, configuration, - fillSourceSnapshot + previousDictionaries ); // AI calls in flight at once (translateJSON + metadata audit) @@ -214,7 +226,6 @@ export const fill = async (options?: FillOptions): Promise => { const taskLimiter = getTaskLimiter(nbConcurrentTasks); let writtenCount = 0; - const writtenTasks: TranslationTask[] = []; const runners = translationTasks.map((task) => taskLimiter(async () => { @@ -295,11 +306,9 @@ export const fill = async (options?: FillOptions): Promise => { configuration ); writtenCount++; - writtenTasks.push(task); } else { await writeContentDeclaration(dictionaryOutput, configuration); writtenCount++; - writtenTasks.push(task); if (dictionaryOutput.filePath) { appLogger( @@ -314,36 +323,6 @@ export const fill = async (options?: FillOptions): Promise => { await Promise.all(runners); await (globalLimiter as any).onIdle(); - // Locales filled before the snapshot existed are taken as up to date - let nextFillSourceSnapshot = fillSourceSnapshot; - - for (const dictionary of targetUnmergedDictionaries) { - const sourceLocale = (dictionary.locale ?? baseLocale) as Locale; - - if (dictionary.filled || !dictionary.localId || sourceLocale !== baseLocale) - continue; - - nextFillSourceSnapshot = setFillSourceSnapshot( - nextFillSourceSnapshot, - dictionary.localId, - outputLocales.filter((locale) => locale !== sourceLocale), - getSourceLocaleContent(dictionary, sourceLocale), - false - ); - } - - for (const task of writtenTasks) { - nextFillSourceSnapshot = setFillSourceSnapshot( - nextFillSourceSnapshot, - task.dictionaryLocalId, - task.targetLocales, - task.sourceContent, - true - ); - } - - await writeFillSourceSnapshot(configuration, nextFillSourceSnapshot); - return { status: 'completed', dictionaryKeys: keysToProcess, diff --git a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts deleted file mode 100644 index 913bb385f2..0000000000 --- a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.test.ts +++ /dev/null @@ -1,257 +0,0 @@ -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { excludeObjectFormat } from '@intlayer/engine/utils'; -import type { IntlayerConfig } from '@intlayer/types/config'; -import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { - getChangedSourceContent, - getSourceLocaleContent, - hashSourceContent, - omitChangedContent, - readFillSourceSnapshot, - setFillSourceSnapshot, - writeFillSourceSnapshot, -} from './fillSourceSnapshot'; - -const dictionaryLocalId: LocalDictionaryId = 'home::local::src/home.content.ts'; - -describe('hashSourceContent', () => { - it('hashes each value and keeps the object shape', () => { - const hashes = hashSourceContent({ - title: 'Hi', - nested: { label: 'Save' }, - items: ['a', 'b'], - }) as Record; - - expect(Object.keys(hashes)).toEqual(['title', 'nested', 'items']); - expect(typeof hashes.title).toBe('string'); - expect(hashes.title).not.toBe('Hi'); - expect(Object.keys(hashes.nested as object)).toEqual(['label']); - expect(typeof hashes.items).toBe('string'); - }); - - it('is stable for the same value', () => { - expect(hashSourceContent({ title: 'Hi' })).toEqual( - hashSourceContent({ title: 'Hi' }) - ); - expect(hashSourceContent({ title: 'Hi' })).not.toEqual( - hashSourceContent({ title: 'Hello' }) - ); - }); -}); - -describe('getChangedSourceContent', () => { - it('returns only the changed values', () => { - expect( - getChangedSourceContent( - hashSourceContent({ - title: 'Hi', - nested: { label: 'Save', hint: 'Click' }, - }), - { title: 'Hello', nested: { label: 'Save', hint: 'Tap' } } - ) - ).toEqual({ title: 'Hello', nested: { hint: 'Tap' } }); - }); - - it('returns undefined when nothing changed', () => { - expect( - getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { - title: 'Hi', - }) - ).toBeUndefined(); - }); - - it('ignores keys with no previous value', () => { - expect( - getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { - title: 'Hi', - added: 'New', - }) - ).toBeUndefined(); - expect(getChangedSourceContent(undefined, { title: 'Hi' })).toBeUndefined(); - }); - - it('returns a changed array as a whole', () => { - expect( - getChangedSourceContent(hashSourceContent({ items: ['a', 'b'] }), { - items: ['a', 'c'], - }) - ).toEqual({ items: ['a', 'c'] }); - }); - - it('returns a value whose shape changed', () => { - expect( - getChangedSourceContent(hashSourceContent({ title: 'Hi' }), { - title: { short: 'Hi' }, - }) - ).toEqual({ title: { short: 'Hi' } }); - }); -}); - -describe('omitChangedContent', () => { - it('removes translations of changed values', () => { - expect( - omitChangedContent( - { title: 'Salut', nested: { label: 'Sauver', hint: 'Cliquer' } }, - { title: 'Hello', nested: { hint: 'Tap' } } - ) - ).toEqual({ nested: { label: 'Sauver' } }); - }); - - it('keeps the target as is when nothing changed', () => { - const targetContent = { title: 'Salut' }; - - expect(omitChangedContent(targetContent, undefined)).toBe(targetContent); - }); - - it('leaves changed and missing values to translate', () => { - const sourceContent = { title: 'Hello', subtitle: 'Welcome', cta: 'Go' }; - const targetContent = { title: 'Salut', subtitle: 'Bienvenue' }; - const changedContent = getChangedSourceContent( - hashSourceContent({ title: 'Hi', subtitle: 'Welcome' }), - sourceContent - ); - - expect( - excludeObjectFormat( - sourceContent, - omitChangedContent(targetContent, changedContent) - ) - ).toEqual({ title: 'Hello', cta: 'Go' }); - }); - - it('removes a changed array as a whole', () => { - expect( - omitChangedContent( - { items: ['a', 'b', 'c'], title: 'Salut' }, - { items: ['x'] } - ) - ).toEqual({ title: 'Salut' }); - }); -}); - -describe('setFillSourceSnapshot', () => { - const hiHashes = hashSourceContent({ title: 'Hi' }); - const helloHashes = hashSourceContent({ title: 'Hello' }); - - it('overwrites the recorded source of the given locales', () => { - const snapshot = setFillSourceSnapshot( - { [dictionaryLocalId]: { fr: hiHashes, es: hiHashes } }, - dictionaryLocalId, - ['fr'], - { title: 'Hello' }, - true - ); - - expect(snapshot[dictionaryLocalId]).toEqual({ - fr: helloHashes, - es: hiHashes, - }); - }); - - it('only fills unrecorded locales when not overwriting', () => { - const previousSnapshot = { - [dictionaryLocalId]: { fr: hiHashes }, - }; - - const snapshot = setFillSourceSnapshot( - previousSnapshot, - dictionaryLocalId, - ['fr', 'es'], - { title: 'Hello' }, - false - ); - - expect(snapshot[dictionaryLocalId]).toEqual({ - fr: hiHashes, - es: helloHashes, - }); - expect(previousSnapshot[dictionaryLocalId]).toEqual({ - fr: hiHashes, - }); - }); -}); - -describe('getSourceLocaleContent', () => { - it('returns the source locale values of a multilingual dictionary', () => { - const dictionary: Dictionary = { - key: 'home', - content: { - title: { - nodeType: 'translation', - translation: { en: 'Hello', fr: 'Bonjour' }, - }, - }, - }; - - expect(getSourceLocaleContent(dictionary, 'en')).toEqual({ - title: 'Hello', - }); - }); - - it('returns the content of a per-locale dictionary', () => { - const dictionary: Dictionary = { - key: 'home', - locale: 'en', - content: { title: 'Hello' }, - }; - - expect(getSourceLocaleContent(dictionary, 'en')).toEqual({ - title: 'Hello', - }); - }); -}); - -describe('readFillSourceSnapshot / writeFillSourceSnapshot', () => { - let baseDir: string; - let configuration: IntlayerConfig; - - beforeEach(async () => { - baseDir = await mkdtemp(join(tmpdir(), 'intlayer-fill-')); - configuration = { system: { baseDir } } as IntlayerConfig; - }); - - afterEach(async () => { - await rm(baseDir, { recursive: true, force: true }); - }); - - it('returns an empty snapshot when none was written', async () => { - expect(await readFillSourceSnapshot(configuration)).toEqual({}); - }); - - it('reads back the written snapshot', async () => { - const snapshot = { - [dictionaryLocalId]: { fr: hashSourceContent({ title: 'Hello' }) }, - }; - - await writeFillSourceSnapshot(configuration, snapshot); - - expect(await readFillSourceSnapshot(configuration)).toEqual(snapshot); - }); - - it('writes a sorted file at the project root', async () => { - const hashes = hashSourceContent({ title: 'Hello' }); - - await writeFillSourceSnapshot(configuration, { - 'b::local::src/b.content.ts': { fr: hashes, es: hashes }, - 'a::local::src/a.content.ts': { fr: hashes }, - }); - - const fileContent = await readFile( - join(baseDir, 'intlayer.journal.json'), - 'utf-8' - ); - const parsedContent = JSON.parse(fileContent); - - expect(Object.keys(parsedContent)).toEqual([ - 'a::local::src/a.content.ts', - 'b::local::src/b.content.ts', - ]); - expect(Object.keys(parsedContent['b::local::src/b.content.ts'])).toEqual([ - 'es', - 'fr', - ]); - }); -}); diff --git a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts b/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts deleted file mode 100644 index 33ebe9df3b..0000000000 --- a/packages/@intlayer/cli/src/fill/fillSourceSnapshot.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { getPerLocaleDictionary } from '@intlayer/core/plugins'; -import { - getPathHash, - type Recursive, - sortAlphabetically, -} from '@intlayer/engine/utils'; -import type { Locale } from '@intlayer/types/allLocales'; -import type { IntlayerConfig } from '@intlayer/types/config'; -import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; - -/** - * Hash of each source value a target locale was last filled from, per - * dictionary. Lets `fill` detect source values edited after translation. - */ -export type FillSourceSnapshot = Partial< - Record>> ->; - -/** - * Stored at the project root, not in `.intlayer`, so it can be committed and - * shared with CI and other clones. - */ -const FILL_SOURCE_SNAPSHOT_FILE_NAME = 'intlayer.journal.json'; - -const getFillSourceSnapshotPath = (configuration: IntlayerConfig): string => - join(configuration.system.baseDir, FILL_SOURCE_SNAPSHOT_FILE_NAME); - -/** - * Reads the snapshot written by the previous `fill` run. - * Returns an empty snapshot if none exists or it cannot be parsed. - */ -export const readFillSourceSnapshot = async ( - configuration: IntlayerConfig -): Promise => { - try { - const fileContent = await readFile( - getFillSourceSnapshotPath(configuration), - 'utf-8' - ); - - return JSON.parse(fileContent) as FillSourceSnapshot; - } catch { - return {}; - } -}; - -/** - * Persists the snapshot for the next `fill` run. - * Dictionaries and locales are sorted to keep the file diff stable. - */ -export const writeFillSourceSnapshot = async ( - configuration: IntlayerConfig, - snapshot: FillSourceSnapshot -): Promise => { - const sortedSnapshot = Object.fromEntries( - Object.entries(snapshot) - .sort(([keyA], [keyB]) => sortAlphabetically(keyA, keyB)) - .map(([dictionaryLocalId, localeSnapshot]) => [ - dictionaryLocalId, - Object.fromEntries( - Object.entries(localeSnapshot ?? {}).sort(([localeA], [localeB]) => - sortAlphabetically(localeA, localeB) - ) - ), - ]) - ); - - await writeFile( - getFillSourceSnapshotPath(configuration), - `${JSON.stringify(sortedSnapshot, null, 2)}\n` - ); -}; - -/** - * Content of a dictionary in its source locale, in the same shape `fill` - * sends to translation (plain values, no translation nodes). - */ -export const getSourceLocaleContent = ( - dictionary: Dictionary, - sourceLocale: Locale -): Recursive => getPerLocaleDictionary(dictionary, sourceLocale).content; - -const isPlainObject = ( - value: Recursive -): value is { [key: string]: Recursive } => - typeof value === 'object' && value !== null && !Array.isArray(value); - -/** - * Replaces each value of `content` by a short hash, keeping the object shape. - * Arrays are hashed as a whole. - */ -export const hashSourceContent = (content: Recursive): Recursive => { - if (isPlainObject(content)) { - return Object.fromEntries( - Object.entries(content).map(([key, value]) => [ - key, - hashSourceContent(value), - ]) - ); - } - - return getPathHash(JSON.stringify(content) ?? ''); -}; - -/** - * Returns the parts of `currentContent` whose hash differs from - * `previousHashes` (built with `hashSourceContent`). - * - * Keys absent from `previousHashes` are ignored, as they are already handled - * as missing translations. Arrays are compared as a whole. - * - * @example - * getChangedSourceContent( - * hashSourceContent({ a: 'Hi', b: 'Bye' }), - * { a: 'Hello', b: 'Bye' } - * ) - * // => { a: 'Hello' } - */ -export const getChangedSourceContent = ( - previousHashes: Recursive, - currentContent: Recursive -): Recursive | undefined => { - if (previousHashes === undefined) return undefined; - - if (isPlainObject(previousHashes) && isPlainObject(currentContent)) { - const changedContent: { [key: string]: Recursive } = {}; - - for (const [key, value] of Object.entries(currentContent)) { - const changedValue = getChangedSourceContent(previousHashes[key], value); - - if (changedValue !== undefined) { - changedContent[key] = changedValue; - } - } - - return Object.keys(changedContent).length > 0 ? changedContent : undefined; - } - - const isSameContent = - JSON.stringify(previousHashes) === - JSON.stringify(hashSourceContent(currentContent)); - - return isSameContent ? undefined : currentContent; -}; - -/** - * Removes from `targetContent` every value whose source changed, so it is - * treated as untranslated. Changed arrays are removed as a whole. - * - * @example - * omitChangedContent({ a: 'Salut', b: 'Au revoir' }, { a: 'Hello' }) - * // => { b: 'Au revoir' } - */ -export const omitChangedContent = ( - targetContent: Recursive, - changedContent: Recursive | undefined -): Recursive | undefined => { - if (changedContent === undefined) return targetContent; - - if (!isPlainObject(targetContent) || !isPlainObject(changedContent)) { - return undefined; - } - - const remainingContent: { [key: string]: Recursive } = {}; - - for (const [key, value] of Object.entries(targetContent)) { - const remainingValue = omitChangedContent(value, changedContent[key]); - - if (remainingValue !== undefined) { - remainingContent[key] = remainingValue; - } - } - - return remainingContent; -}; - -/** - * Records the hashes of `sourceContent` as the source of `locales` for a - * dictionary. - * - * @param overwrite - When false, only locales with no recorded source are set - * (used to take a baseline of translations made before the snapshot existed). - * @returns A new snapshot; the input is not mutated. - */ -export const setFillSourceSnapshot = ( - snapshot: FillSourceSnapshot, - dictionaryLocalId: LocalDictionaryId, - locales: Locale[], - sourceContent: Recursive, - overwrite: boolean -): FillSourceSnapshot => { - const localeSnapshot = { ...snapshot[dictionaryLocalId] }; - const sourceHashes = hashSourceContent(sourceContent); - - for (const locale of locales) { - if (overwrite || !Object.hasOwn(localeSnapshot, locale)) { - localeSnapshot[locale] = sourceHashes; - } - } - - return { ...snapshot, [dictionaryLocalId]: localeSnapshot }; -}; diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts index 332513de54..07778fa86d 100644 --- a/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.test.ts @@ -4,17 +4,16 @@ import type { IntlayerConfig } from '@intlayer/types/config'; import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { listMissingTranslationsWithConfig } from '../test'; -import { - type FillSourceSnapshot, - hashSourceContent, -} from './fillSourceSnapshot'; import { listTranslationsTasks } from './listTranslationsTasks'; +import type { PreviousDictionaries } from './sourceChanges'; vi.mock('@intlayer/dictionaries-entry', () => ({ getDictionaries: vi.fn() })); vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ getUnmergedDictionaries: vi.fn(), })); vi.mock('../test', () => ({ listMissingTranslationsWithConfig: vi.fn() })); +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({})); vi.mock('@intlayer/config/logger', async (importOriginal) => ({ ...(await importOriginal()), getAppLogger: () => () => undefined, @@ -43,14 +42,23 @@ const configuration = { dictionary: { fill: true }, } as IntlayerConfig; -const listTasks = (fillSourceSnapshot?: FillSourceSnapshot) => +/** `dictionary` as it was in git, with the title and translations given. */ +const getPreviousDictionary = (title: Record): Dictionary => ({ + ...dictionary, + content: { + ...dictionary.content, + title: { nodeType: 'translation', translation: title }, + }, +}); + +const listTasks = (previousDictionaries?: PreviousDictionaries) => listTranslationsTasks( [dictionaryLocalId], ['en', 'fr', 'es'], 'complete', 'en', configuration, - fillSourceSnapshot + previousDictionaries ); describe('listTranslationsTasks', () => { @@ -63,33 +71,51 @@ describe('listTranslationsTasks', () => { }); it('skips a fully translated dictionary whose source did not change', () => { - const sourceHashes = hashSourceContent({ - title: 'Hello', - subtitle: 'Welcome', - }); - expect(listTasks()).toEqual([]); expect( listTasks({ - [dictionaryLocalId]: { fr: sourceHashes, es: sourceHashes }, + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hello', + fr: 'Salut', + es: 'Hola', + }), }) ).toEqual([]); }); - it('re-translates locales filled from a since changed source value', () => { + it('re-translates locales whose source value changed in git', () => { const tasks = listTasks({ - [dictionaryLocalId]: { - fr: hashSourceContent({ title: 'Hi', subtitle: 'Welcome' }), - es: hashSourceContent({ title: 'Hello', subtitle: 'Welcome' }), - }, + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hi', + fr: 'Salut', + es: 'Hola', + }), }); expect(tasks).toHaveLength(1); expect(tasks[0]).toMatchObject({ dictionaryKey: 'home', sourceLocale: 'en', + targetLocales: ['fr', 'es'], + changedSourceContent: { + fr: { title: 'Hello' }, + es: { title: 'Hello' }, + }, + }); + }); + + it('keeps a translation edited in the same change as its source', () => { + const tasks = listTasks({ + [dictionaryLocalId]: getPreviousDictionary({ + en: 'Hi', + fr: 'Salut', + es: 'Holi', + }), + }); + + expect(tasks).toHaveLength(1); + expect(tasks[0]).toMatchObject({ targetLocales: ['fr'], - sourceContent: { title: 'Hello', subtitle: 'Welcome' }, changedSourceContent: { fr: { title: 'Hello' } }, }); }); diff --git a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts index ff5fc26185..4414489ca1 100644 --- a/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts +++ b/packages/@intlayer/cli/src/fill/listTranslationsTasks.ts @@ -16,10 +16,9 @@ import type { IntlayerConfig } from '@intlayer/types/config'; import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; import { listMissingTranslationsWithConfig } from '../test'; import { - type FillSourceSnapshot, - getChangedSourceContent, - getSourceLocaleContent, -} from './fillSourceSnapshot'; + getStaleSourceContent, + type PreviousDictionaries, +} from './sourceChanges'; export type TranslationTask = { dictionaryKey: string; @@ -28,9 +27,7 @@ export type TranslationTask = { targetLocales: Locale[]; dictionaryPreset: string; dictionaryFilePath: string; - /** Current source locale content, recorded once the task succeeds. */ - sourceContent: Recursive; - /** Per target locale, source values changed since that locale was filled. */ + /** Per target locale, source values changed since the git ref. */ changedSourceContent: Partial>; }; @@ -40,7 +37,7 @@ export const listTranslationsTasks = ( mode: 'complete' | 'review', baseLocale: Locale, configuration: IntlayerConfig, - fillSourceSnapshot: FillSourceSnapshot = {} + previousDictionaries: PreviousDictionaries = {} ): TranslationTask[] => { const appLogger = getAppLogger(configuration); @@ -124,14 +121,10 @@ export const listTranslationsTasks = ( continue; } - const sourceContent = getSourceLocaleContent( - targetUnmergedDictionary, - sourceLocale - ); - /** - * Source values edited since each locale was last filled. Only the base - * locale is a source of truth: other per-locale files are fill outputs. + * Source values edited since the git ref, with a translation that was not + * updated. Only the base locale is a source of truth: other per-locale + * files are fill outputs. */ const changedSourceContent: Partial> = {}; @@ -139,9 +132,11 @@ export const listTranslationsTasks = ( for (const locale of outputLocales) { if (locale === sourceLocale) continue; - const changedContent = getChangedSourceContent( - fillSourceSnapshot[dictionaryLocalId]?.[locale], - sourceContent + const changedContent = getStaleSourceContent( + previousDictionaries[dictionaryLocalId], + targetUnmergedDictionary, + sourceLocale, + locale ); if (changedContent !== undefined) { @@ -152,7 +147,7 @@ export const listTranslationsTasks = ( /** * In 'complete' mode, filter only the locales with missing translations - * or with source values changed since they were filled + * or with source values changed since the git ref * * Skip the dictionary if there are no locales to translate */ @@ -188,7 +183,6 @@ export const listTranslationsTasks = ( targetLocales: outputLocalesList, dictionaryPreset, dictionaryFilePath: targetUnmergedDictionary.filePath, - sourceContent, changedSourceContent, }); } diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.test.ts b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts new file mode 100644 index 0000000000..6520a0a298 --- /dev/null +++ b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts @@ -0,0 +1,122 @@ +import type { Dictionary } from '@intlayer/types/dictionary'; +import { describe, expect, it, vi } from 'vitest'; +import { + getChangedContent, + getSourceChangesRef, + getStaleSourceContent, + omitChangedContent, +} from './sourceChanges'; + +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({ + getGitMergeBase: vi.fn(async () => 'abc123'), +})); + +const getDictionary = ( + title: Record, + subtitle: Record +): Dictionary => ({ + key: 'home', + content: { + title: { nodeType: 'translation', translation: title }, + subtitle: { nodeType: 'translation', translation: subtitle }, + }, +}); + +describe('getChangedContent', () => { + it('returns only the changed values', () => { + expect( + getChangedContent( + { a: 'Hi', b: 'Bye', c: { d: 'Same' } }, + { a: 'Hello', b: 'Bye', c: { d: 'Same' } } + ) + ).toEqual({ a: 'Hello' }); + }); + + it('returns undefined when nothing changed', () => { + expect(getChangedContent({ a: ['x', 'y'] }, { a: ['x', 'y'] })).toBe( + undefined + ); + }); + + it('compares arrays as a whole', () => { + expect(getChangedContent({ a: ['x', 'y'] }, { a: ['x', 'z'] })).toEqual({ + a: ['x', 'z'], + }); + }); +}); + +describe('omitChangedContent', () => { + it('removes changed values from the target content', () => { + expect( + omitChangedContent( + { a: 'Salut', b: 'Au revoir', c: { d: 'x', e: 'y' } }, + { a: 'Hello', c: { d: 'z' } } + ) + ).toEqual({ b: 'Au revoir', c: { e: 'y' } }); + }); +}); + +describe('getStaleSourceContent', () => { + const current = getDictionary( + { en: 'Hello', fr: 'Salut' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + it('returns source values changed since the previous version', () => { + const previous = getDictionary( + { en: 'Hi', fr: 'Salut' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + expect(getStaleSourceContent(previous, current, 'en', 'fr')).toEqual({ + title: 'Hello', + }); + }); + + it('keeps translations updated in the same change', () => { + const previous = getDictionary( + { en: 'Hi', fr: 'Coucou' }, + { en: 'Welcome', fr: 'Bienvenue' } + ); + + expect(getStaleSourceContent(previous, current, 'en', 'fr')).toBe( + undefined + ); + }); + + it('returns undefined without a previous version', () => { + expect(getStaleSourceContent(undefined, current, 'en', 'fr')).toBe( + undefined + ); + }); + + it('compares per-locale files on their source only', () => { + const perLocale = (title: string): Dictionary => ({ + key: 'home', + locale: 'en', + content: { title }, + }); + + expect( + getStaleSourceContent(perLocale('Hi'), perLocale('Hello'), 'en', 'fr') + ).toEqual({ title: 'Hello' }); + }); +}); + +describe('getSourceChangesRef', () => { + it('compares to the last commit by default', async () => { + expect(await getSourceChangesRef()).toBe('HEAD'); + expect(await getSourceChangesRef({ mode: ['uncommitted'] })).toBe('HEAD'); + }); + + it('compares to the upstream for unpushed changes', async () => { + expect(await getSourceChangesRef({ mode: ['unpushed'] })).toBe('@{push}'); + }); + + it('compares to the merge base with --git-diff', async () => { + expect( + await getSourceChangesRef({ mode: ['gitDiff'], baseRef: 'origin/main' }) + ).toBe('abc123'); + }); +}); diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.ts b/packages/@intlayer/cli/src/fill/sourceChanges.ts new file mode 100644 index 0000000000..9067fd589b --- /dev/null +++ b/packages/@intlayer/cli/src/fill/sourceChanges.ts @@ -0,0 +1,207 @@ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { getPerLocaleDictionary } from '@intlayer/core/plugins'; +import { + loadContentDeclaration, + processContentDeclaration, +} from '@intlayer/engine/build'; +import { + getGitMergeBase, + type ListGitFilesOptions, + readGitFile, +} from '@intlayer/engine/cli'; +import type { Recursive } from '@intlayer/engine/utils'; +import type { Locale } from '@intlayer/types/allLocales'; +import type { IntlayerConfig } from '@intlayer/types/config'; +import type { Dictionary, LocalDictionaryId } from '@intlayer/types/dictionary'; + +/** Previous version of each content declaration, by dictionary local id. */ +export type PreviousDictionaries = Partial< + Record +>; + +/** + * Git ref the source values are compared against. The base branch with + * `--git-diff`, the upstream with `--unpushed`, the last commit otherwise. + */ +export const getSourceChangesRef = async ( + gitOptions?: ListGitFilesOptions +): Promise => { + if (gitOptions?.mode.includes('gitDiff')) { + return getGitMergeBase(gitOptions.baseRef, gitOptions.currentRef); + } + + if (gitOptions?.mode.includes('unpushed')) return '@{push}'; + + return 'HEAD'; +}; + +/** + * Loads each local content declaration as it is at `ref` in git. + * Files that are unchanged, new, or not tracked by git are skipped. + */ +export const loadPreviousDictionaries = async ( + dictionaries: Dictionary[], + configuration: IntlayerConfig, + ref: string +): Promise => { + const previousDictionaries: PreviousDictionaries = {}; + + for (const dictionary of dictionaries) { + if (!dictionary.localId || !dictionary.filePath) continue; + if (!['local', 'hybrid'].includes(dictionary.location ?? 'local')) continue; + + const filePath = join(configuration.system.baseDir, dictionary.filePath); + const previousCode = await readGitFile(filePath, ref); + + if (previousCode === undefined) continue; + + const currentCode = await readFile(filePath, 'utf-8').catch( + () => undefined + ); + + if (previousCode === currentCode) continue; + + const loaded = await loadContentDeclaration( + filePath, + configuration, + undefined, + { logError: false, code: previousCode } + ); + + // A file can declare several dictionaries + const previousDeclaration = [loaded] + .flat() + .find((declaration) => declaration?.key === dictionary.key); + + if (!previousDeclaration) continue; + + const previousDictionary = await processContentDeclaration( + previousDeclaration, + configuration + ); + + if (previousDictionary) { + previousDictionaries[dictionary.localId] = previousDictionary; + } + } + + return previousDictionaries; +}; + +/** + * Content of a dictionary in one locale, in the same shape `fill` sends to + * translation (plain values, no translation nodes). + */ +export const getLocaleContent = ( + dictionary: Dictionary, + locale: Locale +): Recursive => getPerLocaleDictionary(dictionary, locale).content; + +const isPlainObject = ( + value: Recursive +): value is { [key: string]: Recursive } => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isEmptyObject = (value: Recursive): boolean => + isPlainObject(value) && Object.keys(value).length === 0; + +/** + * Returns the parts of `currentContent` that differ from `previousContent`. + * Arrays are compared as a whole. + * + * @example + * getChangedContent({ a: 'Hi', b: 'Bye' }, { a: 'Hello', b: 'Bye' }) + * // => { a: 'Hello' } + */ +export const getChangedContent = ( + previousContent: Recursive, + currentContent: Recursive +): Recursive | undefined => { + if (isPlainObject(previousContent) && isPlainObject(currentContent)) { + const changedContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(currentContent)) { + const changedValue = getChangedContent(previousContent[key], value); + + if (changedValue !== undefined) { + changedContent[key] = changedValue; + } + } + + return Object.keys(changedContent).length > 0 ? changedContent : undefined; + } + + return JSON.stringify(previousContent) === JSON.stringify(currentContent) + ? undefined + : currentContent; +}; + +/** + * Removes from `targetContent` every value present in `changedContent`. + * Changed arrays are removed as a whole. + * + * @example + * omitChangedContent({ a: 'Salut', b: 'Au revoir' }, { a: 'Hello' }) + * // => { b: 'Au revoir' } + */ +export const omitChangedContent = ( + targetContent: Recursive, + changedContent: Recursive | undefined +): Recursive | undefined => { + if (changedContent === undefined) return targetContent; + + if (!isPlainObject(targetContent) || !isPlainObject(changedContent)) { + return undefined; + } + + const remainingContent: { [key: string]: Recursive } = {}; + + for (const [key, value] of Object.entries(targetContent)) { + const remainingValue = omitChangedContent(value, changedContent[key]); + + if (remainingValue !== undefined) { + remainingContent[key] = remainingValue; + } + } + + return remainingContent; +}; + +/** + * Source values edited since `previousDictionary` whose `targetLocale` + * translation was left as is, so it is now stale. In multilingual + * dictionaries, translations edited in the same change are kept. + */ +export const getStaleSourceContent = ( + previousDictionary: Dictionary | undefined, + currentDictionary: Dictionary, + sourceLocale: Locale, + targetLocale: Locale +): Recursive | undefined => { + if (!previousDictionary) return undefined; + + const changedSourceContent = getChangedContent( + getLocaleContent(previousDictionary, sourceLocale), + getLocaleContent(currentDictionary, sourceLocale) + ); + + if (changedSourceContent === undefined) return undefined; + + // Per-locale files hold no translation to compare with + if (typeof currentDictionary.locale === 'string') return changedSourceContent; + + const editedTargetContent = getChangedContent( + getLocaleContent(previousDictionary, targetLocale), + getLocaleContent(currentDictionary, targetLocale) + ); + + const staleContent = omitChangedContent( + changedSourceContent, + editedTargetContent + ); + + return staleContent === undefined || isEmptyObject(staleContent) + ? undefined + : staleContent; +}; diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.test.ts b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts index c38e45e689..ab754c8226 100644 --- a/packages/@intlayer/cli/src/fill/translateDictionary.test.ts +++ b/packages/@intlayer/cli/src/fill/translateDictionary.test.ts @@ -10,6 +10,8 @@ import { translateDictionary } from './translateDictionary'; vi.mock('@intlayer/dictionaries-entry/unmerged', () => ({ getUnmergedDictionaries: vi.fn(), })); +vi.mock('@intlayer/engine/build', () => ({})); +vi.mock('@intlayer/engine/cli', () => ({})); vi.mock('../utils/checkAccess', () => ({ getAuthenticatedAPI: async () => ({}), })); @@ -69,7 +71,6 @@ describe('translateDictionary', () => { targetLocales: ['fr'], dictionaryPreset: '', dictionaryFilePath: 'src/home.content.ts', - sourceContent: { title: 'Hello', subtitle: 'Welcome' }, changedSourceContent: { fr: { title: 'Hello' } }, }; diff --git a/packages/@intlayer/cli/src/fill/translateDictionary.ts b/packages/@intlayer/cli/src/fill/translateDictionary.ts index 5690d084c3..8217f91e82 100644 --- a/packages/@intlayer/cli/src/fill/translateDictionary.ts +++ b/packages/@intlayer/cli/src/fill/translateDictionary.ts @@ -35,8 +35,8 @@ import { extractTranslatableContent, reinsertTranslatedContent, } from './extractTranslatableContent'; -import { omitChangedContent } from './fillSourceSnapshot'; import type { TranslationTask } from './listTranslationsTasks'; +import { omitChangedContent } from './sourceChanges'; type TranslateDictionaryResult = TranslationTask & { dictionaryOutput: Dictionary | null; @@ -191,7 +191,7 @@ export const translateDictionary = async ( // Reset to base dictionary for each locale to ensure we filter from the original let dictionaryToProcess = structuredClone(baseUnmergedDictionary); - // Source values edited since this locale was last filled + // Source values edited since the git ref const changedSourceContent = task.changedSourceContent[targetLocale]; let targetLocaleDictionary: Dictionary; diff --git a/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts b/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts index 2421f0da72..0313901cd9 100644 --- a/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts +++ b/packages/@intlayer/config/src/loadExternalFile/loadExternalFile.ts @@ -29,6 +29,12 @@ export type LoadExternalFileOptions = { * Values are restored (or deleted) after `runInNewContext` returns. */ preloadGlobals?: Record; + /** + * Content to load instead of reading `filePath` from disk (for example an + * older version of the file read from git). `filePath` is still used to + * pick the loader and resolve imports. + */ + code?: string; } & SandBoxContextOptions; /** @@ -146,12 +152,13 @@ export const loadExternalFile = async ( fileExtension === '.jsonc' ) { // Remove cache to force getting fresh content - const fileContent = await readFile(filePath, 'utf-8'); + const fileContent = + options?.code ?? (await readFile(filePath, 'utf-8')); return parseJSON5(fileContent); } // Rest is JS, MJS or TS - const code = await readFile(filePath, 'utf-8'); + const code = options?.code ?? (await readFile(filePath, 'utf-8')); const moduleResultString: string | undefined = await transpileTSToCJS( code, diff --git a/packages/@intlayer/engine/src/listGitFiles.ts b/packages/@intlayer/engine/src/listGitFiles.ts index d6b79d5728..5d82e8b57e 100644 --- a/packages/@intlayer/engine/src/listGitFiles.ts +++ b/packages/@intlayer/engine/src/listGitFiles.ts @@ -1,5 +1,5 @@ import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import { log } from '@intlayer/config/built'; import { getAppLogger } from '@intlayer/config/logger'; import { simpleGit } from 'simple-git'; @@ -199,3 +199,39 @@ export const listGitLines = async ( // Return the list sorted for convenience return Array.from(changedLines).sort((a, b) => a - b); }; + +/** + * Returns the content of a file at a git ref, or `undefined` if the file does + * not exist at that ref or git is not available. + */ +export const readGitFile = async ( + filePath: string, + ref = 'HEAD' +): Promise => { + try { + const git = simpleGit(dirname(filePath)); + + return await git.show([`${ref}:./${basename(filePath)}`]); + } catch { + return undefined; + } +}; + +/** + * Returns the commit `currentRef` forked from `baseRef`, which is the side + * `git diff baseRef...currentRef` compares against. + */ +export const getGitMergeBase = async ( + baseRef = 'origin/main', + currentRef = 'HEAD' +): Promise => { + try { + const git = simpleGit(); + await git.fetch(baseRef).catch(() => undefined); + const mergeBase = await git.raw(['merge-base', baseRef, currentRef]); + + return mergeBase.trim() || undefined; + } catch { + return undefined; + } +}; diff --git a/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts b/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts index 2b35af0318..da5b62704d 100644 --- a/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts +++ b/packages/@intlayer/engine/src/loadDictionaries/loadContentDeclaration.ts @@ -51,6 +51,11 @@ export const ensureIntlayerBundle = async ( type LoadContentDeclarationOptions = { logError?: boolean; + /** + * Content to load instead of reading the file from disk. Only supported + * for JSON, JS and TS content declarations. + */ + code?: string; }; // Initialize a module-level cache @@ -103,11 +108,18 @@ export const loadContentDeclaration = async ( bundleFilePath?: string, options?: LoadContentDeclarationOptions ): Promise => { - if (extname(path) === '.md' || extname(path) === '.mdx') { + const isMarkdown = extname(path) === '.md' || extname(path) === '.mdx'; + const isYaml = extname(path) === '.yaml' || extname(path) === '.yml'; + + if (typeof options?.code === 'string' && (isMarkdown || isYaml)) { + return undefined; + } + + if (isMarkdown) { return loadMarkdownContentDeclaration(path, configuration); } - if (extname(path) === '.yaml' || extname(path) === '.yml') { + if (isYaml) { return loadYamlContentDeclaration(path); } @@ -122,6 +134,7 @@ export const loadContentDeclaration = async ( try { const dictionary = await loadExternalFile(path, { logError: options?.logError, + code: options?.code, projectRequire: build.require ?? getProjectRequire(), buildOptions: { packages: undefined, // It fixes the import of ESM packages in the content declaration From f3ad740e2132766a89c88335921ad230b1768b19 Mon Sep 17 00:00:00 2001 From: theluckystrike <51033404+theluckystrike@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:59:12 +0700 Subject: [PATCH 3/3] fix(cli): log why fill skips source change detection when git is unavailable Fill already fell back to filling missing translations when git could not give the previous version of a file, but it did so silently. It now checks the ref once before reading any file and logs the reason: no git repository, no commit, no upstream branch, or no merge base with --git-diff (for example a shallow clone from actions/checkout with its default fetch-depth: 1). Adds hasGitRef to the engine git helpers, with tests against real repositories (no repository, no commit, shallow clone), and a test that remote and plugin dictionaries are never read from git. --- docs/docs/en/cli/fill.md | 4 +- packages/@intlayer/cli/src/fill/fill.ts | 14 ++- .../cli/src/fill/sourceChanges.test.ts | 85 ++++++++++++++++-- .../@intlayer/cli/src/fill/sourceChanges.ts | 31 +++++-- .../@intlayer/engine/src/listGitFiles.test.ts | 87 +++++++++++++++++++ packages/@intlayer/engine/src/listGitFiles.ts | 22 +++++ 6 files changed, 228 insertions(+), 15 deletions(-) create mode 100644 packages/@intlayer/engine/src/listGitFiles.test.ts diff --git a/docs/docs/en/cli/fill.md b/docs/docs/en/cli/fill.md index 049b312295..a8542edc95 100644 --- a/docs/docs/en/cli/fill.md +++ b/docs/docs/en/cli/fill.md @@ -1,6 +1,6 @@ --- createdAt: 2024-08-11 -updatedAt: 2026-10-05 +updatedAt: 2026-10-06 priority: 5 title: "intlayer fill: Translate Dictionaries with AI" description: "Fill missing translations, audit existing ones and translate your Intlayer dictionaries with AI from the CLI, locally or in CI." @@ -119,7 +119,7 @@ Affected dictionary keys for processing: app, comp-test, hello-world, lang-switc - **`--mode [mode]`**: Translation mode: `complete`, `review`. Default is `complete`. `complete` will fill all missing content and re-translate values whose source changed in git, `review` will fill missing content and review existing keys. - > In `complete` mode, source values are compared with the last commit (`HEAD`), or with the base branch when `--git-diff` is set (with `--unpushed`, with the upstream branch). A translation is re-translated when its source value changed and the translation itself did not. No extra file is written. This applies to JSON, JS and TS content declaration files tracked by git. + > In `complete` mode, source values are compared with the last commit (`HEAD`), or with the base branch when `--git-diff` is set (with `--unpushed`, with the upstream branch). A translation is re-translated when its source value changed and the translation itself did not. No extra file is written. This applies to JSON, JS and TS content declaration files tracked by git. If that git ref can't be read (no git repository, no commit, or a shallow clone without the base branch), `fill` logs why and only fills missing translations. In CI, check out the full history (`fetch-depth: 0` with `actions/checkout`) to use `--git-diff`. **Git options:** diff --git a/packages/@intlayer/cli/src/fill/fill.ts b/packages/@intlayer/cli/src/fill/fill.ts index 516bc5ee2d..b23f4c06c4 100644 --- a/packages/@intlayer/cli/src/fill/fill.ts +++ b/packages/@intlayer/cli/src/fill/fill.ts @@ -189,13 +189,21 @@ export const fill = async (options?: FillOptions): Promise => { let previousDictionaries: PreviousDictionaries = {}; if (mode === 'complete') { - const sourceChangesRef = await getSourceChangesRef(options?.gitOptions); + const { ref, skipReason } = await getSourceChangesRef( + options?.gitOptions, + configuration.system.baseDir + ); - if (sourceChangesRef) { + if (ref) { previousDictionaries = await loadPreviousDictionaries( targetUnmergedDictionaries, configuration, - sourceChangesRef + ref + ); + } else { + appLogger( + `Source changes not checked: ${skipReason}. Only missing translations are filled.`, + { level: options?.gitOptions ? 'warn' : 'info' } ); } } diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.test.ts b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts index 6520a0a298..c6af8fe960 100644 --- a/packages/@intlayer/cli/src/fill/sourceChanges.test.ts +++ b/packages/@intlayer/cli/src/fill/sourceChanges.test.ts @@ -1,15 +1,20 @@ +import { getGitMergeBase, hasGitRef, readGitFile } from '@intlayer/engine/cli'; +import type { IntlayerConfig } from '@intlayer/types/config'; import type { Dictionary } from '@intlayer/types/dictionary'; import { describe, expect, it, vi } from 'vitest'; import { getChangedContent, getSourceChangesRef, getStaleSourceContent, + loadPreviousDictionaries, omitChangedContent, } from './sourceChanges'; vi.mock('@intlayer/engine/build', () => ({})); vi.mock('@intlayer/engine/cli', () => ({ - getGitMergeBase: vi.fn(async () => 'abc123'), + getGitMergeBase: vi.fn(), + hasGitRef: vi.fn(), + readGitFile: vi.fn(), })); const getDictionary = ( @@ -106,17 +111,87 @@ describe('getStaleSourceContent', () => { describe('getSourceChangesRef', () => { it('compares to the last commit by default', async () => { - expect(await getSourceChangesRef()).toBe('HEAD'); - expect(await getSourceChangesRef({ mode: ['uncommitted'] })).toBe('HEAD'); + vi.mocked(hasGitRef).mockResolvedValue(true); + + expect(await getSourceChangesRef()).toEqual({ ref: 'HEAD' }); + expect(await getSourceChangesRef({ mode: ['uncommitted'] })).toEqual({ + ref: 'HEAD', + }); }); it('compares to the upstream for unpushed changes', async () => { - expect(await getSourceChangesRef({ mode: ['unpushed'] })).toBe('@{push}'); + vi.mocked(hasGitRef).mockResolvedValue(true); + + expect(await getSourceChangesRef({ mode: ['unpushed'] })).toEqual({ + ref: '@{push}', + }); }); it('compares to the merge base with --git-diff', async () => { + vi.mocked(getGitMergeBase).mockResolvedValue('abc123'); + expect( await getSourceChangesRef({ mode: ['gitDiff'], baseRef: 'origin/main' }) - ).toBe('abc123'); + ).toEqual({ ref: 'abc123' }); + }); + + it('skips when git or the last commit is not available', async () => { + vi.mocked(hasGitRef).mockResolvedValue(false); + + const result = await getSourceChangesRef(undefined, '/project'); + + expect(hasGitRef).toHaveBeenCalledWith('HEAD', '/project'); + expect(result.ref).toBe(undefined); + expect(result.skipReason).toContain('git ref HEAD not found'); + }); + + it('skips with --git-diff when the merge base is missing', async () => { + // Shallow clone (e.g. actions/checkout with its default fetch-depth: 1) + vi.mocked(getGitMergeBase).mockResolvedValue(undefined); + + const result = await getSourceChangesRef({ + mode: ['gitDiff'], + baseRef: 'origin/main', + }); + + expect(result.ref).toBe(undefined); + expect(result.skipReason).toContain('fetch-depth: 0'); + }); +}); + +describe('loadPreviousDictionaries', () => { + const configuration = { + system: { baseDir: '/project' }, + } as IntlayerConfig; + + const getEntry = (location: string): Dictionary => ({ + key: 'home', + localId: `home::${location}::src/home.content.json`, + location, + filePath: 'src/home.content.json', + content: {}, + }); + + it('reads git only for local and hybrid dictionaries', async () => { + vi.mocked(readGitFile).mockResolvedValue(undefined); + + const previousDictionaries = await loadPreviousDictionaries( + [ + getEntry('local'), + getEntry('hybrid'), + getEntry('remote'), + getEntry('plugin'), + getEntry('sync-json::messages/{{locale}}.json'), + ], + configuration, + 'HEAD' + ); + + expect(previousDictionaries).toEqual({}); + expect(readGitFile).toHaveBeenCalledTimes(2); + expect(readGitFile).toHaveBeenCalledWith( + '/project/src/home.content.json', + 'HEAD' + ); }); }); diff --git a/packages/@intlayer/cli/src/fill/sourceChanges.ts b/packages/@intlayer/cli/src/fill/sourceChanges.ts index 9067fd589b..55c5d1d051 100644 --- a/packages/@intlayer/cli/src/fill/sourceChanges.ts +++ b/packages/@intlayer/cli/src/fill/sourceChanges.ts @@ -7,6 +7,7 @@ import { } from '@intlayer/engine/build'; import { getGitMergeBase, + hasGitRef, type ListGitFilesOptions, readGitFile, } from '@intlayer/engine/cli'; @@ -20,20 +21,40 @@ export type PreviousDictionaries = Partial< Record >; +export type SourceChangesRef = + | { ref: string; skipReason?: undefined } + | { ref?: undefined; skipReason: string }; + /** * Git ref the source values are compared against. The base branch with * `--git-diff`, the upstream with `--unpushed`, the last commit otherwise. + * + * When the ref can't be read (no git, no commit, no upstream, or a shallow + * clone without the base branch), returns why instead, and `fill` only fills + * missing translations, as it did before source change detection. */ export const getSourceChangesRef = async ( - gitOptions?: ListGitFilesOptions -): Promise => { + gitOptions?: ListGitFilesOptions, + cwd?: string +): Promise => { if (gitOptions?.mode.includes('gitDiff')) { - return getGitMergeBase(gitOptions.baseRef, gitOptions.currentRef); + const { baseRef = 'origin/main', currentRef = 'HEAD' } = gitOptions; + const mergeBase = await getGitMergeBase(baseRef, currentRef); + + if (mergeBase) return { ref: mergeBase }; + + return { + skipReason: `no merge base found between ${baseRef} and ${currentRef}. In CI, check out the full git history (fetch-depth: 0)`, + }; } - if (gitOptions?.mode.includes('unpushed')) return '@{push}'; + const ref = gitOptions?.mode.includes('unpushed') ? '@{push}' : 'HEAD'; + + if (await hasGitRef(ref, cwd)) return { ref }; - return 'HEAD'; + return { + skipReason: `git ref ${ref} not found (no git repository, no commit or no upstream branch)`, + }; }; /** diff --git a/packages/@intlayer/engine/src/listGitFiles.test.ts b/packages/@intlayer/engine/src/listGitFiles.test.ts new file mode 100644 index 0000000000..56f8289d9d --- /dev/null +++ b/packages/@intlayer/engine/src/listGitFiles.test.ts @@ -0,0 +1,87 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { hasGitRef, readGitFile } from './listGitFiles'; + +const git = (cwd: string, ...args: string[]) => + execFileSync( + 'git', + [ + '-c', + 'user.name=test', + '-c', + 'user.email=test@example.com', + '-c', + 'commit.gpgsign=false', + ...args, + ], + { cwd, stdio: 'pipe' } + ); + +describe('git helpers used by fill', () => { + let testDir: string; + + beforeEach(async () => { + testDir = await mkdtemp(join(tmpdir(), 'list-git-files-')); + }); + + afterEach(async () => { + await rm(testDir, { recursive: true, force: true }); + }); + + it('returns nothing outside a git repository', async () => { + const filePath = join(testDir, 'home.content.json'); + await writeFile(filePath, '{"key":"home"}'); + + expect(await hasGitRef('HEAD', testDir)).toBe(false); + expect(await readGitFile(filePath)).toBe(undefined); + }); + + it('returns nothing in a repository without commits', async () => { + git(testDir, 'init', '-q'); + + expect(await hasGitRef('HEAD', testDir)).toBe(false); + }); + + it('reads the committed version of a file', async () => { + const filePath = join(testDir, 'home.content.json'); + git(testDir, 'init', '-q'); + await writeFile(filePath, '{"title":"Hi"}'); + git(testDir, 'add', '.'); + git(testDir, 'commit', '-q', '-m', 'init'); + await writeFile(filePath, '{"title":"Hello"}'); + + expect(await hasGitRef('HEAD', testDir)).toBe(true); + expect(await readGitFile(filePath)).toBe('{"title":"Hi"}'); + }); + + it('does not find the base branch in a shallow clone', async () => { + const originDir = join(testDir, 'origin'); + const cloneDir = join(testDir, 'clone'); + git(testDir, 'init', '-q', '-b', 'main', originDir); + await writeFile(join(originDir, 'home.content.json'), '{"title":"Hi"}'); + git(originDir, 'add', '.'); + git(originDir, 'commit', '-q', '-m', 'init'); + git(originDir, 'checkout', '-q', '-b', 'feature'); + await writeFile(join(originDir, 'home.content.json'), '{"title":"Hello"}'); + git(originDir, 'commit', '-q', '-am', 'edit'); + + // Same history as actions/checkout with its default fetch-depth: 1 + git( + testDir, + 'clone', + '-q', + '--depth', + '1', + '--branch', + 'feature', + `file://${originDir}`, + cloneDir + ); + + expect(await hasGitRef('HEAD', cloneDir)).toBe(true); + expect(await hasGitRef('origin/main', cloneDir)).toBe(false); + }); +}); diff --git a/packages/@intlayer/engine/src/listGitFiles.ts b/packages/@intlayer/engine/src/listGitFiles.ts index 5d82e8b57e..5c64b0a3bb 100644 --- a/packages/@intlayer/engine/src/listGitFiles.ts +++ b/packages/@intlayer/engine/src/listGitFiles.ts @@ -217,6 +217,28 @@ export const readGitFile = async ( } }; +/** + * Returns `true` if `ref` points to a commit. Returns `false` if git is not + * installed, `cwd` is not in a git repository, or the ref is missing (no + * commit yet, no upstream branch, or a shallow clone without that history). + */ +export const hasGitRef = async ( + ref: string, + cwd?: string +): Promise => { + try { + const commit = await simpleGit(cwd).raw([ + 'rev-parse', + '--verify', + `${ref}^{commit}`, + ]); + + return commit.trim() !== ''; + } catch { + return false; + } +}; + /** * Returns the commit `currentRef` forked from `baseRef`, which is the side * `git diff baseRef...currentRef` compares against.