diff --git a/.gitignore b/.gitignore index 21e267287..4983d1056 100644 --- a/.gitignore +++ b/.gitignore @@ -373,3 +373,6 @@ app.json # Test coverage (global) coverage/ export-stack/ +allien/ +aem_data_structure/ +.claude/ \ No newline at end of file diff --git a/api/src/controllers/projects.contentMapper.controller.ts b/api/src/controllers/projects.contentMapper.controller.ts index 1481a87ad..fb1c4e3dd 100644 --- a/api/src/controllers/projects.contentMapper.controller.ts +++ b/api/src/controllers/projects.contentMapper.controller.ts @@ -212,6 +212,18 @@ const updateAssetStatus = async (req: Request, res: Response): Promise => res.status(resp?.status).json(resp); }; +/** + * Re-attempts the download for one asset that failed during the last migration run. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const retryAssetDownload = async (req: Request, res: Response): Promise => { + const resp = await contentMapperService.retryAssetDownload(req); + res.status(resp?.status).json(resp); +}; + export const contentMapperController = { getContentTypes, getFieldMapping, @@ -229,5 +241,6 @@ export const contentMapperController = { getEntryMapping, updateEntryStatus, getAssetMapping, - updateAssetStatus + updateAssetStatus, + retryAssetDownload }; diff --git a/api/src/routes/contentMapper.routes.ts b/api/src/routes/contentMapper.routes.ts index d0391fdf7..8a6577cae 100644 --- a/api/src/routes/contentMapper.routes.ts +++ b/api/src/routes/contentMapper.routes.ts @@ -136,6 +136,15 @@ router.put( asyncRouter(contentMapperController.updateAssetStatus) ); +/** + * Retry downloading a single asset that failed during the last migration run + * @route PUT /retryAsset/:projectId/:assetUid + */ +router.put( + "/retryAsset/:projectId/:assetUid", + asyncRouter(contentMapperController.retryAssetDownload) +); + /** * Get Single Global Field data * @route GET /:projectId/:globalFieldUid diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 24f28d709..59572336a 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -15,6 +15,7 @@ import { CONTENT_TYPE_STATUS, VALIDATION_ERRORS, MIGRATION_DATA_CONFIG, + CMS, } from '../constants/index.js'; import logger from '../utils/logger.js'; import { config } from '../config/index.js'; @@ -34,6 +35,9 @@ import getUidMapperDb from "../models/uidMapper.js"; import { isDuplicateEntry } from '../utils/entry-duplicate.utils.js'; import { getSourceLocaleForDestination } from '../utils/locale-migration.utils.js'; import { loadPreviousAssetMetadata } from '../utils/asset-update.utils.js'; +import { flattenNestedUidMap } from '../utils/uid-mapper.utils.js'; +import { contentfulService } from './contentful.service.js'; +import { sanitizeStackId, assertResolvedPathUnderBase } from '../utils/sanitize-path.utils.js'; const idCorrector = ({ id }: { id: string }) => { @@ -175,11 +179,7 @@ const putTestData = async (req: Request) => { const uidMapperCurrent = getUidMapperDb(projectId, iteration); await uidMapperCurrent.read(); - let uidMapperPrev: any = null; - if (iteration > 1) { - uidMapperPrev = getUidMapperDb(projectId, iteration - 1); - await uidMapperPrev.read(); - } + const uidMapperPrev: any = iteration > 1 ? await getNearestPriorUidMapper(projectId, iteration) : null; const mergeEntry = (base: any, incoming: any) => { const keep = { ...(base ?? {}) }; @@ -468,15 +468,29 @@ const getContentTypes = async (req: Request) => { ); // Delta migration: from iteration 2 onwards, split content types into new vs old - // relative to the previous iteration so Step 3 (field mapping) shows only new types - // and Step 4 (entry mapping) shows only already-migrated types. Iteration 1 is untouched. + // relative to EVERY prior iteration (1..N-1) so Step 3 (field mapping) shows only + // genuinely-never-seen types and Step 4 (entry mapping) shows every already-migrated + // type. Comparing only against iteration N-1 misclassified any content type that + // was migrated in iteration 1 but absent from the iteration-2 source as "new" on + // iteration 3 — sending already-migrated types back to Map Content Fields and + // hiding them from Map Entry. Iteration 1 is untouched. if (iteration > 1) { - const PrevContentTypesMapperModelLowdb = getContentTypesMapperDb(projectId, iteration - 1); - await PrevContentTypesMapperModelLowdb.read(); - const prevContentMapper = - PrevContentTypesMapperModelLowdb.chain.get('ContentTypesMappers').value() ?? []; + const seenPrevCts: ContentTypesMapper[] = []; + const seenPrevUids = new Set(); + for (let i = 1; i < iteration; i++) { + const priorModel = getContentTypesMapperDb(projectId, i); + await priorModel.read(); + const cts = priorModel.chain.get('ContentTypesMappers').value() ?? []; + for (const ct of cts) { + const uid = ct?.otherCmsUid; + if (uid && !seenPrevUids.has(uid)) { + seenPrevUids.add(uid); + seenPrevCts.push(ct); + } + } + } - const filtered = filterContentTypesByIteration(content_mapper, prevContentMapper, filter); + const filtered = filterContentTypesByIteration(content_mapper, seenPrevCts, filter); content_mapper.length = 0; content_mapper.push(...filtered); @@ -1970,7 +1984,7 @@ const getExistingExtensions = async ({existingStackId, token_payload}: any) => { const updateEntryStatus = async (req: Request) => { const { projectId } = req?.params; - const { ids } = req?.body; + const { ids, locale } = req?.body; const validatedUids: string[] = Array.isArray(ids) ? ids : []; const srcFunc = "updateEntryMapping"; if (isEmpty(validatedUids)) { @@ -1994,21 +2008,42 @@ const updateEntryStatus = async (req: Request) => { .find({ id: projectId }) .value(); const iteration = projectData?.iteration || 1; - const EntryMapperModel = getEntryMapperDb(projectId, iteration); - await EntryMapperModel.read(); - const foundEntry: EntryMapper[] = []; - // Rows in entry_mapper are already per-(entry × source-locale), so each id uniquely - // identifies one locale variant; toggling isUpdate directly is correct. - await EntryMapperModel.update((data: any) => { - data?.entry_mapper?.forEach((entry: any) => { - if (validatedUids.includes(entry?.id)) { + // Rows in entry_mapper are per-(entry × source-locale); each id is unique per row. + // Also scope the toggle by source-locale as a safety net so a same-id collision + // (if it ever happens) can't flip a sibling locale's row and clobber the user's + // selection state on the other locale. Only enforced when the row actually carries + // a language — legacy rows created before language-tagging existed have none, and + // requiring a match against them would make them permanently untoggleable. + const sourceLocale = locale + ? getSourceLocaleForDestination(projectData ?? {}, locale) + : null; + + const toggleInModel = async (iter: number): Promise => { + const model = getEntryMapperDb(projectId, iter); + await model.read(); + const matched: EntryMapper[] = []; + await model.update((data: any) => { + data?.entry_mapper?.forEach((entry: any) => { + if (!validatedUids.includes(entry?.id)) return; + if (sourceLocale && entry?.language && entry.language !== sourceLocale) return; entry.isUpdate = !entry.isUpdate; - foundEntry.push(entry); - } + matched.push(entry); + }); }); - }); + return matched; + }; + + let foundEntry = await toggleInModel(iteration); - if (foundEntry) { + // Fallback: mirrors getEntryMapping's read-side fallback (contentMapper.service.ts + // ~2119-2131) — right after a restart, before iteration N's entry-mapper rows exist, + // Map Entry renders rows sourced from iteration N-1. Without this, saving those rows + // 404s here even though the user is looking at exactly what the read path showed them. + if (!foundEntry.length && iteration > 1) { + foundEntry = await toggleInModel(iteration - 1); + } + + if (foundEntry.length) { return { status: HTTP_CODES?.OK, data: foundEntry @@ -2203,6 +2238,29 @@ const lookupContentstackEntryUidFromUidMap = ( return String(resolved).trim() || undefined; }; +/** + * Loads the nearest prior iteration's uid-mapper model, walking backward from + * `iteration - 1` down to 1 — not just `iteration - 1` alone. `writeUidMapping` already + * merges each successful run's uid-mapper.json forward from the one before it, so the + * nearest prior iteration that actually has a file already carries everything from every + * iteration before it; we only need to skip iterations where the file is simply absent + * (e.g. a restart that skipped an actual "Start Migration" run for that iteration — see + * CMG-1095, the same gap for content types). Without this, a single skipped iteration + * permanently breaks uid resolution for every iteration after it. + */ +const getNearestPriorUidMapper = async (projectId: string, iteration: number): Promise => { + for (let i = iteration - 1; i >= 1; i--) { + const model = getUidMapperDb(projectId, i); + await model.read(); + const data = model?.data as any; + const hasData = + Object.keys(data?.entry ?? {}).length > 0 || + Object.keys(data?.assets ?? {}).length > 0; + if (hasData) return model; + } + return null; +}; + const getEntryUidMap = (uidMapperModel: any): Record => { const d = uidMapperModel?.data ?? {}; const pick = (x: unknown): Record => { @@ -2221,17 +2279,6 @@ const getEntryUidMap = (uidMapperModel: any): Record => { return {}; }; -const flattenNestedUidMap = (raw: Record): Record => { - const keys = Object?.keys(raw ?? {}); - if (keys?.length === 0) return {}; - const nested = keys?.every((k) => { - const v = raw[k]; - return v != null && typeof v === 'object' && !Array.isArray(v); - }); - if (!nested) return { ...raw }; - return keys.reduce>((acc, k) => ({ ...acc, ...raw[k] }), {}); -}; - /** * Fill missing contentstackEntryUid from uid-mapper. Uses **current** iteration first * (where the latest CLI import writes), then iteration-1 so step 3 still works right @@ -2247,11 +2294,7 @@ const enrichEntriesWithUidMapper = async ( const currentModel = getUidMapperDb(projectId, iteration); await currentModel.read(); - let prevModel: any = null; - if (iteration > 1) { - prevModel = getUidMapperDb(projectId, iteration - 1); - await prevModel.read(); - } + const prevModel: any = iteration > 1 ? await getNearestPriorUidMapper(projectId, iteration) : null; return entries?.map((item: any) => { if (!item) return item; @@ -2338,12 +2381,83 @@ const updateAssetStatus = async (req: Request) => { } }; +/** + * Re-attempts the download for one asset that failed during the last migration run + * (currently CMS Contentful only — the `cs_failed.json` file this reads is written by + * contentfulService.createAssets). Only re-stages the asset locally; it lands in the + * destination stack on the next migration run (Start Migration), same as any other asset. + */ +const retryAssetDownload = async (req: Request) => { + const srcFunc = "retryAssetDownload"; + const projectId = req?.params?.projectId; + const assetUid = req?.params?.assetUid; + + if (!assetUid) { + return { + status: HTTP_CODES?.BAD_REQUEST, + data: { message: "Missing assetUid" }, + }; + } + + try { + await ProjectModelLowdb.read(); + const projectData: any = ProjectModelLowdb.chain + .get("projects") + .find({ id: projectId }) + .value(); + + const destinationStackId = projectData?.destination_stack_id; + const filePath = projectData?.legacy_cms?.file_path; + const cms = projectData?.legacy_cms?.cms; + + if (!destinationStackId || !filePath) { + return { + status: HTTP_CODES?.BAD_REQUEST, + data: { message: "Project is missing a destination stack or source file path." }, + }; + } + if (cms !== CMS.CONTENTFUL) { + return { + status: HTTP_CODES?.BAD_REQUEST, + data: { message: "Asset retry is only supported for Contentful projects." }, + }; + } + + const cleanLocalPath = filePath.replace(/\/$/, ''); + const result = await contentfulService.retryFailedAsset( + cleanLocalPath, + destinationStackId, + projectId, + assetUid, + ); + + return { + status: result.success ? HTTP_CODES?.OK : HTTP_CODES?.BAD_REQUEST, + data: result, + }; + } catch (error: any) { + logger.error( + getLogMessage( + srcFunc, + "Error occurred while retrying asset download", + error + ) + ); + throw new ExceptionFunction( + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR, + ); + } +}; + const getAssetMapping = async (req: Request) => { const srcFunc = "getAssetMapping"; const projectId = req?.params?.projectId; const skip: any = req?.params?.skip; const limit: any = req?.params?.limit; const search: string = req?.params?.searchText?.toLowerCase(); + // Optional status filter: ?status=failed | missing | ok. Absent/anything else = no filter. + const statusFilter = req?.query?.status as string | undefined; let result: any[] = []; let filteredResult = []; @@ -2376,15 +2490,27 @@ const getAssetMapping = async (req: Request) => { } // Fill missing contentstackAssetUid from uid-mapper (current first, then - // the previous iteration) so rows saved before the import resolve later. + // the nearest prior iteration that actually has one) so rows saved before + // the import resolve later. const uidMapperCurrent = getUidMapperDb(projectId, iteration); await uidMapperCurrent.read(); - let uidMapperPrev: any = null; - if (iteration > 1) { - uidMapperPrev = getUidMapperDb(projectId, iteration - 1); - await uidMapperPrev.read(); - } - const enrichedMapping = (assetMapping ?? []).map((item: any) => { + const uidMapperPrev: any = iteration > 1 ? await getNearestPriorUidMapper(projectId, iteration) : null; + + // Whether we actually have any uid data to resolve against yet. getUidMapperDb creates + // the file with an empty `assets: {}` default, so a fresh iteration directory (visited + // right after a restart, before this iteration's CLI import has run and written + // writeUidMapping's output) legitimately has none — distinct from "this project simply + // has no previously-migrated assets". Also true if any row already carries a + // pre-resolved uid from creation time (putTestData resolves it then, see ~line 280). + const hasAnyUidData = + Object.keys((uidMapperCurrent?.data as any)?.assets ?? {}).length > 0 || + Object.keys((uidMapperPrev?.data as any)?.assets ?? {}).length > 0 || + (assetMapping ?? []).some((item: any) => { + const uid = item?.contentstackAssetUid; + return uid != null && String(uid).trim() !== ''; + }); + + let uidEnriched = (assetMapping ?? []).map((item: any) => { if (!item) return item; const existing = item?.contentstackAssetUid; if (existing != null && String(existing).trim() !== '') { @@ -2396,23 +2522,94 @@ const getAssetMapping = async (req: Request) => { return resolved ? { ...item, contentstackAssetUid: resolved } : item; }); - if (!isEmpty(enrichedMapping)) { + // Status per row for the UI: 'missing' (no url/upload in the source at all — nothing + // to retry), 'failed' (had a source but the last migration run's download attempt + // threw — retriable), or 'ok'. Read once per request; cs_failed.json is only written + // after an actual migration run, so it's absent (empty status) before that. + let failedAssets: Record = {}; + const destinationStackId = projectData?.destination_stack_id; + // destination_stack_id is DB-stored, but Snyk's taint tracker still traces it back to + // the HTTP projectId param via the lowdb lookup above — sanitize it the same way the + // rest of this codebase does (sanitizeStackId strips it to an allowlisted charset, + // assertResolvedPathUnderBase re-confirms the joined path can't escape the data dir) + // before it reaches a readFileSync sink. + const safeDestinationStackId = sanitizeStackId(destinationStackId); + if (safeDestinationStackId) { + const assetsBase = path.resolve(MIGRATION_DATA_CONFIG.DATA); + const failedPath = path.join(assetsBase, safeDestinationStackId, MIGRATION_DATA_CONFIG.ASSETS_DIR_NAME, MIGRATION_DATA_CONFIG.ASSETS_FAILED_FILE); + try { + assertResolvedPathUnderBase(assetsBase, failedPath); + if (fs.existsSync(failedPath)) { + failedAssets = JSON.parse(fs.readFileSync(failedPath, 'utf-8')) || {}; + } + } catch { + failedAssets = {}; + } + } + const enrichedMapping = uidEnriched.map((item: any) => { + if (!item) return item; + if (item?.hasSource === false) { + return { ...item, status: 'missing', errorMessage: 'No source file found for this asset — nothing to migrate.' }; + } + const failure = failedAssets?.[item?.otherCmsAssetUid]; + if (failure) { + return { ...item, status: 'failed', errorMessage: failure?.reason_for_error || 'Failed to download this asset during the last migration run.' }; + } + return { ...item, status: 'ok' }; + }); + + // Delta migration intent: on iteration 2+ the Assets tab lists ONLY assets that + // were already migrated in a prior iteration — i.e. those with a Contentstack + // uid. The user selects which of those to update with the current file's newer + // version. Brand-new assets in this iteration have no prior uid; they upload + // automatically during the run and don't need a Map Entry row (nothing to + // select or update yet). Iteration 1 is untouched — everything is new then. + // + // Exception: always surface 'failed'/'missing' rows even without a uid. A brand-new + // asset that fails to download NEVER gets a Contentstack uid (it never successfully + // migrates), so the has-uid check alone would hide it from view forever — the user + // would have no way to discover or retry it. + // Only apply the delta filter once we actually have uid data to filter with — + // otherwise a race right after restart (this iteration's uid-mapper.json not written + // yet) would filter out EVERY row and render an empty tab indistinguishable from "no + // previously-migrated assets", which could be mistaken for correct behavior since + // CMG-1097 already gives that empty state a legitimate-looking layout. + const displayMapping = iteration > 1 && hasAnyUidData + ? enrichedMapping.filter((item: any) => { + const uid = item?.contentstackAssetUid; + const hasUid = uid != null && String(uid).trim() !== ''; + return hasUid || item?.status === 'failed' || item?.status === 'missing'; + }) + : enrichedMapping; + + // Aggregate counts across the FULL (unpaginated, unsearched) visible set — the banner + // needs "3 assets won't migrate" regardless of which page or search term is active. + const missingCount = displayMapping.filter((item: any) => item?.status === 'missing').length; + const failedCount = displayMapping.filter((item: any) => item?.status === 'failed').length; + + const statusFiltered = statusFilter && ['ok', 'missing', 'failed'].includes(statusFilter) + ? displayMapping.filter((item: any) => item?.status === statusFilter) + : displayMapping; + + if (!isEmpty(statusFiltered)) { if (search) { - filteredResult = enrichedMapping?.filter?.((item: any) => + filteredResult = statusFiltered?.filter?.((item: any) => item?.filename?.toLowerCase().includes(search) || item?.title?.toLowerCase().includes(search) ); totalCount = filteredResult?.length; result = filteredResult?.slice(skip, Number(skip) + Number(limit)); } else { - totalCount = enrichedMapping?.length; - result = enrichedMapping?.slice(skip, Number(skip) + Number(limit)); + totalCount = statusFiltered?.length; + result = statusFiltered?.slice(skip, Number(skip) + Number(limit)); } } return { status: HTTP_CODES?.OK, count: totalCount, - assetMapping: result + assetMapping: result, + missingCount, + failedCount, }; } catch (error: any) { @@ -2451,4 +2648,5 @@ export const contentMapperService = { updateEntryStatus, getAssetMapping, updateAssetStatus, + retryAssetDownload, }; diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 070d89f67..159f3af62 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -436,7 +436,7 @@ const processField = ( return refs; } const id = lang_value?.sys?.id; - if(Array?.isArray(entryId?.id)){ + if(Array.isArray(entryId?.[id])){ return entryId?.[id]; } else{ @@ -662,13 +662,39 @@ const saveAsset = async ( } }); - const fileUrl = `https:${(Object.values(assets?.fields?.file)[0] as { url: string }).url - }`; + // Contentful emits `fields.file[locale].url` (CDN-processed path, starts with "//") for + // assets whose CDN entry is ready, and `fields.file[locale].upload` (absolute fetch URL) + // for assets that were just added to the space but not yet processed. Real deltas hit + // the second shape on "newly added asset" exports; reading only `.url` silently drops + // those (see CMG-1106). Prefer `.url`, fall back to `.upload`, skip if neither exists. + const fileMeta = Object.values(assets?.fields?.file)[0] as { url?: string; upload?: string; contentType?: string; details?: { size?: string }; fileName?: string }; + let fileUrl = ''; + if (typeof fileMeta?.url === 'string' && fileMeta.url) { + fileUrl = fileMeta.url.startsWith('//') ? `https:${fileMeta.url}` : fileMeta.url; + } else if (typeof fileMeta?.upload === 'string' && fileMeta.upload) { + fileUrl = fileMeta.upload; + } else { + // No downloadable source — record and continue so the run doesn't hit axios on `https:undefined`. + failedJSON[assets.sys.id] = { + failedUid: assets.sys.id, + name: Object.values(assets?.fields?.title ?? {})[0], + url: '', + file_size: `${fileMeta?.details?.size ?? ''}`, + reason_for_error: 'Asset has no file.url or file.upload — nothing to download', + }; + return assets.sys.id; + } const assetTitle = Object.values(assets?.fields?.title)[0]; - const fileName = path.basename( - (Object.values(assets?.fields?.file)[0] as { fileName: string }) - .fileName - ); + // Assets that only have `.upload` (not yet CDN-processed) often have no `fileName` + // or `details` yet — Contentful only populates those after processing. Fall back to + // the asset's sys.id so path.basename never throws, and derive size/content-type + // defensively so a still-processing asset doesn't crash mid-download. + const rawFileName = typeof fileMeta?.fileName === 'string' && fileMeta.fileName + ? fileMeta.fileName + : `${assets.sys.id}`; + const fileName = path.basename(rawFileName); + const fileSize = `${fileMeta?.details?.size ?? ''}`; + const fileContentType = fileMeta?.contentType ?? ''; const description = Object.values( assets?.fields as { [key: string]: unknown } ) @@ -695,15 +721,8 @@ const saveAsset = async ( uid: assets.sys.id, urlPath: `/assets/${assets.sys.id}`, status: true, - content_type: ( - Object.values(assets?.fields?.file)[0] as { contentType: string } - ).contentType, - file_size: `${( - Object.values(assets?.fields?.file)[0] as { - details: { size: string }; - } - )?.details.size - }`, + content_type: fileContentType, + file_size: fileSize, tag: assets?.metadata?.tags, filename: fileName, url: fileUrl, @@ -732,12 +751,7 @@ const saveAsset = async ( failedUid: assets.sys.id, name: assetTitle, url: fileUrl, - file_size: `${( - Object.values(assets?.fields?.file)[0] as { - details: { size: string }; - } - ).details.size - }`, + file_size: fileSize, reason_for_error: err?.message, }; } else { @@ -795,7 +809,6 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj await Promise.all(tasks); await fs.promises.mkdir(assetsSave, { recursive: true }); - const assetMasterFolderPath = path.join(assetsSave, ASSETS_FAILED_FILE); await writeOneFile(path.join(assetsSave, ASSETS_SCHEMA_FILE), assetData); // This code is intentionally commented out @@ -813,7 +826,11 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), fileMeta); // await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata); - failedJSON && await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON); + // Was double-joining ASSETS_FAILED_FILE (writeFile already appends the filename to its + // dirPath arg), which wrote to `/cs_failed.json/cs_failed.json` — a directory + // named cs_failed.json containing a file of the same name — instead of the intended + // `/cs_failed.json`. Pass the directory alone. + failedJSON && await writeFile(assetsSave, ASSETS_FAILED_FILE, failedJSON); } else { const message = getLogMessage( srcFunc, @@ -834,6 +851,84 @@ const createAssets = async (packagePath: any, destination_stack_id: string, proj } }; +/** + * Re-attempts the download for a single asset that failed during the last migration run + * (recorded in `cs_failed.json` by `saveAsset` above). Reads the current on-disk asset + * index + failed-assets file, re-runs the same `saveAsset` download for just this one + * source asset id, and persists the result back to both files. + * + * This only re-stages the asset locally (downloads the binary, updates index.json) — it + * does not push to the destination Contentstack stack directly. Like every other asset in + * this connector, it lands in the stack the next time the CLI import runs (Start Migration + * on this or a later iteration), since assets are only pushed via that CLI import step. + * + * @returns `success: true` once the asset re-downloads (message notes it needs a migration + * run to land in the stack); `success: false` with the failure reason if it fails again. + */ +const retryFailedAsset = async ( + packagePath: string, + destination_stack_id: string, + projectId: string, + assetSourceId: string, +): Promise<{ success: boolean; message: string }> => { + const srcFunc = 'retryFailedAsset'; + try { + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + const failedPath = path.join(assetsSave, ASSETS_FAILED_FILE); + const indexPath = path.join(assetsSave, ASSETS_SCHEMA_FILE); + + const packageData = await fs.promises.readFile(packagePath, 'utf8'); + const sourceAssets = JSON.parse(packageData)?.assets ?? []; + const targetAsset = sourceAssets.find((a: any) => a?.sys?.id === assetSourceId); + if (!targetAsset) { + return { success: false, message: 'Asset not found in the source export.' }; + } + + let failedJSON: Record = {}; + if (fs.existsSync(failedPath)) { + try { + failedJSON = JSON.parse(await fs.promises.readFile(failedPath, 'utf8')) || {}; + } catch { + failedJSON = {}; + } + } + let assetData: Record = {}; + if (fs.existsSync(indexPath)) { + try { + assetData = JSON.parse(await fs.promises.readFile(indexPath, 'utf8')) || {}; + } catch { + assetData = {}; + } + } + + await saveAsset(targetAsset, failedJSON, assetData, [], projectId, destination_stack_id, 0); + + await fs.promises.mkdir(assetsSave, { recursive: true }); + await writeOneFile(indexPath, assetData); + await writeFile(assetsSave, ASSETS_FAILED_FILE, failedJSON); + + if (failedJSON[assetSourceId]) { + return { + success: false, + message: failedJSON[assetSourceId]?.reason_for_error || 'Retry failed.', + }; + } + return { + success: true, + message: 'Asset downloaded successfully. It will be included in the next migration run.', + }; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error retrying asset "${assetSourceId}".`, + {}, + error, + ); + await customLogger(projectId, destination_stack_id, 'error', message); + return { success: false, message: error?.message || 'Retry failed.' }; + } +}; + /** * Creates environment configurations from a given package file and saves them to the destination stack directory. * @@ -1658,4 +1753,5 @@ export const contentfulService = { createWebhooks, createVersionFile, createTaxonomy: createContentfulTaxonomyFromExport, + retryFailedAsset, }; diff --git a/api/src/services/contentful/jsonRTE.ts b/api/src/services/contentful/jsonRTE.ts index ec998be50..254089d1b 100755 --- a/api/src/services/contentful/jsonRTE.ts +++ b/api/src/services/contentful/jsonRTE.ts @@ -315,8 +315,14 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack } -function parseBlockquote(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +// lang/destination_stack_id are forwarded (not just obj) so a hyperlink nested inside a +// blockquote or heading — entry-hyperlink, asset-hyperlink, or plain hyperlink — can still +// resolve. Every sibling container (parseDocument, parseParagraph, parseLI, table parsers) +// already does this; these seven were missed, silently degrading CMG-1103's fix for that +// specific nesting (entry-hyperlink falls back to plain text, asset-hyperlink drops the +// node entirely since its null return gets filtered out by the caller). +function parseBlockquote(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'blockquote', attrs: {}, @@ -325,8 +331,8 @@ function parseBlockquote(obj: any): any { }; } -function parseHeading1(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading1(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 1 }, @@ -335,8 +341,8 @@ function parseHeading1(obj: any): any { }; } -function parseHeading2(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading2(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 2 }, @@ -345,8 +351,8 @@ function parseHeading2(obj: any): any { }; } -function parseHeading3(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading3(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 3 }, @@ -355,8 +361,8 @@ function parseHeading3(obj: any): any { }; } -function parseHeading4(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading4(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 4 }, @@ -365,8 +371,8 @@ function parseHeading4(obj: any): any { }; } -function parseHeading5(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading5(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 5 }, @@ -375,8 +381,8 @@ function parseHeading5(obj: any): any { }; } -function parseHeading6(obj: any): any { - const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean); +function parseHeading6(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean); return { type: 'heading', attrs: { level: 6 }, @@ -385,34 +391,76 @@ function parseHeading6(obj: any): any { }; } -function parseEntryHyperlink(obj: any, lang?: LangType): any { +// Contentstack JSON RTE uses: +// - plain hyperlink → type: 'a', attrs.url +// - entry hyperlink (link ref) → type: 'reference', display-type: 'link', type: 'entry' +// - asset hyperlink (link ref) → type: 'reference', display-type: 'link', type: 'asset' +// The previous implementation used non-standard types ('hyperlink', 'entry-hyperlink', +// 'asset-hyperlink') that Contentstack's JSON RTE reader silently dropped, so URLs +// vanished in the destination stack even though the surrounding text migrated. + +function parseEntryHyperlink(obj: any, lang?: LangType, destination_stack_id?: StackId): any { + const targetId = obj?.data?.target?.sys?.id ?? ''; + // Prefer the anchor text from `obj.content` (Contentful nests the link label + // as a text child); fall back to a stale `target.title` if present. + const text = obj?.content?.[0]?.value ?? obj?.data?.target?.title ?? ''; + + // A Contentful export's entry-hyperlink target is an unresolved Link — it never + // carries `sys.contentType`. The destination content-type uid has to come from + // the rte-references file (the same source parseBlockReference/parseInlineReference + // use), keyed by locale then by target entry id. + const rteRefs: { [key: string]: any } | undefined = + destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME)); + const entry = rteRefs && Object.entries(rteRefs).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue?.[targetId]); + const contentTypeUid = entry?.[1]?.[targetId]?._content_type_uid; + + if (!targetId || !contentTypeUid) { + // Can't resolve a destination content type for this entry — emit plain text + // so the anchor label still survives, instead of a reference node that can + // never resolve on the destination stack. + return { text }; + } + return { - type: 'entry-hyperlink', - attrs: { href: `/${lang}/${obj.data.uri}` }, - uid: generateUID('entry-hyperlink'), - children: [{ text: obj.data.target.title }], + type: 'reference', + attrs: { + type: 'entry', + 'entry-uid': targetId, + 'content-type-uid': contentTypeUid, + 'display-type': 'link', + locale: lang, + style: {}, + }, + uid: generateUID('reference'), + children: [{ text }], }; } function parseAssetHyperlink(obj: any, lang?: LangType, destination_stack_id?: StackId): any { const assetId = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, ASSETS_DIR_NAME, ASSETS_SCHEMA_FILE)); - const asset = assetId[obj.data.target.sys.id]; - if (asset) { - return { - type: 'asset-hyperlink', - attrs: { href: asset.url }, - uid: generateUID('asset-hyperlink'), - children: [{ text: asset.title }], - }; - } - return null; + const asset = assetId?.[obj?.data?.target?.sys?.id]; + if (!asset) return null; + return { + type: 'reference', + attrs: { + type: 'asset', + 'asset-uid': asset.uid, + 'asset-link': asset.url, + 'asset-name': asset.filename ?? asset.title, + 'asset-type': asset.content_type ?? asset.contentType ?? '', + 'content-type-uid': 'sys_assets', + 'display-type': 'link', + }, + uid: generateUID('reference'), + children: [{ text: obj?.content?.[0]?.value ?? asset.title ?? '' }], + }; } function parseHyperlink(obj: any): any { return { - type: 'hyperlink', - attrs: { href: obj.data.uri }, - uid: generateUID('hyperlink'), - children: [{ text: obj.content[0].value }], + type: 'a', + attrs: { url: obj?.data?.uri ?? '' }, + uid: generateUID('a'), + children: [{ text: obj?.content?.[0]?.value ?? '' }], }; } diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index c7821a8b5..1bc57d78a 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -4,6 +4,7 @@ import { Request } from 'express'; import path from 'path'; import ProjectModelLowdb from '../models/project-lowdb.js'; +import getUidMapperDb from '../models/uidMapper.js'; import { config } from '../config/index.js'; import { safePromise, getLogMessage } from '../utils/index.js'; import https from '../utils/https.utils.js'; @@ -18,6 +19,7 @@ import { CMS, GET_AUDIT_DATA, MIGRATION_DATA_CONFIG, + DATABASE_FILES, } from '../constants/index.js'; import { BadRequestError, @@ -50,8 +52,9 @@ import { import { aemService } from './aem.service.js'; import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js'; import { utilsUpdateCli } from './updateEntryCli.service.js'; -import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; +import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithEntryMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js'; import { removeExistingAssets, saveAssetMetadata, AssetUpdate } from '../utils/asset-update.utils.js'; +import { extractLocalesFromUpdateConfig, recordMigratedLocales } from '../utils/locale-migration.utils.js'; /** * Creates a test stack. @@ -1262,6 +1265,12 @@ const startMigration = async (req: Request): Promise => { iteration, safeDeltaMigrationLogPath ); + enrichConfigWithEntryMapping( + configFilePath, + safePid, + iteration, + safeDeltaMigrationLogPath + ); enrichConfigWithAssetUpdates( configFilePath, assetUpdates, @@ -1274,10 +1283,118 @@ const startMigration = async (req: Request): Promise => { safeDeltaMigrationLogPath || '', configFilePath ); + + // Record every locale that ACTUALLY ran this iteration, AFTER the update/localize CLI + // resolves — moved out of runCli.service.ts because the previous position recorded + // locales before this step wrote them, so a silent failure here (updateEntryCli + // swallows errors — see updateEntryCli.service.ts:240-249) would permanently skip the + // affected locales on every future restart. + await recordDeltaMigratedLocales( + projectId, + safePid, + iteration, + project, + destinationStackId, + configFilePath, + ); } else{ await customLogger(projectId, destinationStackId, 'warn', 'No config file generated for delta migration; skipping update CLI step.'); + // No update CLI ran (nothing to localize/update this iteration), but runCli's bulk + // import above may still have created brand-new locales/entries. Record those too — + // otherwise this locale never appears in migrated_locales, isFullMigrationForLocale + // keeps returning true for it, and every later restart re-routes its entries through + // the localize path forever (same failure class this PR fixes via other triggers). + await recordDeltaMigratedLocales( + projectId, + safePid, + iteration, + project, + destinationStackId, + null, + ); + } + + // Guaranteed terminal signal for the delta path, written unconditionally regardless of + // which branch above ran or whether updateEntryCli succeeded. MigrationLogViewer.tsx + // requires exactly 'Entry Update Process Completed' on iteration > 1 to leave the + // execution-logs spinner — but that string is only ever written by updateEntryCli's own + // success path (updateEntryCli.service.ts:235). Two real delta scenarios never reach it: + // no config file at all (nothing selected to update, no asset updates — the `else` + // branch above), and updateEntryCli throwing internally (it catches its own error and + // only logs 'Failed to update entries...', never rethrows). Without this, the user gets + // stuck on Execution Logs forever after an otherwise-successful migration. Writing this + // here, after both branches, means the client's check is satisfied every time regardless + // of which path executed. + if (safeDeltaMigrationLogPath) { + try { + const terminalLogEntry = { + level: 'info', + message: 'Entry Update Process Completed', + methodName: 'startMigration', + timestamp: new Date().toISOString(), + }; + fs.appendFileSync(safeDeltaMigrationLogPath, JSON.stringify(terminalLogEntry) + '\n'); + } catch (err) { + console.error('Failed to write delta completion marker:', err); + } + } + } +}; + +/** + * Records every locale that actually ran in this delta iteration — union of master + * locale, locales present in the update config (entries the update CLI just localized), + * and locales present in this iteration's uid-mapper `entryByLocale` (brand-new entries + * created by runCli's bulk import, which never appear in the update config since they + * have no prior csEntryUid to localize). + */ +const recordDeltaMigratedLocales = async ( + projectId: string, + safePid: string, + iteration: number, + project: any, + destinationStackId: string, + configFilePath: string | null, +): Promise => { + try { + const dbBase = path.resolve(process.cwd(), DATABASE_FILES.DIRECTORY); + let updateConfig: Record | null = null; + if (configFilePath) { + try { + // configFilePath came from removeEntriesFromDatabase / ensureUpdateConfigFile + // (path.join'd against safePid + iteration) — re-assert it resolves under the + // database dir before reading, so Snyk sees an explicit sink check. + assertResolvedPathUnderBase(dbBase, configFilePath); + updateConfig = JSON.parse(fs.readFileSync(configFilePath, 'utf-8')); + } catch (err) { + updateConfig = null; + await customLogger(projectId, destinationStackId, 'warn', `Failed to read update config for locale recording: ${(err as Error)?.message}`); + } } + let entryByLocaleKeys: string[] = []; + try { + // Read via the lowdb model rather than raw fs — the same read path + // used by writeUidMapping / writePerLocaleEntryUidMapping. Keeps the + // taint out of a direct readFileSync sink so Snyk's SAST stays clean. + const UidMapperModelLowdb = getUidMapperDb(safePid, iteration); + await UidMapperModelLowdb.read(); + entryByLocaleKeys = Object.keys( + (UidMapperModelLowdb.data as any)?.entryByLocale ?? {} + ); + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to read uid-mapper for locale recording: ${(err as Error)?.message}`); + } + const ranLocales = Array.from( + new Set([ + ...Object.keys(project?.master_locale ?? {}), + ...extractLocalesFromUpdateConfig(updateConfig), + ...entryByLocaleKeys, + ]), + ); + await recordMigratedLocales(projectId, ranLocales); + } catch (err) { + await customLogger(projectId, destinationStackId, 'warn', `Failed to record migrated locales: ${(err as Error)?.message}`); } }; const getAuditData = async (req: Request): Promise => { diff --git a/api/src/services/runCli.service.ts b/api/src/services/runCli.service.ts index 8f90b01a2..3984e0ab6 100644 --- a/api/src/services/runCli.service.ts +++ b/api/src/services/runCli.service.ts @@ -20,6 +20,7 @@ interface TestStack { } import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js'; import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js'; +import { recordMigratedLocales } from '../utils/locale-migration.utils.js'; /** * Determines log level based on message content without removing ANSI codes @@ -322,20 +323,27 @@ export const runCli = async ( ProjectModelLowdb.data.projects[projectIndex].current_step = getStepperSteps(ProjectModelLowdb.data.projects[projectIndex]?.iteration).MIGRATION; ProjectModelLowdb.data.projects[projectIndex].status = 5; - // Record every locale that just successfully migrated so the next delta restart can - // tell which locales need a full pass vs delta. Set-union with prior value. - const proj: any = ProjectModelLowdb.data.projects[projectIndex]; - const ranLocales = Array.from( - new Set([ - ...Object.keys(proj?.master_locale ?? {}), - ...Object.keys(proj?.locales ?? {}), - ]), - ); - const existing: string[] = Array.isArray(proj?.migrated_locales) - ? proj.migrated_locales - : []; - proj.migrated_locales = Array.from(new Set([...existing, ...ranLocales])); await ProjectModelLowdb.write(); + + // On iteration 1 the full configured locale set genuinely gets migrated in a single + // bulk import — this CLI IS the terminal step, so recording here is safe. + // For iteration 2+, recording is deliberately deferred to migration.service.ts, AFTER + // the update/localize CLI (`utilsUpdateCli.updateEntryCli`) actually completes. If we + // recorded here, a locale queued in updated-entries.json would be marked migrated + // even when the subsequent update CLI never wrote it (it swallows failures — see + // updateEntryCli.service.ts:240-249), and would then be silently skipped on the next + // restart — the very bug this PR fixes, just via a different trigger. + const proj: any = ProjectModelLowdb.data.projects[projectIndex]; + const currentIteration = proj?.iteration || 1; + if (currentIteration <= 1) { + const ranLocales = Array.from( + new Set([ + ...Object.keys(proj?.master_locale ?? {}), + ...Object.keys(proj?.locales ?? {}), + ]), + ); + await recordMigratedLocales(projectId, ranLocales); + } } } else { console.info('User not found.'); diff --git a/api/src/utils/entry-update-script.cjs b/api/src/utils/entry-update-script.cjs index a14d0bdb3..5caac832b 100644 --- a/api/src/utils/entry-update-script.cjs +++ b/api/src/utils/entry-update-script.cjs @@ -4,6 +4,104 @@ const isAssetField = (value) => value && typeof value === 'object' && !Array.isArray(value) && 'urlPath' in value && 'filename' in value; +/** Shape produced by processField's 'reference' case: { uid, _content_type_uid }. */ +const isReferenceValue = (value) => + value && typeof value === 'object' && !Array.isArray(value) && + 'uid' in value && '_content_type_uid' in value; + +// Loosened from `.every(...)` to `.some(...)`: producers can legitimately emit mixed arrays +// — `processArrayFields` (contentful.service.ts) pushes the raw Contentful Link object when +// the target isn't in the references map, and the single-reference path can inject +// `[undefined]` — so one non-reference element would otherwise disable resolution for the +// whole field. Per-item remap happens in resolveReferenceField. +const isReferenceArray = (value) => + Array.isArray(value) && value.length > 0 && value.some(isReferenceValue); + +/** + * Resolves a source-side entry uid to its real Contentstack destination uid. + * + * The export JSON's reference fields carry the SOURCE cms entry id (see + * `contentful.service.ts`'s `createRefrence`), which only happens to equal the + * Contentstack uid when entries are imported preserving source ids. The + * bulk/master-locale import resolves this correctly via the CLI's own + * reference pass; this update path does not, so it needs the same uid-mapper + * data the asset resolution above already uses (see `entryMapping`). + * + * Preference order: per-locale mapping (most precise — handles entries that + * ended up as distinct Contentstack uids per locale across iterations) → + * flat mapping → identity fallback (keeps existing behavior when no mapping + * data exists, e.g. simple setups where source id equals destination uid). + */ +const resolveReferenceUid = (sourceUid, locale, entryMapping) => { + if (!sourceUid) return sourceUid; + const newByLocale = entryMapping?.new?.byLocale?.[locale]?.[sourceUid]; + if (newByLocale) return newByLocale; + const oldByLocale = entryMapping?.old?.byLocale?.[locale]?.[sourceUid]; + if (oldByLocale) return oldByLocale; + const newFlat = entryMapping?.new?.flat?.[sourceUid]; + if (newFlat) return newFlat; + const oldFlat = entryMapping?.old?.flat?.[sourceUid]; + if (oldFlat) return oldFlat; + return sourceUid; +}; + +/** + * Remaps the uid(s) inside a reference field value (single link object or + * array of link objects) to their Contentstack destination uids. + */ +const resolveReferenceField = (fieldName, entryUid, value, locale, entryMapping) => { + if (isReferenceValue(value)) { + const resolved = resolveReferenceUid(value.uid, locale, entryMapping); + if (resolved !== value.uid) { + console.info(`[${entryUid}] "${fieldName}"${locale ? ` (${locale})` : ''}: resolved reference uid "${value.uid}" → "${resolved}"`); + } + return { ...value, uid: resolved }; + } + if (isReferenceArray(value)) { + // Pass non-reference items through untouched so a stray non-link element (e.g. a raw + // Contentful link that wasn't in the references map, or `undefined` from an earlier + // failed resolve) doesn't crash and doesn't corrupt neighboring references. + return value.map((item) => { + if (!isReferenceValue(item)) return item; + const resolved = resolveReferenceUid(item.uid, locale, entryMapping); + return { ...item, uid: resolved }; + }); + } + return value; +}; + +/** + * Recursively walks a field value and resolves any reference shape found at any + * depth — group and modular-block fields nest references one or more levels deep + * (see processField's 'group' branch and processArrayFields in contentful.service.ts), + * so a shallow top-level-only check misses them and they keep their source-CMS uid on + * the delta/localize path. Asset field objects are left untouched (they need + * resolveAssetField's 3-way stack comparison, not a uid remap) so this only ever + * rewrites reference shapes, nothing else. + */ +const resolveReferencesDeep = (fieldName, entryUid, value, locale, entryMapping) => { + if (isReferenceValue(value)) { + return resolveReferenceField(fieldName, entryUid, value, locale, entryMapping); + } + // Recurse per-element rather than delegating the whole array to isReferenceArray + + // resolveReferenceField's shallow array handling. That shallow path only remaps + // elements matching isReferenceValue and passes everything else through byte-for-byte + // — so a MIXED array (a bare reference next to an object with a reference nested + // inside, e.g. a modular-block array) would leave the nested one unresolved. Recursing + // into every element here — reference, container, or scalar — covers that case too. + if (Array.isArray(value)) { + return value.map((item) => resolveReferencesDeep(fieldName, entryUid, item, locale, entryMapping)); + } + if (value && typeof value === 'object' && !isAssetField(value)) { + const out = {}; + for (const [key, val] of Object.entries(value)) { + out[key] = resolveReferencesDeep(`${fieldName}.${key}`, entryUid, val, locale, entryMapping); + } + return out; + } + return value; +}; + /** Export JSON metadata — not Contentstack content-type field UIDs (WordPress entries are flat). */ const FLAT_PAYLOAD_SKIP = new Set([ 'uid', @@ -68,7 +166,7 @@ const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapp * WordPress (and similar) write migration JSON with fields at the root (email, url, …). * Fetched stack entries keep custom fields under entry.content — merge flat updateData there. */ -const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts) => { +const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping) => { for (const field of Object.keys(updateData)) { if (FLAT_PAYLOAD_SKIP.has(field)) { continue; @@ -89,6 +187,8 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping oldMapping, newMapping ); + } else { + nextVal = resolveReferencesDeep(field, entryUid, nextVal, locale, entryMapping); } entry.content[field] = nextVal; } @@ -103,6 +203,9 @@ module.exports = async ({ const assetMapping = config.__assetMapping__ || { old: {}, new: {} }; delete config.__assetMapping__; + const entryMapping = config.__entryMapping__ || { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: {} } }; + delete config.__entryMapping__; + // Assets the user chose to update in place (same UID, new file). const assetUpdates = Array.isArray(config.__assetUpdates__) ? config.__assetUpdates__ : []; delete config.__assetUpdates__; @@ -110,6 +213,7 @@ module.exports = async ({ const oldMapping = assetMapping.old || {}; const newMapping = assetMapping.new || {}; console.info(`Asset mappings loaded — old: ${Object.keys(oldMapping).length}, new: ${Object.keys(newMapping).length}`); + console.info(`Entry mappings loaded — old: ${Object.keys(entryMapping?.old?.flat || {}).length} flat / ${Object.keys(entryMapping?.old?.byLocale || {}).length} locales, new: ${Object.keys(entryMapping?.new?.flat || {}).length} flat / ${Object.keys(entryMapping?.new?.byLocale || {}).length} locales`); console.info(`Asset updates to replace in place: ${assetUpdates.length}`); const contentTypes = Object.keys(config); @@ -187,13 +291,21 @@ module.exports = async ({ oldMapping, newMapping ); + } else { + updateData.content[field] = resolveReferencesDeep( + field, + entryUid, + updateData?.content[field], + locale, + entryMapping + ); } } Object.assign(entry?.content, updateData?.content); await entry.update(updateOpts); } else if (hasStackContent) { console.info(`[${realEntryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)${locale ? ` for locale "${locale}"` : ''}`); - await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts); + await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping); } else { if (updateData && entry) { for (const field of Object.keys(updateData)) { @@ -206,6 +318,14 @@ module.exports = async ({ oldMapping, newMapping ); + } else { + updateData[field] = resolveReferencesDeep( + field, + entryUid, + updateData[field], + locale, + entryMapping + ); } } } @@ -237,3 +357,8 @@ module.exports = async ({ module.exports.isAssetField = isAssetField; module.exports.resolveAssetField = resolveAssetField; module.exports.mergeFlatPayloadIntoEntry = mergeFlatPayloadIntoEntry; +module.exports.isReferenceValue = isReferenceValue; +module.exports.isReferenceArray = isReferenceArray; +module.exports.resolveReferenceUid = resolveReferenceUid; +module.exports.resolveReferenceField = resolveReferenceField; +module.exports.resolveReferencesDeep = resolveReferencesDeep; diff --git a/api/src/utils/entry-update.utils.ts b/api/src/utils/entry-update.utils.ts index 79cf3a641..cd17af676 100644 --- a/api/src/utils/entry-update.utils.ts +++ b/api/src/utils/entry-update.utils.ts @@ -9,6 +9,7 @@ import { getSourceLocaleForDestination, } from "./locale-migration.utils.js"; import type { AssetUpdate } from "./asset-update.utils.js"; +import { flattenNestedUidMap } from "./uid-mapper.utils.js"; /** * Helper function to write log entries to file @@ -297,6 +298,74 @@ export const enrichConfigWithAssetMapping = ( writeLogEntry(`Asset references will be resolved using combined old and new mappings`, "enrichConfigWithAssetMapping", loggerPath); }; +/** + * Reads old (previous iteration) and new (current iteration) entry uid mappings + * — both the flat source→dest map and the per-locale map written by + * `writePerLocaleEntryUidMapping` — and merges them into the updated-entries + * config file under `__entryMapping__`. + * + * This lets the entry-update-script resolve Link(Entry)/reference field values + * to their real Contentstack destination uid before writing them onto a + * localized (non-master) copy of an entry. Without this, reference fields + * written during a locale-add restart keep the export's source-side uid, + * which only happens to work when source and destination uids are identical — + * the master-locale bulk import resolves this correctly via the Contentstack + * CLI's own reference pass, but this update path does not, unless we prime it + * with the same uid-mapper data (mirrors `enrichConfigWithAssetMapping`). + */ +export const enrichConfigWithEntryMapping = ( + configFilePath: string, + projectId: string, + iteration: number, + loggerPath?: string +): void => { + const dbBase = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, projectId); + + const readEntryMapper = (iter: number): { flat: Record; byLocale: Record> } => { + const p = path.join(dbBase, iter.toString(), DATABASE_FILES.UID_MAPPER); + if (!fs.existsSync(p)) return { flat: {}, byLocale: {} }; + try { + const data = JSON.parse(fs.readFileSync(p, "utf-8")); + // Uid-mapper's `entry` key can be either the flat `{ sourceUid: destUid }` shape + // or the nested `{ [ctUid]: { sourceUid: destUid } }` shape (see + // uid-mapper.utils.ts:mergeUidMaps). contentMapper.service also merges in an + // `entryUid` variant — do the same here so both readers stay in sync and the + // resolver never silently falls through to the identity fallback. + const fromEntry = flattenNestedUidMap(data?.entry); + const fromEntryUid = flattenNestedUidMap(data?.entryUid); + return { + flat: { ...fromEntry, ...fromEntryUid }, + byLocale: data?.entryByLocale || {}, + }; + } catch (err) { + console.error(`Failed to read uid-mapper for iteration ${iter}:`, err); + return { flat: {}, byLocale: {} }; + } + }; + + const oldEntryMapping = iteration > 1 ? readEntryMapper(iteration - 1) : { flat: {}, byLocale: {} }; + const newEntryMapping = readEntryMapper(iteration); + + writeLogEntry( + `Loaded entry uid mappings — old: ${Object.keys(oldEntryMapping.flat).length} flat / ${Object.keys(oldEntryMapping.byLocale).length} locales, ` + + `new: ${Object.keys(newEntryMapping.flat).length} flat / ${Object.keys(newEntryMapping.byLocale).length} locales`, + "enrichConfigWithEntryMapping", + loggerPath, + ); + + try { + const config = JSON.parse(fs.readFileSync(configFilePath, "utf-8")); + config.__entryMapping__ = { old: oldEntryMapping, new: newEntryMapping }; + fs.writeFileSync(configFilePath, JSON.stringify(config), "utf-8"); + } catch (err) { + console.error("Failed to write entry mapping into update config:", err); + writeLogEntry(`Failed to write __entryMapping__ into ${configFilePath}: ${(err as Error)?.message}`, "enrichConfigWithEntryMapping", loggerPath); + return; + } + + writeLogEntry(`Entry mapping enriched into config for iteration ${iteration}`, "enrichConfigWithEntryMapping", loggerPath); +}; + /** * Ensures an update config file exists for this iteration and returns its path. * Used when there are asset updates but no entry updates produced a config, so diff --git a/api/src/utils/locale-migration.utils.ts b/api/src/utils/locale-migration.utils.ts index af5941c56..5095d94af 100644 --- a/api/src/utils/locale-migration.utils.ts +++ b/api/src/utils/locale-migration.utils.ts @@ -64,6 +64,38 @@ export const isFullMigrationForLocale = ( return !getMigratedLocales(project).includes(localeCode); }; +/** + * Extracts destination locale codes that were ACTUALLY targeted by a delta + * run, from an `updated-entries.json` config object. + * + * Per-entry keys in that config are `${csUid}::${localeCode}` (see + * `removeEntriesFromDatabase` in entry-update.utils.ts) — this reads the + * locale suffix back out. Bookkeeping keys added by the enrich* helpers + * (`__assetMapping__`, `__entryMapping__`, `__assetUpdates__`) are skipped. + * + * This exists to fix a bug where a locale got marked "migrated" as soon as + * ANY locale finished a delta run, instead of only the locale(s) that run + * actually processed — which permanently skipped locales configured ahead of + * when they were meant to be migrated (see `runCli.service.ts`). + */ +export const extractLocalesFromUpdateConfig = ( + config: Record | null | undefined, +): string[] => { + if (!config || typeof config !== 'object') return []; + const locales = new Set(); + for (const [ctKey, entries] of Object.entries(config)) { + if (ctKey.startsWith('__')) continue; + if (!entries || typeof entries !== 'object') continue; + for (const entryKey of Object.keys(entries)) { + const sep = entryKey.lastIndexOf('::'); + if (sep === -1) continue; + const locale = entryKey.slice(sep + 2); + if (locale) locales.add(locale); + } + } + return Array.from(locales); +}; + /** * Set-union the given locales into project.migrated_locales and persist. * Idempotent. diff --git a/api/src/utils/uid-mapper.utils.ts b/api/src/utils/uid-mapper.utils.ts index e9c2dd86a..affeaaf10 100644 --- a/api/src/utils/uid-mapper.utils.ts +++ b/api/src/utils/uid-mapper.utils.ts @@ -5,6 +5,31 @@ import customLogger from "./custom-logger.utils"; import fs from "fs"; import projectModelLowdb from "../models/project-lowdb"; +/** + * Normalises a uid map that may be either flat `{ sourceUid: destUid }` or nested + * per-content-type `{ [ctUid]: { sourceUid: destUid } }` into a single flat map. + * Checked per-key rather than all-or-nothing, so a MIXED map (some flat string + * values, some nested objects — which mergeUidMaps below can produce when a prior + * iteration stored the flat shape and the current run wrote the nested one) is + * handled correctly: nested keys get unpacked, flat keys pass through as-is. + * Kept in this module so both entry-mapping consumers (`contentMapper.service` + * and `entry-update.utils`) share one authoritative implementation. + */ +export const flattenNestedUidMap = (raw: Record | undefined | null): Record => { + const keys = Object?.keys(raw ?? {}); + if (keys?.length === 0) return {}; + const out: Record = {}; + for (const k of keys) { + const v = (raw as Record)[k]; + if (v != null && typeof v === 'object' && !Array.isArray(v)) { + Object.assign(out, v); + } else { + out[k] = v; + } + } + return out; +}; + /** * Merges a previous iteration's uid map under the current run's map (current * wins on conflict). Values can be plain strings (flat old→new maps) or diff --git a/api/tests/unit/routes/contentMapper.routes.test.ts b/api/tests/unit/routes/contentMapper.routes.test.ts index 734907599..b76ea02ab 100644 --- a/api/tests/unit/routes/contentMapper.routes.test.ts +++ b/api/tests/unit/routes/contentMapper.routes.test.ts @@ -18,6 +18,7 @@ vi.mock('../../../src/controllers/projects.contentMapper.controller.js', () => ( getSingleGlobalField: vi.fn((_req: any, res: any) => res.status(200).json({})), getAssetMapping: vi.fn((_req: any, res: any) => res.status(200).json({})), updateAssetStatus: vi.fn((_req: any, res: any) => res.status(200).json({})), + retryAssetDownload: vi.fn((_req: any, res: any) => res.status(200).json({})), }, })); diff --git a/api/tests/unit/services/contentMapper.service.test.ts b/api/tests/unit/services/contentMapper.service.test.ts index 25b0d1535..52c60d550 100644 --- a/api/tests/unit/services/contentMapper.service.test.ts +++ b/api/tests/unit/services/contentMapper.service.test.ts @@ -22,6 +22,11 @@ const { mockUidMapperDb, getEntryMapperDbMock, getUidMapperDbMock, + mockRetryFailedAsset, + mockAssetMapperDb, + getAssetMapperDbMock, + mockFsExistsSync, + mockFsReadFileSync, } = vi.hoisted(() => { const mockContentTypesMapperRead = vi.fn(); const mockContentTypesMapperUpdate = vi.fn(); @@ -63,6 +68,13 @@ const { chain: { get: mockUidMapperChainGet }, data: { entry: {} as Record, assets: {} as Record }, }; + const mockAssetMapperRead = vi.fn(); + const mockAssetMapperChainGet = vi.fn(); + const mockAssetMapperDb = { + read: mockAssetMapperRead, + chain: { get: mockAssetMapperChainGet }, + data: { asset_mapper: [] as unknown[] }, + }; return { mockHttps: vi.fn(), mockGetAuthToken: vi.fn(), @@ -85,6 +97,11 @@ const { mockUidMapperDb, getEntryMapperDbMock: vi.fn(() => mockEntryMapperDb), getUidMapperDbMock: vi.fn(() => mockUidMapperDb), + mockRetryFailedAsset: vi.fn(), + mockAssetMapperDb, + getAssetMapperDbMock: vi.fn(() => mockAssetMapperDb), + mockFsExistsSync: vi.fn(() => false), + mockFsReadFileSync: vi.fn(() => '{}'), }; }); @@ -131,11 +148,26 @@ vi.mock('../../../src/models/uidMapper.js', () => ({ default: getUidMapperDbMock, })); +vi.mock('../../../src/models/assetMapper.js', () => ({ + default: getAssetMapperDbMock, +})); + +vi.mock('../../../src/services/contentful.service.js', () => ({ + contentfulService: { retryFailedAsset: mockRetryFailedAsset }, +})); + vi.mock('fs', () => { const mkdirSync = vi.fn(); return { - default: { promises: mockFsPromises, mkdirSync }, + default: { + promises: mockFsPromises, + mkdirSync, + existsSync: mockFsExistsSync, + readFileSync: mockFsReadFileSync, + }, mkdirSync, + existsSync: mockFsExistsSync, + readFileSync: mockFsReadFileSync, promises: mockFsPromises, }; }); @@ -147,13 +179,15 @@ const createChain = (opts: { find?: unknown; findIndex?: number; value?: unknown; + filter?: unknown[]; }) => { const findValue = opts.find !== undefined ? opts.find : null; const findIndexValue = opts.findIndex !== undefined ? opts.findIndex : -1; + const filterValue = opts.filter !== undefined ? opts.filter : []; return { find: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(findValue) }), findIndex: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(findIndexValue) }), - filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue([]) }), + filter: vi.fn().mockReturnValue({ value: vi.fn().mockReturnValue(filterValue) }), }; }; @@ -165,12 +199,16 @@ describe('contentMapper.service', () => { (mockFieldDb.chain.get as ReturnType).mockReset(); (mockEntryMapperDb.chain.get as ReturnType).mockReset(); (mockUidMapperDb.chain.get as ReturnType).mockReset(); + (mockAssetMapperDb.chain.get as ReturnType).mockReset(); mockGetAuthToken.mockResolvedValue('cs-auth-token'); mockProjectRead.mockResolvedValue(undefined); mockContentTypesMapperRead.mockResolvedValue(undefined); mockFieldMapperRead.mockResolvedValue(undefined); (mockEntryMapperDb.read as ReturnType).mockResolvedValue(undefined); (mockUidMapperDb.read as ReturnType).mockResolvedValue(undefined); + (mockAssetMapperDb.read as ReturnType).mockResolvedValue(undefined); + mockFsExistsSync.mockReturnValue(false); + mockFsReadFileSync.mockReturnValue('{}'); mockProjectUpdate.mockImplementation(async (fn: (d: any) => void) => { const data = ProjectModelLowdb.data as any; if (!data.projects) data.projects = []; @@ -202,11 +240,13 @@ describe('contentMapper.service', () => { mockFieldDb.data = { field_mapper: [] }; mockEntryMapperDb.data = { entry_mapper: [] }; mockUidMapperDb.data = { entry: {}, assets: {} }; + mockAssetMapperDb.data = { asset_mapper: [] }; (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); (mockContentTypesDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); (mockFieldDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); (mockEntryMapperDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); (mockUidMapperDb.chain.get as ReturnType).mockReturnValue(createChain({ find: null, findIndex: -1 })); + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue(createChain({ filter: [] })); }); describe('putTestData', () => { @@ -887,4 +927,191 @@ describe('contentMapper.service', () => { expect(result.data).toBe('Project not found'); }); }); + + describe('retryAssetDownload', () => { + it('returns 400 when assetUid is missing', async () => { + const req = { params: { projectId: 'proj-1' } } as any; + + const result = await contentMapperService.retryAssetDownload(req); + + expect(result.status).toBe(400); + expect(result.data.message).toMatch(/assetUid/i); + }); + + it('returns 400 when the project has no destination stack or source file path', async () => { + const project = { id: 'proj-1', legacy_cms: { cms: 'contentful' } }; + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project }) + ); + + const req = { params: { projectId: 'proj-1', assetUid: 'src-1' } } as any; + const result = await contentMapperService.retryAssetDownload(req); + + expect(result.status).toBe(400); + expect(result.data.message).toMatch(/destination stack|source file/i); + }); + + it('returns 400 for a non-Contentful project', async () => { + const project = { + id: 'proj-1', + destination_stack_id: 'stack-1', + legacy_cms: { cms: 'wordpress', file_path: '/tmp/export.xml' }, + }; + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project }) + ); + + const req = { params: { projectId: 'proj-1', assetUid: 'src-1' } } as any; + const result = await contentMapperService.retryAssetDownload(req); + + expect(result.status).toBe(400); + expect(result.data.message).toMatch(/Contentful/i); + }); + + it('returns 200 with success:true when the retry succeeds', async () => { + const project = { + id: 'proj-1', + destination_stack_id: 'stack-1', + legacy_cms: { cms: 'contentful', file_path: '/tmp/export.json' }, + }; + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project }) + ); + mockRetryFailedAsset.mockResolvedValue({ + success: true, + message: 'Asset downloaded successfully. It will be included in the next migration run.', + }); + + const req = { params: { projectId: 'proj-1', assetUid: 'src-1' } } as any; + const result = await contentMapperService.retryAssetDownload(req); + + expect(mockRetryFailedAsset).toHaveBeenCalledWith('/tmp/export.json', 'stack-1', 'proj-1', 'src-1'); + expect(result.status).toBe(200); + expect(result.data.success).toBe(true); + }); + + it('returns 400 with the failure reason when the retry fails again', async () => { + const project = { + id: 'proj-1', + destination_stack_id: 'stack-1', + legacy_cms: { cms: 'contentful', file_path: '/tmp/export.json' }, + }; + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project }) + ); + mockRetryFailedAsset.mockResolvedValue({ success: false, message: 'DNS lookup failed.' }); + + const req = { params: { projectId: 'proj-1', assetUid: 'src-1' } } as any; + const result = await contentMapperService.retryAssetDownload(req); + + expect(result.status).toBe(400); + expect(result.data.success).toBe(false); + expect(result.data.message).toBe('DNS lookup failed.'); + }); + }); + + describe('getAssetMapping', () => { + const project = { id: 'proj-1', destination_stack_id: 'stack1', iteration: 1 }; + const baseReq = (overrides: Record = {}) => + ({ + params: { projectId: 'proj-1', skip: '0', limit: '10', searchText: '', ...overrides }, + query: {}, + }) as any; + + beforeEach(() => { + (ProjectModelLowdb.chain.get as ReturnType).mockReturnValue( + createChain({ find: project }) + ); + }); + + it('returns assets with an ok status when nothing failed', async () => { + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue( + createChain({ + filter: [ + { projectId: 'proj-1', otherCmsAssetUid: 'src-1', contentstackAssetUid: 'cs-1', filename: 'a.jpg', title: 'A' }, + ], + }) + ); + + const result = await contentMapperService.getAssetMapping(baseReq()); + + expect(result.status).toBe(200); + expect(result.count).toBe(1); + expect(result.assetMapping[0].status).toBe('ok'); + expect(result.missingCount).toBe(0); + expect(result.failedCount).toBe(0); + }); + + it('marks assets with no source file as missing', async () => { + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue( + createChain({ + filter: [ + { projectId: 'proj-1', otherCmsAssetUid: 'src-2', hasSource: false, filename: 'b.jpg', title: 'B' }, + ], + }) + ); + + const result = await contentMapperService.getAssetMapping(baseReq()); + + expect(result.assetMapping[0].status).toBe('missing'); + expect(result.missingCount).toBe(1); + }); + + it('marks assets recorded in cs_failed.json as failed', async () => { + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue( + createChain({ + filter: [ + { projectId: 'proj-1', otherCmsAssetUid: 'src-3', filename: 'c.jpg', title: 'C' }, + ], + }) + ); + mockFsExistsSync.mockReturnValue(true); + mockFsReadFileSync.mockReturnValue( + JSON.stringify({ 'src-3': { reason_for_error: 'Timeout downloading asset.' } }) + ); + + const result = await contentMapperService.getAssetMapping(baseReq()); + + expect(result.assetMapping[0].status).toBe('failed'); + expect(result.assetMapping[0].errorMessage).toBe('Timeout downloading asset.'); + expect(result.failedCount).toBe(1); + }); + + it('filters by the requested status', async () => { + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue( + createChain({ + filter: [ + { projectId: 'proj-1', otherCmsAssetUid: 'src-1', contentstackAssetUid: 'cs-1', filename: 'a.jpg', title: 'A' }, + { projectId: 'proj-1', otherCmsAssetUid: 'src-2', hasSource: false, filename: 'b.jpg', title: 'B' }, + ], + }) + ); + + const result = await contentMapperService.getAssetMapping(baseReq({})); + const filteredResult = await contentMapperService.getAssetMapping({ + params: { projectId: 'proj-1', skip: '0', limit: '10', searchText: '' }, + query: { status: 'missing' }, + } as any); + + expect(result.count).toBe(2); + expect(filteredResult.count).toBe(1); + expect(filteredResult.assetMapping[0].status).toBe('missing'); + }); + + it('filters by search text against filename and title', async () => { + (mockAssetMapperDb.chain.get as ReturnType).mockReturnValue( + createChain({ + filter: [ + { projectId: 'proj-1', otherCmsAssetUid: 'src-1', contentstackAssetUid: 'cs-1', filename: 'windmill.jpg', title: 'Windmill' }, + { projectId: 'proj-1', otherCmsAssetUid: 'src-2', contentstackAssetUid: 'cs-2', filename: 'sunset.jpg', title: 'Sunset' }, + ], + }) + ); + + const result = await contentMapperService.getAssetMapping(baseReq({ searchText: 'wind' })); + + expect(result.count).toBe(1); + expect(result.assetMapping[0].filename).toBe('windmill.jpg'); + }); + }); }); diff --git a/api/tests/unit/services/contentful.service.retryFailedAsset.test.ts b/api/tests/unit/services/contentful.service.retryFailedAsset.test.ts new file mode 100644 index 000000000..a8a1e7e4d --- /dev/null +++ b/api/tests/unit/services/contentful.service.retryFailedAsset.test.ts @@ -0,0 +1,46 @@ +import fs from 'fs'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('../../../src/utils/custom-logger.utils.js', () => ({ + default: vi.fn().mockResolvedValue(undefined), +})); + +const { contentfulService } = await import('../../../src/services/contentful.service.js'); + +describe('contentfulService.retryFailedAsset', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('returns failure when the asset is not present in the source export', async () => { + vi.spyOn(fs.promises, 'readFile').mockResolvedValue( + JSON.stringify({ assets: [{ sys: { id: 'some-other-asset' } }] }), + ); + + const result = await contentfulService.retryFailedAsset( + '/fake/package/path.json', + 'stack123', + 'project123', + 'missing-asset-id', + ); + + expect(result).toEqual({ + success: false, + message: 'Asset not found in the source export.', + }); + }); + + it('returns a failure message when reading the package file throws', async () => { + vi.spyOn(fs.promises, 'readFile').mockRejectedValue(new Error('ENOENT: no such file')); + + const result = await contentfulService.retryFailedAsset( + '/fake/package/path.json', + 'stack123', + 'project123', + 'asset-id', + ); + + expect(result.success).toBe(false); + expect(result.message).toContain('ENOENT'); + }); +}); diff --git a/api/tests/unit/utils/entry-update-script.test.ts b/api/tests/unit/utils/entry-update-script.test.ts index ae0582a7a..068e74845 100644 --- a/api/tests/unit/utils/entry-update-script.test.ts +++ b/api/tests/unit/utils/entry-update-script.test.ts @@ -7,7 +7,16 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; // default function export for testing. const require = createRequire(import.meta.url); const script = require('../../../src/utils/entry-update-script.cjs'); -const { isAssetField, resolveAssetField, mergeFlatPayloadIntoEntry } = script; +const { + isAssetField, + resolveAssetField, + mergeFlatPayloadIntoEntry, + isReferenceValue, + isReferenceArray, + resolveReferenceUid, + resolveReferenceField, + resolveReferencesDeep, +} = script; describe('entry-update-script — isAssetField', () => { it('is true only for objects carrying urlPath + filename', () => { @@ -66,6 +75,141 @@ describe('entry-update-script — resolveAssetField (3-way resolution)', () => { }); }); +describe('entry-update-script — isReferenceValue / isReferenceArray', () => { + it('recognizes the { uid, _content_type_uid } shape produced by processField', () => { + expect(isReferenceValue({ uid: 'src-1', _content_type_uid: 'author' })).toBe(true); + }); + + it('is false for asset shapes, primitives, and arrays', () => { + expect(isReferenceValue({ urlPath: '/x', filename: 'f.jpg' })).toBe(false); + expect(isReferenceValue(null)).toBeFalsy(); + expect(isReferenceValue('str')).toBeFalsy(); + expect(isReferenceValue([{ uid: 'a', _content_type_uid: 'b' }])).toBe(false); + expect(isReferenceValue({ uid: 'src-1' })).toBe(false); // missing _content_type_uid + }); + + it('recognizes a non-empty array of reference values (multi-reference field)', () => { + expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, { uid: 'b', _content_type_uid: 'article' }])).toBe(true); + }); + + it('is false for an empty array', () => { + expect(isReferenceArray([])).toBe(false); + }); + + it('is true for a mixed array so per-item remap still runs — non-ref items pass through in resolveReferenceField', () => { + expect(isReferenceArray([{ uid: 'a', _content_type_uid: 'article' }, 'not-a-ref'])).toBe(true); + }); +}); + +describe('entry-update-script — resolveReferenceUid', () => { + const locale = 'en-in'; + const entryMapping = { + old: { flat: { 'src-1': 'cs-old-flat' }, byLocale: { 'en-in': { 'src-2': 'cs-old-locale' } } }, + new: { flat: { 'src-3': 'cs-new-flat' }, byLocale: { 'en-in': { 'src-1': 'cs-new-locale' } } }, + }; + + it('prefers the new per-locale mapping over everything else', () => { + expect(resolveReferenceUid('src-1', locale, entryMapping)).toBe('cs-new-locale'); + }); + + it('falls back to the old per-locale mapping when no new per-locale entry exists', () => { + expect(resolveReferenceUid('src-2', locale, entryMapping)).toBe('cs-old-locale'); + }); + + it('falls back to the flat new mapping when no per-locale entry exists at all', () => { + expect(resolveReferenceUid('src-3', locale, entryMapping)).toBe('cs-new-flat'); + }); + + it('falls back to the flat old mapping as a last resort', () => { + const mapping = { old: { flat: { 'src-4': 'cs-old-flat-only' }, byLocale: {} }, new: { flat: {}, byLocale: {} } }; + expect(resolveReferenceUid('src-4', locale, mapping)).toBe('cs-old-flat-only'); + }); + + it('falls back to identity (source uid unchanged) when no mapping exists at all', () => { + expect(resolveReferenceUid('unmapped-src', locale, { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: {} } })).toBe('unmapped-src'); + }); + + it('handles a missing/undefined entryMapping gracefully', () => { + expect(resolveReferenceUid('src-1', locale, undefined)).toBe('src-1'); + }); +}); + +describe('entry-update-script — resolveReferencesDeep', () => { + const locale = 'en-in'; + const entryMapping = { + old: { flat: {}, byLocale: {} }, + new: { flat: { 'src-1': 'cs-1', 'src-2': 'cs-2' }, byLocale: {} }, + }; + + it('resolves a top-level reference value', () => { + const out = resolveReferencesDeep('field', 'entry-1', { uid: 'src-1', _content_type_uid: 'author' }, locale, entryMapping); + expect(out).toEqual({ uid: 'cs-1', _content_type_uid: 'author' }); + }); + + it('resolves every reference nested inside a plain object (group field)', () => { + const value = { heroBlock: { author: { uid: 'src-1', _content_type_uid: 'author' } } }; + const out = resolveReferencesDeep('field', 'entry-1', value, locale, entryMapping); + expect(out).toEqual({ heroBlock: { author: { uid: 'cs-1', _content_type_uid: 'author' } } }); + }); + + it('resolves a MIXED array — a bare reference alongside an object with a reference nested inside', () => { + // Regression case: isReferenceArray's .some() used to route the whole array through + // resolveReferenceField's shallow per-item remap, which passes non-reference-shaped + // items through untouched — so the nested reference inside the block object never got + // resolved. resolveReferencesDeep must recurse into every element instead. + const value = [ + { uid: 'src-1', _content_type_uid: 'author' }, + { heroBlock: { uid: 'src-2', _content_type_uid: 'category' } }, + ]; + const out = resolveReferencesDeep('field', 'entry-1', value, locale, entryMapping); + expect(out).toEqual([ + { uid: 'cs-1', _content_type_uid: 'author' }, + { heroBlock: { uid: 'cs-2', _content_type_uid: 'category' } }, + ]); + }); + + it('leaves asset field objects untouched (they need resolveAssetField, not a uid remap)', () => { + const value = { urlPath: '/assets/1', filename: 'f.jpg', uid: 'src-1' }; + const out = resolveReferencesDeep('field', 'entry-1', value, locale, entryMapping); + expect(out).toBe(value); + }); + + it('passes scalars and unresolvable shapes through unchanged', () => { + expect(resolveReferencesDeep('field', 'entry-1', 'plain-string', locale, entryMapping)).toBe('plain-string'); + expect(resolveReferencesDeep('field', 'entry-1', null, locale, entryMapping)).toBe(null); + expect(resolveReferencesDeep('field', 'entry-1', 42, locale, entryMapping)).toBe(42); + }); +}); + +describe('entry-update-script — resolveReferenceField', () => { + const locale = 'en-gb'; + const entryMapping = { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: { 'en-gb': { 'author-src': 'author-cs' } } } }; + + it('remaps a single reference value uid', () => { + const out = resolveReferenceField('author', 'e1', { uid: 'author-src', _content_type_uid: 'author' }, locale, entryMapping); + expect(out).toEqual({ uid: 'author-cs', _content_type_uid: 'author' }); + }); + + it('remaps every uid in a multi-reference array', () => { + const mapping = { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: { 'en-gb': { a1: 'a1-cs', a2: 'a2-cs' } } } }; + const out = resolveReferenceField( + 'relatedArticles', + 'e1', + [{ uid: 'a1', _content_type_uid: 'article' }, { uid: 'a2', _content_type_uid: 'article' }], + locale, + mapping + ); + expect(out).toEqual([ + { uid: 'a1-cs', _content_type_uid: 'article' }, + { uid: 'a2-cs', _content_type_uid: 'article' }, + ]); + }); + + it('passes non-reference values through unchanged', () => { + expect(resolveReferenceField('title', 'e1', 'plain string', locale, entryMapping)).toBe('plain string'); + }); +}); + describe('entry-update-script — mergeFlatPayloadIntoEntry', () => { it('merges flat fields into entry.content, resolves assets, and skips reserved keys', async () => { const update = vi.fn().mockResolvedValue(undefined); @@ -88,6 +232,40 @@ describe('entry-update-script — mergeFlatPayloadIntoEntry', () => { expect(entry.content._version).toBeUndefined(); expect(update).toHaveBeenCalledTimes(1); }); + + it('resolves a reference field uid using entryMapping when localizing an existing entry (CMG delta bug)', async () => { + // Reproduces the bug: an export's reference field carries the SOURCE cms + // entry id (e.g. Contentful's id), which only equals the Contentstack uid + // by coincidence. Without entryMapping this used to be written verbatim, + // silently pointing at a non-existent uid on any locale added via restart. + const update = vi.fn().mockResolvedValue(undefined); + const entry: any = { title: 'old', content: {}, update }; + + const updateData = { + uid: 'should-be-skipped', + title: 'Article 1', + author: { uid: 'contentful-author-src-id', _content_type_uid: 'author' }, + }; + + const entryMapping = { + old: { flat: {}, byLocale: {} }, + new: { flat: {}, byLocale: { 'en-in': { 'contentful-author-src-id': 'real-cs-author-uid' } } }, + }; + + await mergeFlatPayloadIntoEntry(entry, 'e1', updateData, {}, {}, undefined, 'en-in', entryMapping); + + expect(entry.content.author).toEqual({ uid: 'real-cs-author-uid', _content_type_uid: 'author' }); + }); + + it('falls back to the source uid unchanged when no entryMapping is supplied (back-compat)', async () => { + const update = vi.fn().mockResolvedValue(undefined); + const entry: any = { title: 'old', content: {}, update }; + const updateData = { author: { uid: 'src-id', _content_type_uid: 'author' } }; + + await mergeFlatPayloadIntoEntry(entry, 'e1', updateData, {}, {}, undefined, 'en-in', undefined); + + expect(entry.content.author).toEqual({ uid: 'src-id', _content_type_uid: 'author' }); + }); }); describe('entry-update-script — main task runner', () => { diff --git a/api/tests/unit/utils/entry-update.enrich.test.ts b/api/tests/unit/utils/entry-update.enrich.test.ts index c4ee7e099..90c1f0272 100644 --- a/api/tests/unit/utils/entry-update.enrich.test.ts +++ b/api/tests/unit/utils/entry-update.enrich.test.ts @@ -120,3 +120,79 @@ describe('entry-update.utils — enrichConfigWithAssetMapping (extra branches)', expect(written.__assetMapping__).toEqual({ old: {}, new: {} }); }); }); + +// Covers the fix for the "reference fields blank on localized entries" bug: +// entry-update-script.cjs needs entry uid-mapper data (flat + per-locale) +// threaded into the config under __entryMapping__, the same way asset uids +// already are under __assetMapping__. +describe('entry-update.utils — enrichConfigWithEntryMapping', () => { + beforeEach(() => vi.clearAllMocks()); + + it('writes empty old/new entry mappings when no uid-mapper files exist', async () => { + mockExistsSync.mockReturnValue(false); + mockReadFileSync.mockReturnValue(JSON.stringify({ page: {} })); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 1, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__).toEqual({ + old: { flat: {}, byLocale: {} }, + new: { flat: {}, byLocale: {} }, + }); + }); + + it('reads new-iteration entry + entryByLocale maps from uid-mapper.json', async () => { + mockExistsSync.mockImplementation((p: string) => p.includes('/2/uid-mapper.json')); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('/2/uid-mapper.json')) { + return JSON.stringify({ + entry: { 'src-a': 'cs-a' }, + entryByLocale: { 'en-in': { 'src-a': 'cs-a-in' } }, + }); + } + return JSON.stringify({ page: {} }); // config file + }); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 2, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__.new).toEqual({ + flat: { 'src-a': 'cs-a' }, + byLocale: { 'en-in': { 'src-a': 'cs-a-in' } }, + }); + expect(written.__entryMapping__.old).toEqual({ flat: {}, byLocale: {} }); + }); + + it('reads both old (iteration-1) and new mappings when iteration > 1', async () => { + mockExistsSync.mockReturnValue(true); + mockReadFileSync.mockImplementation((p: string) => { + if (p.includes('/1/uid-mapper.json')) { + return JSON.stringify({ entry: { 'src-old': 'cs-old' }, entryByLocale: {} }); + } + if (p.includes('/2/uid-mapper.json')) { + return JSON.stringify({ entry: { 'src-new': 'cs-new' }, entryByLocale: {} }); + } + return JSON.stringify({ page: {} }); + }); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 2, '/tmp/x.log'); + + const write = mockWriteFileSync.mock.calls.find((c) => c[0] === '/tmp/config.json'); + const written = JSON.parse(String(write?.[1])); + expect(written.__entryMapping__.old.flat).toEqual({ 'src-old': 'cs-old' }); + expect(written.__entryMapping__.new.flat).toEqual({ 'src-new': 'cs-new' }); + }); + + it('swallows a read/parse error on the config file without throwing', async () => { + mockExistsSync.mockReturnValue(false); + mockReadFileSync.mockReturnValue('{ not json'); + + const { enrichConfigWithEntryMapping } = await import('../../../src/utils/entry-update.utils.js'); + expect(() => enrichConfigWithEntryMapping('/tmp/config.json', 'p1', 1, '/tmp/x.log')).not.toThrow(); + }); +}); diff --git a/api/tests/unit/utils/locale-migration.utils.test.ts b/api/tests/unit/utils/locale-migration.utils.test.ts index 38d39a9ab..4d2ca5810 100644 --- a/api/tests/unit/utils/locale-migration.utils.test.ts +++ b/api/tests/unit/utils/locale-migration.utils.test.ts @@ -18,6 +18,7 @@ import { getMigratedLocales, isFullMigrationForLocale, recordMigratedLocales, + extractLocalesFromUpdateConfig, } from '../../../src/utils/locale-migration.utils'; describe('locale-migration.utils', () => { @@ -183,4 +184,60 @@ describe('locale-migration.utils', () => { expect((data.projects[0] as any).migrated_locales).toBeUndefined(); }); }); + + // Covers the fix for the "locale marked migrated before it was ever + // actually processed" bug: runCli.service.ts used to compute the migrated + // locale set from the project's FULL configured locale list, which + // permanently skipped any locale configured ahead of when it was meant to + // be migrated. This helper extracts ONLY the locale(s) an iteration's delta + // pass actually queued, from updated-entries.json's compound + // `${csUid}::${localeCode}` keys. + describe('extractLocalesFromUpdateConfig', () => { + it('extracts locale codes from compound entry keys across content types', () => { + const config = { + article: { 'blt-1::en-in': {}, 'blt-2::en-in': {} }, + author: { 'blt-3::en-in': {} }, + }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-in']); + }); + + it('dedupes locales seen across multiple entries', () => { + const config = { + article: { 'blt-1::en-gb': {}, 'blt-2::en-gb': {}, 'blt-3::en-in': {} }, + }; + const result = extractLocalesFromUpdateConfig(config); + expect(result).toEqual(expect.arrayContaining(['en-gb', 'en-in'])); + expect(result).toHaveLength(2); + }); + + it('ignores bookkeeping keys (__assetMapping__, __entryMapping__, __assetUpdates__)', () => { + const config = { + __assetMapping__: { old: {}, new: {} }, + __entryMapping__: { old: {}, new: {} }, + __assetUpdates__: [{ uid: 'a' }], + article: { 'blt-1::en-gb': {} }, + }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-gb']); + }); + + it('ignores legacy keys with no locale suffix', () => { + const config = { page: { 'cs-1': { title: 'T' } } }; + expect(extractLocalesFromUpdateConfig(config)).toEqual([]); + }); + + it('returns [] for null, undefined, or non-object input', () => { + expect(extractLocalesFromUpdateConfig(null)).toEqual([]); + expect(extractLocalesFromUpdateConfig(undefined)).toEqual([]); + expect(extractLocalesFromUpdateConfig('not an object' as any)).toEqual([]); + }); + + it('returns [] for an empty config object', () => { + expect(extractLocalesFromUpdateConfig({})).toEqual([]); + }); + + it('skips content types whose value is not an object', () => { + const config = { article: null, author: { 'blt-1::en-in': {} } }; + expect(extractLocalesFromUpdateConfig(config)).toEqual(['en-in']); + }); + }); }); diff --git a/ui/index.html b/ui/index.html index 9fc3e0dfe..54d6b764e 100644 --- a/ui/index.html +++ b/ui/index.html @@ -36,19 +36,6 @@ rel="stylesheet" /> - - - -