From e4a944a00bee932281a6128afa0fe9becfd579ee Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:40:51 -0400 Subject: [PATCH 01/12] add soft reads for saved items --- .../scripts/performance/client-audit-model.ts | 65 ++- .../feed/view-lists/ItemDisplay.tsx | 13 +- .../feed/view-lists/RenderViewItems.tsx | 166 +++++--- .../components/feed/view-lists/softReads.ts | 132 ++++++ .../feed/view-lists/useSoftReads.ts | 131 ++++++ .../src/lib/hooks/useFeedItemNavigation.ts | 15 +- apps/app/tests/e2e/fixtures/seed-db.ts | 11 + .../e2e/self-hosted/item-actions.spec.ts | 14 +- .../sidebar-navigation-snapshot.spec.ts | 2 +- .../tests/e2e/self-hosted/soft-reads.spec.ts | 239 +++++++++++ .../view-mixed-content-matrix.spec.ts | 15 +- .../tests/unit/components/soft-reads.test.ts | 392 ++++++++++++++++++ .../performance/client-audit-model.test.ts | 14 + 13 files changed, 1125 insertions(+), 84 deletions(-) create mode 100644 apps/app/src/components/feed/view-lists/softReads.ts create mode 100644 apps/app/src/components/feed/view-lists/useSoftReads.ts create mode 100644 apps/app/tests/e2e/self-hosted/soft-reads.spec.ts create mode 100644 apps/app/tests/unit/components/soft-reads.test.ts diff --git a/apps/app/scripts/performance/client-audit-model.ts b/apps/app/scripts/performance/client-audit-model.ts index 24f4e016e..f4a36b9eb 100644 --- a/apps/app/scripts/performance/client-audit-model.ts +++ b/apps/app/scripts/performance/client-audit-model.ts @@ -25,6 +25,11 @@ import { selectPersistedPages, } from "~/lib/data/page-retention"; import { projectLocalMixedContentOrder } from "~/lib/data/mixed-content/bookmarkProjection"; +import { + getEligibleSoftReadIds, + retainEligibleSoftReadPositions, + retainSoftReadPositions, +} from "~/components/feed/view-lists/softReads"; const PAGE_SIZE = 30; const CONTENT_STATUSES: readonly ContentStatusFilter[] = CONTENT_STATUS_FILTERS; @@ -85,6 +90,7 @@ export type ClientAuditResult = { bookmarkBurstSingleFrame: ClientAuditOperation; bookmarkBurstSeparateFrames: ClientAuditOperation; localViewProjection: ClientAuditOperation; + softReadProjection: ClientAuditOperation; normalizedPersistenceMutation: ClientAuditOperation; }; }; @@ -612,11 +618,65 @@ export function runClientAuditProfile( ): ClientAuditResult { let fixture = seedClientFixture(profileName); const profile = fixture.profile; + const savedIds = [ + ...fixture.feedItems + .filter((item) => item.isWatchLater) + .map((item) => item.id), + ...fixture.bookmarks.filter((item) => item.isSaved).map((item) => item.id), + ]; + const softReadPositions = new Map( + savedIds.flatMap((id, index) => + index % 2 === 0 + ? [[id, { sectionKey: "uncategorized", index }] as const] + : [], + ), + ); + const softReadView = { + ...makeView(0), + feedIds: [...new Set(fixture.feedItems.map((item) => item.feedId))], + categoryIds: fixture.views.map((view) => view.id), + }; + const softReadViews = fixture.views.map((view) => + view.id === softReadView.id ? softReadView : view, + ); + const softReadProjection = measure(() => { + const eligibleIds = getEligibleSoftReadIds({ + positions: softReadPositions, + bookmarksById: bookmarksStore.getState().snapshot(), + feedItemsById: feedItemsStore.getState().feedItemsDict, + feedCategories: [], + views: softReadViews, + currentView: softReadView, + categoryFilter: -1, + feedFilter: -1, + saveStatus: "saved", + }); + const section = { + name: "Saved", + layout: "list" as const, + startIndex: 0, + isUncategorized: true, + placement: null, + }; + const activePositions = retainEligibleSoftReadPositions(softReadPositions, [ + { ...section, items: eligibleIds }, + ]); + retainSoftReadPositions( + [ + { + ...section, + items: savedIds.filter((id) => !softReadPositions.has(id)), + }, + ], + activePositions, + ); + return 0; + }); const localProjectionView = fixture.views.at(-1)!; + const localProjectionFeedIds = new Set(localProjectionView.feedIds); const localProjectionFeedItemIds = fixture.feedItems .filter( - (item) => - item.isWatchLater && localProjectionView.feedIds.includes(item.feedId), + (item) => item.isWatchLater && localProjectionFeedIds.has(item.feedId), ) .map((item) => item.id); const localViewProjection = measure(() => { @@ -790,6 +850,7 @@ export function runClientAuditProfile( bookmarkBurstSingleFrame, bookmarkBurstSeparateFrames, localViewProjection, + softReadProjection, normalizedPersistenceMutation, }, }; diff --git a/apps/app/src/components/feed/view-lists/ItemDisplay.tsx b/apps/app/src/components/feed/view-lists/ItemDisplay.tsx index dc5c9fff3..341ede7b6 100644 --- a/apps/app/src/components/feed/view-lists/ItemDisplay.tsx +++ b/apps/app/src/components/feed/view-lists/ItemDisplay.tsx @@ -48,6 +48,11 @@ import { useBookmarkCaptureValue } from "~/lib/data/bookmarks/capture-store"; export type ItemSize = "standard" | "large"; +function itemOpacity(canOpen: boolean, isRead: boolean) { + if (!canOpen) return "opacity-50"; + return isRead ? "opacity-75" : undefined; +} + // Typography components for consistent styling across layouts interface ItemTitleProps { @@ -716,7 +721,7 @@ function BookmarkGridItem({ onMouseEnter={onSelect} className={clsx( "group relative flex h-full w-full flex-col", - !canOpen && "opacity-50", + itemOpacity(canOpen, bookmark.isRead), )} > void; showPaginationLoader: boolean; showPaginationEnd: boolean; + toggleRead: (id: string, mutate: () => boolean) => boolean; }) { const navigationItems = useMemo( () => fullComputedSections.flatMap((section) => section.items), @@ -281,6 +285,7 @@ function ContentStatusSectionList({ navigationItems, navigationIsGridLayout, navigationSectionInfo, + toggleRead, ); const handleSectionMarkAsRead = useCallback( (sectionIndex: number) => { @@ -323,104 +328,129 @@ function ContentStatusSectionList({ } export function RenderViewItems() { - const { feeds, hasFetchedFeeds } = useFeeds(); - const { hasFetchedFeedCategories } = useFeedCategories(); - - const hasInitialData = useHasInitialData(); - - const filteredFeedItemsOrder = useFilteredContentOrder(); - const currentView = useAtomValue(viewFilterAtom); const feedFilter = useAtomValue(feedFilterAtom); const categoryFilter = useAtomValue(categoryFilterAtom); - const { - sentinelRef, - paginationState, - visibleItems: visibleFilteredFeedItemsOrder, - hasRenderedAllItems, - } = useViewListScroll(filteredFeedItemsOrder); + const dateFilter = useAtomValue(dateFilterAtom); + const contentStatusFilter = useAtomValue(contentStatusFilterAtom); + const viewListKey = `view-${currentView?.id ?? "none"}-${buildContentStatusKey(contentStatusFilter)}`; + const scopeKey = `${viewListKey}-feed-${feedFilter}-tag-${categoryFilter}-date-${dateFilter}`; + + return ; +} + +function ViewListSkeleton({ layout }: { layout: ViewSection["layout"] }) { + switch (layout) { + case VIEW_LAYOUT.LARGE_LIST: + return ; + case VIEW_LAYOUT.GRID: + return ; + case VIEW_LAYOUT.LARGE_GRID: + return ; + default: + return ; + } +} + +function getViewPlaceholder({ + hasInitialData, + hasFetchedFeeds, + hasFetchedFeedCategories, + hasFeeds, + hasItems, + paginationState, + layout, +}: { + hasInitialData: boolean; + hasFetchedFeeds: boolean; + hasFetchedFeedCategories: boolean; + hasFeeds: boolean; + hasItems: boolean; + paginationState: ReturnType["paginationState"]; + layout: ViewSection["layout"]; +}) { + if (!hasInitialData) return ; + if (hasItems) return null; + if ( + paginationState.isLoaded && + hasFetchedFeeds && + !hasFeeds && + Object.keys(bookmarksStore.getState().snapshot()).length === 0 + ) { + return ; + } + if (!paginationState.isLoaded || paginationState.isFetching) { + return ; + } + if (hasFetchedFeeds && hasFetchedFeedCategories) return ; + return null; +} - const { computedSections: fullComputedSections, baseLayout } = - useViewSections(currentView, filteredFeedItemsOrder); - const { computedSections: visibleComputedSections } = useViewSections( +function ViewVisit({ viewListKey }: { viewListKey: string }) { + const { feeds, hasFetchedFeeds } = useFeeds(); + const { hasFetchedFeedCategories } = useFeedCategories(); + const hasInitialData = useHasInitialData(); + const filteredItemIds = useFilteredContentOrder(); + const currentView = useAtomValue(viewFilterAtom); + const { computedSections, baseLayout } = useViewSections( currentView, - visibleFilteredFeedItemsOrder, + filteredItemIds, ); - const contentStatusFilter = useAtomValue(contentStatusFilterAtom); - const selectedItemId = useAtomValue(selectedItemIdAtom); - const setSelectedItemId = useSetAtom(selectedItemIdAtom); - const navigationItems = useMemo( + const { sections: fullComputedSections, toggleRead } = useSoftReads( + computedSections, + currentView, + ); + const filteredFeedItemsOrder = useMemo( () => fullComputedSections.flatMap((section) => section.items), [fullComputedSections], ); + const { sentinelRef, paginationState, visibleItems, hasRenderedAllItems } = + useViewListScroll(filteredFeedItemsOrder); + const visibleComputedSections = useMemo(() => { + const visibleIds = new Set(visibleItems); + return fullComputedSections.map((section) => ({ + ...section, + items: section.items.filter((id) => visibleIds.has(id)), + })); + }, [fullComputedSections, visibleItems]); + const selectedItemId = useAtomValue(selectedItemIdAtom); + const setSelectedItemId = useSetAtom(selectedItemIdAtom); const rootListReady = hasInitialData && - (navigationItems.length > 0 || + (filteredFeedItemsOrder.length > 0 || (hasFetchedFeeds && hasFetchedFeedCategories && (paginationState.isLoaded || feeds.length === 0))); useRootItemScrollRestoration({ - activeItemIds: navigationItems, + activeItemIds: filteredFeedItemsOrder, selectedItemId, setSelectedItemId, ready: rootListReady, }); - const contentStatusKey = buildContentStatusKey(contentStatusFilter); - const viewListKey = `view-${currentView?.id ?? "none"}-${contentStatusKey}`; - const contentStatusContextKey = `${viewListKey}-feed-${feedFilter}-tag-${categoryFilter}`; const shouldShowPaginationEnd = hasRenderedAllItems && - paginationState?.hasMore === false && + paginationState.hasMore === false && paginationState.isFetching !== true; - if (!hasInitialData) { - return ; - } - - if ( - paginationState.isLoaded && - hasFetchedFeeds && - !feeds.length && - filteredFeedItemsOrder.length === 0 && - Object.keys(bookmarksStore.getState().snapshot()).length === 0 - ) { - return ; - } - - // Show skeletons while feed items are being fetched - if ( - (!paginationState.isLoaded || paginationState.isFetching) && - filteredFeedItemsOrder.length === 0 - ) { - switch (baseLayout) { - case VIEW_LAYOUT.LARGE_LIST: - return ; - case VIEW_LAYOUT.GRID: - return ; - case VIEW_LAYOUT.LARGE_GRID: - return ; - default: - return ; - } - } - - if ( - hasFetchedFeeds && - paginationState.isLoaded && - hasFetchedFeedCategories && - !filteredFeedItemsOrder.length - ) { - return ; - } + const placeholder = getViewPlaceholder({ + hasInitialData, + hasFetchedFeeds, + hasFetchedFeedCategories, + hasFeeds: feeds.length > 0, + hasItems: filteredFeedItemsOrder.length > 0, + paginationState, + layout: baseLayout, + }); + if (placeholder) return placeholder; return ( ); diff --git a/apps/app/src/components/feed/view-lists/softReads.ts b/apps/app/src/components/feed/view-lists/softReads.ts new file mode 100644 index 000000000..71d85999c --- /dev/null +++ b/apps/app/src/components/feed/view-lists/softReads.ts @@ -0,0 +1,132 @@ +import type { ViewSection } from "./useViewSections"; +import type { ApplicationBookmark } from "~/server/mixed-content/projection"; +import type { ApplicationView, DatabaseFeedCategory } from "~/server/db/schema"; +import type { FeedItemListProjection } from "~/lib/data/feed-items/listProjection"; +import type { ContentStatusFilter } from "~/lib/content-status"; +import { + createFeedItemFilterIndex, + createFeedItemFilterPredicate, +} from "~/lib/data/feed-items/listProjection"; +import { matchesScope } from "~/lib/data/mixed-content/bookmarkProjection"; + +export type SoftReadPosition = { sectionKey: string; index: number }; + +export function getEligibleSoftReadIds({ + positions, + bookmarksById, + feedItemsById, + feedCategories, + views, + currentView, + categoryFilter, + feedFilter, + saveStatus, +}: { + positions: ReadonlyMap; + bookmarksById: Record; + feedItemsById: Record; + feedCategories: DatabaseFeedCategory[]; + views: ApplicationView[]; + currentView: ApplicationView | null; + categoryFilter: number; + feedFilter: number; + saveStatus: ContentStatusFilter["saveStatus"]; +}) { + if (saveStatus !== "saved" || positions.size === 0) return []; + + const filterIndex = createFeedItemFilterIndex(feedCategories, views); + const makeFeedPredicate = (archiveStatus: "unread" | "archived") => + createFeedItemFilterPredicate({ + contentStatusFilter: { saveStatus, archiveStatus }, + categoryFilter, + feedFilter, + viewFilter: currentView, + filterIndex, + }); + const acceptsUnreadFeedItem = makeFeedPredicate("unread"); + const acceptsArchivedFeedItem = makeFeedPredicate("archived"); + const bookmarkScope = + feedFilter >= 0 + ? ({ type: "feed", feedId: feedFilter } as const) + : categoryFilter >= 0 + ? ({ type: "tag", tagId: categoryFilter } as const) + : currentView + ? ({ type: "view", viewId: currentView.id } as const) + : null; + + return [...positions.keys()].filter((id) => { + const bookmark = bookmarksById[id]; + if (bookmark) { + return ( + bookmark.isSaved && + bookmarkScope !== null && + matchesScope(bookmark, bookmarkScope, views) + ); + } + const feedItem = feedItemsById[id]; + return Boolean( + feedItem && + (feedItem.isWatched + ? acceptsArchivedFeedItem(feedItem) + : acceptsUnreadFeedItem(feedItem)), + ); + }); +} + +export function softReadSectionKey(section: ViewSection) { + return section.isUncategorized + ? "uncategorized" + : `${section.itemType}:${section.itemId}`; +} + +export function retainEligibleSoftReadPositions( + positions: ReadonlyMap, + eligibleSections: ViewSection[], +) { + const eligibleSectionKeyById = new Map(); + for (const section of eligibleSections) { + const sectionKey = softReadSectionKey(section); + for (const id of section.items) eligibleSectionKeyById.set(id, sectionKey); + } + + return new Map( + [...positions].filter( + ([id, position]) => + eligibleSectionKeyById.get(id) === position.sectionKey, + ), + ); +} + +/** Restore individually toggled items without freezing new pages or other removals. */ +export function retainSoftReadPositions( + sections: ViewSection[], + positions: ReadonlyMap, +): ViewSection[] { + if (positions.size === 0) return sections; + + const retainedBySection = new Map>(); + for (const [id, position] of positions) { + const entries = retainedBySection.get(position.sectionKey) ?? []; + entries.push([id, position.index]); + retainedBySection.set(position.sectionKey, entries); + } + + let startIndex = 0; + return sections.map((section) => { + const retained = retainedBySection.get(softReadSectionKey(section)) ?? []; + retained.sort((left, right) => left[1] - right[1]); + const liveItems = section.items.filter((id) => !positions.has(id)); + const items: string[] = []; + let liveIndex = 0; + for (const [id, index] of retained) { + while (items.length < index && liveIndex < liveItems.length) { + items.push(liveItems[liveIndex++]!); + } + items.push(id); + } + items.push(...liveItems.slice(liveIndex)); + const result = { ...section, items, startIndex }; + startIndex += items.length; + return result; + }); +} diff --git a/apps/app/src/components/feed/view-lists/useSoftReads.ts b/apps/app/src/components/feed/view-lists/useSoftReads.ts new file mode 100644 index 000000000..91f8eabcd --- /dev/null +++ b/apps/app/src/components/feed/view-lists/useSoftReads.ts @@ -0,0 +1,131 @@ +import { useCallback, useEffect, useId, useMemo, useState } from "react"; +import { useAtomValue } from "jotai"; +import { + getEligibleSoftReadIds, + retainEligibleSoftReadPositions, + retainSoftReadPositions, + softReadSectionKey, +} from "./softReads"; +import { useViewSections } from "./useViewSections"; +import type { SoftReadPosition } from "./softReads"; +import type { ViewSection } from "./useViewSections"; +import type { ApplicationView } from "~/server/db/schema"; +import { + categoryFilterAtom, + contentStatusFilterAtom, + feedFilterAtom, +} from "~/lib/data/atoms"; +import { bookmarksStore } from "~/lib/data/bookmarks/store"; +import { useFeedCategories } from "~/lib/data/feed-categories"; +import { feedItemsStore, useFeedItemsListProjection } from "~/lib/data/store"; +import { useViews } from "~/lib/data/views"; +import { + clearRetainedEntityPins, + setRetainedEntityPins, +} from "~/lib/data/page-retention"; + +export function useSoftReads( + sections: ViewSection[], + currentView: ApplicationView | null, +) { + const contentStatusFilter = useAtomValue(contentStatusFilterAtom); + const { saveStatus } = contentStatusFilter; + const categoryFilter = useAtomValue(categoryFilterAtom); + const feedFilter = useAtomValue(feedFilterAtom); + const owner = useId(); + const [positions, setPositions] = useState( + new Map(), + ); + const feedItems = useFeedItemsListProjection(); + const { feedCategories } = useFeedCategories(); + const { views } = useViews(); + const bookmarkRevision = bookmarksStore.useRevision(); + const eligibleIds = useMemo(() => { + void bookmarkRevision; + return getEligibleSoftReadIds({ + positions, + bookmarksById: bookmarksStore.getState().snapshot(), + feedItemsById: feedItems.getItems(), + feedCategories, + views, + currentView, + categoryFilter, + feedFilter, + saveStatus, + }); + }, [ + bookmarkRevision, + categoryFilter, + currentView, + feedFilter, + feedCategories, + feedItems, + positions, + saveStatus, + views, + ]); + const { computedSections: eligibleSections } = useViewSections( + currentView, + eligibleIds, + ); + const activePositions = useMemo( + () => retainEligibleSoftReadPositions(positions, eligibleSections), + [positions, eligibleSections], + ); + const retainedSections = useMemo( + () => retainSoftReadPositions(sections, activePositions), + [sections, activePositions], + ); + + const pinPositions = useCallback( + (nextPositions: ReadonlyMap) => { + const bookmarkIds: string[] = []; + const feedItemIds: string[] = []; + for (const id of nextPositions.keys()) { + if (bookmarksStore.getState().getBookmark(id)) bookmarkIds.push(id); + else if (feedItemsStore.getState().feedItemsDict[id]) + feedItemIds.push(id); + } + setRetainedEntityPins(owner, { bookmarkIds, feedItemIds }); + }, + [owner], + ); + + if (activePositions.size !== positions.size) setPositions(activePositions); + useEffect( + () => pinPositions(activePositions), + [activePositions, pinPositions], + ); + useEffect(() => () => clearRetainedEntityPins(owner), [owner]); + + const toggleRead = useCallback( + (id: string, mutate: () => boolean) => { + if (saveStatus !== "saved" || activePositions.has(id)) return mutate(); + const section = retainedSections.find((candidate) => + candidate.items.includes(id), + ); + if (!section) return mutate(); + const nextPositions = new Map(activePositions); + nextPositions.set(id, { + sectionKey: softReadSectionKey(section), + index: section.items.indexOf(id), + }); + // Pin before the optimistic mutation can remove the last page reference. + pinPositions(nextPositions); + try { + if (!mutate()) { + pinPositions(activePositions); + return false; + } + } catch (error) { + pinPositions(activePositions); + throw error; + } + setPositions(nextPositions); + return true; + }, + [activePositions, pinPositions, retainedSections, saveStatus], + ); + + return { sections: retainedSections, toggleRead }; +} diff --git a/apps/app/src/lib/hooks/useFeedItemNavigation.ts b/apps/app/src/lib/hooks/useFeedItemNavigation.ts index ffa3d9f73..14f488934 100644 --- a/apps/app/src/lib/hooks/useFeedItemNavigation.ts +++ b/apps/app/src/lib/hooks/useFeedItemNavigation.ts @@ -153,6 +153,7 @@ export function useFeedItemNavigation( items: string[], isGridLayout: boolean = false, sections?: SectionInfo[], + toggleReadInView?: (id: string, mutate: () => boolean) => boolean, ) { const [selectedItemId, setSelectedItemId] = useAtom(selectedItemIdAtom); const viewFilterId = useAtomValue(viewFilterIdAtom); @@ -567,7 +568,12 @@ export function useFeedItemNavigation( const toggleReadWithAdvance = useCallback( (contentId: string, toggleRead: () => boolean) => { const idx = items.indexOf(contentId); - if (!toggleRead()) return; + if ( + !(toggleReadInView + ? toggleReadInView(contentId, toggleRead) + : toggleRead()) + ) + return; if (idx === -1) return; if ( @@ -578,7 +584,12 @@ export function useFeedItemNavigation( selectItemAfterCurrentItemLeavesView(idx); } }, - [items, contentStatusFilter, selectItemAfterCurrentItemLeavesView], + [ + items, + contentStatusFilter, + selectItemAfterCurrentItemLeavesView, + toggleReadInView, + ], ); const toggleSavedWithAdvance = useCallback( diff --git a/apps/app/tests/e2e/fixtures/seed-db.ts b/apps/app/tests/e2e/fixtures/seed-db.ts index 2508a4975..f10d4643a 100644 --- a/apps/app/tests/e2e/fixtures/seed-db.ts +++ b/apps/app/tests/e2e/fixtures/seed-db.ts @@ -226,6 +226,17 @@ export async function getFeedItemProgress(tursoPort: number, id: string) { return feedItem?.progress ?? null; } +export async function getFeedItemWatchedState(tursoPort: number, id: string) { + const { db, client } = getDb(tursoPort); + const item = await db + .select({ isWatched: schema.feedItems.isWatched }) + .from(schema.feedItems) + .where(eq(schema.feedItems.id, id)) + .get(); + client.close(); + return item?.isWatched ?? null; +} + export async function getFeedItemWatchLaterState( tursoPort: number, id: string, diff --git a/apps/app/tests/e2e/self-hosted/item-actions.spec.ts b/apps/app/tests/e2e/self-hosted/item-actions.spec.ts index 5532b15a8..da7f5de5b 100644 --- a/apps/app/tests/e2e/self-hosted/item-actions.spec.ts +++ b/apps/app/tests/e2e/self-hosted/item-actions.spec.ts @@ -318,7 +318,9 @@ test.describe("feed item actions", () => { }); await page.keyboard.press("e"); - await expect(itemLink(firstSavedItemId)).toHaveCount(0); + await expect( + page.locator(`article[data-item-id="${firstSavedItemId}"]`), + ).toHaveCSS("opacity", "0.75"); await page .getByRole("tab", { @@ -421,8 +423,8 @@ test.describe("feed item actions", () => { } for (const bookmarkCase of [ - // Archiving moves the bookmark to saved/archived; unsaving moves it to - // inbox/unread. Both leave the saved list, so both must advance. + // Archiving retains the Saved row for this visit; unsaving removes it. + // Both actions still advance selection to the successor. { action: "archiving", buttonName: "Archive", destinationKey: "y" }, { action: "unsaving", buttonName: "Unsave", destinationKey: "i" }, ] as const) { @@ -484,7 +486,11 @@ test.describe("feed item actions", () => { .click(); await page.mouse.move(5, 5); - await expect(bookmarkItem).toHaveCount(0, { timeout: 10000 }); + if (bookmarkCase.action === "archiving") { + await expect(bookmarkItem).toHaveCSS("opacity", "0.75"); + } else { + await expect(bookmarkItem).toHaveCount(0, { timeout: 10000 }); + } const successorItem = page.locator( `article[data-item-id="${successorBookmarkId}"]`, ); diff --git a/apps/app/tests/e2e/self-hosted/sidebar-navigation-snapshot.spec.ts b/apps/app/tests/e2e/self-hosted/sidebar-navigation-snapshot.spec.ts index 55faedcfc..42ff4e235 100644 --- a/apps/app/tests/e2e/self-hosted/sidebar-navigation-snapshot.spec.ts +++ b/apps/app/tests/e2e/self-hosted/sidebar-navigation-snapshot.spec.ts @@ -198,7 +198,7 @@ test.describe("authoritative sidebar navigation", () => { await item.getByRole("link").hover(); await page.keyboard.press("e"); - await expect(item).toHaveCount(0); + await expect(item).toHaveCSS("opacity", "0.75"); await expect(viewButton.locator(".bg-sidebar-accent")).toHaveCount(0); await page diff --git a/apps/app/tests/e2e/self-hosted/soft-reads.spec.ts b/apps/app/tests/e2e/self-hosted/soft-reads.spec.ts new file mode 100644 index 000000000..4db1fcdda --- /dev/null +++ b/apps/app/tests/e2e/self-hosted/soft-reads.spec.ts @@ -0,0 +1,239 @@ +import { expect, test } from "@playwright/test"; +import { signIn } from "../fixtures/auth"; +import { + SELF_HOSTED_APP_PORT, + SELF_HOSTED_TURSO_PORT, +} from "../fixtures/ports"; +import { + cleanupUser, + getBookmarkState, + getFeedItemWatchedState, + seedArticleData, + seedBookmarkProjectionData, + seedMixedViewSectionCase, + setFeedItemWatchLater, +} from "../fixtures/seed-db"; + +test.describe("Saved soft reads", () => { + test.use({ viewport: { width: 1920, height: 1080 } }); + let email = ""; + test.afterEach(async () => { + if (email) await cleanupUser(SELF_HOSTED_TURSO_PORT, email); + }); + + for (const scope of ["Tag", "View"] as const) { + test(`removes a retained bookmark when its ${scope} assignment is removed`, async ({ + page, + }) => { + const fixture = await seedMixedViewSectionCase( + SELF_HOSTED_TURSO_PORT, + SELF_HOSTED_APP_PORT, + { + feedSectionFeedItem: false, + tagSectionFeedItem: false, + tagSectionBookmark: scope === "Tag", + uncategorizedFeedItem: false, + uncategorizedBookmark: scope === "View", + }, + ); + email = fixture.email; + await signIn({ page, email, password: fixture.password }); + await page.getByRole("tab", { name: /^Saved/ }).click(); + if (scope === "View") { + await page + .getByRole("radio", { name: fixture.viewName, exact: true }) + .click(); + } else { + await page.getByRole("button", { name: "Menu", exact: true }).click(); + await page + .locator('[data-sidebar="group"]') + .filter({ + has: page.locator('[data-sidebar="group-label"]', { + hasText: "Tags", + }), + }) + .locator('[data-sidebar="menu-button"]') + .filter({ hasText: fixture.tagName }) + .click(); + } + const id = + scope === "Tag" + ? fixture.items.tagSectionBookmark + : fixture.items.uncategorizedBookmark; + const row = page.locator(`article[data-item-id="${id}"]`); + await expect(row).toBeVisible({ timeout: 30_000 }); + await row.hover(); + await row.getByRole("button", { name: /^Archive/ }).click(); + await expect + .poll( + async () => + (await getBookmarkState(SELF_HOSTED_TURSO_PORT, id))?.isRead, + ) + .toBe(true); + await expect(row).toHaveCSS("opacity", "0.75"); + await row.hover(); + await row.getByRole("button", { name: "Edit Bookmark" }).click(); + const dialog = page.getByRole("dialog"); + const assignment = dialog.getByRole("button", { + name: scope === "Tag" ? fixture.tagName : fixture.viewName, + exact: true, + }); + await expect(assignment).toHaveAttribute("aria-pressed", "true"); + const update = page.waitForResponse((response) => + response.url().includes(`/bookmark/set${scope}`), + ); + await assignment.click(); + expect((await update).ok()).toBe(true); + await expect(assignment).toHaveAttribute("aria-pressed", "false"); + await expect(row).toHaveCount(0); + }); + } + + for (const scope of ["View", "Tag", "Feed"] as const) { + test(`clears retention when changing ${scope} scope and returning`, async ({ + page, + }) => { + const fixture = await seedMixedViewSectionCase( + SELF_HOSTED_TURSO_PORT, + SELF_HOSTED_APP_PORT, + { + feedSectionFeedItem: false, + tagSectionFeedItem: true, + tagSectionBookmark: false, + uncategorizedFeedItem: false, + uncategorizedBookmark: false, + }, + ); + email = fixture.email; + await signIn({ page, email, password: fixture.password }); + await page.getByRole("tab", { name: /^Saved/ }).click(); + const view = page.getByRole("radio", { + name: fixture.viewName, + exact: true, + }); + await view.click(); + const id = fixture.items.tagSectionFeedItem; + const row = page.locator(`article[data-item-id="${id}"]`); + await expect(row).toBeVisible({ timeout: 30_000 }); + await row.hover(); + await row.getByRole("button", { name: /^Archive/ }).click(); + await expect + .poll(() => getFeedItemWatchedState(SELF_HOSTED_TURSO_PORT, id)) + .toBe(true); + await expect(row).toHaveCSS("opacity", "0.75"); + if (scope === "View") { + await page + .getByRole("radio", { name: fixture.emptyViewName, exact: true }) + .click(); + } else { + await page.getByRole("button", { name: "Menu", exact: true }).click(); + const group = page.locator('[data-sidebar="group"]').filter({ + has: page.locator('[data-sidebar="group-label"]', { + hasText: `${scope}s`, + }), + }); + await group + .locator('[data-sidebar="menu-button"]') + .filter({ + hasText: + scope === "Tag" ? fixture.tagName : fixture.feeds.tagSection.name, + }) + .click(); + } + await expect(row).toHaveCount(0); + await view.click(); + await expect(row).toHaveCount(0); + }); + } + + for (const kind of ["feed", "bookmark"] as const) { + test(`${kind} retains individual toggles until status navigation or reader departure`, async ({ + page, + }) => { + const fixture = await seedArticleData( + SELF_HOSTED_TURSO_PORT, + SELF_HOSTED_APP_PORT, + ); + email = fixture.email; + const id = + kind === "bookmark" + ? ( + await seedBookmarkProjectionData( + SELF_HOSTED_TURSO_PORT, + email, + fixture.feedItemId, + ) + ).bookmarkId + : fixture.feedItemId; + if (kind === "feed") + await setFeedItemWatchLater(SELF_HOSTED_TURSO_PORT, id, true); + const isRead = async () => + kind === "bookmark" + ? (await getBookmarkState(SELF_HOSTED_TURSO_PORT, id))?.isRead + : await getFeedItemWatchedState(SELF_HOSTED_TURSO_PORT, id); + await signIn({ page, email, password: fixture.password }); + await page.getByRole("tab", { name: /^Saved/ }).click(); + const row = page.locator(`article[data-item-id="${id}"]`); + await expect(row).toBeVisible({ timeout: 30_000 }); + await row.hover(); + await row.getByRole("button", { name: /^Archive/ }).click(); + await expect.poll(isRead).toBe(true); + await expect(row).toHaveCSS("opacity", "0.75"); + + // A retained row remains actionable and its actual status drives opacity. + await row.hover(); + await row.getByRole("button", { name: /^Unarchive/ }).click(); + await expect.poll(isRead).toBe(false); + await expect(row).toHaveCSS("opacity", "1"); + await page.getByRole("heading", { name: "Serial", exact: true }).hover(); + await row.hover(); + await page.keyboard.press("e"); + await expect.poll(isRead).toBe(true); + await expect(row).toHaveCSS("opacity", "0.75"); + + await page + .getByRole("tab", { name: "Switch to archived content" }) + .click(); + await expect(row).toHaveCSS("opacity", "0.75"); + await page.getByRole("tab", { name: "Switch to unread content" }).click(); + await expect(row).toHaveCount(0); + + await page + .getByRole("tab", { name: "Switch to archived content" }) + .click(); + await expect(row).toBeVisible(); + await row.hover(); + await row.getByRole("button", { name: /^Unarchive/ }).click(); + await expect.poll(isRead).toBe(false); + await expect(row).toHaveCSS("opacity", "1"); + await row.getByRole("link").click(); + await expect(page).toHaveURL(`/read/${id}`); + await page.goBack(); + await expect(page).toHaveURL("/"); + await expect(row).toHaveCount(0); + await page.getByRole("tab", { name: "Switch to unread content" }).click(); + await expect(row).toBeVisible(); + + // Unsave still removes a soft-retained row immediately. + await row.hover(); + await row.getByRole("button", { name: /^Archive/ }).click(); + await expect.poll(isRead).toBe(true); + await expect(row).toHaveCSS("opacity", "0.75"); + await row.hover(); + await row.getByRole("button", { name: /^Unsave/ }).click(); + await expect(row).toHaveCount(0); + await page.getByRole("tab", { name: /^Inbox/ }).click(); + await page + .getByRole("tab", { name: "Switch to archived content" }) + .click(); + await expect(row).toHaveCSS("opacity", "0.75"); + await row.hover(); + await row.getByRole("button", { name: /^Unarchive/ }).click(); + await expect(row).toHaveCount(0); + await page.getByRole("tab", { name: "Switch to unread content" }).click(); + await expect(row).toBeVisible(); + await page.getByRole("button", { name: "Mark all as read" }).click(); + await expect(row).toHaveCount(0); + }); + } +}); diff --git a/apps/app/tests/e2e/self-hosted/view-mixed-content-matrix.spec.ts b/apps/app/tests/e2e/self-hosted/view-mixed-content-matrix.spec.ts index 55b8ccd7d..eb3eb9004 100644 --- a/apps/app/tests/e2e/self-hosted/view-mixed-content-matrix.spec.ts +++ b/apps/app/tests/e2e/self-hosted/view-mixed-content-matrix.spec.ts @@ -441,7 +441,7 @@ test.describe("exhaustive mixed-content View section matrix", () => { }); } - test("advances in Saved Unread and exposes the item in Saved Archived", async ({ + test("advances while retaining Saved Unread items until status navigation", async ({ page, }) => { test.setTimeout(60_000); @@ -484,16 +484,25 @@ test.describe("exhaustive mixed-content View section matrix", () => { ); await expect(feedItem).toBeVisible({ timeout: 30_000 }); + const initialOrder = await renderedItemIdsInOrder( + feedMain.locator("article[data-item-id]"), + ); + await feedItem.getByRole("link").hover(); await page.keyboard.press("e"); - await expect(feedItem).toHaveCount(0); + await expect(feedItem).toHaveCSS("opacity", "0.75"); + expect( + await renderedItemIdsInOrder(feedMain.locator("article[data-item-id]")), + ).toEqual(initialOrder); await expect(nextFeedItem.getByRole("link")).toHaveClass(/md:bg-muted/); await expect( feedMain.getByRole("heading", { name: "Test Blog", exact: true }), - ).toHaveCount(0); + ).toBeVisible(); await contentStatusTab(page, "Archived").click(); await expect(feedItem).toBeVisible(); + await contentStatusTab(page, "Unread").click(); + await expect(feedItem).toHaveCount(0); }); test("collapses Saved Archived into one View section", async ({ page }) => { diff --git a/apps/app/tests/unit/components/soft-reads.test.ts b/apps/app/tests/unit/components/soft-reads.test.ts new file mode 100644 index 000000000..945829bf6 --- /dev/null +++ b/apps/app/tests/unit/components/soft-reads.test.ts @@ -0,0 +1,392 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ViewSection } from "~/components/feed/view-lists/useViewSections"; +import type { ApplicationBookmark } from "~/server/mixed-content/projection"; +import type { ApplicationFeedItem, ApplicationView } from "~/server/db/schema"; +import { useSoftReads } from "~/components/feed/view-lists/useSoftReads"; +import { + getEligibleSoftReadIds, + retainEligibleSoftReadPositions, + retainSoftReadPositions, +} from "~/components/feed/view-lists/softReads"; +import { + clearRetainedEntityPins, + getRetainedEntityPins, +} from "~/lib/data/page-retention"; + +const state = vi.hoisted( + (): { + saveStatus: string; + bookmarks: Record; + feeds: Record; + revision: number; + categoryFilter: number; + feedFilter: number; + } => ({ + saveStatus: "saved", + bookmarks: {}, + feeds: {}, + revision: 0, + categoryFilter: -1, + feedFilter: -1, + }), +); +const currentView = vi.hoisted(() => ({ + id: 1, + userId: "user", + name: "Saved", + categoryIds: [10], + feedIds: [1], + contentFilter: 7, + daysWindow: 0, + readStatus: 0, + layout: "list", + placement: 0, + createdAt: new Date(0), + updatedAt: new Date(0), + isDefault: false, + viewSections: [], +})) as ApplicationView; +const atoms = vi.hoisted(() => ({ + contentStatus: {}, + category: {}, + feed: {}, +})); +vi.mock("jotai", () => ({ + useAtomValue: (atom: object) => + atom === atoms.contentStatus + ? { saveStatus: state.saveStatus, archiveStatus: "unread" } + : atom === atoms.category + ? state.categoryFilter + : state.feedFilter, +})); +vi.mock("~/lib/data/atoms", () => ({ + contentStatusFilterAtom: atoms.contentStatus, + categoryFilterAtom: atoms.category, + feedFilterAtom: atoms.feed, +})); +vi.mock("~/lib/data/bookmarks/store", () => ({ + bookmarksStore: { + useRevision: () => state.revision, + getState: () => ({ + getBookmark: (id: string) => state.bookmarks[id], + snapshot: () => state.bookmarks, + }), + }, +})); +vi.mock("~/lib/data/store", () => ({ + useFeedItemsListProjection: () => ({ getItems: () => state.feeds }), + feedItemsStore: { getState: () => ({ feedItemsDict: state.feeds }) }, +})); +vi.mock("~/lib/data/feed-categories", () => ({ + useFeedCategories: () => ({ feedCategories: [] }), +})); +vi.mock("~/lib/data/views", () => ({ + useViews: () => ({ views: [currentView] }), +})); +vi.mock("~/components/feed/view-lists/useViewSections", () => ({ + useViewSections: (_view: ApplicationView | null, ids: string[]) => ({ + computedSections: [section(ids)], + }), +})); +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +function section( + items: string[], + placement: number | null = null, +): ViewSection { + return { + name: "Saved", + items, + layout: "list", + startIndex: 0, + isUncategorized: placement === null, + itemType: placement === null ? undefined : "tag", + itemId: placement ?? undefined, + placement, + }; +} + +const roots: Array> = []; +function mount() { + const root = createRoot(document.createElement("div")); + roots.push(root); + let current: ReturnType; + function List({ sections }: { sections: ViewSection[] }) { + current = useSoftReads(sections, currentView); + return null; + } + return { + render: (ids: string[], key = "saved:unread") => + act(() => + root.render(createElement(List, { key, sections: [section(ids)] })), + ), + get items() { + return current.sections.flatMap((entry) => entry.items); + }, + toggle: (id: string, mutate = () => true) => + act(() => { + current.toggleRead(id, mutate); + }), + unmount: () => act(() => root.unmount()), + }; +} + +beforeEach(() => { + state.saveStatus = "saved"; + state.bookmarks = { + bookmark: { + id: "bookmark", + isSaved: true, + isRead: false, + viewIds: [currentView.id], + tagIds: [], + contentType: "article", + createdAt: new Date(), + } as unknown as ApplicationBookmark, + }; + state.feeds = { + feed: { + id: "feed", + feedId: 1, + isWatchLater: true, + isWatched: false, + platform: "website", + contentType: "article", + orientation: null, + postedAt: new Date(), + } as unknown as ApplicationFeedItem, + }; + state.revision++; + state.categoryFilter = -1; + state.feedFilter = -1; +}); +afterEach(() => { + for (const root of roots.splice(0)) act(() => root.unmount()); + clearRetainedEntityPins("other"); +}); + +describe("Saved soft reads for a view visit", () => { + it.each(["feed", "bookmark"])( + "retains %s through toggles, reordering, and rollback without duplicates", + (id) => { + const list = mount(); + list.render(["before", id, "after"]); + list.toggle(id, () => { + expect( + getRetainedEntityPins(id === "feed" ? "feed-item" : "bookmark").has( + id, + ), + ).toBe(true); + return true; + }); + list.render(["before", "after"]); + expect(list.items).toEqual(["before", id, "after"]); + list.toggle(id); + list.render([id, "before", "after"]); + expect(list.items).toEqual(["before", id, "after"]); + list.render(["before", id, "after"]); + expect(list.items).toEqual(["before", id, "after"]); + }, + ); + + it("does not retain Inbox items or items removed by another action", () => { + state.saveStatus = "inbox"; + const list = mount(); + list.render(["feed", "bookmark"]); + list.toggle("feed"); + list.render([]); + expect(list.items).toEqual([]); + expect(getRetainedEntityPins("feed-item").size).toBe(0); + }); + + it("discards retention on scope changes and on leaving the root route", () => { + const list = mount(); + list.render(["bookmark"]); + list.toggle("bookmark"); + list.render([]); + expect(list.items).toEqual(["bookmark"]); + list.render([], "other-view"); + expect(list.items).toEqual([]); + expect(getRetainedEntityPins("bookmark").size).toBe(0); + list.render([], "saved:unread"); + expect(list.items).toEqual([]); + list.render(["feed"]); + list.toggle("feed"); + list.unmount(); + expect(getRetainedEntityPins("feed-item").size).toBe(0); + }); + + it("does not retain rejected mutations", () => { + const list = mount(); + list.render(["bookmark"]); + list.toggle("bookmark", () => false); + list.render([]); + expect(list.items).toEqual([]); + expect(getRetainedEntityPins("bookmark").size).toBe(0); + }); + + it.each(["unsave", "delete"])( + "releases retained items after %s", + (action) => { + const list = mount(); + list.render(["feed", "bookmark"]); + list.toggle("feed"); + list.toggle("bookmark"); + list.render([]); + expect(list.items).toEqual(["feed", "bookmark"]); + state.bookmarks = + action === "delete" + ? {} + : { + bookmark: { + ...state.bookmarks.bookmark!, + isSaved: false, + }, + }; + state.feeds = + action === "delete" + ? {} + : { + feed: { ...state.feeds.feed!, isWatchLater: false }, + }; + state.revision++; + list.render([]); + expect(list.items).toEqual([]); + expect(getRetainedEntityPins("bookmark").size).toBe(0); + expect(getRetainedEntityPins("feed-item").size).toBe(0); + }, + ); + + it("releases a retained bookmark that leaves the selected Tag", () => { + state.categoryFilter = 10; + state.bookmarks.bookmark = { + ...state.bookmarks.bookmark!, + tagIds: [10], + }; + const list = mount(); + list.render(["bookmark"]); + list.toggle("bookmark"); + list.render([]); + expect(list.items).toEqual(["bookmark"]); + expect(getRetainedEntityPins("bookmark").has("bookmark")).toBe(true); + + state.bookmarks.bookmark = { + ...state.bookmarks.bookmark, + tagIds: [], + }; + state.revision++; + list.render([]); + + expect(list.items).toEqual([]); + expect(getRetainedEntityPins("bookmark").has("bookmark")).toBe(false); + }); + + it("retains positions in multiple sections while accepting new pages", () => { + const sections = [section(["a", "c"], 1), section(["e", "new"], 2)]; + const result = retainSoftReadPositions( + sections, + new Map([ + ["b", { sectionKey: "tag:1", index: 1 }], + ["d", { sectionKey: "tag:2", index: 0 }], + ]), + ); + expect(result.map((entry) => entry.items)).toEqual([ + ["a", "b", "c"], + ["d", "e", "new"], + ]); + expect(result.map((entry) => entry.startIndex)).toEqual([0, 3]); + }); + + it("drops bookmarks that leave the selected Tag or View", () => { + const bookmark = { + ...state.bookmarks.bookmark!, + tagIds: [10], + viewIds: [currentView.id], + }; + const positions = new Map([ + ["bookmark", { sectionKey: "uncategorized", index: 0 }], + ]); + const input = { + positions, + bookmarksById: { bookmark }, + feedItemsById: {}, + feedCategories: [], + views: [currentView], + currentView, + feedFilter: -1, + saveStatus: "saved" as const, + }; + + expect(getEligibleSoftReadIds({ ...input, categoryFilter: 10 })).toEqual([ + "bookmark", + ]); + expect( + getEligibleSoftReadIds({ + ...input, + categoryFilter: 10, + bookmarksById: { bookmark: { ...bookmark, tagIds: [] } }, + }), + ).toEqual([]); + expect( + getEligibleSoftReadIds({ + ...input, + categoryFilter: -1, + bookmarksById: { bookmark: { ...bookmark, tagIds: [] } }, + }), + ).toEqual(["bookmark"]); + expect( + getEligibleSoftReadIds({ + ...input, + categoryFilter: -1, + bookmarksById: { + bookmark: { ...bookmark, tagIds: [], viewIds: [] }, + }, + }), + ).toEqual([]); + }); + + it("drops feed items that leave the selected View", () => { + const positions = new Map([ + ["feed", { sectionKey: "uncategorized", index: 0 }], + ]); + const input = { + positions, + bookmarksById: {}, + feedItemsById: state.feeds, + feedCategories: [], + currentView, + categoryFilter: -1, + feedFilter: -1, + saveStatus: "saved" as const, + }; + + expect(getEligibleSoftReadIds({ ...input, views: [currentView] })).toEqual([ + "feed", + ]); + const removedFeedView = { ...currentView, feedIds: [] }; + expect( + getEligibleSoftReadIds({ + ...input, + currentView: removedFeedView, + views: [removedFeedView], + }), + ).toEqual([]); + }); + + it("drops positions whose assigned section changed or disappeared", () => { + const positions = new Map([ + ["moved", { sectionKey: "tag:1", index: 0 }], + ["missing", { sectionKey: "tag:3", index: 1 }], + ["stable", { sectionKey: "tag:2", index: 2 }], + ]); + const eligibleSections = [section([], 1), section(["moved", "stable"], 2)]; + + expect([ + ...retainEligibleSoftReadPositions(positions, eligibleSections).keys(), + ]).toEqual(["stable"]); + }); +}); diff --git a/apps/app/tests/unit/performance/client-audit-model.test.ts b/apps/app/tests/unit/performance/client-audit-model.test.ts index 3cf25e75d..d2612176a 100644 --- a/apps/app/tests/unit/performance/client-audit-model.test.ts +++ b/apps/app/tests/unit/performance/client-audit-model.test.ts @@ -53,6 +53,12 @@ describe("client performance audit model", () => { mixedStoreNotifications: 0, authoritativeRefills: 0, }); + expect(result.operations.softReadProjection).toMatchObject({ + bookmarkStoreNotifications: 0, + feedItemStoreNotifications: 0, + mixedStoreNotifications: 0, + authoritativeRefills: 0, + }); }, 30_000, ); @@ -84,6 +90,14 @@ describe("client performance audit model", () => { ); }); + it("includes soft-read projection in the operation duration gate", () => { + const result = runClientAuditProfile("small"); + result.operations.softReadProjection.durationMs = 51; + expect(evaluateClientAuditOperationBudgets(result)).toContain( + "softReadProjection: 51.0ms > 50ms", + ); + }); + it("keeps normalized persistence mutations within their explicit budget", () => { const result = runClientAuditProfile("stress"); From c19ca8a8a7f3e9f95965604f80d173213b278729 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:38:40 -0400 Subject: [PATCH 02/12] fix stale reconciliation retries after mutations --- .../components/feed/RefetchItemsButton.tsx | 2 + apps/app/src/lib/data/loading-machine.ts | 3 + apps/app/src/lib/data/reconciliation.ts | 23 +- apps/app/src/lib/reconciliation/contracts.ts | 2 + .../app/src/lib/reconciliation/coordinator.ts | 16 +- apps/app/src/lib/reconciliation/runtime.ts | 68 +++- .../reconciliation/client-runtime.test.ts | 201 ++++++++++ .../mutation-reconciliation.test.ts | 367 ++++++++++++++++++ .../reconciliation/refresh-recovery.test.ts | 97 +++++ 9 files changed, 754 insertions(+), 25 deletions(-) create mode 100644 apps/app/tests/unit/reconciliation/mutation-reconciliation.test.ts create mode 100644 apps/app/tests/unit/reconciliation/refresh-recovery.test.ts diff --git a/apps/app/src/components/feed/RefetchItemsButton.tsx b/apps/app/src/components/feed/RefetchItemsButton.tsx index 07a7a0edd..dac756ea2 100644 --- a/apps/app/src/components/feed/RefetchItemsButton.tsx +++ b/apps/app/src/components/feed/RefetchItemsButton.tsx @@ -137,6 +137,8 @@ export function RefetchItemsButton() { "Checking for newer data" ) : reconciliationStatus === "retrying" ? ( "Data may be stale. Retrying automatically" + ) : reconciliationStatus === "stale" ? ( + "Data may be stale. Refresh to try again." ) : ( <> Refresh available in{" "} diff --git a/apps/app/src/lib/data/loading-machine.ts b/apps/app/src/lib/data/loading-machine.ts index c50a0e64d..942de50e3 100644 --- a/apps/app/src/lib/data/loading-machine.ts +++ b/apps/app/src/lib/data/loading-machine.ts @@ -9,6 +9,7 @@ type LoadingMachineEvent = // Initial reconciliation | { type: "INITIAL_LOAD_START" } | { type: "RECONCILIATION_COMPLETE" } + | { type: "RECONCILIATION_FAILED" } // Background RSS refresh | { type: "BACKGROUND_REFRESH_START"; totalFeeds: number } @@ -168,6 +169,7 @@ export const loadingMachine = setup({ initialLoad: { on: { RECONCILIATION_COMPLETE: { target: "idle" }, + RECONCILIATION_FAILED: { target: "idle" }, // Import can interrupt initial load IMPORT_START: { target: "importing", @@ -219,6 +221,7 @@ export const loadingMachine = setup({ manualRefresh: { on: { RECONCILIATION_COMPLETE: { target: "idle" }, + RECONCILIATION_FAILED: { target: "idle" }, BACKGROUND_REFRESH_START: { target: "backgroundRefresh", actions: assign({ diff --git a/apps/app/src/lib/data/reconciliation.ts b/apps/app/src/lib/data/reconciliation.ts index 247351c2c..7ba4323c5 100644 --- a/apps/app/src/lib/data/reconciliation.ts +++ b/apps/app/src/lib/data/reconciliation.ts @@ -46,6 +46,7 @@ import { getReconciliationTargetKey, MAX_TARGETED_RECONCILIATION_TARGETS, } from "~/lib/reconciliation"; +import { ITEMS_PER_PAGE } from "~/server/api/constants"; import { DEFAULT_CONTENT_STATUS_FILTER } from "~/lib/content-status"; type PersistedStore = { @@ -114,7 +115,12 @@ function firstPageManifest( ); const feedItems: ReconciliationEntityManifestEntry[] = []; const bookmarks: ReconciliationEntityManifestEntry[] = []; - for (const referenceKey of rootPage.value.referenceKeys) { + // Optimistic membership can exceed a server page. Omitted versions are + // returned as upserts; cached references and pagination remain intact. + for (const referenceKey of rootPage.value.referenceKeys.slice( + 0, + ITEMS_PER_PAGE, + )) { const reference = referencesByKey.get(referenceKey); if (!reference) continue; if (reference.entityKind === "feed-item") { @@ -161,7 +167,9 @@ function buildInput( type: "selected", scope: selectedScope.scope, contentStatus: selectedScope.contentStatus, - pageManifest: firstPageManifest(selectedScope), + pageManifest: request.intent.discardManifest + ? { feedItems: [], bookmarks: [] } + : firstPageManifest(selectedScope), membershipRevision: getMixedContentMembershipRevision(), }, }; @@ -544,6 +552,9 @@ const runtime = createReconciliationRuntime({ }); } }, + onRecoveryFailed: () => { + loadingActor.send({ type: "RECONCILIATION_FAILED" }); + }, onFullReconciliationFailed: ({ reconciliationId }) => { if (reconciliationId !== supersededManualFullId) { rejectManualFull?.(new Error("Manual reconciliation failed")); @@ -687,13 +698,15 @@ export function getReconciliationTargetStatus( return runtime.getState().targets[getReconciliationTargetKey(target)]?.status; } -export type ReconciliationDisplayStatus = "idle" | "syncing" | "retrying"; +export type ReconciliationDisplayStatus = + "idle" | "syncing" | "retrying" | "stale"; function reconciliationDisplayStatus(): ReconciliationDisplayStatus { const state = runtime.getState(); - if (state.cacheUsableAt === null) return "idle"; + if (state.cacheUsableAt === null && !state.recoveryFailed) return "idle"; if (state.inFlight) return "syncing"; - return state.retryPending ? "retrying" : "idle"; + if (state.retryPending) return "retrying"; + return state.recoveryFailed ? "stale" : "idle"; } export function useReconciliationDisplayStatus() { diff --git a/apps/app/src/lib/reconciliation/contracts.ts b/apps/app/src/lib/reconciliation/contracts.ts index 4bee2ab75..b6ca84dca 100644 --- a/apps/app/src/lib/reconciliation/contracts.ts +++ b/apps/app/src/lib/reconciliation/contracts.ts @@ -181,10 +181,12 @@ export type ReconciliationStreamEvent = { export type ReconciliationRequestIntent = | { type: "full"; + discardManifest?: boolean; selectedScope: ReconciliationScopeTarget; } | { type: "full"; + discardManifest?: boolean; coldContentStatus: ContentStatusFilter; } | { diff --git a/apps/app/src/lib/reconciliation/coordinator.ts b/apps/app/src/lib/reconciliation/coordinator.ts index dd1b91eb8..b4af67cab 100644 --- a/apps/app/src/lib/reconciliation/coordinator.ts +++ b/apps/app/src/lib/reconciliation/coordinator.ts @@ -98,6 +98,7 @@ export type ReconciliationCoordinatorState< serverParityAppliedAt: number | null; sseConnected: boolean; retryPending: boolean; + recoveryFailed: boolean; retryAt: number | null; automaticRssOwner: AutomaticRssOwner | null; trustedUpToDate: boolean; @@ -175,6 +176,9 @@ export type ReconciliationCoordinatorEvent = failed?: boolean; failedTargets?: ReconciliationTarget[]; epochComplete?: boolean; + inputRejected?: boolean; + recoveryIntent?: ReconciliationRequestIntent; + recoveryFailed?: boolean; } | { type: "automatic-rss-owner-resolved"; @@ -232,6 +236,7 @@ export function createReconciliationCoordinatorState< serverParityAppliedAt: null, sseConnected: false, retryPending: false, + recoveryFailed: false, retryAt: null, automaticRssOwner: null, trustedUpToDate: false, @@ -666,6 +671,7 @@ function withEstablishedFullParity( ...state, latestFullEpoch: { ...state.latestFullEpoch, established: true }, serverParityAppliedAt: at, + recoveryFailed: false, }; } @@ -687,6 +693,7 @@ function deriveTrustedUpToDate( targetKeysOverlap(targetKeys, requiredTargetKeys), ); return Boolean( + !state.recoveryFailed && state.cacheUsableAt !== null && state.latestFullEpoch?.established && state.serverParityAppliedAt !== null && @@ -977,8 +984,15 @@ export function transitionReconciliation( ...nextState, inFlight: null, trailingIntent: null, + recoveryFailed: event.recoveryFailed ?? nextState.recoveryFailed, }; - return trailingIntent + if (event.recoveryIntent) { + return startRequest( + { ...nextState, trailingIntent }, + event.recoveryIntent, + ); + } + return trailingIntent && !event.inputRejected ? startRequest(nextState, trailingIntent) : { state: withDerivedTrust(nextState), commands: [] }; } diff --git a/apps/app/src/lib/reconciliation/runtime.ts b/apps/app/src/lib/reconciliation/runtime.ts index 2cd63ad6b..2ce21b9fc 100644 --- a/apps/app/src/lib/reconciliation/runtime.ts +++ b/apps/app/src/lib/reconciliation/runtime.ts @@ -1,3 +1,4 @@ +import { ORPCError } from "@orpc/client"; import { createReconciliationCoordinatorState, transitionReconciliation, @@ -68,6 +69,7 @@ export type ReconciliationRuntimeDependencies = { automaticRssOwner: ReconciliationCoordinatorState["automaticRssOwner"]; }) => void; onFullReconciliationFailed?: (input: { reconciliationId: string }) => void; + onRecoveryFailed?: () => void; }; export type ReconciliationRuntime = ReturnType< @@ -128,6 +130,7 @@ export function createReconciliationRuntime( let completedBeforeFirstConnection = false; let liveSequence = 0; let retryDelayMs = 1_000; + let recoveringInvalidInput = false; let retryIntent: ReconciliationRequestIntent | null = null; let retryTimer: ReturnType | null = null; const listeners = new Set<() => void>(); @@ -219,23 +222,31 @@ export function createReconciliationRuntime( if (successfulRequestCoversRetry(request)) clearRetry(); }; + const currentFullIntent = (): ReconciliationRequestIntent => { + const selectedScope = dependencies.getCurrentSelection(); + const intent: ReconciliationRequestIntent = selectedScope + ? { type: "full", selectedScope } + : { + type: "full", + coldContentStatus: { saveStatus: "inbox", archiveStatus: "unread" }, + }; + return recoveringInvalidInput + ? { ...intent, discardManifest: true } + : intent; + }; + const retryIntentForFailure = ( failedTargets: ReconciliationTarget[] | undefined, forceFull = false, ): ReconciliationRequestIntent => { if ( + recoveringInvalidInput || forceFull || !failedTargets || failedTargets.length === 0 || failedTargets.length > MAX_TARGETED_RECONCILIATION_TARGETS ) { - const selectedScope = dependencies.getCurrentSelection(); - return selectedScope - ? { type: "full", selectedScope } - : { - type: "full", - coldContentStatus: { saveStatus: "inbox", archiveStatus: "unread" }, - }; + return currentFullIntent(); } return { type: "targeted", targets: failedTargets }; }; @@ -311,6 +322,7 @@ export function createReconciliationRuntime( previousParity !== state.serverParityAppliedAt && state.serverParityAppliedAt !== null ) { + recoveringInvalidInput = false; dependencies.mark?.( "serial:server-parity-applied", state.latestFullEpoch?.reconciliationId, @@ -360,6 +372,7 @@ export function createReconciliationRuntime( > = {}; const recoverableFailures = new Map(); let settled = false; + let inputRejected = false; try { const input = dependencies.buildInput(request); const stream = await dependencies.openStream(input, signal); @@ -463,6 +476,13 @@ export function createReconciliationRuntime( } } } catch (error) { + inputRejected = + error instanceof ORPCError && + error.code === "BAD_REQUEST" && + typeof error.data === "object" && + error.data !== null && + "issues" in error.data && + Array.isArray(error.data.issues); if (!signal.aborted) { console.error("Reconciliation request failed", error); } @@ -473,15 +493,33 @@ export function createReconciliationRuntime( completedBeforeFirstConnection = true; } const failedTargets = state.requests[request.reconciliationId]?.targets; + const recoveryFailed = inputRejected && recoveringInvalidInput; + if (inputRejected) { + recoveringInvalidInput = true; + clearRetry(); + } send({ type: "request-settled", reconciliationId: request.reconciliationId, at: dependencies.now(), failed: true, failedTargets, + inputRejected, + recoveryIntent: + inputRejected && !recoveryFailed ? currentFullIntent() : undefined, + recoveryFailed: recoveryFailed || undefined, }); - scheduleRetry(retryIntentForFailure(failedTargets)); - if (request.intent.type === "full") { + if (inputRejected) { + if (recoveryFailed) { + dependencies.onRecoveryFailed?.(); + } + } else { + scheduleRetry(retryIntentForFailure(failedTargets)); + } + if ( + request.intent.type === "full" && + (!inputRejected || recoveryFailed) + ) { dependencies.onFullReconciliationFailed?.({ reconciliationId: request.reconciliationId, }); @@ -492,16 +530,7 @@ export function createReconciliationRuntime( const requestCurrentFull = () => { if (retryIntent) clearRetry(); - const selectedScope = dependencies.getCurrentSelection(); - send({ - type: "request-reconciliation", - intent: selectedScope - ? { type: "full", selectedScope } - : { - type: "full", - coldContentStatus: { saveStatus: "inbox", archiveStatus: "unread" }, - }, - }); + send({ type: "request-reconciliation", intent: currentFullIntent() }); }; return { @@ -516,6 +545,7 @@ export function createReconciliationRuntime( clearRetryTimer(); retryIntent = null; retryDelayMs = 1_000; + recoveringInvalidInput = false; started = false; everConnected = false; reconnectRequired = false; diff --git a/apps/app/tests/unit/reconciliation/client-runtime.test.ts b/apps/app/tests/unit/reconciliation/client-runtime.test.ts index 61b66dbb0..b961683c4 100644 --- a/apps/app/tests/unit/reconciliation/client-runtime.test.ts +++ b/apps/app/tests/unit/reconciliation/client-runtime.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { ORPCError } from "@orpc/client"; import type { ActiveFirstPageResult, OrganizationSnapshot, @@ -1048,3 +1049,203 @@ describe("client reconciliation runtime", () => { vi.useRealTimers(); }); }); + +const invalidInput = () => + new ORPCError("BAD_REQUEST", { + data: { + issues: [ + { code: "too_big", path: ["selection", "pageManifest", "feedItems"] }, + ], + }, + }); + +describe("rejected reconciliation recovery", () => { + it("recovers once with a full request preserving the current selection", async () => { + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + const test = harness((request) => { + if (request.reconciliationId.endsWith("-1")) throw invalidInput(); + return completeEpoch(request.reconciliationId); + }); + test.setSelection(ACTIVE_SCOPE); + hydrate(test.runtime); + test.runtime.cacheUsable(); + test.runtime.sseConnectionChanged(true); + try { + test.runtime.start(); + await vi.waitFor(() => + expect(test.runtime.getState().trustedUpToDate).toBe(true), + ); + expect(test.requests).toHaveLength(2); + expect(test.requests[1]?.intent).toEqual({ + type: "full", + selectedScope: ACTIVE_SCOPE, + discardManifest: true, + }); + expect(test.runtime.getState().retryPending).toBe(false); + } finally { + test.runtime.stop(); + log.mockRestore(); + } + }); + + it("stops after rejected recovery, discards trailing work, and permits manual recovery", async () => { + vi.useFakeTimers(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + let rejects = true; + let queueTrailing = true; + const test = harness((request) => { + if (rejects) { + if (queueTrailing) { + queueTrailing = false; + test.runtime.requestFull(); + } + throw invalidInput(); + } + return completeEpoch(request.reconciliationId); + }); + test.setSelection(ACTIVE_SCOPE); + hydrate(test.runtime); + test.runtime.cacheUsable(); + test.runtime.sseConnectionChanged(true); + try { + test.runtime.start(); + await vi.advanceTimersByTimeAsync(60_000); + expect(test.requests).toHaveLength(2); + expect(test.runtime.getState()).toMatchObject({ + recoveryFailed: true, + retryPending: false, + inFlight: null, + trustedUpToDate: false, + }); + rejects = false; + test.runtime.requestFull(); + await vi.advanceTimersByTimeAsync(0); + expect(test.requests).toHaveLength(3); + expect(test.requests[2]?.intent).toMatchObject({ discardManifest: true }); + expect(test.runtime.getState()).toMatchObject({ + recoveryFailed: false, + trustedUpToDate: true, + }); + } finally { + test.runtime.stop(); + log.mockRestore(); + vi.useRealTimers(); + } + }); + + it("retains empty manifests through a transient recovery failure", async () => { + vi.useFakeTimers(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + const test = harness((request) => { + if (test.requests.length === 1) throw invalidInput(); + if (test.requests.length === 2) throw new Error("network unavailable"); + if (test.requests.length === 3) throw invalidInput(); + return completeEpoch(request.reconciliationId); + }); + test.setSelection(ACTIVE_SCOPE); + hydrate(test.runtime); + try { + test.runtime.start(); + await vi.advanceTimersByTimeAsync(60_000); + expect(test.requests).toHaveLength(3); + expect(test.requests[2]?.intent).toMatchObject({ + type: "full", + discardManifest: true, + }); + expect(test.runtime.getState()).toMatchObject({ + recoveryFailed: true, + retryPending: false, + }); + } finally { + test.runtime.stop(); + log.mockRestore(); + vi.useRealTimers(); + } + }); +}); + +describe("recovery target ownership", () => { + it("keeps terminal stale state after an unrelated targeted success", async () => { + vi.useFakeTimers(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + let rejectFull = true; + const test = harness((request) => { + if (rejectFull) throw invalidInput(); + return completeTargetedScope( + request.reconciliationId, + INACTIVE_VIEW_SCOPE, + ); + }); + hydrate(test.runtime); + test.setSelection(ACTIVE_SCOPE); + test.runtime.cacheUsable(); + test.runtime.sseConnectionChanged(true); + try { + test.runtime.start(); + await vi.advanceTimersByTimeAsync(0); + expect(test.runtime.getState().recoveryFailed).toBe(true); + rejectFull = false; + test.runtime.activateScope(INACTIVE_VIEW_SCOPE); + await vi.advanceTimersByTimeAsync(0); + expect(test.requests).toHaveLength(3); + expect(test.runtime.getState()).toMatchObject({ + recoveryFailed: true, + retryPending: false, + inFlight: null, + serverParityAppliedAt: null, + }); + } finally { + test.runtime.stop(); + log.mockRestore(); + vi.useRealTimers(); + } + }); + + it.each([false, true])( + "retains queued repairs when full recovery has a transient failure: %s", + async (transientFailure) => { + vi.useFakeTimers(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + const feedTarget: ReconciliationScopeTarget = { + ...ACTIVE_SCOPE, + scope: { type: "feed", feedId: 42 }, + }; + const test = harness( + (request) => { + if (test.requests.length === 1) { + test.runtime.receiveLiveEvent(["feed-changed"]); + throw invalidInput(); + } + if (transientFailure && test.requests.length === 2) + throw new Error("network unavailable"); + return request.intent.type === "full" + ? completeEpoch(request.reconciliationId) + : completeTargetedScope(request.reconciliationId, feedTarget); + }, + () => [feedTarget], + { liveEventTargets: () => ({ targets: [feedTarget] }) }, + ); + hydrate(test.runtime); + test.setSelection(ACTIVE_SCOPE); + try { + test.runtime.start(); + await vi.advanceTimersByTimeAsync(60_000); + expect( + test.runtime.getState().targets[ + getReconciliationTargetKey(feedTarget) + ]?.status, + ).toBe("verified"); + expect(test.requests.map(({ intent }) => intent.type)).toEqual( + transientFailure + ? ["full", "full", "targeted", "full"] + : ["full", "full", "targeted"], + ); + expect(test.runtime.getState().retryPending).toBe(false); + } finally { + test.runtime.stop(); + log.mockRestore(); + vi.useRealTimers(); + } + }, + ); +}); diff --git a/apps/app/tests/unit/reconciliation/mutation-reconciliation.test.ts b/apps/app/tests/unit/reconciliation/mutation-reconciliation.test.ts new file mode 100644 index 000000000..bf8eb6a8d --- /dev/null +++ b/apps/app/tests/unit/reconciliation/mutation-reconciliation.test.ts @@ -0,0 +1,367 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createBookmarkTestDatabase } from "../bookmarks/database"; +import type { ORPCContext } from "~/server/orpc/base"; +import { feedItems, feeds, user, viewFeeds, views } from "~/server/db/schema"; + +const testState = vi.hoisted( + (): { + database: unknown; + fetchDueSources: ReturnType; + } => ({ + database: undefined, + fetchDueSources: vi.fn(), + }), +); + +vi.mock("~/server/db", () => ({ + get db() { + return testState.database; + }, +})); +vi.mock("~/server/auth", () => ({ auth: {} })); +vi.mock("~/env", () => ({ + env: { + BACKGROUND_REFRESH_ENABLED: false, + DATABASE_URL: "file::memory:", + KV_STORE: "none", + PUBLIC_BASE_URL: "http://localhost:3000", + TRUSTED_ORIGINS: [], + }, +})); +vi.mock("~/lib/orpc", async () => { + const { createRouterClient: createClient } = await import("@orpc/server"); + const { orpcRouter: router } = await import("~/server/orpc/router"); + const real = createClient(router, { + context: () => + ({ + headers: new Headers(), + session: { id: "probe-session" }, + user: { id: USER_ID }, + db: testState.database, + }) as ORPCContext, + }); + return { + orpc: new Proxy( + {}, + { get: () => new Proxy({}, { get: () => () => ({}) }) }, + ), + orpcRouterClient: { + feedItem: { setWatchedValue: real.feedItem.setWatchedValue }, + bookmark: { getCaptures: real.bookmark.getCaptures }, + initial: { + reconcileApplicationState: real.initial.reconcileApplicationState, + requestFullTextForItems: real.initial.requestFullTextForItems, + getNavigationSnapshot: real.initial.getNavigationSnapshot, + fetchDueSources: testState.fetchDueSources, + }, + }, + }; +}); + +const USER_ID = "probe-user"; +const VIEW_ID = 10; +const FEED_ID = 20; +const NOW = new Date("2026-09-11T12:00:00.000Z"); +const UNREAD_COUNT = 30; +const ARCHIVED_COUNT = 30; + +type TestDatabase = Awaited< + ReturnType +>["database"]; +type Cleanup = Awaited< + ReturnType +>["cleanup"]; + +let database: TestDatabase; +let cleanup: Cleanup; + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitFor( + predicate: () => boolean, + { timeoutMs = 10_000, label = "condition" } = {}, +) { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > timeoutMs) { + throw new Error(`Timed out waiting for ${label}`); + } + await sleep(20); + } +} + +beforeEach(async () => { + vi.resetModules(); + ({ database, cleanup } = await createBookmarkTestDatabase()); + testState.database = database; + testState.fetchDueSources.mockResolvedValue({ + status: "cooldown", + nextRefreshAt: new Date(Date.now() + 60_000), + }); + await database.insert(user).values({ + id: USER_ID, + name: "Probe user", + email: "probe@example.com", + emailVerified: true, + createdAt: NOW, + updatedAt: NOW, + }); + await database.insert(views).values({ + id: VIEW_ID, + userId: USER_ID, + name: "Reading", + contentFilter: 3, + placement: 1, + }); + await database.insert(feeds).values({ + id: FEED_ID, + userId: USER_ID, + name: "Reading feed", + url: "https://example.com/feed.xml", + platform: "website", + }); + await database.insert(viewFeeds).values({ viewId: VIEW_ID, feedId: FEED_ID }); + const rows = []; + for (let index = 0; index < UNREAD_COUNT; index++) { + rows.push({ + id: `unread-${index}`, + feedId: FEED_ID, + contentId: `unread-${index}`, + title: `Unread ${index}`, + author: "Author", + url: `https://example.com/unread-${index}`, + postedAt: new Date(NOW.getTime() - index * 60_000), + createdAt: NOW, + updatedAt: NOW, + isWatched: false, + }); + } + for (let index = 0; index < ARCHIVED_COUNT; index++) { + rows.push({ + id: `archived-${index}`, + feedId: FEED_ID, + contentId: `archived-${index}`, + title: `Archived ${index}`, + author: "Author", + url: `https://example.com/archived-${index}`, + postedAt: new Date(NOW.getTime() - index * 60_000), + createdAt: NOW, + updatedAt: NOW, + isWatched: true, + isWatchedUpdatedAt: NOW, + }); + } + if (rows.length > 0) await database.insert(feedItems).values(rows); + await import("~/lib/data/reconciliation"); +}, 30_000); + +afterEach(async () => { + const { dataReconciliation } = await import("~/lib/data/reconciliation"); + dataReconciliation.stop(); + const { loadingActor } = await import("~/lib/data/loading-machine"); + loadingActor.stop(); + cleanup(); + vi.restoreAllMocks(); +}); + +describe("mutation reconciliation", () => { + it.each([ + { mutation: "archive", duringLoad: false }, + { mutation: "unarchive", duringLoad: false }, + { mutation: "archive", duringLoad: true }, + { mutation: "unarchive", duringLoad: true }, + ])( + "repairs a full destination page after $mutation, during load: $duringLoad", + async ({ mutation, duringLoad }) => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { dataReconciliation } = await import("~/lib/data/reconciliation"); + const { publisher } = await import("~/server/api/publisher"); + const { feedItemsStore } = await import("~/lib/data/store"); + const { mixedContentStore, getMixedScopeKey } = + await import("~/lib/data/mixed-content/store"); + const { orpcRouterClient } = await import("~/lib/orpc"); + const { applyOptimisticWatchedValue, resolveOptimisticWatchedValue } = + await import("~/lib/data/feed-items/mutations"); + const originalReconcile = + orpcRouterClient.initial.reconcileApplicationState; + let resumeLoad = () => {}; + const loadGate = new Promise((resolve) => { + resumeLoad = resolve; + }); + const reconcile = vi + .spyOn(orpcRouterClient.initial, "reconcileApplicationState") + .mockImplementation(async (...args) => { + const stream = await originalReconcile(...args); + return (async function* () { + for await (const event of stream) { + yield event; + if ( + duringLoad && + event.chunk.type === "domain-complete" && + event.chunk.domain === "active-scope" && + event.chunk.target?.contentStatus.saveStatus === "inbox" && + event.chunk.target.contentStatus.archiveStatus === "archived" + ) + await loadGate; + } + })(); + }); + const controller = new AbortController(); + const subscription = publisher.subscribe(`user:${USER_ID}`, { + signal: controller.signal, + }); + const forwarding = (async () => { + try { + for await (const payload of subscription) + dataReconciliation.receivePublishedChunks([payload]); + } catch { + // Subscription abort during cleanup. + } + })(); + try { + await sleep(0); + dataReconciliation.start(); + dataReconciliation.sseConnectionChanged(true); + await waitFor(() => + duringLoad + ? mixedContentStore + .getState() + .scopes[ + getMixedScopeKey( + { type: "view", viewId: VIEW_ID }, + { saveStatus: "inbox", archiveStatus: "archived" }, + ) + ]?.pages.some( + (page) => + page.requestCursorKey === "root" && + page.value.referenceKeys.length === 30, + ) === true + : dataReconciliation.getState().serverParityAppliedAt !== null && + dataReconciliation.getState().inFlight === null, + ); + if (duringLoad) + expect(dataReconciliation.getState().inFlight?.intent.type).toBe( + "full", + ); + const requestsBeforeMutation = reconcile.mock.calls.length; + const itemId = mutation === "archive" ? "unread-0" : "archived-0"; + const isWatched = mutation === "archive"; + const context = applyOptimisticWatchedValue(itemId, isWatched); + const serverValue = await orpcRouterClient.feedItem.setWatchedValue({ + id: itemId, + feedId: FEED_ID, + isWatched, + }); + resolveOptimisticWatchedValue(context, serverValue); + resumeLoad(); + await waitFor( + () => + reconcile.mock.calls.length > requestsBeforeMutation && + dataReconciliation.getState().inFlight === null, + ); + expect(errors.mock.calls).toEqual([]); + expect(dataReconciliation.getState().retryPending).toBe(false); + expect(dataReconciliation.getState().trustedUpToDate).toBe(true); + const destination = + mixedContentStore.getState().scopes[ + getMixedScopeKey( + { type: "view", viewId: VIEW_ID }, + { + saveStatus: "inbox", + archiveStatus: isWatched ? "archived" : "unread", + }, + ) + ]; + expect( + destination?.pages.find((page) => page.requestCursorKey === "root") + ?.value.referenceKeys, + ).toHaveLength(30); + expect(feedItemsStore.getState().feedItemsDict[itemId]?.isWatched).toBe( + isWatched, + ); + } finally { + resumeLoad(); + controller.abort(); + await forwarding; + } + }, + ); +}); + +describe("invalid request recovery through the transport", () => { + it("keeps manual refresh pending until recovery restores the selected page", async () => { + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + const { dataReconciliation } = await import("~/lib/data/reconciliation"); + const { orpcRouterClient } = await import("~/lib/orpc"); + dataReconciliation.start(); + dataReconciliation.sseConnectionChanged(true); + await waitFor( + () => + dataReconciliation.getState().serverParityAppliedAt !== null && + dataReconciliation.getState().inFlight === null, + ); + const original = orpcRouterClient.initial.reconcileApplicationState; + const reconcile = vi + .spyOn(orpcRouterClient.initial, "reconcileApplicationState") + .mockImplementationOnce((input, options) => + original({ ...input, reconciliationId: "" }, options), + ); + await dataReconciliation.requestManualFull(); + expect(reconcile).toHaveBeenCalledTimes(2); + const originalInput = reconcile.mock.calls[0]![0]; + const recoveryInput = reconcile.mock.calls[1]![0]; + expect(originalInput).toMatchObject({ + type: "full", + selection: { type: "selected", scope: { type: "view", viewId: VIEW_ID } }, + }); + if (originalInput.type !== "full") + throw new Error("Expected full reconciliation"); + expect(recoveryInput).toMatchObject({ + type: "full", + selection: { + type: "selected", + scope: { type: "view", viewId: VIEW_ID }, + contentStatus: originalInput.selection.contentStatus, + pageManifest: { feedItems: [], bookmarks: [] }, + }, + }); + expect(log).toHaveBeenCalledTimes(1); + expect(dataReconciliation.getState()).toMatchObject({ + trustedUpToDate: true, + retryPending: false, + recoveryFailed: false, + }); + }); + + it("releases initial loading after terminal rejection and supports a new manual attempt", async () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + const { dataReconciliation } = await import("~/lib/data/reconciliation"); + const { loadingActor } = await import("~/lib/data/loading-machine"); + const { orpcRouterClient } = await import("~/lib/orpc"); + const original = orpcRouterClient.initial.reconcileApplicationState; + const reconcile = vi + .spyOn(orpcRouterClient.initial, "reconcileApplicationState") + .mockImplementation((input, options) => + original({ ...input, reconciliationId: "" }, options), + ); + dataReconciliation.start(); + dataReconciliation.sseConnectionChanged(true); + await waitFor(() => dataReconciliation.getState().recoveryFailed); + expect(reconcile).toHaveBeenCalledTimes(2); + expect(dataReconciliation.getState()).toMatchObject({ + trustedUpToDate: false, + serverParityAppliedAt: null, + retryPending: false, + inFlight: null, + }); + expect(loadingActor.getSnapshot().matches("idle")).toBe(true); + reconcile.mockRestore(); + await dataReconciliation.requestManualFull(); + expect(dataReconciliation.getState()).toMatchObject({ + trustedUpToDate: true, + recoveryFailed: false, + }); + }); +}); diff --git a/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts b/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts new file mode 100644 index 000000000..9c8046ba2 --- /dev/null +++ b/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts @@ -0,0 +1,97 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { RefetchItemsButton } from "~/components/feed/RefetchItemsButton"; + +const state = vi.hoisted(() => ({ + refresh: vi.fn().mockResolvedValue(undefined), + nextRefreshAt: null as number | null, +})); +vi.mock("@tanstack/react-router", () => ({ + useLocation: () => ({ pathname: "/" }), +})); +vi.mock("~/lib/data/store", () => ({ useFetchNewData: () => state.refresh })); +vi.mock("~/lib/data/reconciliation", () => ({ + useReconciliationDisplayStatus: () => "stale", +})); +vi.mock("~/lib/data/loading-machine", () => ({ + useLoadingMode: () => ({ mode: "idle" }), + useIsLoadingActive: () => false, + useNextRefreshAt: () => state.nextRefreshAt, +})); +vi.mock("~/components/feed/dialogStore", () => ({ + useDialogStore: () => false, +})); +vi.mock("~/lib/hooks/useShowShortcuts", () => ({ + useShowShortcuts: () => false, +})); +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +let unmount: () => void; +beforeEach(() => { + state.refresh.mockClear(); + state.nextRefreshAt = null; + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + }, + ); +}); +afterEach(() => { + unmount(); + document.body.innerHTML = ""; + vi.unstubAllGlobals(); +}); + +function mount() { + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + act(() => root.render(createElement(RefetchItemsButton))); + unmount = () => act(() => root.unmount()); + return container.querySelector("button")!; +} + +describe("Refresh after rejected recovery", () => { + it("shows stale data and permits click and keyboard recovery", async () => { + const button = mount(); + expect(button.disabled).toBe(false); + expect(button.getAttribute("aria-label")).toBe("Refresh"); + await act(async () => { + button.parentElement!.focus(); + }); + expect(document.querySelector('[role="tooltip"]')?.textContent).toBe( + "Data may be stale. Refresh to try again.", + ); + await act(async () => { + button.click(); + }); + expect(state.refresh).toHaveBeenCalledTimes(1); + await act(async () => { + window.dispatchEvent(new KeyboardEvent("keydown", { key: "r" })); + }); + expect(state.refresh).toHaveBeenCalledTimes(2); + }); + + it("preserves the refresh cooldown for both click and keyboard recovery", async () => { + state.nextRefreshAt = Date.now() + 60_000; + const button = mount(); + expect(button.disabled).toBe(true); + await act(async () => { + button.click(); + window.dispatchEvent(new KeyboardEvent("keydown", { key: "r" })); + }); + expect(state.refresh).not.toHaveBeenCalled(); + await act(async () => { + button.parentElement!.focus(); + }); + expect(document.querySelector('[role="tooltip"]')?.textContent).toBe( + "Data may be stale. Refresh to try again.", + ); + }); +}); From 2b362e8ced602df0f62f274a502bfeb14a99fe92 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:41:29 -0400 Subject: [PATCH 03/12] simplify reconciliation tooltip messages --- .../app/src/components/feed/RefetchItemsButton.tsx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/app/src/components/feed/RefetchItemsButton.tsx b/apps/app/src/components/feed/RefetchItemsButton.tsx index dac756ea2..da9ce4cd5 100644 --- a/apps/app/src/components/feed/RefetchItemsButton.tsx +++ b/apps/app/src/components/feed/RefetchItemsButton.tsx @@ -19,6 +19,12 @@ import { useFetchNewData } from "~/lib/data/store"; import { useReconciliationDisplayStatus } from "~/lib/data/reconciliation"; import { useShortcut } from "~/lib/hooks/useShortcut"; +const reconciliationMessages = { + syncing: "Checking for newer data", + retrying: "Data may be stale. Retrying automatically", + stale: "Data may be stale. Refresh to try again.", +}; + function formatRelativeTime(targetMs: number, now: number): string { const diffMs = targetMs - now; if (diffMs <= 0) return "now"; @@ -133,12 +139,8 @@ export function RefetchItemsButton() { {button} - {reconciliationStatus === "syncing" ? ( - "Checking for newer data" - ) : reconciliationStatus === "retrying" ? ( - "Data may be stale. Retrying automatically" - ) : reconciliationStatus === "stale" ? ( - "Data may be stale. Refresh to try again." + {reconciliationStatus !== "idle" ? ( + reconciliationMessages[reconciliationStatus] ) : ( <> Refresh available in{" "} From 1239c5ed6e2b6b60812188afb3191c75b19eb815 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:45:21 -0400 Subject: [PATCH 04/12] isolate refresh cooldown timing --- .../components/feed/RefetchItemsButton.tsx | 76 ++++++++++--------- .../reconciliation/refresh-recovery.test.ts | 15 ++++ 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/apps/app/src/components/feed/RefetchItemsButton.tsx b/apps/app/src/components/feed/RefetchItemsButton.tsx index da9ce4cd5..b6d5f9e3c 100644 --- a/apps/app/src/components/feed/RefetchItemsButton.tsx +++ b/apps/app/src/components/feed/RefetchItemsButton.tsx @@ -36,44 +36,15 @@ function formatRelativeTime(targetMs: number, now: number): string { return `${diffHr}h`; } -export function RefetchItemsButton() { - const location = useLocation(); - const fetchNewData = useFetchNewData(); - const loading = useLoadingMode(); +function useRefreshCooldown(active: boolean) { const nextRefreshAt = useNextRefreshAt(); - const reconciliationStatus = useReconciliationDisplayStatus(); - - // Track current time in state so the cooldown check is pure during render. - // Only updated via the timeout callback (async) to satisfy the lint rule. const [now, setNow] = useState(() => Date.now()); - const isMachineActive = useIsLoadingActive(); - const isRateLimited = nextRefreshAt !== null && nextRefreshAt > now; - - const isReconciling = reconciliationStatus !== "idle"; - const isRefreshing = isMachineActive || reconciliationStatus === "syncing"; - const isDisabled = - isRefreshing || - reconciliationStatus === "retrying" || - isRateLimited || - loading.mode === "initialLoad" || - loading.mode === "backgroundRefresh"; - - // Show check icon when the user is up to date (cooldown active), - // refresh icon when they can refresh again (cooldown expired/absent). - const showCheck = isRateLimited && !isMachineActive && !isReconciling; - const statusLabel = - reconciliationStatus === "syncing" - ? "Syncing" - : reconciliationStatus === "retrying" - ? "Retrying" - : "Refresh"; - // Tick `now` so the tooltip text updates live and the button re-enables // when the cooldown expires. Chained timeouts stop at expiry and use fewer // updates for longer cooldowns. useEffect(() => { - if (nextRefreshAt === null || location.pathname !== "/") return; + if (nextRefreshAt === null || !active) return; let timeout: ReturnType | undefined; @@ -97,7 +68,44 @@ export function RefetchItemsButton() { scheduleTick(); return () => clearTimeout(timeout); - }, [location.pathname, nextRefreshAt]); + }, [active, nextRefreshAt]); + + return { + isRateLimited: nextRefreshAt !== null && nextRefreshAt > now, + remainingTime: + nextRefreshAt === null ? null : formatRelativeTime(nextRefreshAt, now), + }; +} + +export function RefetchItemsButton() { + const location = useLocation(); + const fetchNewData = useFetchNewData(); + const loading = useLoadingMode(); + const reconciliationStatus = useReconciliationDisplayStatus(); + + const isMachineActive = useIsLoadingActive(); + const { isRateLimited, remainingTime } = useRefreshCooldown( + location.pathname === "/", + ); + + const isReconciling = reconciliationStatus !== "idle"; + const isRefreshing = isMachineActive || reconciliationStatus === "syncing"; + const isDisabled = + isRefreshing || + reconciliationStatus === "retrying" || + isRateLimited || + loading.mode === "initialLoad" || + loading.mode === "backgroundRefresh"; + + // Show check icon when the user is up to date (cooldown active), + // refresh icon when they can refresh again (cooldown expired/absent). + const showCheck = isRateLimited && !isMachineActive && !isReconciling; + const statusLabel = + reconciliationStatus === "syncing" + ? "Syncing" + : reconciliationStatus === "retrying" + ? "Retrying" + : "Refresh"; const onClick = useCallback(async () => { if (isDisabled) return; @@ -144,9 +152,7 @@ export function RefetchItemsButton() { ) : ( <> Refresh available in{" "} - - {formatRelativeTime(nextRefreshAt!, now)} - + {remainingTime} )} diff --git a/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts b/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts index 9c8046ba2..9c9141806 100644 --- a/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts +++ b/apps/app/tests/unit/reconciliation/refresh-recovery.test.ts @@ -46,6 +46,7 @@ afterEach(() => { unmount(); document.body.innerHTML = ""; vi.unstubAllGlobals(); + vi.useRealTimers(); }); function mount() { @@ -94,4 +95,18 @@ describe("Refresh after rejected recovery", () => { "Data may be stale. Refresh to try again.", ); }); + it("allows recovery when the existing cooldown expires", async () => { + vi.useFakeTimers(); + state.nextRefreshAt = Date.now() + 1_000; + const button = mount(); + expect(button.disabled).toBe(true); + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000); + }); + expect(button.disabled).toBe(false); + await act(async () => { + button.click(); + }); + expect(state.refresh).toHaveBeenCalledTimes(1); + }); }); From 037e033161a75cf3b54be9cd04c783e199ce3e57 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 04:00:45 -0400 Subject: [PATCH 05/12] stop terminal recovery retries on reconnect --- apps/app/src/lib/reconciliation/runtime.ts | 6 +- .../reconciliation/client-runtime.test.ts | 60 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/apps/app/src/lib/reconciliation/runtime.ts b/apps/app/src/lib/reconciliation/runtime.ts index 2ce21b9fc..ece9fe1bc 100644 --- a/apps/app/src/lib/reconciliation/runtime.ts +++ b/apps/app/src/lib/reconciliation/runtime.ts @@ -572,10 +572,12 @@ export function createReconciliationRuntime( reconnectRequired || (!everConnected && completedBeforeFirstConnection) ) { - clearRetry(); reconnectRequired = false; completedBeforeFirstConnection = false; - requestCurrentFull(); + if (!state.recoveryFailed) { + clearRetry(); + requestCurrentFull(); + } } everConnected = true; }, diff --git a/apps/app/tests/unit/reconciliation/client-runtime.test.ts b/apps/app/tests/unit/reconciliation/client-runtime.test.ts index b961683c4..e20dbe0f6 100644 --- a/apps/app/tests/unit/reconciliation/client-runtime.test.ts +++ b/apps/app/tests/unit/reconciliation/client-runtime.test.ts @@ -1133,6 +1133,66 @@ describe("rejected reconciliation recovery", () => { } }); + it.each([false, true])( + "keeps terminal recovery stopped across SSE connections, initially connected: %s", + async (initiallyConnected) => { + vi.useFakeTimers(); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + let rejects = true; + const test = harness((request) => { + if (rejects) throw invalidInput(); + return completeEpoch(request.reconciliationId); + }); + test.setSelection(ACTIVE_SCOPE); + hydrate(test.runtime); + test.runtime.cacheUsable(); + if (initiallyConnected) test.runtime.sseConnectionChanged(true); + try { + test.runtime.start(); + await vi.advanceTimersByTimeAsync(0); + expect(test.requests).toHaveLength(2); + expect(test.runtime.getState().recoveryFailed).toBe(true); + + test.runtime.sseConnectionChanged(false); + test.runtime.sseConnectionChanged(true); + await vi.advanceTimersByTimeAsync(60_000); + test.runtime.sseConnectionChanged(false); + test.runtime.sseConnectionChanged(true); + await vi.advanceTimersByTimeAsync(60_000); + expect(test.requests).toHaveLength(2); + expect(test.runtime.getState()).toMatchObject({ + recoveryFailed: true, + retryPending: false, + inFlight: null, + trustedUpToDate: false, + }); + + rejects = false; + test.runtime.requestFull(); + await vi.advanceTimersByTimeAsync(0); + expect(test.requests).toHaveLength(3); + expect(test.requests[2]?.intent).toMatchObject({ + discardManifest: true, + }); + expect(test.runtime.getState().trustedUpToDate).toBe(true); + + test.runtime.sseConnectionChanged(false); + test.runtime.sseConnectionChanged(true); + await vi.advanceTimersByTimeAsync(0); + expect(test.requests).toHaveLength(4); + expect(test.requests[3]?.intent).toEqual({ + type: "full", + selectedScope: ACTIVE_SCOPE, + }); + expect(test.runtime.getState().trustedUpToDate).toBe(true); + } finally { + test.runtime.stop(); + log.mockRestore(); + vi.useRealTimers(); + } + }, + ); + it("retains empty manifests through a transient recovery failure", async () => { vi.useFakeTimers(); const log = vi.spyOn(console, "error").mockImplementation(() => {}); From 5097d6019e84af00683258455ec746541d91dc55 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:51:27 -0400 Subject: [PATCH 06/12] narrow standard.site publication to releases --- .github/workflows/standard-site-sync.yml | 3 +- apps/www/README.md | 2 +- apps/www/scripts/lib/standard-site-content.ts | 38 +++++ apps/www/scripts/sync-standard-site.ts | 65 +------- apps/www/src/lib/releases.ts | 5 + apps/www/src/lib/standard-site/index.ts | 42 +---- .../releases.ts} | 4 +- apps/www/src/pages/guides/[slug].astro | 24 --- apps/www/src/pages/releases/rss.xml.js | 11 +- apps/www/tests/standard-site.test.ts | 157 ++++++++++++++++++ 10 files changed, 224 insertions(+), 127 deletions(-) create mode 100644 apps/www/scripts/lib/standard-site-content.ts create mode 100644 apps/www/src/lib/releases.ts rename apps/www/src/pages/.well-known/{site.standard.publication.ts => site.standard.publication/releases.ts} (72%) create mode 100644 apps/www/tests/standard-site.test.ts diff --git a/.github/workflows/standard-site-sync.yml b/.github/workflows/standard-site-sync.yml index 45675ac3f..b33199751 100644 --- a/.github/workflows/standard-site-sync.yml +++ b/.github/workflows/standard-site-sync.yml @@ -4,10 +4,11 @@ on: push: branches: [main] paths: - - "apps/www/src/content/guides/**" - "apps/www/src/content/releases/**" + - "apps/www/src/lib/releases.ts" - "apps/www/src/lib/standard-site/**" - "apps/www/scripts/sync-standard-site.ts" + - "apps/www/scripts/lib/standard-site-content.ts" - ".github/workflows/standard-site-sync.yml" workflow_dispatch: inputs: diff --git a/apps/www/README.md b/apps/www/README.md index 910de2b30..668f6fa38 100644 --- a/apps/www/README.md +++ b/apps/www/README.md @@ -38,4 +38,4 @@ Environment variables are documented in `.env.example`. ## Standard.Site sync -`pnpm --filter @serial/www standard-site:sync` publishes guides and release notes as `site.standard.document` records to an AT Protocol PDS, reading the markdown in `src/content` directly. Use `--dry-run` to preview and `--allow-large-delete` to override the delete guard. Requires the `WWW_STANDARD_SITE_*` variables from `.env.example`. The GitHub workflow always supports manual runs when credentials are configured; automatic push-triggered syncs also require the repository variable `WWW_STANDARD_SITE_SYNC_ENABLED=true`. +`pnpm --filter @serial/www standard-site:sync` publishes the same public release notes as RSS under the `Serial Releases` publication at `https://www.serial.tube/releases`, reading the markdown in `src/content/releases` directly. Guides are excluded; sync removes previously published guide documents while retaining the publication and release record identities. Publication verification is served at `/.well-known/site.standard.publication/releases`. Use `--dry-run` to preview and `--allow-large-delete` to override the delete guard. Requires the `WWW_STANDARD_SITE_*` variables from `.env.example`. The GitHub workflow always supports manual runs when credentials are configured; automatic push-triggered syncs also require the repository variable `WWW_STANDARD_SITE_SYNC_ENABLED=true`. diff --git a/apps/www/scripts/lib/standard-site-content.ts b/apps/www/scripts/lib/standard-site-content.ts new file mode 100644 index 000000000..03561eef6 --- /dev/null +++ b/apps/www/scripts/lib/standard-site-content.ts @@ -0,0 +1,38 @@ +import { readdir, readFile } from "node:fs/promises"; +import matter from "gray-matter"; +import { z } from "zod"; +import { buildReleaseDocumentSource } from "../../src/lib/standard-site"; + +const frontmatterSchema = z.object({ + title: z.string(), + description: z.string().optional(), + publish_date: z.iso.date(), + public: z.boolean(), +}); + +const RELEASES_DIR = new URL("../../src/content/releases/", import.meta.url); + +export async function loadReleaseDocuments(directory = RELEASES_DIR) { + const fileNames = await readdir(directory); + const documents = []; + + for (const fileName of fileNames) { + if (!fileName.endsWith(".md")) continue; + + const raw = await readFile(new URL(fileName, directory), "utf8"); + const { data, content } = matter(raw); + const parsed = frontmatterSchema.parse(data); + + if (!parsed.public) continue; + + documents.push( + buildReleaseDocumentSource({ + slug: fileName.replace(/\.md$/, ""), + ...parsed, + content: content.trim(), + }), + ); + } + + return documents.sort((a, b) => a.publishedAt.localeCompare(b.publishedAt)); +} diff --git a/apps/www/scripts/sync-standard-site.ts b/apps/www/scripts/sync-standard-site.ts index 2052f24a3..71bbca9fb 100644 --- a/apps/www/scripts/sync-standard-site.ts +++ b/apps/www/scripts/sync-standard-site.ts @@ -1,14 +1,8 @@ -import { readdir, readFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { AtpAgent } from "@atproto/api"; -import matter from "gray-matter"; import { z } from "zod"; -import { - buildGuideDocumentSource, - buildReleaseDocumentSource, - parsePublicationUri, - STANDARD_SITE, -} from "../src/lib/standard-site"; -import type { StandardSiteContent } from "../src/lib/standard-site"; +import { parsePublicationUri, STANDARD_SITE } from "../src/lib/standard-site"; +import { loadReleaseDocuments } from "./lib/standard-site-content"; import { assertStandardSiteSyncPlanIsSafe, planStandardSiteSync, @@ -24,17 +18,6 @@ const syncEnvSchema = z.object({ const syncEnv = syncEnvSchema.parse(process.env); -const frontmatterSchema = z.object({ - title: z.string(), - description: z.string().optional(), - publish_date: z.string(), - updated_at: z.string().optional(), - public: z.boolean(), -}); - -const GUIDES_DIR = new URL("../src/content/guides/", import.meta.url); -const RELEASES_DIR = new URL("../src/content/releases/", import.meta.url); - const isDryRun = process.argv.includes("--dry-run"); const allowLargeDelete = process.argv.includes("--allow-large-delete"); const publicationUri = syncEnv.WWW_STANDARD_SITE_PUBLICATION_URI; @@ -44,46 +27,6 @@ const publicationIcon = { url: new URL("../public/icon-256.png", import.meta.url), } as const; -async function loadContentDirectory( - directory: URL, -): Promise { - const fileNames = await readdir(directory); - const documents: StandardSiteContent[] = []; - - for (const fileName of fileNames) { - if (!fileName.endsWith(".md")) continue; - - const raw = await readFile(new URL(fileName, directory), "utf8"); - const { data, content } = matter(raw); - const parsed = frontmatterSchema.parse(data); - - if (!parsed.public) continue; - - documents.push({ - slug: fileName.replace(/\.md$/, ""), - title: parsed.title, - content: content.trim(), - publish_date: parsed.publish_date, - description: parsed.description, - updated_at: parsed.updated_at, - }); - } - - return documents; -} - -async function loadDocuments() { - const releases = await loadContentDirectory(RELEASES_DIR); - const guides = await loadContentDirectory(GUIDES_DIR); - - const releaseDocuments = releases.map(buildReleaseDocumentSource); - const guideDocuments = guides.map(buildGuideDocumentSource); - - return [...releaseDocuments, ...guideDocuments].sort((a, b) => - a.publishedAt.localeCompare(b.publishedAt), - ); -} - async function listRecords(agent: AtpAgent, repo: string, collection: string) { const records: StandardSiteRecord[] = []; let cursor: string | undefined; @@ -104,7 +47,7 @@ async function listRecords(agent: AtpAgent, repo: string, collection: string) { } async function syncStandardSite() { - const documents = await loadDocuments(); + const documents = await loadReleaseDocuments(); const agent = new AtpAgent({ service: syncEnv.WWW_STANDARD_SITE_PDS_URL }); await agent.login({ identifier: syncEnv.WWW_STANDARD_SITE_IDENTIFIER, diff --git a/apps/www/src/lib/releases.ts b/apps/www/src/lib/releases.ts new file mode 100644 index 000000000..9b71d1665 --- /dev/null +++ b/apps/www/src/lib/releases.ts @@ -0,0 +1,5 @@ +export const RELEASES = { + name: "Serial Releases", + description: "Release notes and product updates from Serial.", + url: "https://www.serial.tube/releases", +} as const; diff --git a/apps/www/src/lib/standard-site/index.ts b/apps/www/src/lib/standard-site/index.ts index 68b264681..71908ba3e 100644 --- a/apps/www/src/lib/standard-site/index.ts +++ b/apps/www/src/lib/standard-site/index.ts @@ -1,25 +1,14 @@ +import { RELEASES } from "../releases"; + const SORTABLE_BASE32_CHARACTERS = "234567abcdefghijklmnopqrstuvwxyz"; const TID_LENGTH = 13; export const STANDARD_SITE = { documentCollection: "site.standard.document", publicationCollection: "site.standard.publication", - publicationDescription: - "Guides, release notes, and product updates for Serial, a calm and customizable RSS reader.", - publicationName: "Serial", - publicationUrl: "https://www.serial.tube", - documentTypes: { - guide: { - keyPrefix: "guides", - pathPrefix: "/guides", - tags: ["guide"], - }, - release: { - keyPrefix: "", - pathPrefix: "/releases", - tags: ["release"], - }, - }, + publicationDescription: RELEASES.description, + publicationName: RELEASES.name, + publicationUrl: RELEASES.url, } as const; export type StandardSiteContent = { @@ -81,20 +70,15 @@ function buildTid(timestampMicroseconds: number, clockId: number) { return tid; } -function buildDocumentSource( +export function buildReleaseDocumentSource( content: StandardSiteContent, - documentType: (typeof STANDARD_SITE.documentTypes)[keyof typeof STANDARD_SITE.documentTypes], ): StandardSiteDocumentSource { - const key = documentType.keyPrefix - ? `${documentType.keyPrefix}/${content.slug}` - : content.slug; - return { - key, + key: content.slug, title: content.title, - path: `${documentType.pathPrefix}/${content.slug}`, + path: `/${content.slug}/`, publishedAt: `${content.publish_date}T00:00:00.000Z`, - tags: [...documentType.tags], + tags: ["release"], markdownContent: content.content, ...(content.description ? { description: content.description } : {}), ...(content.updated_at @@ -103,14 +87,6 @@ function buildDocumentSource( }; } -export function buildReleaseDocumentSource(release: StandardSiteContent) { - return buildDocumentSource(release, STANDARD_SITE.documentTypes.release); -} - -export function buildGuideDocumentSource(guide: StandardSiteContent) { - return buildDocumentSource(guide, STANDARD_SITE.documentTypes.guide); -} - export function getDocumentRkey( document: Pick, ) { diff --git a/apps/www/src/pages/.well-known/site.standard.publication.ts b/apps/www/src/pages/.well-known/site.standard.publication/releases.ts similarity index 72% rename from apps/www/src/pages/.well-known/site.standard.publication.ts rename to apps/www/src/pages/.well-known/site.standard.publication/releases.ts index 30eb3a4d1..df2e7ca16 100644 --- a/apps/www/src/pages/.well-known/site.standard.publication.ts +++ b/apps/www/src/pages/.well-known/site.standard.publication/releases.ts @@ -1,6 +1,6 @@ import type { APIRoute } from "astro"; -import { STANDARD_SITE_PUBLICATION_URI } from "../../lib/site"; -import { getConfiguredPublicationUri } from "../../lib/standard-site"; +import { STANDARD_SITE_PUBLICATION_URI } from "../../../lib/site"; +import { getConfiguredPublicationUri } from "../../../lib/standard-site"; export const GET: APIRoute = () => { const publicationUri = getConfiguredPublicationUri( diff --git a/apps/www/src/pages/guides/[slug].astro b/apps/www/src/pages/guides/[slug].astro index 08a3ced30..a89986676 100644 --- a/apps/www/src/pages/guides/[slug].astro +++ b/apps/www/src/pages/guides/[slug].astro @@ -5,12 +5,6 @@ import FooterCTA from "../../components/FooterCTA.astro"; import Icon from "../../components/Icon.astro"; import Site from "../../layouts/Site.astro"; import { getAllGuides } from "../../lib/content"; -import { STANDARD_SITE_PUBLICATION_URI } from "../../lib/site"; -import { - buildDocumentUri, - buildGuideDocumentSource, - getConfiguredPublicationUri, -} from "../../lib/standard-site"; export async function getStaticPaths() { const guides = await getAllGuides(); @@ -33,23 +27,6 @@ const { Content } = await render(guide); const iconName = guide.data.icon ? GUIDE_ICONS[guide.data.icon] : undefined; const description = guide.data.description ?? `Guide for ${guide.data.title}`; - -const publicationUri = getConfiguredPublicationUri( - STANDARD_SITE_PUBLICATION_URI, -); -const documentUri = publicationUri - ? buildDocumentUri( - publicationUri, - buildGuideDocumentSource({ - slug: guide.id, - title: guide.data.title, - content: guide.body ?? "", - publish_date: guide.data.publish_date, - description: guide.data.description, - updated_at: guide.data.updated_at, - }), - ) - : undefined; ---
diff --git a/apps/www/src/pages/releases/rss.xml.js b/apps/www/src/pages/releases/rss.xml.js index a83883bf9..b70ca75d6 100644 --- a/apps/www/src/pages/releases/rss.xml.js +++ b/apps/www/src/pages/releases/rss.xml.js @@ -3,6 +3,7 @@ import { experimental_AstroContainer as AstroContainer } from "astro/container"; import { render } from "astro:content"; import sanitizeHtml from "sanitize-html"; import { getAllReleases } from "../../lib/content"; +import { RELEASES } from "../../lib/releases"; function resolveSiteUrl(site, value) { if (!value || !value.startsWith("/")) return value; @@ -22,7 +23,7 @@ function resolveAttribute(site, attribute) { export async function GET(context) { const container = await AstroContainer.create(); const releases = await getAllReleases(); - const releasesUrl = new URL("/releases/", context.site).toString(); + const releasesUrl = `${RELEASES.url}/`; const iconUrl = new URL("/icon-256.png", context.site).toString(); const items = await Promise.all( @@ -34,7 +35,7 @@ export async function GET(context) { title: release.data.title, description: release.data.description, pubDate: new Date(`${release.data.publish_date}T00:00:00Z`), - link: new URL(`/releases/${release.id}/`, context.site).toString(), + link: `${releasesUrl}${release.id}/`, content: sanitizeHtml(content, { allowedTags: sanitizeHtml.defaults.allowedTags.concat([ "img", @@ -57,15 +58,15 @@ export async function GET(context) { ); return rss({ - title: "Serial Releases", - description: "Release notes and product updates from Serial.", + title: RELEASES.name, + description: RELEASES.description, site: releasesUrl, items, customData: ` en-us ${iconUrl} - Serial Releases + ${RELEASES.name} ${releasesUrl} `, diff --git a/apps/www/tests/standard-site.test.ts b/apps/www/tests/standard-site.test.ts new file mode 100644 index 000000000..55a5e369e --- /dev/null +++ b/apps/www/tests/standard-site.test.ts @@ -0,0 +1,157 @@ +// @vitest-environment node +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { BlobRef } from "@atproto/api"; +import { describe, expect, it } from "vitest"; +import { loadReleaseDocuments } from "../scripts/lib/standard-site-content"; +import { + buildDocumentUri, + buildReleaseDocumentSource, + getDocumentRkey, + STANDARD_SITE, +} from "../src/lib/standard-site"; +import { + assertStandardSiteSyncPlanIsSafe, + buildDocumentRecord, + buildPublicationRecord, + planStandardSiteSync, +} from "../src/lib/standard-site/records"; + +const publicationUri = + "at://did:plc:serial/site.standard.publication/3m22222222222"; +const publicationIcon = BlobRef.fromJsonRef({ + cid: "bafkreigh2akiscaildc4lmyyzy3jxuvkwqezbszr77grdgo4mpfwu5zbmq", + mimeType: "image/png", +}); +const release = buildReleaseDocumentSource({ + slug: "2026-09-11", + title: "September update", + publish_date: "2026-09-11", + content: "A **release** update.", +}); + +describe("release publication", () => { + it("keeps the release identity while moving its URL under the publication", () => { + const oldDocument = { ...release, path: "/releases/2026-09-11" }; + expect(buildDocumentUri(publicationUri, release)).toBe( + buildDocumentUri(publicationUri, oldDocument), + ); + expect(`${STANDARD_SITE.publicationUrl}${release.path}`).toBe( + "https://www.serial.tube/releases/2026-09-11/", + ); + expect(buildPublicationRecord(publicationIcon)).toMatchObject({ + name: "Serial Releases", + description: "Release notes and product updates from Serial.", + url: "https://www.serial.tube/releases", + }); + }); + + it("updates existing records, removes guides, and leaves other publications alone", () => { + const rkey = getDocumentRkey(release); + const plan = planStandardSiteSync({ + documents: [release], + publicationUri, + publicationIcon, + existingPublications: [ + { + uri: publicationUri, + value: { + ...buildPublicationRecord(publicationIcon), + name: "Serial", + url: "https://www.serial.tube", + }, + }, + ], + existingDocuments: [ + { + uri: buildDocumentUri(publicationUri, release), + value: { + ...buildDocumentRecord(release, publicationUri), + path: "/releases/2026-09-11", + }, + }, + { + uri: "at://did:plc:serial/site.standard.document/3m22222222223", + value: { + site: publicationUri, + path: "/guides/getting-started", + tags: ["guide"], + }, + }, + { + uri: "at://did:plc:serial/site.standard.document/3m22222222224", + value: { + site: "at://did:plc:serial/site.standard.publication/3m22222222225", + }, + }, + ], + }); + + expect(plan).toMatchObject({ creates: 0, updates: 2, deletes: 1 }); + expect(plan.writes).toMatchObject([ + { $type: "com.atproto.repo.applyWrites#update", rkey: "3m22222222222" }, + { + $type: "com.atproto.repo.applyWrites#update", + rkey, + value: { path: "/2026-09-11/" }, + }, + { $type: "com.atproto.repo.applyWrites#delete", rkey: "3m22222222223" }, + ]); + expect(() => + assertStandardSiteSyncPlanIsSafe(plan, { allowLargeDelete: false }), + ).not.toThrow(); + }); + + it("does no writes after the publication matches the releases", () => { + const plan = planStandardSiteSync({ + documents: [release], + publicationUri, + publicationIcon, + existingPublications: [ + { uri: publicationUri, value: buildPublicationRecord(publicationIcon) }, + ], + existingDocuments: [ + { + uri: buildDocumentUri(publicationUri, release), + value: buildDocumentRecord(release, publicationUri), + }, + ], + }); + expect(plan.writes).toEqual([]); + }); + + it("loads only public top-level release Markdown, including future-dated RSS entries", async () => { + const root = await mkdtemp(join(tmpdir(), "serial-releases-")); + const releases = join(root, "releases"); + const guides = join(root, "guides"); + const markdown = (isPublic: boolean) => + `---\ntitle: Test\npublish_date: "2099-01-01"\npublic: ${isPublic}\n---\nRelease body.`; + try { + await mkdir(releases); + await mkdir(guides); + await mkdir(join(releases, "nested")); + await Promise.all([ + writeFile(join(releases, "public.md"), markdown(true)), + writeFile(join(releases, "draft.md"), markdown(false)), + writeFile(join(releases, "ignored.txt"), markdown(true)), + writeFile(join(releases, "nested", "nested.md"), markdown(true)), + writeFile(join(guides, "guide.md"), markdown(true)), + ]); + const documents = await loadReleaseDocuments( + pathToFileURL(`${releases}/`), + ); + expect(documents).toMatchObject([ + { + key: "public", + path: "/public/", + publishedAt: "2099-01-01T00:00:00.000Z", + markdownContent: "Release body.", + }, + ]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); From 7c0b6cb3b940d9e70cd1577810e3de224a8d25d7 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:54:22 -0400 Subject: [PATCH 07/12] lock the existing release record identity --- apps/www/tests/standard-site.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/www/tests/standard-site.test.ts b/apps/www/tests/standard-site.test.ts index 55a5e369e..7f806c1f3 100644 --- a/apps/www/tests/standard-site.test.ts +++ b/apps/www/tests/standard-site.test.ts @@ -34,9 +34,8 @@ const release = buildReleaseDocumentSource({ describe("release publication", () => { it("keeps the release identity while moving its URL under the publication", () => { - const oldDocument = { ...release, path: "/releases/2026-09-11" }; expect(buildDocumentUri(publicationUri, release)).toBe( - buildDocumentUri(publicationUri, oldDocument), + "at://did:plc:serial/site.standard.document/3mv7b42cwqisi", ); expect(`${STANDARD_SITE.publicationUrl}${release.path}`).toBe( "https://www.serial.tube/releases/2026-09-11/", From 44d17cd8f503d1f71e9034190661ffe639c0259f Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:22:37 -0400 Subject: [PATCH 08/12] highlight article code and widen image placeholders --- apps/app/package.json | 2 + apps/app/src/app/_app.read.$id.tsx | 17 +-- .../bookmarks/BookmarkArticleContent.tsx | 3 + .../content-reader/BookmarkReader.tsx | 2 +- .../content-reader/ReaderCodeBlock.tsx | 100 +++++++++++++ .../content-reader/codeHighlightingPolicy.ts | 7 + .../content-reader/highlightCode.ts | 57 +++++++ .../content-reader/highlightCode.worker.ts | 8 + .../content-reader/highlightCodeClient.ts | 51 +++++++ .../components/feed/read/ArticleContent.tsx | 12 +- .../feed/read/ArticleImageLightbox.tsx | 4 +- .../components/feed/read/article.module.css | 80 +++++++++- apps/app/tests/e2e/fixtures/seed-db.ts | 5 +- .../self-hosted/article-code-blocks.spec.ts | 140 ++++++++++++++++++ .../code-highlighting-worker.test.ts | 86 +++++++++++ .../tests/unit/components/offline-ui.test.ts | 5 +- .../components/reader-code-blocks.test.ts | 121 +++++++++++++++ .../performance/reader-highlighting.test.ts | 41 +++++ pnpm-lock.yaml | 9 ++ 19 files changed, 727 insertions(+), 23 deletions(-) create mode 100644 apps/app/src/components/content-reader/ReaderCodeBlock.tsx create mode 100644 apps/app/src/components/content-reader/codeHighlightingPolicy.ts create mode 100644 apps/app/src/components/content-reader/highlightCode.ts create mode 100644 apps/app/src/components/content-reader/highlightCode.worker.ts create mode 100644 apps/app/src/components/content-reader/highlightCodeClient.ts create mode 100644 apps/app/tests/e2e/self-hosted/article-code-blocks.spec.ts create mode 100644 apps/app/tests/unit/components/code-highlighting-worker.test.ts create mode 100644 apps/app/tests/unit/components/reader-code-blocks.test.ts create mode 100644 apps/app/tests/unit/performance/reader-highlighting.test.ts diff --git a/apps/app/package.json b/apps/app/package.json index c65d7cf06..27bcb41ce 100644 --- a/apps/app/package.json +++ b/apps/app/package.json @@ -65,6 +65,7 @@ "benchmark:view-matrix:gate": "dotenv -e .env.test -- node --expose-gc --import=tsx scripts/performance/run-view-matrix-benchmark.ts --gate --output benchmarks/results/view-matrix-gate.json", "benchmark:client": "dotenv -e .env.test -- node --import=tsx scripts/performance/run-client-audit.ts", "benchmark:client:smoke": "dotenv -e .env.test -- node --import=tsx scripts/performance/run-client-audit.ts --profile small --output benchmarks/results/client-small.json", + "benchmark:client:browser": "SERIAL_RUN_CLIENT_PERFORMANCE=1 pnpm test:e2e:self-hosted client-performance-audit.spec.ts --project self-hosted --workers 1", "benchmark:client:gate": "pnpm benchmark:client --profile small --gate && pnpm benchmark:client --profile representative --gate && pnpm benchmark:client --profile stress --gate", "benchmark:client:coverage": "node --import=tsx scripts/performance/inventory-client-audit.ts", "benchmark:client:coverage:check": "node --import=tsx scripts/performance/inventory-client-audit.ts --check", @@ -133,6 +134,7 @@ "drizzle-zod": "^0.8.3", "fast-xml-parser": "^5.10.1", "feedscout": "^1.10.0", + "highlight.js": "^11.12.0", "html-react-parser": "^6.1.4", "idb-keyval": "^6.3.0", "ioredis": "^5.11.1", diff --git a/apps/app/src/app/_app.read.$id.tsx b/apps/app/src/app/_app.read.$id.tsx index a05be274e..a88f45bac 100644 --- a/apps/app/src/app/_app.read.$id.tsx +++ b/apps/app/src/app/_app.read.$id.tsx @@ -177,7 +177,7 @@ function FeedReader({ return (

{feedItem?.title}

{feedItem?.author || feed?.name || ""}
- {articleStyle === "simplified" ? ( - // Content is sanitized by the module-level rehype pipeline above. - // react-doctor-disable-next-line react-doctor/dangerous-html-sink -
- ) : ( - - )} +
{shouldShowTruncationAlert && ( diff --git a/apps/app/src/components/bookmarks/BookmarkArticleContent.tsx b/apps/app/src/components/bookmarks/BookmarkArticleContent.tsx index c45139dd4..345049aad 100644 --- a/apps/app/src/components/bookmarks/BookmarkArticleContent.tsx +++ b/apps/app/src/components/bookmarks/BookmarkArticleContent.tsx @@ -6,6 +6,7 @@ import { useMemo, useSyncExternalStore } from "react"; import type { HTMLReactParserOptions } from "html-react-parser"; import { CustomVideoPlayer } from "~/components/CustomVideoPlayer"; import { flattenReaderImages } from "~/components/content-reader/flattenReaderImages"; +import { replaceReaderCodeBlock } from "~/components/content-reader/ReaderCodeBlock"; import { ArticleImageLightbox } from "~/components/feed/read/ArticleImageLightbox"; import { useFlagState } from "~/lib/hooks/useFlagState"; import { @@ -43,6 +44,8 @@ export function BookmarkArticleContent({ content }: { content: string }) { const options: HTMLReactParserOptions = { replace: (node) => { + const codeBlock = replaceReaderCodeBlock(node); + if (codeBlock) return codeBlock; if (!(node instanceof Element)) return; if (node.name === "a" && node.attribs.href?.startsWith("http")) { node.attribs.target = "_blank"; diff --git a/apps/app/src/components/content-reader/BookmarkReader.tsx b/apps/app/src/components/content-reader/BookmarkReader.tsx index 34b7546e5..335fcc6e3 100644 --- a/apps/app/src/components/content-reader/BookmarkReader.tsx +++ b/apps/app/src/components/content-reader/BookmarkReader.tsx @@ -176,7 +176,7 @@ export function BookmarkReader({ id }: { id: string }) { return (
codeText(child as DOMNode)).join(""); +} + +function codeLanguage(node: Element): string | undefined { + const classes = node.attribs.class ?? ""; + if (/(?:^|\s)(?:no-highlight|nohighlight)(?:\s|$)/i.test(classes)) + return "text"; + return ( + /(?:^|\s)lang(?:uage)?-([\w+-]+)/i.exec(classes)?.[1]?.toLowerCase() ?? + node.attribs["data-language"]?.toLowerCase() + ); +} + +function ReaderCodeBlock({ + code, + language, + id, + dir, + label, +}: { + code: string; + language?: string; + id?: string; + dir?: string; + label?: string; +}) { + const [highlighted, setHighlighted] = useState<{ + code: string; + language?: string; + html: string | null; + }>(); + + useEffect(() => { + let active = true; + let cancel = () => {}; + if (!canHighlightCode(code, language)) return; + // Load only for articles with code. The plain block is readable immediately. + void import("./highlightCodeClient") + .then(({ requestCodeHighlight }) => { + if (active) + cancel = requestCodeHighlight(code, language, (html) => { + setHighlighted({ code, language, html }); + }); + }) + .catch(() => { + /* Keep plain text when offline or a chunk cannot load. */ + }); + return () => { + active = false; + cancel(); + }; + }, [code, language]); + + const html = + highlighted?.code === code && highlighted.language === language + ? highlighted.html + : null; + const content = useMemo(() => (html ? parse(html) : code), [html, code]); + + return ( +
+      {content}
+    
+ ); +} + +export function replaceReaderCodeBlock(node: DOMNode) { + if (!(node instanceof Element) || node.name !== "pre") return; + const code = node.children.find( + (child) => child instanceof Element && child.name === "code", + ); + return ( + + ); +} diff --git a/apps/app/src/components/content-reader/codeHighlightingPolicy.ts b/apps/app/src/components/content-reader/codeHighlightingPolicy.ts new file mode 100644 index 000000000..e920151af --- /dev/null +++ b/apps/app/src/components/content-reader/codeHighlightingPolicy.ts @@ -0,0 +1,7 @@ +export function canHighlightCode(code: string, language?: string): boolean { + return ( + Boolean(code.trim()) && + code.length <= (language ? 20_000 : 5_000) && + !/^(?:text|plain|plaintext|none)$/.test(language ?? "") + ); +} diff --git a/apps/app/src/components/content-reader/highlightCode.ts b/apps/app/src/components/content-reader/highlightCode.ts new file mode 100644 index 000000000..5c95b6b88 --- /dev/null +++ b/apps/app/src/components/content-reader/highlightCode.ts @@ -0,0 +1,57 @@ +import hljs from "highlight.js/lib/core"; +import bash from "highlight.js/lib/languages/bash"; +import css from "highlight.js/lib/languages/css"; +import go from "highlight.js/lib/languages/go"; +import java from "highlight.js/lib/languages/java"; +import javascript from "highlight.js/lib/languages/javascript"; +import json from "highlight.js/lib/languages/json"; +import python from "highlight.js/lib/languages/python"; +import rust from "highlight.js/lib/languages/rust"; +import sql from "highlight.js/lib/languages/sql"; +import typescript from "highlight.js/lib/languages/typescript"; +import xml from "highlight.js/lib/languages/xml"; +import yaml from "highlight.js/lib/languages/yaml"; +import { canHighlightCode } from "./codeHighlightingPolicy"; + +for (const [name, grammar] of Object.entries({ + bash, + css, + go, + java, + javascript, + json, + python, + rust, + sql, + typescript, + xml, + yaml, +})) { + hljs.registerLanguage(name, grammar); +} + +const DETECT_LANGUAGES = [ + "javascript", + "python", + "bash", + "css", + "xml", + "json", + "sql", +]; + +export function highlightCode(code: string, language?: string): string | null { + // Bound work on imported content. Oversized blocks remain selectable text. + if (!canHighlightCode(code, language)) return null; + if (language && !hljs.getLanguage(language)) return null; + + try { + const result = language + ? hljs.highlight(code, { language, ignoreIllegals: true }) + : hljs.highlightAuto(code, DETECT_LANGUAGES); + if (!language && result.relevance < 2) return null; + return result.value; + } catch { + return null; + } +} diff --git a/apps/app/src/components/content-reader/highlightCode.worker.ts b/apps/app/src/components/content-reader/highlightCode.worker.ts new file mode 100644 index 000000000..21fbee4e5 --- /dev/null +++ b/apps/app/src/components/content-reader/highlightCode.worker.ts @@ -0,0 +1,8 @@ +import { highlightCode } from "./highlightCode"; + +self.onmessage = ( + event: MessageEvent<{ id: number; code: string; language?: string }>, +) => { + const { id, code, language } = event.data; + self.postMessage({ id, html: highlightCode(code, language) }); +}; diff --git a/apps/app/src/components/content-reader/highlightCodeClient.ts b/apps/app/src/components/content-reader/highlightCodeClient.ts new file mode 100644 index 000000000..f9d9040f5 --- /dev/null +++ b/apps/app/src/components/content-reader/highlightCodeClient.ts @@ -0,0 +1,51 @@ +let worker: Worker | undefined; +let nextId = 0; +const pending = new Map void>(); + +function getWorker() { + if (worker) return worker; + const currentWorker = new Worker( + new URL("./highlightCode.worker.ts", import.meta.url), + { + type: "module", + }, + ); + worker = currentWorker; + currentWorker.onmessage = ( + event: MessageEvent<{ id: number; html: string | null }>, + ) => { + const callback = pending.get(event.data.id); + pending.delete(event.data.id); + callback?.(event.data.html); + }; + currentWorker.onerror = () => { + if (worker !== currentWorker) return; + currentWorker.terminate(); + worker = undefined; + for (const callback of pending.values()) callback(null); + pending.clear(); + }; + return worker; +} + +export function requestCodeHighlight( + code: string, + language: string | undefined, + onResult: (html: string | null) => void, +): () => void { + const id = nextId++; + try { + const highlighter = getWorker(); + pending.set(id, onResult); + highlighter.postMessage({ id, code, language }); + } catch { + pending.delete(id); + onResult(null); + } + return () => { + if (pending.delete(id) && pending.size === 0) { + worker?.terminate(); + worker = undefined; + } + }; +} diff --git a/apps/app/src/components/feed/read/ArticleContent.tsx b/apps/app/src/components/feed/read/ArticleContent.tsx index 80237e8f3..a938e7f8b 100644 --- a/apps/app/src/components/feed/read/ArticleContent.tsx +++ b/apps/app/src/components/feed/read/ArticleContent.tsx @@ -4,6 +4,7 @@ import parse, { Element } from "html-react-parser"; import type { HTMLReactParserOptions } from "html-react-parser"; import { CustomVideoPlayer } from "~/components/CustomVideoPlayer"; import { flattenReaderImages } from "~/components/content-reader/flattenReaderImages"; +import { replaceReaderCodeBlock } from "~/components/content-reader/ReaderCodeBlock"; import { ArticleImageLightbox } from "~/components/feed/read/ArticleImageLightbox"; import { useFlagState } from "~/lib/hooks/useFlagState"; import classes from "~/components/feed/read/article.module.css"; @@ -38,11 +39,20 @@ function isImageContainer(node: Element): boolean { return false; } -export function ArticleContent({ content }: { content: string }) { +export function ArticleContent({ + content, + simplified = false, +}: { + content: string; + simplified?: boolean; +}) { const [videoPlayer] = useFlagState("CUSTOM_VIDEO_PLAYER"); const options: HTMLReactParserOptions = { replace: (domNode) => { + const codeBlock = replaceReaderCodeBlock(domNode); + if (codeBlock) return codeBlock; + if (simplified) return; if (!(domNode instanceof Element)) return; // Open external links in new tabs. In-page links include footnote refs. diff --git a/apps/app/src/components/feed/read/ArticleImageLightbox.tsx b/apps/app/src/components/feed/read/ArticleImageLightbox.tsx index 7a134a6d2..4ebbb28af 100644 --- a/apps/app/src/components/feed/read/ArticleImageLightbox.tsx +++ b/apps/app/src/components/feed/read/ArticleImageLightbox.tsx @@ -32,6 +32,8 @@ export function ArticleImageLightbox({ aria-disabled={failed} style={{ display: "block", + width: failed ? "100%" : undefined, + maxWidth: failed ? "100%" : undefined, cursor: failed ? "default" : "zoom-in", }} onClick={toggle} @@ -41,7 +43,7 @@ export function ArticleImageLightbox({ data-image-fallback role="img" aria-label={alt} - className="bg-muted block aspect-square size-48 max-w-full rounded" + className="bg-muted block aspect-video w-full rounded" /> ) : ( Captured Bookmark body

+ contentHtml: + contentHtml ?? + `

Captured Bookmark body

External reader link

Reader image diff --git a/apps/app/tests/e2e/self-hosted/article-code-blocks.spec.ts b/apps/app/tests/e2e/self-hosted/article-code-blocks.spec.ts new file mode 100644 index 000000000..82142888f --- /dev/null +++ b/apps/app/tests/e2e/self-hosted/article-code-blocks.spec.ts @@ -0,0 +1,140 @@ +import { expect, test } from "@playwright/test"; +import { signIn } from "../fixtures/auth"; +import { + SELF_HOSTED_APP_PORT, + SELF_HOSTED_TURSO_PORT, +} from "../fixtures/ports"; +import { + cleanupUser, + seedArticleData, + seedBookmarkProjectionData, + setFeedItemContent, +} from "../fixtures/seed-db"; + +const longLine = `const message = "${"A long line of code. ".repeat(18)}";`; +const articleHtml = `

Code examples retain their formatting in the reader.

+
const answer = 42;\nconsole.log(answer);
+
${longLine}
+

Images that cannot load keep their place in the article.

+ Unavailable illustration`; + +test.describe("article code and image layout", () => { + test.use({ serviceWorkers: "block" }); + let email: string; + test.afterEach(async () => { + if (email) await cleanupUser(SELF_HOSTED_TURSO_PORT, email); + }); + + for (const kind of ["full", "simplified", "bookmark"] as const) { + test(`${kind} code fills the reader and scrolls within its block`, async ({ + page, + }) => { + const fixture = await seedArticleData( + SELF_HOSTED_TURSO_PORT, + SELF_HOSTED_APP_PORT, + ); + email = fixture.email; + await setFeedItemContent( + SELF_HOSTED_TURSO_PORT, + fixture.feedItemId, + articleHtml, + ); + const contentId = + kind === "bookmark" + ? ( + await seedBookmarkProjectionData( + SELF_HOSTED_TURSO_PORT, + email, + fixture.feedItemId, + articleHtml, + ) + ).bookmarkId + : fixture.feedItemId; + await page.addInitScript( + (style) => { + localStorage.setItem( + "serial-flag-article-style", + JSON.stringify(style), + ); + }, + kind === "simplified" ? "simplified" : "full", + ); + await page.route("https://images.example.com/**", (route) => + route.abort(), + ); + await signIn({ page, email, password: fixture.password }); + await page.goto(`/read/${contentId}`); + const blocks = page.locator("pre"); + await expect(blocks).toHaveCount(2); + await expect(blocks.first().locator(".hljs-keyword")).toBeVisible(); + await expect(blocks.last()).toHaveText(longLine); + + for (const width of [1280, 390]) { + await page.setViewportSize({ width, height: 900 }); + const geometry = await blocks.first().evaluate((short) => { + const wide = document.querySelectorAll("pre")[1]!; + const parent = short.parentElement!; + const style = getComputedStyle(parent); + const available = + parent.clientWidth - + parseFloat(style.paddingLeft) - + parseFloat(style.paddingRight); + wide.scrollLeft = 100; + const scrollContainer = document.querySelector( + '[data-slot="sidebar-inset"]', + )!; + return { + short: short.getBoundingClientRect().width, + wide: wide.getBoundingClientRect().width, + available, + scrollLeft: wide.scrollLeft, + pageOverflow: + scrollContainer.scrollWidth - scrollContainer.clientWidth, + whiteSpace: getComputedStyle(wide).whiteSpace, + }; + }); + expect( + Math.abs(geometry.short - geometry.available), + ).toBeLessThanOrEqual(1); + expect(geometry.wide).toBe(geometry.short); + expect(geometry.scrollLeft).toBeGreaterThan(0); + expect(geometry.pageOverflow).toBeLessThanOrEqual(1); + expect(geometry.whiteSpace).toBe("pre"); + + if (kind !== "simplified") { + const placeholder = page.locator("[data-image-fallback]"); + await placeholder.scrollIntoViewIfNeeded(); + await expect(placeholder).toBeVisible(); + const box = await placeholder.boundingBox(); + expect(box).not.toBeNull(); + expect(Math.abs(box!.width - geometry.available)).toBeLessThanOrEqual( + 1, + ); + expect(Math.abs(box!.width / box!.height - 16 / 9)).toBeLessThan( + 0.02, + ); + } + } + const token = blocks.first().locator(".hljs-keyword"); + await page.evaluate(() => + document.documentElement.classList.remove("dark"), + ); + const light = await token.evaluate( + (node) => getComputedStyle(node).color, + ); + await page.evaluate(() => document.documentElement.classList.add("dark")); + expect( + await token.evaluate((node) => getComputedStyle(node).color), + ).not.toBe(light); + + await blocks.last().evaluate((node) => { + node.scrollLeft = 0; + }); + await blocks.last().focus(); + await page.keyboard.press("ArrowRight"); + await expect + .poll(() => blocks.last().evaluate((node) => node.scrollLeft)) + .toBeGreaterThan(0); + }); + } +}); diff --git a/apps/app/tests/unit/components/code-highlighting-worker.test.ts b/apps/app/tests/unit/components/code-highlighting-worker.test.ts new file mode 100644 index 000000000..a44de2bac --- /dev/null +++ b/apps/app/tests/unit/components/code-highlighting-worker.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +class TestWorker { + static instances: TestWorker[] = []; + onmessage?: (event: { data: { id: number; html: string | null } }) => void; + onerror?: () => void; + postMessage = vi.fn(); + terminate = vi.fn(); + constructor() { + TestWorker.instances.push(this); + } +} + +beforeEach(() => { + vi.resetModules(); + TestWorker.instances = []; + vi.stubGlobal("Worker", TestWorker); +}); +afterEach(() => vi.unstubAllGlobals()); + +describe("code highlighting worker", () => { + it("shares one worker and delivers each result to its block", async () => { + const { requestCodeHighlight } = + await import("~/components/content-reader/highlightCodeClient"); + const first = vi.fn(); + const second = vi.fn(); + requestCodeHighlight("first", "js", first); + requestCodeHighlight("second", undefined, second); + expect(TestWorker.instances).toHaveLength(1); + const worker = TestWorker.instances[0]!; + const [firstMessage, secondMessage] = worker.postMessage.mock.calls.map( + ([message]) => message as { id: number }, + ); + worker.onmessage?.({ + data: { id: secondMessage!.id, html: "second result" }, + }); + worker.onmessage?.({ + data: { id: firstMessage!.id, html: "first result" }, + }); + expect(first).toHaveBeenCalledWith("first result"); + expect(second).toHaveBeenCalledWith("second result"); + }); + + it("drops cancelled results and resolves pending blocks as plain text on failure", async () => { + const { requestCodeHighlight } = + await import("~/components/content-reader/highlightCodeClient"); + const cancelled = vi.fn(); + const pending = vi.fn(); + const cancel = requestCodeHighlight("old", "js", cancelled); + requestCodeHighlight("current", "js", pending); + cancel(); + const worker = TestWorker.instances[0]!; + const message = worker.postMessage.mock.calls[0]![0] as { id: number }; + worker.onmessage?.({ data: { id: message.id, html: "stale" } }); + expect(cancelled).not.toHaveBeenCalled(); + worker.onerror?.(); + expect(pending).toHaveBeenCalledWith(null); + expect(worker.terminate).toHaveBeenCalledOnce(); + requestCodeHighlight("retry", "js", vi.fn()); + expect(TestWorker.instances).toHaveLength(2); + }); + + it("keeps plain code when workers are unavailable", async () => { + vi.stubGlobal("Worker", undefined); + const { requestCodeHighlight } = + await import("~/components/content-reader/highlightCodeClient"); + const callback = vi.fn(); + requestCodeHighlight("const x = 1;", "js", callback); + expect(callback).toHaveBeenCalledWith(null); + }); + + it("terminates abandoned work before highlighting the next article", async () => { + const { requestCodeHighlight } = + await import("~/components/content-reader/highlightCodeClient"); + const cancel = requestCodeHighlight("old", "js", vi.fn()); + const abandonedWorker = TestWorker.instances[0]!; + cancel(); + expect(abandonedWorker.terminate).toHaveBeenCalledOnce(); + const next = vi.fn(); + requestCodeHighlight("new", "js", next); + expect(TestWorker.instances).toHaveLength(2); + abandonedWorker.onerror?.(); + expect(next).not.toHaveBeenCalled(); + expect(TestWorker.instances[1]!.terminate).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/app/tests/unit/components/offline-ui.test.ts b/apps/app/tests/unit/components/offline-ui.test.ts index 667309327..708ee18a7 100644 --- a/apps/app/tests/unit/components/offline-ui.test.ts +++ b/apps/app/tests/unit/components/offline-ui.test.ts @@ -46,7 +46,7 @@ describe("Offline banner", () => { }); describe("article image fallback", () => { - it("keeps a failed image in flow as a square muted block", () => { + it("keeps a failed image in flow as a full-width landscape block", () => { const container = render( createElement(ArticleImageLightbox, { src: "https://example.com/unavailable.jpg", @@ -59,7 +59,8 @@ describe("article image fallback", () => { act(() => image?.dispatchEvent(new Event("error"))); const fallback = container.querySelector("[data-image-fallback]"); - expect(fallback?.className).toContain("aspect-square"); + expect(fallback?.className).toContain("aspect-video"); + expect(fallback?.className).toContain("w-full"); expect(fallback?.className).toContain("bg-muted"); expect(fallback?.getAttribute("aria-label")).toBe( "Unavailable illustration", diff --git a/apps/app/tests/unit/components/reader-code-blocks.test.ts b/apps/app/tests/unit/components/reader-code-blocks.test.ts new file mode 100644 index 000000000..d66ac62e9 --- /dev/null +++ b/apps/app/tests/unit/components/reader-code-blocks.test.ts @@ -0,0 +1,121 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ArticleContent } from "~/components/feed/read/ArticleContent"; +import { BookmarkArticleContent } from "~/components/bookmarks/BookmarkArticleContent"; +import * as highlighter from "~/components/content-reader/highlightCode"; + +vi.mock("~/components/content-reader/highlightCodeClient", () => ({ + requestCodeHighlight: ( + code: string, + language: string | undefined, + onResult: (html: string | null) => void, + ) => { + onResult(highlighter.highlightCode(code, language)); + return () => {}; + }, +})); + +vi.mock("~/lib/hooks/useFlagState", () => ({ useFlagState: () => ["iframe"] })); +vi.mock("~/components/CustomVideoPlayer", () => ({ + CustomVideoPlayer: () => null, +})); +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const roots: Array> = []; +afterEach(() => { + for (const root of roots.splice(0)) act(() => root.unmount()); + vi.restoreAllMocks(); +}); + +async function render(content: string, kind = "full") { + const container = document.createElement("div"); + const root = createRoot(container); + roots.push(root); + const element = + kind === "bookmark" + ? createElement(BookmarkArticleContent, { content }) + : createElement(ArticleContent, { + content, + simplified: kind === "simplified", + }); + await act(async () => { + root.render(element); + }); + return { container, root, element }; +} + +describe("article code blocks", () => { + it("preserves code direction and an explicit accessible label", async () => { + const { container } = await render( + '
const value = 42;
', + ); + expect(container.querySelector("pre")?.dir).toBe("ltr"); + expect(container.querySelector("pre")?.getAttribute("aria-label")).toBe( + "Example", + ); + }); + it.each(["full", "simplified", "bookmark"])( + "highlights %s code without changing text or inline code", + async (kind) => { + const { container } = await render( + '

Use value.

const value = "<img src=x onerror=alert(1)>&";\n  console.log(value);
', + kind, + ); + expect(container.querySelector("pre")?.textContent).toBe( + 'const value = "&";\n console.log(value);', + ); + expect(container.querySelector("pre .hljs-keyword")).not.toBeNull(); + expect(container.querySelector("img")).toBeNull(); + expect(container.querySelector("p code")?.innerHTML).toBe("value"); + expect(container.querySelector("pre")?.tabIndex).toBe(0); + }, + ); + + it("reads language aliases from pre and strips existing token markup", async () => { + const { container } = await render( + '
def greet(name):
return "Hello " + name
', + ); + expect(container.querySelector("pre")?.textContent).toBe( + 'def greet(name):\n return "Hello " + name', + ); + expect(container.querySelector(".hljs-keyword")?.textContent).toBe("def"); + }); + + it.each(["language-unknown", "nohighlight", "language-text"])( + "keeps %s blocks as plain text", + async (className) => { + const { container } = await render( + `
const value = 42;
`, + ); + expect(container.querySelector("pre")?.textContent).toBe( + "const value = 42;", + ); + expect(container.querySelector("pre span")).toBeNull(); + }, + ); + + it("does not tokenize again on unrelated rerenders and discards stale highlighted content", async () => { + const spy = vi.spyOn(highlighter, "highlightCode"); + const { container, root, element } = await render( + '
const first = 42;
', + ); + expect(spy).toHaveBeenCalledTimes(1); + await act(async () => { + root.render({ ...element }); + }); + expect(spy).toHaveBeenCalledTimes(1); + await act(async () => { + root.render( + createElement(ArticleContent, { + content: '
second
', + }), + ); + }); + expect(container.querySelector("pre")?.innerHTML).toBe( + "second", + ); + }); +}); diff --git a/apps/app/tests/unit/performance/reader-highlighting.test.ts b/apps/app/tests/unit/performance/reader-highlighting.test.ts new file mode 100644 index 000000000..77f1ae3b9 --- /dev/null +++ b/apps/app/tests/unit/performance/reader-highlighting.test.ts @@ -0,0 +1,41 @@ +import hljs from "highlight.js/lib/core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { canHighlightCode } from "~/components/content-reader/codeHighlightingPolicy"; +import { highlightCode } from "~/components/content-reader/highlightCode"; + +afterEach(() => vi.restoreAllMocks()); + +describe("reader highlighting work bounds", () => { + it("skips oversized, empty and explicitly plain code before tokenization", () => { + const explicit = vi.spyOn(hljs, "highlight"); + const automatic = vi.spyOn(hljs, "highlightAuto"); + for (const [code, language] of [ + ["x".repeat(20_001), "js"], + ["x".repeat(5_001), undefined], + ["const x = 1", "text"], + [" ", undefined], + ] as const) { + expect(canHighlightCode(code, language)).toBe(false); + expect(highlightCode(code, language)).toBeNull(); + } + expect(explicit).not.toHaveBeenCalled(); + expect(automatic).not.toHaveBeenCalled(); + }); + + it("restricts detection to a fixed grammar set and leaves ambiguous prose plain", () => { + const automatic = vi.spyOn(hljs, "highlightAuto"); + expect(highlightCode("hello world")).toBeNull(); + expect(automatic.mock.calls[0]?.[1]).toHaveLength(7); + expect(highlightCode("const value = 42;\nconsole.log(value);")).toContain( + "hljs", + ); + }); + + it("falls back when a grammar throws or a language is unavailable", () => { + expect(highlightCode("unknown", "nonexistent")).toBeNull(); + vi.spyOn(hljs, "highlight").mockImplementation(() => { + throw new Error("grammar failure"); + }); + expect(highlightCode("const x = 1;", "js")).toBeNull(); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be94e654c..4c7148596 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -220,6 +220,9 @@ importers: feedscout: specifier: ^1.10.0 version: 1.10.0 + highlight.js: + specifier: ^11.12.0 + version: 11.12.0 html-react-parser: specifier: ^6.1.4 version: 6.1.4(@types/react@19.2.17)(react@19.2.7) @@ -6670,6 +6673,10 @@ packages: resolution: {integrity: sha512-Ox1pJVrDCyGHMG9CFg1tmrRUMRPRsAWYc/PinY0XzJU4K7y7vjNoLKIQ7BR5UJMCxNN8EM1MNDmHWA/B3aZUuw==} engines: {node: '>=6'} + highlight.js@11.12.0: + resolution: {integrity: sha512-nbfWpyRMcMrPMmDwJB+dhX/eiaPKtc2RB+0QZskqJ3WjRA/FDS0e9hZrx8EC/lbEv8gXy98FcDbNa/dspAaJMg==} + engines: {node: '>=12.0.0'} + hookable@6.1.1: resolution: {integrity: sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ==} @@ -16537,6 +16544,8 @@ snapshots: hex-rgb@4.3.0: {} + highlight.js@11.12.0: {} + hookable@6.1.1: {} html-dom-parser@8.0.0: From 4a0a2fa74684de7db7939045cc85a68b687ac081 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:34:26 -0400 Subject: [PATCH 09/12] reduce code text and preserve code block boundaries --- .../components/content-reader/ReaderCodeBlock.tsx | 2 +- .../src/components/feed/read/article.module.css | 1 + .../unit/components/reader-code-blocks.test.ts | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/app/src/components/content-reader/ReaderCodeBlock.tsx b/apps/app/src/components/content-reader/ReaderCodeBlock.tsx index b71e7fa1d..397d2685d 100644 --- a/apps/app/src/components/content-reader/ReaderCodeBlock.tsx +++ b/apps/app/src/components/content-reader/ReaderCodeBlock.tsx @@ -87,7 +87,7 @@ export function replaceReaderCodeBlock(node: DOMNode) { ); return ( { + it.each(["full", "simplified", "bookmark"])( + "excludes surrounding controls and formatting from %s code", + async (kind) => { + const { container } = await render( + '
\n  const value = 42;\n\n
', + kind, + ); + expect(container.querySelector("pre")?.textContent).toBe( + " const value = 42;\n", + ); + expect(container.querySelector("button")).toBeNull(); + }, + ); + it("preserves code direction and an explicit accessible label", async () => { const { container } = await render( '
const value = 42;
', From 31991ce4bbf56aacf53261abc3b03235c2677abf Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Fri, 18 Sep 2026 22:05:31 -0400 Subject: [PATCH 10/12] show alt text in article image placeholders --- .../components/feed/read/ArticleContent.tsx | 38 ++++++++++--------- .../feed/read/ArticleImageLightbox.tsx | 12 +++++- .../self-hosted/article-code-blocks.spec.ts | 17 +++++++++ .../tests/unit/components/offline-ui.test.ts | 1 + .../components/reader-content-images.test.ts | 20 +++++++++- 5 files changed, 68 insertions(+), 20 deletions(-) diff --git a/apps/app/src/components/feed/read/ArticleContent.tsx b/apps/app/src/components/feed/read/ArticleContent.tsx index a938e7f8b..9b0078c76 100644 --- a/apps/app/src/components/feed/read/ArticleContent.tsx +++ b/apps/app/src/components/feed/read/ArticleContent.tsx @@ -16,14 +16,25 @@ function extractYouTubeVideoId(src: string): string | null { return match?.[1] ?? null; } -function findImageSrc(node: Element): string | null { - if (node.name === "img") return node.attribs.src ?? null; - if (node.name === "source") - return node.attribs.srcset?.split(/\s/)[0] ?? null; +function findImage(node: Element): { src: string; alt?: string } | null { + if (node.name === "img") { + return node.attribs.src + ? { src: node.attribs.src, alt: node.attribs.alt } + : null; + } + if (node.name === "source") { + const src = node.attribs.srcset?.split(/\s/)[0]; + const image = node.parent?.children.find( + (child) => child instanceof Element && child.name === "img", + ); + return src + ? { src, alt: image instanceof Element ? image.attribs.alt : undefined } + : null; + } for (const child of node.children) { if (child instanceof Element) { - const src = findImageSrc(child); - if (src) return src; + const image = findImage(child); + if (image) return image; } } return null; @@ -34,7 +45,7 @@ function isImageContainer(node: Element): boolean { const cls = node.attribs.class ?? ""; if (cls.includes("image-link") || cls.includes("image2")) return true; } - if (node.name === "figure") return !!findImageSrc(node); + if (node.name === "figure") return true; if (node.attribs.class?.includes("captioned-image-container")) return true; return false; } @@ -65,16 +76,9 @@ export function ArticleContent({ domNode.attribs.rel = "noopener noreferrer"; } - if (domNode.name === "img") { - const src = domNode.attribs.src ?? ""; - const alt = domNode.attribs.alt ?? ""; - if (!src) return; - return ; - } - - if (isImageContainer(domNode)) { - const src = findImageSrc(domNode); - if (src) return ; + if (domNode.name === "img" || isImageContainer(domNode)) { + const image = findImage(domNode); + if (image) return ; } if (domNode.name !== "iframe") return; diff --git a/apps/app/src/components/feed/read/ArticleImageLightbox.tsx b/apps/app/src/components/feed/read/ArticleImageLightbox.tsx index 4ebbb28af..be324cc87 100644 --- a/apps/app/src/components/feed/read/ArticleImageLightbox.tsx +++ b/apps/app/src/components/feed/read/ArticleImageLightbox.tsx @@ -43,8 +43,16 @@ export function ArticleImageLightbox({ data-image-fallback role="img" aria-label={alt} - className="bg-muted block aspect-video w-full rounded" - /> + className="bg-muted relative block aspect-video w-full overflow-hidden rounded" + > + {alt && ( + + + {alt} + + + )} + ) : ( { const placeholder = page.locator("[data-image-fallback]"); await placeholder.scrollIntoViewIfNeeded(); await expect(placeholder).toBeVisible(); + await expect(placeholder).toHaveText("Unavailable illustration"); + const label = placeholder.locator("span").first(); + expect( + await label.evaluate((node) => + parseFloat(getComputedStyle(node).paddingLeft), + ), + ).toBe(width < 640 ? 32 : 64); + const textBox = await label.locator("span").boundingBox(); const box = await placeholder.boundingBox(); expect(box).not.toBeNull(); + expect(textBox).not.toBeNull(); + expect( + Math.abs(textBox!.x + textBox!.width / 2 - box!.x - box!.width / 2), + ).toBeLessThanOrEqual(1); + expect( + Math.abs( + textBox!.y + textBox!.height / 2 - box!.y - box!.height / 2, + ), + ).toBeLessThanOrEqual(1); expect(Math.abs(box!.width - geometry.available)).toBeLessThanOrEqual( 1, ); diff --git a/apps/app/tests/unit/components/offline-ui.test.ts b/apps/app/tests/unit/components/offline-ui.test.ts index 708ee18a7..0220b5968 100644 --- a/apps/app/tests/unit/components/offline-ui.test.ts +++ b/apps/app/tests/unit/components/offline-ui.test.ts @@ -65,5 +65,6 @@ describe("article image fallback", () => { expect(fallback?.getAttribute("aria-label")).toBe( "Unavailable illustration", ); + expect(fallback?.textContent).toBe("Unavailable illustration"); }); }); diff --git a/apps/app/tests/unit/components/reader-content-images.test.ts b/apps/app/tests/unit/components/reader-content-images.test.ts index fe09ba6a4..ce67cbff2 100644 --- a/apps/app/tests/unit/components/reader-content-images.test.ts +++ b/apps/app/tests/unit/components/reader-content-images.test.ts @@ -1,9 +1,15 @@ import { createElement } from "react"; import { renderToStaticMarkup } from "react-dom/server"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import type { ReactNode } from "react"; import { flattenReaderImages } from "~/components/content-reader/flattenReaderImages"; import { ArticleImageLightbox } from "~/components/feed/read/ArticleImageLightbox"; +import { ArticleContent } from "~/components/feed/read/ArticleContent"; + +vi.mock("~/lib/hooks/useFlagState", () => ({ useFlagState: () => ["iframe"] })); +vi.mock("~/components/CustomVideoPlayer", () => ({ + CustomVideoPlayer: () => null, +})); function renderReaderNodes(...nodes: ReactNode[]) { return renderToStaticMarkup( @@ -19,6 +25,18 @@ function expectInOrder(markup: string, ...values: string[]) { } describe("reader content images", () => { + it.each([ + '
Garden path
', + '
Garden path
', + '
Garden path
', + ])("preserves alt text when replacing an image wrapper: %s", (content) => { + const markup = renderToStaticMarkup( + createElement(ArticleContent, { content }), + ); + expect(markup).toContain('alt="Garden path"'); + expect(markup).toContain('aria-label="Open image preview: Garden path"'); + }); + it("preserves an image between paragraphs in a wrapped article", () => { const markup = renderReaderNodes( createElement( From 73d151425c9277aad83444ae457c2538121f11ec Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:06:36 -0400 Subject: [PATCH 11/12] initial release notes draft --- apps/www/src/content/releases/2026-09-23.md | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 apps/www/src/content/releases/2026-09-23.md diff --git a/apps/www/src/content/releases/2026-09-23.md b/apps/www/src/content/releases/2026-09-23.md new file mode 100644 index 000000000..46d260da5 --- /dev/null +++ b/apps/www/src/content/releases/2026-09-23.md @@ -0,0 +1,26 @@ +--- +title: "Surfing the Atmosphere" +publish_date: "2026-09-23" +description: "Serial now supports adding standard.site publications and syncing them to your PDS, with extra support for Pckt, Leaflet, and Offprint documents. Plus, some QOL improvements." +public: false +--- +## Standard.Site Support + +TBD + + + +  + +## Other Improvements + +- Code blocks got some much needed TLC, with much more consistent rendering syntax highlighting +- Image blocks now fallback to displaying alt text (when available) if the image isn’t available or hasn’t loaded yet. This is especially useful while offline! +- Saved items that are marked as read now stay visible until you navigate away from your current view, allowing you to unsave the item if desired (instead of having it vanish immediately to the read section) + + + +## Fixes + +- Fixed some initial data fetching edge cases which would sometimes place the “Refresh” button in a perpetual “Retrying” state + From f6d34c5b6df4ac3c2ec94b53804b731f1f382e00 Mon Sep 17 00:00:00 2001 From: Henry <48483883+hfellerhoff@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:08:36 -0400 Subject: [PATCH 12/12] document the browser performance audit script and the standalone release loader --- apps/app/benchmarks/README.md | 12 ++++++++++++ apps/www/scripts/lib/standard-site-content.ts | 3 +++ 2 files changed, 15 insertions(+) diff --git a/apps/app/benchmarks/README.md b/apps/app/benchmarks/README.md index c6d0b5655..702530363 100644 --- a/apps/app/benchmarks/README.md +++ b/apps/app/benchmarks/README.md @@ -241,3 +241,15 @@ The client audit profiles also measure local mixed-content View projection over their complete Bookmark fixtures. This protects fetch-free View navigation and View-chip status computation with the same 50 ms operation budget as other client state transitions. + +Run the retained browser-performance audit against a production build when a +change touches hydration, synchronization, rendering, or the reader: + +```sh +pnpm benchmark:client:browser +``` + +It runs `tests/e2e/self-hosted/client-performance-audit.spec.ts`, which is +skipped unless `SERIAL_RUN_CLIENT_PERFORMANCE=1`, and evaluates each scenario +against the budgets in `scripts/performance/client-browser-budgets.ts`. Its +generated JSON is written under the ignored `benchmarks/results/` directory. diff --git a/apps/www/scripts/lib/standard-site-content.ts b/apps/www/scripts/lib/standard-site-content.ts index 03561eef6..a43c9c87c 100644 --- a/apps/www/scripts/lib/standard-site-content.ts +++ b/apps/www/scripts/lib/standard-site-content.ts @@ -10,6 +10,9 @@ const frontmatterSchema = z.object({ public: z.boolean(), }); +// The sync script runs under plain Node, so it cannot import the Astro content +// collection loader in src/lib/content.ts (an `astro:content` virtual module). +// Keep this reader aligned with that loader's frontmatter contract. const RELEASES_DIR = new URL("../../src/content/releases/", import.meta.url); export async function loadReleaseDocuments(directory = RELEASES_DIR) {