diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index ae9a572dee..64b81a6ca6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -17,7 +17,7 @@ import { ConnectionHealthProvider, useConnectionHealth } from './hooks/useConnec import { DEFAULT_GLOBAL_LABELS } from './components/Labels/labelDefaults' import { filtersFromSearchParams, filtersToSearchParams } from './components/History/historyFilters' import type { ViewName } from './components/Sidebar/Navigation' -import type { TargetInstance, TargetInfo } from './types' +import type { TargetInstance, TargetInfo, AttackOutcome, ScoreView } from './types' import { targetEndpoint, targetIdentifierHash, @@ -56,6 +56,9 @@ interface LoadedAttack { labels: Record | null target: TargetInfo | null relatedConversationIds: string[] + objective: string + outcome: AttackOutcome | null + lastScore: ScoreView | null status: AttackLoadStatus } @@ -206,6 +209,9 @@ function App() { labels: null, target: null, relatedConversationIds: [], + objective: '', + outcome: null, + lastScore: null, }) attacksApi .getAttack(routeAttackId) @@ -217,6 +223,9 @@ function App() { labels: attack.labels ?? {}, target: attack.target ?? null, relatedConversationIds: attack.related_conversation_ids ?? [], + objective: attack.objective ?? '', + outcome: attack.outcome ?? null, + lastScore: attack.last_score ?? null, status: 'success', }) }) @@ -233,6 +242,9 @@ function App() { labels: null, target: null, relatedConversationIds: [], + objective: '', + outcome: null, + lastScore: null, }) }) // Drop a stale response once the route has moved on to another attack. @@ -300,6 +312,9 @@ function App() { labels: null, target, relatedConversationIds: [], + objective: '', + outcome: null, + lastScore: null, status: 'success', }) // Replace when promoting an empty /chat to its attack url (first message); @@ -339,6 +354,9 @@ function App() { attackTarget={readyAttack ? readyAttack.target : null} isLoadingAttack={isLoadingAttack} relatedConversationCount={readyAttack ? readyAttack.relatedConversationIds.length : 0} + objective={readyAttack ? readyAttack.objective : ''} + outcome={readyAttack ? readyAttack.outcome : null} + lastScore={readyAttack ? readyAttack.lastScore : null} /> ) diff --git a/frontend/src/components/Chat/AttackVerdictChip.styles.ts b/frontend/src/components/Chat/AttackVerdictChip.styles.ts new file mode 100644 index 0000000000..6baed2190c --- /dev/null +++ b/frontend/src/components/Chat/AttackVerdictChip.styles.ts @@ -0,0 +1,42 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +export const useAttackVerdictChipStyles = makeStyles({ + chip: { + display: 'inline-flex', + alignItems: 'center', + columnGap: tokens.spacingHorizontalXS, + }, + chipLabel: { + textTransform: 'capitalize', + }, + surface: { + display: 'flex', + flexDirection: 'column', + rowGap: tokens.spacingVerticalXS, + minWidth: '240px', + maxWidth: '360px', + }, + row: { + display: 'flex', + columnGap: tokens.spacingHorizontalS, + }, + rowLabel: { + minWidth: '72px', + color: tokens.colorNeutralForeground2, + }, + rationaleBlock: { + display: 'flex', + flexDirection: 'column', + rowGap: tokens.spacingVerticalXXS, + marginTop: tokens.spacingVerticalXS, + paddingTop: tokens.spacingVerticalXS, + borderTop: `1px solid ${tokens.colorNeutralStroke2}`, + }, + rationaleText: { + color: tokens.colorNeutralForeground2, + whiteSpace: 'pre-wrap', + wordBreak: 'break-word', + maxHeight: '30vh', + overflowY: 'auto', + }, +}) diff --git a/frontend/src/components/Chat/AttackVerdictChip.test.tsx b/frontend/src/components/Chat/AttackVerdictChip.test.tsx new file mode 100644 index 0000000000..97d8ff1bd4 --- /dev/null +++ b/frontend/src/components/Chat/AttackVerdictChip.test.tsx @@ -0,0 +1,114 @@ +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import AttackVerdictChip from './AttackVerdictChip' +import type { ScoreView } from '../../types' + +const TestWrapper: React.FC<{ children: React.ReactNode }> = ({ children }) => ( + {children} +) + +const sampleScore: ScoreView = { + id: 'score-1', + scorer_type: 'SelfAskRefusalScorer', + score_type: 'true_false', + score_value: 'true', + score_category: ['refusal'], + score_rationale: 'The model refused the request.', + timestamp: '2026-01-15T11:00:00Z', +} + +describe('AttackVerdictChip', () => { + it('renders nothing when there is neither an outcome nor a score', () => { + render( + + + + ) + + expect(screen.queryByTestId('attack-verdict-chip')).not.toBeInTheDocument() + expect(screen.queryByTestId('attack-score-chip')).not.toBeInTheDocument() + }) + + it('renders an outcome-only badge when there is an outcome but no score', () => { + render( + + + + ) + + // No score means no interactive score chip/popover, just the outcome badge. + expect(screen.queryByTestId('attack-score-chip')).not.toBeInTheDocument() + const chip = screen.getByTestId('attack-verdict-chip') + expect(within(chip).getByText('failure')).toBeInTheDocument() + }) + + it('shows a single chip labeled with the outcome, tinted by the score', () => { + render( + + + + ) + + const chip = screen.getByRole('button', { name: /verdict failure, score true/i }) + // The chip shows the outcome; the raw score value lives in the popover + expect(within(chip).getByText('failure')).toBeInTheDocument() + expect(within(chip).queryByText('true')).not.toBeInTheDocument() + }) + + it('opens a popover with full verdict details when clicked', async () => { + const user = userEvent.setup() + render( + + + + ) + + await user.click(screen.getByRole('button', { name: /verdict failure, score true/i })) + + const details = await screen.findByTestId('attack-score-details') + expect(within(details).getByText('true_false')).toBeInTheDocument() + expect(within(details).getByText('SelfAskRefusalScorer')).toBeInTheDocument() + expect(within(details).getByText('refusal')).toBeInTheDocument() + expect(within(details).getByText('The model refused the request.')).toBeInTheDocument() + }) + + it('shows the underlying scale score when a thresholded verdict provides one', async () => { + const user = userEvent.setup() + const thresholdScore: ScoreView = { + id: 'score-2', + scorer_type: 'FloatScaleThresholdScorer', + score_type: 'true_false', + score_value: 'false', + score_category: ['humor'], + score_rationale: 'Normalized scale score: 0.5 < threshold 0.6', + scale_score: 0.5, + timestamp: '2026-01-15T11:00:00Z', + } + render( + + + + ) + + await user.click(screen.getByRole('button', { name: /verdict failure, score false/i })) + + const details = await screen.findByTestId('attack-score-details') + expect(within(details).getByText('Scale score')).toBeInTheDocument() + expect(within(details).getByText('0.50')).toBeInTheDocument() + }) + + it('omits the scale score row for a plain true/false score', async () => { + const user = userEvent.setup() + render( + + + + ) + + await user.click(screen.getByRole('button', { name: /verdict failure, score true/i })) + + const details = await screen.findByTestId('attack-score-details') + expect(within(details).queryByText('Scale score')).not.toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/Chat/AttackVerdictChip.tsx b/frontend/src/components/Chat/AttackVerdictChip.tsx new file mode 100644 index 0000000000..17b61a9d12 --- /dev/null +++ b/frontend/src/components/Chat/AttackVerdictChip.tsx @@ -0,0 +1,115 @@ +import { + Button, + Text, + Badge, + Popover, + PopoverTrigger, + PopoverSurface, +} from '@fluentui/react-components' +import type { AttackOutcome, ScoreView } from '../../types' +import { OUTCOME_COLORS, OUTCOME_ICONS, resolveOutcome } from '../../utils/attackOutcome' +import { getScoreColor } from '../../utils/scoreColor' +import { useAttackVerdictChipStyles } from './AttackVerdictChip.styles' + +interface AttackVerdictChipProps { + outcome?: AttackOutcome | null + score?: ScoreView | null +} + +export default function AttackVerdictChip({ outcome, score }: AttackVerdictChipProps) { + const styles = useAttackVerdictChipStyles() + + // The verdict is the attack's outcome plus its score. If both are missing, there's nothing to show + // (e.g. a manual GUI attack, which can't be scored yet), so the chip is omitted entirely. + if (!outcome && !score) return null + const resolvedOutcome = resolveOutcome(outcome) + const outcomeBadge = ( + + {resolvedOutcome} + + ) + + // Outcome-only verdict: strategies that produce an outcome but no score (e.g. sequential, + // barge-in) still get the outcome badge, matching the history table, with no score popover. + if (!score) { + return ( +
+ {outcomeBadge} +
+ ) + } + + const categories = score.score_category?.filter(Boolean) ?? [] + // A FloatScaleThresholdScorer keeps the raw scale score behind the true/false + // score; surface the float value and let it determine the badge tone. + const scaleScore = score.scale_score ?? null + const scoreColor = getScoreColor(resolvedOutcome, score.score_type, score.score_value, scaleScore) + const scoreBadgeStyle = { + backgroundColor: scoreColor.background, + borderColor: scoreColor.background, + color: scoreColor.foreground, + } + + return ( + + + + + +
+ Verdict +
+ Value + + {score.score_value} + +
+ {scaleScore !== null && ( +
+ Scale score + {scaleScore.toFixed(2)} +
+ )} +
+ Type + {score.score_type} +
+
+ Scorer + {score.scorer_type} +
+ {categories.length > 0 && ( +
+ Category + {categories.join(', ')} +
+ )} +
+ Outcome + {resolvedOutcome} +
+ {score.score_rationale && ( +
+ Rationale + {score.score_rationale} +
+ )} +
+ + + ) +} diff --git a/frontend/src/components/Chat/ChatWindow.tsx b/frontend/src/components/Chat/ChatWindow.tsx index f9d2096c5c..879c16ee03 100644 --- a/frontend/src/components/Chat/ChatWindow.tsx +++ b/frontend/src/components/Chat/ChatWindow.tsx @@ -15,6 +15,8 @@ import ChatInputArea from './ChatInputArea' import ConversationPanel from './ConversationPanel' import ConverterPanel from './ConverterPanel' import TargetBadge from './TargetBadge' +import ObjectiveHeader from './ObjectiveHeader' +import AttackVerdictChip from './AttackVerdictChip' import type { PieceConversion } from './converterTypes' import { PIECE_TYPE_TO_DATA_TYPE, basenameFromValue, buildMediaUrl, dataTypeToAttachmentKind, isPathDataType } from './converterTypes' import LabelsBar from '../Labels/LabelsBar' @@ -22,7 +24,7 @@ import type { ChatInputAreaHandle } from './ChatInputArea' import { attacksApi } from '../../services/api' import { toApiError } from '../../services/errors' import { buildMessagePieces, backendMessagesToFrontend } from '../../utils/messageMapper' -import type { Message, MessageAttachment, TargetInstance, TargetInfo } from '../../types' +import type { Message, MessageAttachment, TargetInstance, TargetInfo, AttackOutcome, ScoreView } from '../../types' import { targetInfoMatchesTarget } from '../../utils/targetIdentity' import type { ViewName } from '../Sidebar/Navigation' import { useChatWindowStyles } from './ChatWindow.styles' @@ -54,6 +56,12 @@ interface ChatWindowProps { isLoadingAttack?: boolean /** Number of related (non-main) conversations in the loaded attack. */ relatedConversationCount?: number + /** The loaded attack's objective (empty for new/manual attacks). */ + objective?: string + /** The loaded attack's outcome. */ + outcome?: AttackOutcome | null + /** The loaded attack's final score, if any. */ + lastScore?: ScoreView | null } export default function ChatWindow({ @@ -71,6 +79,9 @@ export default function ChatWindow({ attackTarget, isLoadingAttack, relatedConversationCount, + objective = '', + outcome, + lastScore, }: ChatWindowProps) { const styles = useChatWindowStyles() const restoreFocusTargetAttributes = useRestoreFocusTarget() @@ -642,6 +653,9 @@ export default function ChatWindow({ )}
+ {activeConversationId === conversationId && ( + + )}
+ {systemMessage && } = ({ children }) => ( + {children} +) + +// jsdom has no layout engine, so scrollWidth/clientWidth are 0 by default (no overflow). +// Force overflow by overriding the prototype getters for the duration of a test. +function mockOverflow(scrollWidth: number, clientWidth: number) { + Object.defineProperty(HTMLElement.prototype, 'scrollWidth', { configurable: true, get: () => scrollWidth }) + Object.defineProperty(HTMLElement.prototype, 'clientWidth', { configurable: true, get: () => clientWidth }) +} + +describe('ObjectiveHeader', () => { + afterEach(() => { + delete (HTMLElement.prototype as { scrollWidth?: number }).scrollWidth + delete (HTMLElement.prototype as { clientWidth?: number }).clientWidth + }) + + it('renders nothing when the objective is empty', () => { + render( + + + + ) + + expect(screen.queryByTestId('objective-header')).not.toBeInTheDocument() + }) + + it('renders the label and the objective text', () => { + render( + + + + ) + + expect(screen.getByText('Objective')).toBeInTheDocument() + expect(screen.getByText('Extract the hidden system prompt')).toBeInTheDocument() + }) + + it('does not render an expand toggle when the objective fits on one line', () => { + render( + + + + ) + + expect(screen.queryByTestId('toggle-objective-header-btn')).not.toBeInTheDocument() + }) + + it('renders a collapsed "Show more" toggle when the objective overflows', () => { + mockOverflow(1000, 200) + render( + + + + ) + + const toggle = screen.getByRole('button', { name: /show more of the objective/i }) + expect(toggle).toHaveTextContent('Show more') + expect(toggle).toHaveAttribute('aria-expanded', 'false') + }) + + it('expands to "Show less" when the overflowing toggle is clicked', async () => { + const user = userEvent.setup() + mockOverflow(1000, 200) + render( + + + + ) + + await user.click(screen.getByRole('button', { name: /show more of the objective/i })) + + const toggle = screen.getByRole('button', { name: /show less of the objective/i }) + expect(toggle).toHaveTextContent('Show less') + expect(toggle).toHaveAttribute('aria-expanded', 'true') + }) +}) diff --git a/frontend/src/components/Chat/ObjectiveHeader.tsx b/frontend/src/components/Chat/ObjectiveHeader.tsx new file mode 100644 index 0000000000..9a05dda24d --- /dev/null +++ b/frontend/src/components/Chat/ObjectiveHeader.tsx @@ -0,0 +1,60 @@ +import { useLayoutEffect, useRef, useState } from 'react' +import { Badge, Button, Text, mergeClasses } from '@fluentui/react-components' +import { ChevronDownRegular, ChevronUpRegular } from '@fluentui/react-icons' +import { useObjectiveHeaderStyles } from './ObjectiveHeader.styles' + +interface ObjectiveHeaderProps { + objective: string +} + +export default function ObjectiveHeader({ objective }: ObjectiveHeaderProps) { + const styles = useObjectiveHeaderStyles() + const [expanded, setExpanded] = useState(false) + const [overflowing, setOverflowing] = useState(false) + const contentRef = useRef(null) + + useLayoutEffect(() => { + const el = contentRef.current + if (!el) return + const measure = () => { + if (expanded) return + setOverflowing(el.scrollWidth > el.clientWidth) + } + measure() + const observer = new ResizeObserver(measure) + observer.observe(el) + return () => observer.disconnect() + }, [objective, expanded]) + + if (!objective) return null + + const showToggle = overflowing || expanded + + return ( +
+ Objective + + {objective} + + {showToggle && ( + + )} +
+ ) +} diff --git a/frontend/src/components/History/AttackHistory.test.tsx b/frontend/src/components/History/AttackHistory.test.tsx index 641431ea9d..3c4be0eb2d 100644 --- a/frontend/src/components/History/AttackHistory.test.tsx +++ b/frontend/src/components/History/AttackHistory.test.tsx @@ -28,6 +28,7 @@ const sampleAttacks = [ conversation_id: 'conv-1', attack_type: 'CrescendoAttack', attack_specific_params: null, + objective: 'Extract the hidden system prompt', target: { target_type: 'OpenAIChatTarget', endpoint: 'https://api.openai.com', model_name: 'gpt-4' }, converters: ['Base64Converter'], outcome: 'success' as const, @@ -43,6 +44,7 @@ const sampleAttacks = [ conversation_id: 'conv-2', attack_type: 'ManualAttack', attack_specific_params: null, + objective: 'Bypass the safety filter', target: { target_type: 'OpenAIImageTarget', endpoint: 'https://api.openai.com', model_name: 'dall-e-3' }, converters: [], outcome: 'failure' as const, diff --git a/frontend/src/components/History/AttackTable.test.tsx b/frontend/src/components/History/AttackTable.test.tsx index 03f7ec6063..212951d0cb 100644 --- a/frontend/src/components/History/AttackTable.test.tsx +++ b/frontend/src/components/History/AttackTable.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, fireEvent } from '@testing-library/react' +import { render, screen, fireEvent, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' import AttackTable from './AttackTable' @@ -17,6 +17,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-1', conversation_id: 'conv-1', attack_type: 'CrescendoAttack', + objective: 'Extract the hidden system prompt', target: { target_type: 'OpenAIChatTarget', endpoint: 'https://api.openai.com', model_name: 'gpt-4' }, converters: ['Base64Converter', 'ROT13Converter', 'UnicodeConverter'], outcome: 'success', @@ -31,6 +32,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-2', conversation_id: 'conv-2', attack_type: 'ManualAttack', + objective: 'Bypass the safety filter', target: null, converters: [], outcome: 'failure', @@ -45,6 +47,7 @@ const sampleAttacks: AttackSummary[] = [ attack_result_id: 'ar-3', conversation_id: 'conv-3', attack_type: 'ManualAttack', + objective: 'Elicit disallowed content', target: { target_type: 'TextTarget', endpoint: null, model_name: null }, converters: [], outcome: undefined, @@ -201,6 +204,67 @@ describe('AttackTable', () => { expect(screen.getAllByText('undetermined')).toHaveLength(1) }) + it('tints the outcome badge by the score when the attack has one', () => { + const scored: AttackSummary[] = [ + { + ...sampleAttacks[0], + attack_result_id: 'ar-scored', + outcome: 'failure', + last_score: { + id: 'score-1', + scorer_type: 'FloatScaleThresholdScorer', + score_type: 'true_false', + score_value: 'false', + scale_score: 0.5, + timestamp: '2026-01-15T11:00:00Z', + }, + }, + ] + render( + + + + ) + + // A scored row is tinted via an inline background (mirroring the verdict + // chip), not the flat Fluent outcome color. + const cell = screen.getByTestId('outcome-badge-ar-scored') + expect(within(cell).getByText('failure')).toHaveStyle({ backgroundColor: 'rgb(161, 62, 64)' }) + }) + + it('reveals the last score in a popover without opening the attack', async () => { + const user = userEvent.setup() + const onOpenAttack = jest.fn() + const scored: AttackSummary[] = [ + { + ...sampleAttacks[0], + attack_result_id: 'ar-scored', + outcome: 'failure', + last_score: { + id: 'score-1', + scorer_type: 'FloatScaleThresholdScorer', + score_type: 'true_false', + score_value: 'false', + scale_score: 0.5, + timestamp: '2026-01-15T11:00:00Z', + }, + }, + ] + render( + + + + ) + + await user.click(screen.getByRole('button', { name: /verdict failure, score false/i })) + + const details = await screen.findByTestId('attack-score-details') + expect(within(details).getByText('Scale score')).toBeInTheDocument() + expect(within(details).getByText('0.50')).toBeInTheDocument() + // Interacting with the chip must not navigate to the attack. + expect(onOpenAttack).not.toHaveBeenCalled() + }) + it('should show target model name as badge when available', () => { render( diff --git a/frontend/src/components/History/AttackTable.tsx b/frontend/src/components/History/AttackTable.tsx index 906be3538b..a07a255cee 100644 --- a/frontend/src/components/History/AttackTable.tsx +++ b/frontend/src/components/History/AttackTable.tsx @@ -13,28 +13,12 @@ import { } from '@fluentui/react-components' import { OpenRegular, - CheckmarkCircleRegular, - DismissCircleRegular, - QuestionCircleRegular, - ErrorCircleRegular, } from '@fluentui/react-icons' import type { AttackSummary } from '../../types' +import { resolveOutcome } from '../../utils/attackOutcome' +import AttackVerdictChip from '../Chat/AttackVerdictChip' import { useAttackHistoryStyles } from './AttackHistory.styles' -const OUTCOME_ICONS: Record = { - success: , - failure: , - error: , - undetermined: , -} - -const OUTCOME_COLORS: Record = { - success: 'success', - failure: 'danger', - error: 'warning', - undetermined: 'informative', -} - interface AttackTableProps { attacks: AttackSummary[] onOpenAttack: (attackResultId: string) => void @@ -80,14 +64,16 @@ export default function AttackTable({ attacks, onOpenAttack, formatDate }: Attac data-testid={`attack-row-${attack.attack_result_id}`} > - e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} > - {attack.outcome ?? 'undetermined'} - + + {attack.attack_type} diff --git a/frontend/src/components/Home/Home.test.tsx b/frontend/src/components/Home/Home.test.tsx index e4776d9421..fe3185cea9 100644 --- a/frontend/src/components/Home/Home.test.tsx +++ b/frontend/src/components/Home/Home.test.tsx @@ -31,6 +31,7 @@ function makeAttack(overrides: Partial = {}): AttackSummary { attack_result_id: "ar-1", conversation_id: "conv-1", attack_type: "TestAttack", + objective: "Test objective", converters: [], outcome: "success", last_message_preview: "preview", diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index a3da01bd0a..a0777e1ecb 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -173,14 +173,33 @@ export interface TargetInfo { identifier_hash: string } +export type AttackOutcome = 'undetermined' | 'success' | 'failure' | 'error' + +// Attack-level verdict score (backend `ScoreView`). Mirrors the display fields +// the API exposes on `AttackSummary.last_score`. +export interface ScoreView { + id: string + scorer_type: string + score_type: string + score_value: string + score_category?: string[] | null + score_rationale?: string | null + // Raw 0-1 scale score behind a thresholded true/false verdict (e.g. from a + // FloatScaleThresholdScorer); null for a plain true/false score. + scale_score?: number | null + timestamp: string +} + export interface AttackSummary { attack_result_id: string conversation_id: string attack_type: string attack_specific_params?: Record | null + objective: string target?: TargetInfo | null converters: string[] - outcome?: 'undetermined' | 'success' | 'failure' | 'error' | null + outcome?: AttackOutcome | null + last_score?: ScoreView | null last_message_preview?: string | null message_count: number related_conversation_ids: string[] diff --git a/frontend/src/utils/attackOutcome.tsx b/frontend/src/utils/attackOutcome.tsx new file mode 100644 index 0000000000..b6a7686f49 --- /dev/null +++ b/frontend/src/utils/attackOutcome.tsx @@ -0,0 +1,29 @@ +import { tokens } from '@fluentui/react-components' +import { + CheckmarkCircleRegular, + DismissCircleRegular, + QuestionCircleRegular, + ErrorCircleRegular, +} from '@fluentui/react-icons' +import type { AttackOutcome } from '../types' + +// Shared presentation for an attack's outcome, used by both the history table +// and the chat verdict chip so the two views stay visually consistent. + +export const OUTCOME_ICONS: Record = { + success: , + failure: , + error: , + undetermined: , +} + +export const OUTCOME_COLORS: Record = { + success: 'success', + failure: 'danger', + error: 'warning', + undetermined: 'informative', +} + +export function resolveOutcome(outcome?: AttackOutcome | null): AttackOutcome { + return outcome ?? 'undetermined' +} diff --git a/frontend/src/utils/scoreColor.test.ts b/frontend/src/utils/scoreColor.test.ts new file mode 100644 index 0000000000..c2c0cf3066 --- /dev/null +++ b/frontend/src/utils/scoreColor.test.ts @@ -0,0 +1,91 @@ +import { getScoreColor, normalizeScoreValue } from './scoreColor' + +const WHITE = '#ffffff' +const GREY = 'rgb(97, 97, 97)' +const RED = 'rgb(188, 47, 50)' +const GREEN = 'rgb(14, 112, 14)' +const AMBER = 'rgb(196, 53, 1)' + +describe('normalizeScoreValue', () => { + it('maps true/false verdicts to the [0, 1] extremes', () => { + expect(normalizeScoreValue('true_false', 'true')).toBe(1) + expect(normalizeScoreValue('true_false', 'false')).toBe(0) + }) + + it('is case- and whitespace-insensitive for true/false', () => { + expect(normalizeScoreValue('true_false', ' TRUE ')).toBe(1) + expect(normalizeScoreValue('true_false', 'False')).toBe(0) + }) + + it('returns null for an uninterpretable true/false value', () => { + expect(normalizeScoreValue('true_false', 'maybe')).toBeNull() + }) + + it('parses float_scale values and clamps them to [0, 1]', () => { + expect(normalizeScoreValue('float_scale', '0')).toBe(0) + expect(normalizeScoreValue('float_scale', '0.75')).toBe(0.75) + expect(normalizeScoreValue('float_scale', '1')).toBe(1) + expect(normalizeScoreValue('float_scale', '1.5')).toBe(1) + expect(normalizeScoreValue('float_scale', '-0.5')).toBe(0) + }) + + it('returns null for non-numeric float_scale values and unknown types', () => { + expect(normalizeScoreValue('float_scale', 'abc')).toBeNull() + expect(normalizeScoreValue('unknown', '0.5')).toBeNull() + }) +}) + +describe('getScoreColor', () => { + it('drives the hue from the outcome, not the score polarity', () => { + // A refusal scorer reports "true" on a *failed* attack; the badge must stay + // red (failure) rather than turning success-green from the raw polarity. + expect(getScoreColor('failure', 'true_false', 'true')).toEqual({ background: RED, foreground: WHITE }) + expect(getScoreColor('success', 'true_false', 'true')).toEqual({ background: GREEN, foreground: WHITE }) + }) + + it('renders a boolean verdict at full, brightest strength for either value', () => { + // A true/false verdict is definitive, so both true and false render the + // full hue -- e.g. a false failure is the brightest red. + expect(getScoreColor('failure', 'true_false', 'false')).toEqual({ background: RED, foreground: WHITE }) + expect(getScoreColor('failure', 'true_false', 'true')).toEqual({ background: RED, foreground: WHITE }) + expect(getScoreColor('success', 'true_false', 'false')).toEqual({ background: GREEN, foreground: WHITE }) + }) + + it('grades a thresholded true/false verdict by its underlying scale score', () => { + // A FloatScaleThresholdScorer keeps the raw 0-1 score in metadata; a false + // failure at 0.5 is a medium red, matching a plain float_scale 0.5. + expect(getScoreColor('failure', 'true_false', 'false', 0.5)).toEqual({ background: 'rgb(161, 62, 64)', foreground: WHITE }) + expect(getScoreColor('failure', 'true_false', 'false', 0.5)).not.toEqual({ background: RED, foreground: WHITE }) + expect(getScoreColor('success', 'true_false', 'true', 1.5)).toEqual({ background: GREEN, foreground: WHITE }) + }) + + it('tints a float by its value: a lower value is lighter', () => { + // A full-value success is the brightest hue; a lower value is a lighter tint of + // the same hue). + expect(getScoreColor('success', 'float_scale', '1')).toEqual({ background: GREEN, foreground: WHITE }) + expect(getScoreColor('success', 'float_scale', '0.75')).toEqual({ background: 'rgb(26, 110, 26)', foreground: WHITE }) + expect(getScoreColor('failure', 'float_scale', '0.5')).toEqual({ background: 'rgb(161, 62, 64)', foreground: WHITE }) + }) + + it('floors the tint', () => { + const lowest = getScoreColor('success', 'float_scale', '0') + expect(lowest).toEqual({ background: 'rgb(64, 103, 64)', foreground: WHITE }) + expect(lowest.background).not.toBe(GREY) + }) + + it('renders an error outcome as amber', () => { + expect(getScoreColor('error', 'true_false', 'true')).toEqual({ background: AMBER, foreground: WHITE }) + }) + + it('renders grey for an undetermined outcome', () => { + expect(getScoreColor('undetermined', 'true_false', 'true')).toEqual({ background: GREY, foreground: WHITE }) + expect(getScoreColor(null, 'float_scale', '0.9')).toEqual({ background: GREY, foreground: WHITE }) + }) + + it('renders grey for an unscored or uninterpretable value', () => { + expect(getScoreColor('success', null, null)).toEqual({ background: GREY, foreground: WHITE }) + expect(getScoreColor('success', 'true_false', null)).toEqual({ background: GREY, foreground: WHITE }) + expect(getScoreColor('success', 'unknown', '0.5')).toEqual({ background: GREY, foreground: WHITE }) + expect(getScoreColor('success', 'float_scale', 'not-a-number')).toEqual({ background: GREY, foreground: WHITE }) + }) +}) diff --git a/frontend/src/utils/scoreColor.ts b/frontend/src/utils/scoreColor.ts new file mode 100644 index 0000000000..f730a81ba8 --- /dev/null +++ b/frontend/src/utils/scoreColor.ts @@ -0,0 +1,106 @@ +// Colors an attack score badge so its verdict is readable at a glance. +// +// The hue comes from the attack OUTCOME (success -> green, failure -> red, +// error -> amber). The intensity grades by score value: a higher value is more +// vivid, a lower value a lighter tint (e.g. a success at 0.75 is lighter than a +// success at 1.0). +// Intensity is taken from, in order: +// 1. `underlyingFloat` -- the raw 0-1 scale score a FloatScaleThresholdScorer +// keeps in metadata, so a thresholded true/false verdict still grades its +// hue by how close the score was. +// 2. a float_scale score's own value. +// 3. otherwise full strength -- a pure true/false verdict is definitive. +// +// The hue endpoints match the Fluent light-theme palette foreground tokens the +// rest of the UI uses (e.g. colorPaletteRedForeground1 = #bc2f32, the same red +// as the "No target selected" banner), so a full-strength badge lines up with +// them. Values are hardcoded because the gradient is interpolated numerically. +// +// Neutral grey is reserved for the *unscored* case: no score, a value we can't +// interpret (unknown scorer type / non-numeric), or an `undetermined` outcome. + +import type { AttackOutcome } from '../types' +import { resolveOutcome } from './attackOutcome' + +export interface ScoreColor { + background: string + foreground: string +} + +type Rgb = readonly [number, number, number] + +const GREY: Rgb = [97, 97, 97] +const FOREGROUND = '#ffffff' + +// Full-saturation hue per outcome, matching Fluent's light-theme palette +// foreground tokens. `undetermined` has no hue (renders grey). +const OUTCOME_HUE: Record = { + success: [14, 112, 14], // #0e700e - colorPaletteGreenForeground1 + failure: [188, 47, 50], // #bc2f32 - colorPaletteRedForeground1 + error: [196, 53, 1], // #c43501 - colorPaletteDarkOrangeForeground1 + undetermined: null, +} + +const INTENSITY_FLOOR = 0.4 + +function rgb([r, g, b]: Rgb): string { + return `rgb(${r}, ${g}, ${b})` +} + +// Picks the shade between two colors +function shadeBetween(from: Rgb, to: Rgb, t: number): Rgb { + return [ + Math.round(from[0] + (to[0] - from[0]) * t), + Math.round(from[1] + (to[1] - from[1]) * t), + Math.round(from[2] + (to[2] - from[2]) * t), + ] +} + +const NEUTRAL: ScoreColor = { background: rgb(GREY), foreground: FOREGROUND } + +// Resolves a score's position on a normalized [0, 1] scale, or null when the +// value can't be interpreted (unknown scorer type or a non-numeric value). +export function normalizeScoreValue(scoreType: string, scoreValue: string): number | null { + if (scoreType === 'true_false') { + const normalized = scoreValue.trim().toLowerCase() + if (normalized === 'true') return 1 + if (normalized === 'false') return 0 + return null + } + if (scoreType === 'float_scale') { + const parsed = Number.parseFloat(scoreValue) + if (Number.isNaN(parsed)) return null + return Math.min(1, Math.max(0, parsed)) + } + return null +} + +// Resolves the badge color for a score. +export function getScoreColor( + outcome?: AttackOutcome | null, + scoreType?: string | null, + scoreValue?: string | null, + underlyingFloat?: number | null +): ScoreColor { + // Unscored or uninterpretable value -> neutral. + if (!scoreType || scoreValue == null) return NEUTRAL + const normalized = normalizeScoreValue(scoreType, scoreValue) + if (normalized === null) return NEUTRAL + + const hue = OUTCOME_HUE[resolveOutcome(outcome)] + if (!hue) return NEUTRAL + + // Intensity: prefer the wrapped scale score (thresholded true/false verdict), + // then a float_scale value, else full strength for a definitive true/false. + const intensity = + underlyingFloat != null + ? Math.min(1, Math.max(0, underlyingFloat)) + : scoreType === 'true_false' + ? 1 + : normalized + const t = INTENSITY_FLOOR + (1 - INTENSITY_FLOOR) * intensity + return { + background: rgb(shadeBetween(GREY, hue, t)), + foreground: FOREGROUND, + } +} diff --git a/pyrit/backend/models/attacks.py b/pyrit/backend/models/attacks.py index 6f320f7bd5..d1a25ff8a4 100644 --- a/pyrit/backend/models/attacks.py +++ b/pyrit/backend/models/attacks.py @@ -23,6 +23,7 @@ MessagePiece, Score, ) +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY class TargetInfo(BaseModel): @@ -51,6 +52,21 @@ def scorer_type(self) -> str: return identifier.class_name return "Unknown" + @computed_field # type: ignore[prop-decorator] + @property + def scale_score(self) -> float | None: + """ + The raw 0-1 float scale score behind a thresholded true/false verdict. + + ``FloatScaleThresholdScorer`` keeps the pre-threshold scale value in + ``score_metadata``; surface it here so clients don't have to know the + metadata key. ``None`` when the score is not a thresholded float scale. + """ + value = (self.score_metadata or {}).get(ORIGINAL_FLOAT_VALUE_KEY) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) + @classmethod def from_domain(cls, score: Score) -> "ScoreView": """ diff --git a/tests/unit/backend/test_response_contracts.py b/tests/unit/backend/test_response_contracts.py index 282ee6ced2..f894efb95e 100644 --- a/tests/unit/backend/test_response_contracts.py +++ b/tests/unit/backend/test_response_contracts.py @@ -31,6 +31,7 @@ RetryEvent, Score, ) +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY def _make_score() -> Score: @@ -88,6 +89,25 @@ def test_schema_builds(self) -> None: """Test that ScoreView's serialization schema includes the computed field.""" assert "scorer_type" in ScoreView.model_json_schema(mode="serialization")["properties"] + def test_scale_score_exposes_thresholded_float(self) -> None: + """A thresholded true/false score surfaces its raw scale score from metadata.""" + score = _make_score() + score.score_type = "true_false" + score.score_value = "False" + score.score_metadata = {ORIGINAL_FLOAT_VALUE_KEY: 0.5} + + view = ScoreView.from_domain(score) + + assert view.scale_score == 0.5 + assert view.model_dump(mode="json")["scale_score"] == 0.5 + + def test_scale_score_is_none_without_a_numeric_metadata_value(self) -> None: + """A plain score (no float metadata) reports no numeric scale score.""" + assert ScoreView.from_domain(_make_score()).scale_score is None + boolean_meta = _make_score() + boolean_meta.score_metadata = {ORIGINAL_FLOAT_VALUE_KEY: True} + assert ScoreView.from_domain(boolean_meta).scale_score is None + class TestMessagePieceViewContract: """JSON contract for MessagePieceView."""