From ee08def04fabc211c6355bfbdd23a44621c019cf Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Wed, 16 Sep 2026 13:34:11 -0700 Subject: [PATCH 1/3] fix(api): omit unmatched learner hints Let unrecognized evaluator formats fall through to their technical output while preserving tailored guidance for known failures.\n\nRefs #186\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- codewit/api/src/utils/learnerHints.spec.ts | 59 ++++++++++++++++++++-- codewit/api/src/utils/learnerHints.ts | 34 +++++-------- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts index dde3f70..e4ab7de 100644 --- a/codewit/api/src/utils/learnerHints.spec.ts +++ b/codewit/api/src/utils/learnerHints.spec.ts @@ -1451,7 +1451,7 @@ E assert 2 == 4 expect(hinted.learner_hint?.kind).toBe('timeout'); }); - it('does not classify import failures from other modules as missing lesson names', () => { + it('does not add a learner hint for import failures from other modules', () => { const evaluation: EvaluationResponse = { state: 'failed', tests_run: 0, @@ -1481,7 +1481,59 @@ E assert 2 == 4 title: 'Value', }); - expect(hinted.failure_details[0].learner_hint?.kind).toBe('unknown'); + expect(hinted.failure_details[0].learner_hint).toBeUndefined(); + expect(hinted.learner_hint).toBeNull(); + }); + + it.each([ + { + framework: 'CxxTest', + error_message: 'Error: Assertion failed: false', + rawout: `Running cxxtest tests (1 test) +test_program.h:15: Error: Assertion failed: false +Failed 1 and Skipped 0 of 1 test`, + }, + { + framework: 'JUnit', + error_message: 'java.lang.IllegalStateException: unexpected state', + rawout: 'java.lang.IllegalStateException: unexpected state\n\tat MainTest.testValue(MainTest.java:12)', + }, + { + framework: 'pytest', + error_message: 'ZeroDivisionError: division by zero', + rawout: 'E ZeroDivisionError: division by zero', + }, + ])('does not add a learner hint for unrecognized $framework output', ({ error_message, rawout }) => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'unrecognized failure', + expected: '', + received: '', + error_message, + rawout, + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + }; + + const hinted = addLearnerHintsToEvaluation(evaluation, { + referenceTest: '', + submittedCode: '', + topic: null, + title: null, + }); + + expect(hinted.failure_details[0].learner_hint).toBeUndefined(); + expect(hinted.learner_hint).toBeNull(); }); it('does not claim a memory limit was enforced from an unsupported flag', () => { @@ -1507,7 +1559,6 @@ E assert 2 == 4 title: 'Value', }); - expect(hinted.learner_hint?.kind).toBe('unknown'); - expect(hinted.learner_hint?.summary).not.toMatch(/memory/i); + expect(hinted.learner_hint).toBeNull(); }); }); diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts index 91b1c96..1852649 100644 --- a/codewit/api/src/utils/learnerHints.ts +++ b/codewit/api/src/utils/learnerHints.ts @@ -531,28 +531,15 @@ function buildRuntimeHint(message: string): LearnerHint { ); } -function buildUnknownHint(): LearnerHint { - return createHint( - 'unknown', - 'low', - 'The lesson found a problem, but it needs the technical details to explain it', - 'I could not safely turn this failure into a more specific beginner hint yet.', - [ - 'Open the Output tab to see the technical error details.', - 'Focus first on the first error shown there, then submit again.' - ] - ); -} - function hasAssertionFailure(diagnosticText: string): boolean { - return /AssertionError\b|Assertion failed:|^\s*assert\b/m.test(diagnosticText); + return /AssertionError\b|Assertion failed:\s*assert\b|^\s*assert\b/m.test(diagnosticText); } function buildFailureHint( detail: FailureDetail, context: LearnerHintContext, includeRawOutput: boolean -): LearnerHint { +): LearnerHint | undefined { const contract = extractExerciseContract(context.referenceTest, context.topic, context.title); const message = detail.error_message || ''; const diagnosticText = buildDiagnosticText(detail, includeRawOutput); @@ -655,7 +642,7 @@ function buildFailureHint( return buildOutputMismatchHint(); } - return buildUnknownHint(); + return undefined; } function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintContext): LearnerHint | null { @@ -708,14 +695,14 @@ function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintC } if (evaluation.memory_exceeded) { - return buildUnknownHint(); + return null; } if (evaluation.state === 'passed') { return null; } - return buildUnknownHint(); + return null; } function addLearnerHintsToEvaluation( @@ -723,10 +710,13 @@ function addLearnerHintsToEvaluation( context: LearnerHintContext ): EvaluationResponse { const includeRawOutput = evaluation.failure_details.length === 1; - const failure_details = evaluation.failure_details.map((detail) => ({ - ...detail, - learner_hint: buildFailureHint(detail, context, includeRawOutput), - })); + const failure_details = evaluation.failure_details.map((detail) => { + const learnerHint = buildFailureHint(detail, context, includeRawOutput); + + return learnerHint + ? { ...detail, learner_hint: learnerHint } + : detail; + }); const hintedEvaluation = { ...evaluation, From 11decd9d40015f2031e5281100fc5e5b72027343 Mon Sep 17 00:00:00 2001 From: Kevin Buffardi Date: Wed, 16 Sep 2026 13:35:39 -0700 Subject: [PATCH 2/3] fix(client): show unmatched evaluation output Render verbatim technical results in Outcome when no tailored hint exists while preserving structured diffs, navigation, and full Output details.\n\nFixes #186\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../codeblock/CodeSubmission.spec.tsx | 133 ++++++++++++++++++ .../components/codeblock/CodeSubmission.tsx | 63 ++++++--- 2 files changed, 179 insertions(+), 17 deletions(-) diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx index 5e76907..359bd7c 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx @@ -235,6 +235,139 @@ describe('CodeSubmission', () => { expect(screen.getByText(/embedded diagnostic/).textContent?.match(/embedded diagnostic/g)).toHaveLength(1); }); + it('shows complete raw output in Outcome when no learner hint matches', () => { + const rawout = `Running cxxtest tests (3 tests) +In codewit_test::testSuccessfulLogin: +Error: Assertion failed: first failure +In codewit_test::testWrongPassword: +Error: Assertion failed: second failure +Failed 2 and Skipped 0 of 3 tests`; + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 3, + passed: 1, + failed: 2, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'Test 1', + expected: '', + received: '', + error_message: 'Error: Assertion failed: first failure', + rawout: 'partial failure output', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + rawout, + learner_hint: null, + }; + + render(); + + expect(screen.queryByText(/The lesson found a problem/)).toBeNull(); + expect(screen.getByText('Technical result')).toBeTruthy(); + expect(screen.getByTestId('outcome-technical-output').textContent).toBe(rawout); + }); + + it('shows structured CxxTest diffs for each issue and keeps the full run in Output', () => { + const rawout = `Running cxxtest tests (3 tests) +In codewit_test::testSuccessfulLogin: +Error: Expected (...), found ("expected first" != "actual first") +In codewit_test::testWrongPassword: +Error: Expected (...), found ("expected second" != "actual second") +Failed 2 and Skipped 0 of 3 tests`; + const hint = { + kind: 'output_mismatch' as const, + confidence: 'medium' as const, + title: 'Your program ran, but its result did not match the lesson', + summary: 'The output differed from what the lesson expected.', + next_steps: ['Compare the expected and actual output.'], + }; + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 3, + passed: 1, + failed: 2, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [ + { + test_case: 'Test 1', + expected: 'expected first', + received: 'actual first', + error_message: 'Error: Expected (...), found ("expected first" != "actual first")', + rawout, + learner_hint: hint, + }, + { + test_case: 'Test 2', + expected: 'expected second', + received: 'actual second', + error_message: 'Error: Expected (...), found ("expected second" != "actual second")', + rawout, + learner_hint: hint, + }, + ], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + rawout, + learner_hint: hint, + }; + + render(); + + expect(screen.getByText('expected first')).toBeTruthy(); + expect(screen.getByText('actual first')).toBeTruthy(); + + fireEvent.click(screen.getByRole('button', { name: 'Next issue' })); + + expect(screen.getByText('expected second')).toBeTruthy(); + expect(screen.getByText('actual second')).toBeTruthy(); + + fireEvent.click(screen.getByRole('button', { name: 'Output' })); + + expect(screen.getByTestId('technical-output').textContent).toBe(rawout); + }); + + it('falls back to structured diagnostics when raw output is unavailable', () => { + const evaluation: EvaluationResponse = { + state: 'failed', + tests_run: 1, + passed: 0, + failed: 1, + errors: 0, + no_tests_collected: false, + exit_code: 1, + failure_details: [{ + test_case: 'unrecognized failure', + expected: '', + received: '', + error_message: 'specific failure message', + rawout: '', + diagnostic: 'scoped technical diagnostic', + stderr: 'runner warning', + }], + compilation_error: '', + runtime_error: '', + execution_time_exceeded: false, + memory_exceeded: false, + learner_hint: null, + }; + + render(); + + expect(screen.getByText('Technical result')).toBeTruthy(); + expect(screen.getByText(/scoped technical diagnostic/)).toBeTruthy(); + expect(screen.getByText(/specific failure message/)).toBeTruthy(); + expect(screen.getByText(/runner warning/)).toBeTruthy(); + }); + it('prioritizes timeout guidance over a partial failure hint', () => { const evaluation: EvaluationResponse = { state: 'failed', diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx index 082bf6d..233896e 100644 --- a/codewit/client/src/components/codeblock/CodeSubmission.tsx +++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx @@ -7,17 +7,6 @@ type EvalProps = { evaluation: EvaluationResponse | null; }; -const fallbackHint: LearnerHint = { - kind: 'unknown', - confidence: 'low', - title: 'The lesson found a problem', - summary: 'Open the Output tab to see the technical details, then fix the first error shown there.', - next_steps: [ - 'Read the first technical error in Output.', - 'Fix that error and submit again.' - ], -}; - const HintCard = ({ hint }: { hint: LearnerHint }): JSX.Element => { return (
@@ -51,6 +40,10 @@ const composeTechnicalOutput = (...values: Array): string => return sections.join('\n\n'); }; +const selectVerbatimOutput = (...values: Array): string => { + return values.find((value) => value?.trim()) ?? ''; +}; + const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const [activeTab, setActiveTab] = useState<'outcome' | 'output'>('outcome'); const [issueIdx, setIssueIdx] = useState(0); @@ -84,7 +77,9 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null; const activeHint = execution_time_exceeded ? topLevelHint - : activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint); + : activeIssue + ? activeIssue.learner_hint ?? null + : topLevelHint; const technicalOutput = composeTechnicalOutput( activeIssue?.rawout, evaluation.rawout, @@ -96,7 +91,23 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => { compilation_error, runtime_error, error, - execution_time_exceeded ? 'Execution time exceeded' : undefined + execution_time_exceeded ? 'Execution time exceeded' : undefined, + memory_exceeded ? 'Memory limit exceeded' : undefined + ); + const outcomeTechnicalOutput = selectVerbatimOutput( + evaluation.rawout, + activeIssue?.rawout + ) || composeTechnicalOutput( + activeIssue?.diagnostic, + activeIssue?.error_message, + activeIssue?.stderr, + evaluation.stdout, + evaluation.stderr, + compilation_error, + runtime_error, + error, + execution_time_exceeded ? 'Execution time exceeded' : undefined, + memory_exceeded ? 'Memory limit exceeded' : undefined ); const hasFailures = failure_details.length > 0; @@ -117,6 +128,7 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {

Issues

{issueIdx + 1} / {failure_details.length}
)} - {hasFailures && activeIssue && (activeHint || activeIssue.expected || activeIssue.received) && ( + {hasFailures && activeIssue && (activeHint || hasComparison) && (
{activeIssue.test_case} - {activeIssue.expected && ( + {hasComparison && (
Expected: - 
{activeIssue.expected}
+ 
+                      {activeIssue.expected || '(empty output)'}
+
)} - {activeIssue.received && ( + {hasComparison && (
Actual: - 
{activeIssue.received}
+ 
+                      {activeIssue.received || '(empty output)'}
+
)}

Open the Output tab to see the technical details for this issue.