diff --git a/codewit/api/src/utils/learnerHints.spec.ts b/codewit/api/src/utils/learnerHints.spec.ts
index dde3f70..e4ab7de 100644
--- a/codewit/api/src/utils/learnerHints.spec.ts
+++ b/codewit/api/src/utils/learnerHints.spec.ts
@@ -1451,7 +1451,7 @@ E assert 2 == 4
expect(hinted.learner_hint?.kind).toBe('timeout');
});
- it('does not classify import failures from other modules as missing lesson names', () => {
+ it('does not add a learner hint for import failures from other modules', () => {
const evaluation: EvaluationResponse = {
state: 'failed',
tests_run: 0,
@@ -1481,7 +1481,59 @@ E assert 2 == 4
title: 'Value',
});
- expect(hinted.failure_details[0].learner_hint?.kind).toBe('unknown');
+ expect(hinted.failure_details[0].learner_hint).toBeUndefined();
+ expect(hinted.learner_hint).toBeNull();
+ });
+
+ it.each([
+ {
+ framework: 'CxxTest',
+ error_message: 'Error: Assertion failed: false',
+ rawout: `Running cxxtest tests (1 test)
+test_program.h:15: Error: Assertion failed: false
+Failed 1 and Skipped 0 of 1 test`,
+ },
+ {
+ framework: 'JUnit',
+ error_message: 'java.lang.IllegalStateException: unexpected state',
+ rawout: 'java.lang.IllegalStateException: unexpected state\n\tat MainTest.testValue(MainTest.java:12)',
+ },
+ {
+ framework: 'pytest',
+ error_message: 'ZeroDivisionError: division by zero',
+ rawout: 'E ZeroDivisionError: division by zero',
+ },
+ ])('does not add a learner hint for unrecognized $framework output', ({ error_message, rawout }) => {
+ const evaluation: EvaluationResponse = {
+ state: 'failed',
+ tests_run: 1,
+ passed: 0,
+ failed: 1,
+ errors: 0,
+ no_tests_collected: false,
+ exit_code: 1,
+ failure_details: [{
+ test_case: 'unrecognized failure',
+ expected: '',
+ received: '',
+ error_message,
+ rawout,
+ }],
+ compilation_error: '',
+ runtime_error: '',
+ execution_time_exceeded: false,
+ memory_exceeded: false,
+ };
+
+ const hinted = addLearnerHintsToEvaluation(evaluation, {
+ referenceTest: '',
+ submittedCode: '',
+ topic: null,
+ title: null,
+ });
+
+ expect(hinted.failure_details[0].learner_hint).toBeUndefined();
+ expect(hinted.learner_hint).toBeNull();
});
it('does not claim a memory limit was enforced from an unsupported flag', () => {
@@ -1507,7 +1559,6 @@ E assert 2 == 4
title: 'Value',
});
- expect(hinted.learner_hint?.kind).toBe('unknown');
- expect(hinted.learner_hint?.summary).not.toMatch(/memory/i);
+ expect(hinted.learner_hint).toBeNull();
});
});
diff --git a/codewit/api/src/utils/learnerHints.ts b/codewit/api/src/utils/learnerHints.ts
index 91b1c96..1852649 100644
--- a/codewit/api/src/utils/learnerHints.ts
+++ b/codewit/api/src/utils/learnerHints.ts
@@ -531,28 +531,15 @@ function buildRuntimeHint(message: string): LearnerHint {
);
}
-function buildUnknownHint(): LearnerHint {
- return createHint(
- 'unknown',
- 'low',
- 'The lesson found a problem, but it needs the technical details to explain it',
- 'I could not safely turn this failure into a more specific beginner hint yet.',
- [
- 'Open the Output tab to see the technical error details.',
- 'Focus first on the first error shown there, then submit again.'
- ]
- );
-}
-
function hasAssertionFailure(diagnosticText: string): boolean {
- return /AssertionError\b|Assertion failed:|^\s*assert\b/m.test(diagnosticText);
+ return /AssertionError\b|Assertion failed:\s*assert\b|^\s*assert\b/m.test(diagnosticText);
}
function buildFailureHint(
detail: FailureDetail,
context: LearnerHintContext,
includeRawOutput: boolean
-): LearnerHint {
+): LearnerHint | undefined {
const contract = extractExerciseContract(context.referenceTest, context.topic, context.title);
const message = detail.error_message || '';
const diagnosticText = buildDiagnosticText(detail, includeRawOutput);
@@ -655,7 +642,7 @@ function buildFailureHint(
return buildOutputMismatchHint();
}
- return buildUnknownHint();
+ return undefined;
}
function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintContext): LearnerHint | null {
@@ -708,14 +695,14 @@ function buildTopLevelHint(evaluation: EvaluationResponse, context: LearnerHintC
}
if (evaluation.memory_exceeded) {
- return buildUnknownHint();
+ return null;
}
if (evaluation.state === 'passed') {
return null;
}
- return buildUnknownHint();
+ return null;
}
function addLearnerHintsToEvaluation(
@@ -723,10 +710,13 @@ function addLearnerHintsToEvaluation(
context: LearnerHintContext
): EvaluationResponse {
const includeRawOutput = evaluation.failure_details.length === 1;
- const failure_details = evaluation.failure_details.map((detail) => ({
- ...detail,
- learner_hint: buildFailureHint(detail, context, includeRawOutput),
- }));
+ const failure_details = evaluation.failure_details.map((detail) => {
+ const learnerHint = buildFailureHint(detail, context, includeRawOutput);
+
+ return learnerHint
+ ? { ...detail, learner_hint: learnerHint }
+ : detail;
+ });
const hintedEvaluation = {
...evaluation,
diff --git a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx
index 5e76907..ac9191d 100644
--- a/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx
+++ b/codewit/client/src/components/codeblock/CodeSubmission.spec.tsx
@@ -235,6 +235,181 @@ describe('CodeSubmission', () => {
expect(screen.getByText(/embedded diagnostic/).textContent?.match(/embedded diagnostic/g)).toHaveLength(1);
});
+ it('shows complete raw output in Outcome when no learner hint matches', () => {
+ const rawout = `Running cxxtest tests (3 tests)
+In codewit_test::testSuccessfulLogin:
+Error: Assertion failed: first failure
+In codewit_test::testWrongPassword:
+Error: Assertion failed: second failure
+Failed 2 and Skipped 0 of 3 tests`;
+ const evaluation: EvaluationResponse = {
+ state: 'failed',
+ tests_run: 3,
+ passed: 1,
+ failed: 2,
+ errors: 0,
+ no_tests_collected: false,
+ exit_code: 1,
+ failure_details: [{
+ test_case: 'Test 1',
+ expected: '',
+ received: '',
+ error_message: 'Error: Assertion failed: first failure',
+ rawout: 'partial failure output',
+ }],
+ compilation_error: '',
+ runtime_error: '',
+ execution_time_exceeded: false,
+ memory_exceeded: false,
+ rawout,
+ learner_hint: null,
+ };
+
+ render();
+
+ expect(screen.queryByText(/The lesson found a problem/)).toBeNull();
+ expect(screen.getByText('Technical result')).toBeTruthy();
+ expect(screen.getByTestId('outcome-technical-output').textContent).toBe(rawout);
+ });
+
+ it('shows structured CxxTest diffs for each issue and keeps the full run in Output', () => {
+ const rawout = `Running cxxtest tests (3 tests)
+In codewit_test::testSuccessfulLogin:
+Error: Expected (...), found ("expected first" != "actual first")
+In codewit_test::testWrongPassword:
+Error: Expected (...), found ("expected second" != "actual second")
+Failed 2 and Skipped 0 of 3 tests`;
+ const hint = {
+ kind: 'output_mismatch' as const,
+ confidence: 'medium' as const,
+ title: 'Your program ran, but its result did not match the lesson',
+ summary: 'The output differed from what the lesson expected.',
+ next_steps: ['Compare the expected and actual output.'],
+ };
+ const evaluation: EvaluationResponse = {
+ state: 'failed',
+ tests_run: 3,
+ passed: 1,
+ failed: 2,
+ errors: 0,
+ no_tests_collected: false,
+ exit_code: 1,
+ failure_details: [
+ {
+ test_case: 'Test 1',
+ expected: 'expected first',
+ received: 'actual first',
+ error_message: 'Error: Expected (...), found ("expected first" != "actual first")',
+ rawout,
+ learner_hint: hint,
+ },
+ {
+ test_case: 'Test 2',
+ expected: 'expected second',
+ received: 'actual second',
+ error_message: 'Error: Expected (...), found ("expected second" != "actual second")',
+ rawout,
+ learner_hint: hint,
+ },
+ ],
+ compilation_error: '',
+ runtime_error: '',
+ execution_time_exceeded: false,
+ memory_exceeded: false,
+ rawout,
+ learner_hint: hint,
+ };
+
+ render();
+
+ expect(screen.getByText('expected first')).toBeTruthy();
+ expect(screen.getByText('actual first')).toBeTruthy();
+
+ fireEvent.click(screen.getByRole('button', { name: 'Next issue' }));
+
+ expect(screen.getByText('expected second')).toBeTruthy();
+ expect(screen.getByText('actual second')).toBeTruthy();
+
+ fireEvent.click(screen.getByRole('button', { name: 'Output' }));
+
+ expect(screen.getByTestId('technical-output').textContent).toBe(rawout);
+ });
+
+ it.each([
+ { expected: 'expected output', received: '', emptyLabel: 'Actual:' },
+ { expected: '', received: 'actual output', emptyLabel: 'Expected:' },
+ ])('shows an explicit empty value when $emptyLabel output is empty', ({
+ expected,
+ received,
+ emptyLabel,
+ }) => {
+ const evaluation: EvaluationResponse = {
+ state: 'failed',
+ tests_run: 1,
+ passed: 0,
+ failed: 1,
+ errors: 0,
+ no_tests_collected: false,
+ exit_code: 1,
+ failure_details: [{
+ test_case: 'Test 1',
+ expected,
+ received,
+ error_message: 'Error: Expected (...), found ("expected output" != "")',
+ rawout: 'CxxTest output mismatch',
+ learner_hint: {
+ kind: 'output_mismatch',
+ confidence: 'medium',
+ title: 'Your program ran, but its result did not match the lesson',
+ summary: 'The output differed from what the lesson expected.',
+ next_steps: [],
+ },
+ }],
+ compilation_error: '',
+ runtime_error: '',
+ execution_time_exceeded: false,
+ memory_exceeded: false,
+ };
+
+ render();
+
+ expect(screen.getByText(emptyLabel)).toBeTruthy();
+ expect(screen.getByText('(empty output)')).toBeTruthy();
+ });
+
+ it('falls back to structured diagnostics when raw output is unavailable', () => {
+ const evaluation: EvaluationResponse = {
+ state: 'failed',
+ tests_run: 1,
+ passed: 0,
+ failed: 1,
+ errors: 0,
+ no_tests_collected: false,
+ exit_code: 1,
+ failure_details: [{
+ test_case: 'unrecognized failure',
+ expected: '',
+ received: '',
+ error_message: 'specific failure message',
+ rawout: '',
+ diagnostic: 'scoped technical diagnostic',
+ stderr: 'runner warning',
+ }],
+ compilation_error: '',
+ runtime_error: '',
+ execution_time_exceeded: false,
+ memory_exceeded: false,
+ learner_hint: null,
+ };
+
+ render();
+
+ expect(screen.getByText('Technical result')).toBeTruthy();
+ expect(screen.getByText(/scoped technical diagnostic/)).toBeTruthy();
+ expect(screen.getByText(/specific failure message/)).toBeTruthy();
+ expect(screen.getByText(/runner warning/)).toBeTruthy();
+ });
+
it('prioritizes timeout guidance over a partial failure hint', () => {
const evaluation: EvaluationResponse = {
state: 'failed',
diff --git a/codewit/client/src/components/codeblock/CodeSubmission.tsx b/codewit/client/src/components/codeblock/CodeSubmission.tsx
index 082bf6d..5a9bbe4 100644
--- a/codewit/client/src/components/codeblock/CodeSubmission.tsx
+++ b/codewit/client/src/components/codeblock/CodeSubmission.tsx
@@ -7,17 +7,6 @@ type EvalProps = {
evaluation: EvaluationResponse | null;
};
-const fallbackHint: LearnerHint = {
- kind: 'unknown',
- confidence: 'low',
- title: 'The lesson found a problem',
- summary: 'Open the Output tab to see the technical details, then fix the first error shown there.',
- next_steps: [
- 'Read the first technical error in Output.',
- 'Fix that error and submit again.'
- ],
-};
-
const HintCard = ({ hint }: { hint: LearnerHint }): JSX.Element => {
return (
@@ -51,6 +40,10 @@ const composeTechnicalOutput = (...values: Array
): string =>
return sections.join('\n\n');
};
+const selectVerbatimOutput = (...values: Array): string => {
+ return values.find((value) => value?.trim()) ?? '';
+};
+
const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
const [activeTab, setActiveTab] = useState<'outcome' | 'output'>('outcome');
const [issueIdx, setIssueIdx] = useState(0);
@@ -84,7 +77,9 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
const topLevelHint = 'learner_hint' in evaluation ? (evaluation.learner_hint ?? null) : null;
const activeHint = execution_time_exceeded
? topLevelHint
- : activeIssue?.learner_hint || topLevelHint || (state === 'passed' ? null : fallbackHint);
+ : activeIssue
+ ? activeIssue.learner_hint ?? null
+ : topLevelHint;
const technicalOutput = composeTechnicalOutput(
activeIssue?.rawout,
evaluation.rawout,
@@ -96,11 +91,28 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
compilation_error,
runtime_error,
error,
- execution_time_exceeded ? 'Execution time exceeded' : undefined
+ execution_time_exceeded ? 'Execution time exceeded' : undefined,
+ memory_exceeded ? 'Memory limit exceeded' : undefined
+ );
+ const outcomeTechnicalOutput = selectVerbatimOutput(
+ evaluation.rawout,
+ activeIssue?.rawout
+ ) || composeTechnicalOutput(
+ activeIssue?.diagnostic,
+ activeIssue?.error_message,
+ activeIssue?.stderr,
+ evaluation.stdout,
+ evaluation.stderr,
+ compilation_error,
+ runtime_error,
+ error,
+ execution_time_exceeded ? 'Execution time exceeded' : undefined,
+ memory_exceeded ? 'Memory limit exceeded' : undefined
);
const hasFailures = failure_details.length > 0;
const hasOutput = technicalOutput.trim().length > 0;
+ const hasComparison = Boolean(activeIssue && (activeIssue.expected || activeIssue.received));
const allPassed = !hasFailures && !compilation_error && !runtime_error && !execution_time_exceeded && !memory_exceeded && !error && state === 'passed';
const showOutcomeTab = true;
@@ -117,6 +129,7 @@ const CodeSubmission = ({ evaluation }: EvalProps): JSX.Element => {
Issues
{issueIdx + 1} / {failure_details.length}