pytest-parser.test.js

const test = require('node:test');
const assert = require('node:assert/strict');

const { parsePytestOutput, extractPytestShortSummaryTarget } = require('./pytest-parser');

test('extracts the real technical message for pytest attribute failures', () => {
  const stdout = `
============================= test session starts ==============================
platform darwin -- Python 3.9.10, pytest-7.4.0, pluggy-1.2.0
collected 1 item

test_program.py F                                                       [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert program.numberOfHats == 9
E       AttributeError: module 'program' has no attribute 'numberOfHats'

/tmp/test_program.py:4: AttributeError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables
============================== 1 failed in 0.02s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failed, 1);
  assert.equal(parsed.failure_details.length, 1);
  assert.equal(parsed.failure_details[0].test_case, 'test_hat_variables');
  assert.equal(
    parsed.failure_details[0].error_message,
    "AttributeError: module 'program' has no attribute 'numberOfHats'"
  );
  assert.match(parsed.failure_details[0].rawout, /FAILED test_program\.py::test_hat_variables/);
});

test('normalizes pytest short summary targets to the test name', () => {
  const stdout = `
FAILED ../../../../../../../var/folders/example/test_program.py::test_hat_variables
  `.trim();

  assert.equal(
    extractPytestShortSummaryTarget(stdout, 'FAILED'),
    'test_hat_variables'
  );
});

test('preserves custom assertion messages for hasattr-style lesson failures', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert hasattr(program,"HatName"), "There should be a variable named exactly HatName"
E       AssertionError: There should be a variable named exactly HatName
E       assert False
E        +  where False = hasattr(program, 'HatName')

test_program.py:8: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables - AssertionError: There should be ...
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failed, 1);
  assert.equal(parsed.failure_details.length, 1);
  assert.equal(
    parsed.failure_details[0].error_message,
    'AssertionError: There should be a variable named exactly HatName'
  );
  assert.equal(parsed.failure_details[0].expected, '');
  assert.equal(parsed.failure_details[0].received, '');
});

test('preserves custom assertion messages for printed dataframe regression cases', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
____________________________ test_describe_printed _____________________________

    def test_describe_printed(capsys):
>       assert part in out, f"Expected '{part}' in output, got:\\n{out}"
E       AssertionError: Expected 'count' in output, got:
E       0  1  2  3

test_program.py:12: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_describe_printed - AssertionError: Expected 'cou...
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failed, 1);
  assert.equal(parsed.failure_details.length, 1);
  assert.equal(parsed.failure_details[0].test_case, 'test_describe_printed');
  assert.equal(
    parsed.failure_details[0].error_message,
    "AssertionError: Expected 'count' in output, got:"
  );
});

test('extracts expected and received values for numeric equality assertions', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert program.NumberOfHats == 9
E       assert 8 == 9

test_program.py:12: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables - assert 8 == 9
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.NumberOfHats == 9');
  assert.equal(parsed.failure_details[0].received, '8');
  assert.equal(parsed.failure_details[0].expected, '9');
});

test('extracts expected and received values for string equality assertions', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert program.HatName == "Veracruz"
E       assert "Oaxaca" == "Veracruz"

test_program.py:10: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables - assert "Oaxaca" == "Veracruz"
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.HatName == "Veracruz"');
  assert.equal(parsed.failure_details[0].received, '"Oaxaca"');
  assert.equal(parsed.failure_details[0].expected, '"Veracruz"');
});

test('ignores comparison operators inside quoted assertion values', () => {
  const comparisons = [
    ["'This is wrong' == 'This is right'", "'This is wrong'", "'This is right'"],
    ["'value is not ready' == 'value is not done'", "'value is not ready'", "'value is not done'"],
    ["'left == side' == 'right == side'", "'left == side'", "'right == side'"],
  ];

  for (const [expression, received, expected] of comparisons) {
    const stdout = `
=================================== FAILURES ===================================
______________________________ test_string_value _______________________________

>       assert actual == expected
E       assert ${expression}

test_program.py:10: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_string_value - assert ${expression}
============================== 1 failed in 0.01s ===============================
    `.trim();

    const parsed = parsePytestOutput(stdout, '', 1);

    assert.equal(parsed.failure_details[0].received, received);
    assert.equal(parsed.failure_details[0].expected, expected);
  }
});

test('extracts expected and received values for float equality assertions', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert program.CostOfHats == 278.91
E       assert 199.99 == 278.91

test_program.py:14: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables - assert 199.99 == 278.91
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.CostOfHats == 278.91');
  assert.equal(parsed.failure_details[0].received, '199.99');
  assert.equal(parsed.failure_details[0].expected, '278.91');
});

test('extracts expected and received values for boolean identity assertions', () => {
  const stdout = `
============================= test session starts ==============================
platform linux -- Python 3.11.12, pytest-9.1.1, pluggy-1.6.0
collected 1 item

test_program.py F                                                        [100%]

=================================== FAILURES ===================================
______________________________ test_hat_variables ______________________________

    def test_hat_variables():
>       assert program.WearingHat is False
E       assert True is False
E        +  where True = program.WearingHat

test_program.py:16: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hat_variables - assert True is False
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details[0].error_message, 'Assertion failed: assert program.WearingHat is False');
  assert.equal(parsed.failure_details[0].received, 'True');
  assert.equal(parsed.failure_details[0].expected, 'False');
});

test('extracts complete captured output values from pytest verbose assertion failures', () => {
  // Captured from pytest 8.2.0 with `pytest -vv`; -vv prevents pytest from
  // abbreviating the compared output with ellipses and a truncation notice.
  const stdout = `
============================= test session starts ==============================
collected 1 item

test_program.py::test_weekly_todo FAILED                                [100%]

=================================== FAILURES ===================================
_______________________________ test_weekly_todo _______________________________

>       assert capsys.readouterr().out == expected_output
E       AssertionError: assert 'Weekly to-do list for the 7 days of the weeke:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n' == 'Weekly to-do list for the 7 days of the week:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n'
E
E         - Weekly to-do list for the 7 days of the week:
E         + Weekly to-do list for the 7 days of the weeke:
E         ?                                             +
E           Mon: Laundry

test_program.py:29: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_weekly_todo - AssertionError: assert 'Weekly to-do list for the 7 days of the weeke:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n' == 'Weekly to-do list for the 7 days of the week:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n'
============================== 1 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details.length, 1);
  assert.equal(parsed.failure_details[0].received, "'Weekly to-do list for the 7 days of the weeke:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n'");
  assert.equal(parsed.failure_details[0].expected, "'Weekly to-do list for the 7 days of the week:\\nMon: Laundry\\nTues: Groceries\\nWed Gym\\nThur: Study\\nFri: Work\\nSat: Hike\\nSun: Rest\\n'");
});

test('keeps a distinct scoped diagnostic for each pytest failure', () => {
  const stdout = `
============================= test session starts ==============================
collected 2 items

test_program.py::test_hats FAILED                                      [ 50%]
test_program.py::TestClothes::test_shirt[color-red] FAILED             [100%]

=================================== FAILURES ===================================
__________________________________ test_hats ___________________________________

>       assert program.NumberOfHats == 9
E       assert 8 == 9

test_program.py:4: AssertionError
_____________________ TestClothes.test_shirt[color-red] ______________________

>       assert program.ShirtColor == "red"
E       assert "blue" == "red"

test_program.py:8: AssertionError
=========================== short test summary info ============================
FAILED test_program.py::test_hats - assert 8 == 9
FAILED test_program.py::TestClothes::test_shirt[color-red] - assert "blue" == "red"
============================== 2 failed in 0.01s ===============================
  `.trim();

  const parsed = parsePytestOutput(stdout, '', 1);

  assert.equal(parsed.failure_details.length, 2);
  assert.match(parsed.failure_details[0].diagnostic, /program\.NumberOfHats/);
  assert.doesNotMatch(parsed.failure_details[0].diagnostic, /program\.ShirtColor/);
  assert.match(parsed.failure_details[1].diagnostic, /program\.ShirtColor/);
  assert.doesNotMatch(parsed.failure_details[1].diagnostic, /program\.NumberOfHats/);
  assert.equal(parsed.failure_details[0].rawout, parsed.failure_details[1].rawout);
});