diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py index d0db5d2e7..a22404c7b 100644 --- a/packtools/sps/formats/pdf/pipeline/xml.py +++ b/packtools/sps/formats/pdf/pipeline/xml.py @@ -164,14 +164,20 @@ def extract_contrib_data(xml_tree): def extract_abstract_data(xml_tree): """ Extracts the title and content of the abstract from the given XML tree. - + + Handles both a plain abstract (

direct children of ) and a + structured one (subsections wrapped in , e.g. Introduction/Methods/ + Results, each with its own and <p>) - see _extract_abstract_paragraphs. + Args: xml_tree (ElementTree): The XML tree to extract the abstract from. - + Returns: dict: A dictionary containing the following keys: - 'title': The text content of the abstract title element, or an empty string if not found. - - 'content': The text content of the abstract paragraphs, concatenated into a single string. + - 'content': The text content of the abstract paragraphs (and, for a + structured abstract, each subsection's title), concatenated into a + single string. """ data = {'title': '', 'content': ''} @@ -182,22 +188,22 @@ def extract_abstract_data(xml_tree): if node_title is not None: data['title'] = ''.join(node_title.itertext()).strip() - abstract = [] - for p in node_abstract.findall('p'): - if p is not None: - abstract.append(''.join(p.itertext()).strip()) - data['content'] = ' '.join(abstract) + data['content'] = ' '.join(_extract_abstract_paragraphs(node_abstract)) return data def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/XML/1998/namespace'}): """ Extracts the title and content of translated abstracts from the given XML tree. - + + Handles both a plain and a structured trans-abstract (subsections wrapped + in <sec>) the same way extract_abstract_data does - see + _extract_abstract_paragraphs. + Args: xml_tree (ElementTree): The XML tree to extract the translated abstracts from. namespaces (dict, optional): A dictionary of XML namespaces to use in the XPath expressions. - + Returns: list: A list of dictionaries, where each dictionary contains the following keys: - 'lang': The language of the translated abstract. @@ -217,11 +223,7 @@ def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/ item['lang'] = node.attrib.get(lang_attrib_name) - abstract = [] - for p in node.findall('p'): - if p is not None: - abstract.append(''.join(p.itertext()).strip()) - item['content'] = ' '.join(abstract) + item['content'] = ' '.join(_extract_abstract_paragraphs(node)) data.append(item) @@ -340,6 +342,11 @@ def extract_body_data(xml_tree, table_layout_overrides=None): """ Extracts the body data from an XML tree, including section titles, paragraphs, and tables. + Excludes any <sec> nested inside <abstract> or <trans-abstract> - those + are structured-abstract subsections handled by extract_abstract_data / + extract_trans_abstract_data, and would otherwise be picked up twice by + a plain './/sec' search. + Args: xml_tree (ElementTree): The XML tree to extract the body data from. table_layout_overrides (dict, optional): Maps a table-wrap @id to a forced @@ -357,7 +364,10 @@ def extract_body_data(xml_tree, table_layout_overrides=None): data = [] seen_fig_keys = set() - for document_section in xml_tree.findall('.//sec'): + body_sections = xml_tree.xpath( + './/sec[not(ancestor::abstract) and not(ancestor::trans-abstract)]' + ) + for document_section in body_sections: sec = {'paragraphs': [], 'tables': [], 'figures': []} sec['level'] = xml_utils.get_node_level(document_section, xml_tree) sec['title'] = document_section.find('title') @@ -797,6 +807,43 @@ def get_table_column_info(headers, rows): # Private helpers # ----------------- +def _extract_abstract_paragraphs(node): + """ + Collects an abstract's readable text as a list of strings, one per + <p> found at any depth. A structured abstract wraps each subsection + in its own <sec> (e.g. <sec><title>Methods:

...

), + so a plain `node.findall('p')` (direct children only) misses every + paragraph and returns an empty abstract. Recursing into finds + them, and including each 's own in the flattened output + preserves the abstract's structure instead of silently merging + distinct subsections together. Some XMLs already carry a trailing + colon in the title (e.g. "Methods:"), others don't (e.g. "Methods"); + a colon is appended only when the title lacks its own closing + punctuation, so it never gets duplicated. + + Args: + node (ElementTree): The <abstract> or <trans-abstract> element + (or a <sec> within one, for the recursive call). + + Returns: + list: Text fragments in document order - <sec> titles and <p> content. + """ + parts = [] + for child in node: + if child.tag == 'p': + parts.append(''.join(child.itertext()).strip()) + elif child.tag == 'sec': + sec_title = child.find('title') + if sec_title is not None: + title_text = ''.join(sec_title.itertext()).strip() + if title_text: + if title_text[-1] not in ':.!?;': + title_text = f'{title_text}:' + parts.append(title_text) + parts.extend(_extract_abstract_paragraphs(child)) + return parts + + def _extract_table_rows_with_merged_cells(table_section, cell_tag): """ Extracts table rows handling merged cells (colspan/rowspan). diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py index 774630aae..ac6b686a8 100644 --- a/tests/sps/formats/pdf/pipeline/test_xml.py +++ b/tests/sps/formats/pdf/pipeline/test_xml.py @@ -83,6 +83,58 @@ def test_extract_abstract_data_with_nested_elements(self): result = xml_pipe.extract_abstract_data(xml) self.assertEqual(result, expected) + def test_extract_abstract_data_structured_with_sections(self): + # Regression for issue #1332: a structured abstract wraps + # each subsection in its own <sec>, so a plain findall('p') (direct + # children only) found nothing and returned an empty content. + xml = etree.fromstring( + '<article><abstract>' + '<title>Abstract' + 'Introduction:

Some introduction text.

' + 'Methods:

Some methods text.

' + '' + ) + expected = { + 'title': 'Abstract', + 'content': 'Introduction: Some introduction text. Methods: Some methods text.', + } + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + + def test_extract_abstract_data_structured_section_title_without_punctuation(self): + # Some XMLs don't carry a trailing colon in the , unlike + # the "Methods:" style above - a colon must be added so the title + # doesn't run into the paragraph text (e.g. "Objetivodescrever..."). + xml = etree.fromstring( + '<article><abstract>' + '<sec><title>Objetivo

Descrever o metodo.

' + '' + ) + expected = {'title': '', 'content': 'Objetivo: Descrever o metodo.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + + def test_extract_abstract_data_structured_section_without_title(self): + xml = etree.fromstring( + '
' + '

Untitled section text.

' + '
' + ) + expected = {'title': '', 'content': 'Untitled section text.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + + def test_extract_abstract_data_mixed_direct_and_sectioned_paragraphs(self): + xml = etree.fromstring( + '
' + '

Lead paragraph.

' + 'Conclusion:

Final remarks.

' + '
' + ) + expected = {'title': '', 'content': 'Lead paragraph. Conclusion: Final remarks.'} + result = xml_pipe.extract_abstract_data(xml) + self.assertEqual(result, expected) + class TestExtractAcknowledgmentData(unittest.TestCase): @@ -303,6 +355,36 @@ def test_extract_body_data_basic(self): result = xml_pipe.extract_body_data(xml) self.assertEqual(result, expected) + def test_extract_body_data_excludes_abstract_and_trans_abstract_sections(self): + # Regression: a structured abstract/trans-abstract wraps each + # subsection in its own (see extract_abstract_data), which a + # plain './/sec' search would also pick up as a body section, + # duplicating the same content in both the abstract and the body. + xml = etree.fromstring( + '
' + '' + 'Background:

Abstract text.

' + '
' + '' + 'Contexto:

Texto do resumo.

' + '
' + '' + 'Introduction

Body text.

' + '' + '
' + ) + expected = [ + { + 'level': 2, + 'title': 'Introduction', + 'paragraphs': ['Body text.'], + 'tables': [], + 'figures': [], + } + ] + result = xml_pipe.extract_body_data(xml) + self.assertEqual(result, expected) + def test_extract_body_data_with_tables(self): xml = etree.fromstring( '
' @@ -1573,6 +1655,30 @@ def test_extract_trans_abstract_data_title_with_inline_markup(self): result = xml_pipe.extract_trans_abstract_data(xml) self.assertEqual(result[0]['title'], 'Resumo*') + def test_extract_trans_abstract_data_structured_with_sections(self): + # Regression for issue #1332: same bug as extract_abstract_data, + # a structured trans-abstract's

nested in was invisible + # to a plain findall('p'), so the translated abstract's content + # came out empty (e.g. a10.xml's RESUMO in the real test corpus). + xml = etree.fromstring( + '

' + '' + 'Resumo' + 'Contexto:

Texto de contexto.

' + 'Métodos:

Texto de métodos.

' + '
' + '
' + ) + expected = [ + { + 'lang': 'pt', + 'title': 'Resumo', + 'content': 'Contexto: Texto de contexto. Métodos: Texto de métodos.', + } + ] + result = xml_pipe.extract_trans_abstract_data(xml) + self.assertEqual(result, expected) + class TestExtractFigureData(unittest.TestCase): """Tests for extract_figure_data's graphic href resolution, including