diff --git a/packtools/sps/formats/pdf/pipeline/xml.py b/packtools/sps/formats/pdf/pipeline/xml.py
index d0db5d2e7..a22404c7b 100644
--- a/packtools/sps/formats/pdf/pipeline/xml.py
+++ b/packtools/sps/formats/pdf/pipeline/xml.py
@@ -164,14 +164,20 @@ def extract_contrib_data(xml_tree):
def extract_abstract_data(xml_tree):
"""
Extracts the title and content of the abstract from the given XML tree.
-
+
+ Handles both a plain abstract (
direct children of ) and a
+ structured one (subsections wrapped in , e.g. Introduction/Methods/
+ Results, each with its own and ) - see _extract_abstract_paragraphs.
+
Args:
xml_tree (ElementTree): The XML tree to extract the abstract from.
-
+
Returns:
dict: A dictionary containing the following keys:
- 'title': The text content of the abstract title element, or an empty string if not found.
- - 'content': The text content of the abstract paragraphs, concatenated into a single string.
+ - 'content': The text content of the abstract paragraphs (and, for a
+ structured abstract, each subsection's title), concatenated into a
+ single string.
"""
data = {'title': '', 'content': ''}
@@ -182,22 +188,22 @@ def extract_abstract_data(xml_tree):
if node_title is not None:
data['title'] = ''.join(node_title.itertext()).strip()
- abstract = []
- for p in node_abstract.findall('p'):
- if p is not None:
- abstract.append(''.join(p.itertext()).strip())
- data['content'] = ' '.join(abstract)
+ data['content'] = ' '.join(_extract_abstract_paragraphs(node_abstract))
return data
def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/XML/1998/namespace'}):
"""
Extracts the title and content of translated abstracts from the given XML tree.
-
+
+ Handles both a plain and a structured trans-abstract (subsections wrapped
+ in ) the same way extract_abstract_data does - see
+ _extract_abstract_paragraphs.
+
Args:
xml_tree (ElementTree): The XML tree to extract the translated abstracts from.
namespaces (dict, optional): A dictionary of XML namespaces to use in the XPath expressions.
-
+
Returns:
list: A list of dictionaries, where each dictionary contains the following keys:
- 'lang': The language of the translated abstract.
@@ -217,11 +223,7 @@ def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/
item['lang'] = node.attrib.get(lang_attrib_name)
- abstract = []
- for p in node.findall('p'):
- if p is not None:
- abstract.append(''.join(p.itertext()).strip())
- item['content'] = ' '.join(abstract)
+ item['content'] = ' '.join(_extract_abstract_paragraphs(node))
data.append(item)
@@ -340,6 +342,11 @@ def extract_body_data(xml_tree, table_layout_overrides=None):
"""
Extracts the body data from an XML tree, including section titles, paragraphs, and tables.
+ Excludes any nested inside or - those
+ are structured-abstract subsections handled by extract_abstract_data /
+ extract_trans_abstract_data, and would otherwise be picked up twice by
+ a plain './/sec' search.
+
Args:
xml_tree (ElementTree): The XML tree to extract the body data from.
table_layout_overrides (dict, optional): Maps a table-wrap @id to a forced
@@ -357,7 +364,10 @@ def extract_body_data(xml_tree, table_layout_overrides=None):
data = []
seen_fig_keys = set()
- for document_section in xml_tree.findall('.//sec'):
+ body_sections = xml_tree.xpath(
+ './/sec[not(ancestor::abstract) and not(ancestor::trans-abstract)]'
+ )
+ for document_section in body_sections:
sec = {'paragraphs': [], 'tables': [], 'figures': []}
sec['level'] = xml_utils.get_node_level(document_section, xml_tree)
sec['title'] = document_section.find('title')
@@ -797,6 +807,43 @@ def get_table_column_info(headers, rows):
# Private helpers
# -----------------
+def _extract_abstract_paragraphs(node):
+ """
+ Collects an abstract's readable text as a list of strings, one per
+ found at any depth. A structured abstract wraps each subsection
+ in its own (e.g. Methods:...
),
+ so a plain `node.findall('p')` (direct children only) misses every
+ paragraph and returns an empty abstract. Recursing into finds
+ them, and including each 's own in the flattened output
+ preserves the abstract's structure instead of silently merging
+ distinct subsections together. Some XMLs already carry a trailing
+ colon in the title (e.g. "Methods:"), others don't (e.g. "Methods");
+ a colon is appended only when the title lacks its own closing
+ punctuation, so it never gets duplicated.
+
+ Args:
+ node (ElementTree): The or element
+ (or a within one, for the recursive call).
+
+ Returns:
+ list: Text fragments in document order - titles and content.
+ """
+ parts = []
+ for child in node:
+ if child.tag == 'p':
+ parts.append(''.join(child.itertext()).strip())
+ elif child.tag == 'sec':
+ sec_title = child.find('title')
+ if sec_title is not None:
+ title_text = ''.join(sec_title.itertext()).strip()
+ if title_text:
+ if title_text[-1] not in ':.!?;':
+ title_text = f'{title_text}:'
+ parts.append(title_text)
+ parts.extend(_extract_abstract_paragraphs(child))
+ return parts
+
+
def _extract_table_rows_with_merged_cells(table_section, cell_tag):
"""
Extracts table rows handling merged cells (colspan/rowspan).
diff --git a/tests/sps/formats/pdf/pipeline/test_xml.py b/tests/sps/formats/pdf/pipeline/test_xml.py
index 774630aae..ac6b686a8 100644
--- a/tests/sps/formats/pdf/pipeline/test_xml.py
+++ b/tests/sps/formats/pdf/pipeline/test_xml.py
@@ -83,6 +83,58 @@ def test_extract_abstract_data_with_nested_elements(self):
result = xml_pipe.extract_abstract_data(xml)
self.assertEqual(result, expected)
+ def test_extract_abstract_data_structured_with_sections(self):
+ # Regression for issue #1332: a structured abstract wraps
+ # each subsection in its own , so a plain findall('p') (direct
+ # children only) found nothing and returned an empty content.
+ xml = etree.fromstring(
+ ''
+ 'Abstract'
+ 'Introduction:Some introduction text.
'
+ 'Methods:Some methods text.
'
+ ''
+ )
+ expected = {
+ 'title': 'Abstract',
+ 'content': 'Introduction: Some introduction text. Methods: Some methods text.',
+ }
+ result = xml_pipe.extract_abstract_data(xml)
+ self.assertEqual(result, expected)
+
+ def test_extract_abstract_data_structured_section_title_without_punctuation(self):
+ # Some XMLs don't carry a trailing colon in the , unlike
+ # the "Methods:" style above - a colon must be added so the title
+ # doesn't run into the paragraph text (e.g. "Objetivodescrever...").
+ xml = etree.fromstring(
+ ''
+ 'ObjetivoDescrever o metodo.
'
+ ''
+ )
+ expected = {'title': '', 'content': 'Objetivo: Descrever o metodo.'}
+ result = xml_pipe.extract_abstract_data(xml)
+ self.assertEqual(result, expected)
+
+ def test_extract_abstract_data_structured_section_without_title(self):
+ xml = etree.fromstring(
+ ''
+ 'Untitled section text.
'
+ ''
+ )
+ expected = {'title': '', 'content': 'Untitled section text.'}
+ result = xml_pipe.extract_abstract_data(xml)
+ self.assertEqual(result, expected)
+
+ def test_extract_abstract_data_mixed_direct_and_sectioned_paragraphs(self):
+ xml = etree.fromstring(
+ ''
+ 'Lead paragraph.
'
+ 'Conclusion:Final remarks.
'
+ ''
+ )
+ expected = {'title': '', 'content': 'Lead paragraph. Conclusion: Final remarks.'}
+ result = xml_pipe.extract_abstract_data(xml)
+ self.assertEqual(result, expected)
+
class TestExtractAcknowledgmentData(unittest.TestCase):
@@ -303,6 +355,36 @@ def test_extract_body_data_basic(self):
result = xml_pipe.extract_body_data(xml)
self.assertEqual(result, expected)
+ def test_extract_body_data_excludes_abstract_and_trans_abstract_sections(self):
+ # Regression: a structured abstract/trans-abstract wraps each
+ # subsection in its own (see extract_abstract_data), which a
+ # plain './/sec' search would also pick up as a body section,
+ # duplicating the same content in both the abstract and the body.
+ xml = etree.fromstring(
+ ''
+ ''
+ 'Background:Abstract text.
'
+ ''
+ ''
+ 'Contexto:Texto do resumo.
'
+ ''
+ ''
+ 'IntroductionBody text.
'
+ ''
+ ''
+ )
+ expected = [
+ {
+ 'level': 2,
+ 'title': 'Introduction',
+ 'paragraphs': ['Body text.'],
+ 'tables': [],
+ 'figures': [],
+ }
+ ]
+ result = xml_pipe.extract_body_data(xml)
+ self.assertEqual(result, expected)
+
def test_extract_body_data_with_tables(self):
xml = etree.fromstring(
''
@@ -1573,6 +1655,30 @@ def test_extract_trans_abstract_data_title_with_inline_markup(self):
result = xml_pipe.extract_trans_abstract_data(xml)
self.assertEqual(result[0]['title'], 'Resumo*')
+ def test_extract_trans_abstract_data_structured_with_sections(self):
+ # Regression for issue #1332: same bug as extract_abstract_data,
+ # a structured trans-abstract's nested in was invisible
+ # to a plain findall('p'), so the translated abstract's content
+ # came out empty (e.g. a10.xml's RESUMO in the real test corpus).
+ xml = etree.fromstring(
+ ''
+ ''
+ 'Resumo'
+ 'Contexto:Texto de contexto.
'
+ 'Métodos:Texto de métodos.
'
+ ''
+ ''
+ )
+ expected = [
+ {
+ 'lang': 'pt',
+ 'title': 'Resumo',
+ 'content': 'Contexto: Texto de contexto. Métodos: Texto de métodos.',
+ }
+ ]
+ result = xml_pipe.extract_trans_abstract_data(xml)
+ self.assertEqual(result, expected)
+
class TestExtractFigureData(unittest.TestCase):
"""Tests for extract_figure_data's graphic href resolution, including