fix: extrai parágrafos de resumo estruturado (sec por subseção)

packtools/sps/formats/pdf/pipeline/xml.py

@@ -164,14 +164,20 @@ def extract_contrib_data(xml_tree):
 def extract_abstract_data(xml_tree):
     """
     Extracts the title and content of the abstract from the given XML tree.
-
+
+    Handles both a plain abstract (<p> direct children of <abstract>) and a
+    structured one (subsections wrapped in <sec>, e.g. Introduction/Methods/
+    Results, each with its own <title> and <p>) - see _extract_abstract_paragraphs.
+
     Args:
         xml_tree (ElementTree): The XML tree to extract the abstract from.
-    
+
     Returns:
         dict: A dictionary containing the following keys:
             - 'title': The text content of the abstract title element, or an empty string if not found.
-            - 'content': The text content of the abstract paragraphs, concatenated into a single string.
+            - 'content': The text content of the abstract paragraphs (and, for a
+              structured abstract, each subsection's title), concatenated into a
+              single string.
     """
     data = {'title': '', 'content': ''}
 
@@ -182,22 +188,22 @@ def extract_abstract_data(xml_tree):
         if node_title is not None:
             data['title'] = ''.join(node_title.itertext()).strip()
 
-        abstract = []
-        for p in node_abstract.findall('p'):
-            if p is not None:
-                abstract.append(''.join(p.itertext()).strip())
-        data['content'] = ' '.join(abstract)
+        data['content'] = ' '.join(_extract_abstract_paragraphs(node_abstract))
 
     return data
 
 def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/XML/1998/namespace'}):
     """
     Extracts the title and content of translated abstracts from the given XML tree.
-
+
+    Handles both a plain and a structured trans-abstract (subsections wrapped
+    in <sec>) the same way extract_abstract_data does - see
+    _extract_abstract_paragraphs.
+
     Args:
         xml_tree (ElementTree): The XML tree to extract the translated abstracts from.
         namespaces (dict, optional): A dictionary of XML namespaces to use in the XPath expressions.
-    
+
     Returns:
         list: A list of dictionaries, where each dictionary contains the following keys:
             - 'lang': The language of the translated abstract.
@@ -217,11 +223,7 @@ def extract_trans_abstract_data(xml_tree, namespaces={'xml': 'http://www.w3.org/
 
         item['lang'] = node.attrib.get(lang_attrib_name)
 
-        abstract = []
-        for p in node.findall('p'):
-            if p is not None:
-                abstract.append(''.join(p.itertext()).strip())
-        item['content'] = ' '.join(abstract)
+        item['content'] = ' '.join(_extract_abstract_paragraphs(node))
 
         data.append(item)
 
@@ -340,6 +342,11 @@ def extract_body_data(xml_tree, table_layout_overrides=None):
     """
     Extracts the body data from an XML tree, including section titles, paragraphs, and tables.
 
+    Excludes any <sec> nested inside <abstract> or <trans-abstract> - those
+    are structured-abstract subsections handled by extract_abstract_data /
+    extract_trans_abstract_data, and would otherwise be picked up twice by
+    a plain './/sec' search.
+
     Args:
         xml_tree (ElementTree): The XML tree to extract the body data from.
         table_layout_overrides (dict, optional): Maps a table-wrap @id to a forced
@@ -357,7 +364,10 @@ def extract_body_data(xml_tree, table_layout_overrides=None):
     data = []
     seen_fig_keys = set()
 
-    for document_section in xml_tree.findall('.//sec'):
+    body_sections = xml_tree.xpath(
+        './/sec[not(ancestor::abstract) and not(ancestor::trans-abstract)]'
+    )
+    for document_section in body_sections:
         sec = {'paragraphs': [], 'tables': [], 'figures': []}
         sec['level'] = xml_utils.get_node_level(document_section, xml_tree)
         sec['title'] = document_section.find('title')
@@ -797,6 +807,43 @@ def get_table_column_info(headers, rows):
 # Private helpers
 # -----------------
 
+def _extract_abstract_paragraphs(node):
+    """
+    Collects an abstract's readable text as a list of strings, one per
+    <p> found at any depth. A structured abstract wraps each subsection
+    in its own <sec> (e.g. <sec><title>Methods:</title><p>...</p></sec>),
+    so a plain `node.findall('p')` (direct children only) misses every
+    paragraph and returns an empty abstract. Recursing into <sec> finds
+    them, and including each <sec>'s own <title> in the flattened output
+    preserves the abstract's structure instead of silently merging
+    distinct subsections together. Some XMLs already carry a trailing
+    colon in the title (e.g. "Methods:"), others don't (e.g. "Methods");
+    a colon is appended only when the title lacks its own closing
+    punctuation, so it never gets duplicated.
+
+    Args:
+        node (ElementTree): The <abstract> or <trans-abstract> element
+            (or a <sec> within one, for the recursive call).
+
+    Returns:
+        list: Text fragments in document order - <sec> titles and <p> content.
+    """
+    parts = []
+    for child in node:
+        if child.tag == 'p':
+            parts.append(''.join(child.itertext()).strip())
+        elif child.tag == 'sec':
+            sec_title = child.find('title')
+            if sec_title is not None:
+                title_text = ''.join(sec_title.itertext()).strip()
+                if title_text:
+                    if title_text[-1] not in ':.!?;':
+                        title_text = f'{title_text}:'
+                    parts.append(title_text)
+            parts.extend(_extract_abstract_paragraphs(child))
+    return parts
+
+
 def _extract_table_rows_with_merged_cells(table_section, cell_tag):
     """
     Extracts table rows handling merged cells (colspan/rowspan).

tests/sps/formats/pdf/pipeline/test_xml.py

@@ -83,6 +83,58 @@ def test_extract_abstract_data_with_nested_elements(self):
         result = xml_pipe.extract_abstract_data(xml)
         self.assertEqual(result, expected)
 
+    def test_extract_abstract_data_structured_with_sections(self):
+        # Regression for issue #1332: a structured abstract wraps
+        # each subsection in its own <sec>, so a plain findall('p') (direct
+        # children only) found nothing and returned an empty content.
+        xml = etree.fromstring(
+            '<article><abstract>'
+            '<title>Abstract</title>'
+            '<sec><title>Introduction:</title><p>Some introduction text.</p></sec>'
+            '<sec><title>Methods:</title><p>Some methods text.</p></sec>'
+            '</abstract></article>'
+        )
+        expected = {
+            'title': 'Abstract',
+            'content': 'Introduction: Some introduction text. Methods: Some methods text.',
+        }
+        result = xml_pipe.extract_abstract_data(xml)
+        self.assertEqual(result, expected)
+
+    def test_extract_abstract_data_structured_section_title_without_punctuation(self):
+        # Some XMLs don't carry a trailing colon in the <sec><title>, unlike
+        # the "Methods:" style above - a colon must be added so the title
+        # doesn't run into the paragraph text (e.g. "Objetivodescrever...").
+        xml = etree.fromstring(
+            '<article><abstract>'
+            '<sec><title>Objetivo</title><p>Descrever o metodo.</p></sec>'
+            '</abstract></article>'
+        )
+        expected = {'title': '', 'content': 'Objetivo: Descrever o metodo.'}
+        result = xml_pipe.extract_abstract_data(xml)
+        self.assertEqual(result, expected)
+
+    def test_extract_abstract_data_structured_section_without_title(self):
+        xml = etree.fromstring(
+            '<article><abstract>'
+            '<sec><p>Untitled section text.</p></sec>'
+            '</abstract></article>'
+        )
+        expected = {'title': '', 'content': 'Untitled section text.'}
+        result = xml_pipe.extract_abstract_data(xml)
+        self.assertEqual(result, expected)
+
+    def test_extract_abstract_data_mixed_direct_and_sectioned_paragraphs(self):
+        xml = etree.fromstring(
+            '<article><abstract>'
+            '<p>Lead paragraph.</p>'
+            '<sec><title>Conclusion:</title><p>Final remarks.</p></sec>'
+            '</abstract></article>'
+        )
+        expected = {'title': '', 'content': 'Lead paragraph. Conclusion: Final remarks.'}
+        result = xml_pipe.extract_abstract_data(xml)
+        self.assertEqual(result, expected)
+
 
 class TestExtractAcknowledgmentData(unittest.TestCase):
 
@@ -303,6 +355,36 @@ def test_extract_body_data_basic(self):
         result = xml_pipe.extract_body_data(xml)
         self.assertEqual(result, expected)
 
+    def test_extract_body_data_excludes_abstract_and_trans_abstract_sections(self):
+        # Regression: a structured abstract/trans-abstract wraps each
+        # subsection in its own <sec> (see extract_abstract_data), which a
+        # plain './/sec' search would also pick up as a body section,
+        # duplicating the same content in both the abstract and the body.
+        xml = etree.fromstring(
+            '<article>'
+            '<abstract>'
+            '<sec><title>Background:</title><p>Abstract text.</p></sec>'
+            '</abstract>'
+            '<trans-abstract>'
+            '<sec><title>Contexto:</title><p>Texto do resumo.</p></sec>'
+            '</trans-abstract>'
+            '<body>'
+            '<sec><title>Introduction</title><p>Body text.</p></sec>'
+            '</body>'
+            '</article>'
+        )
+        expected = [
+            {
+                'level': 2,
+                'title': 'Introduction',
+                'paragraphs': ['Body text.'],
+                'tables': [],
+                'figures': [],
+            }
+        ]
+        result = xml_pipe.extract_body_data(xml)
+        self.assertEqual(result, expected)
+
     def test_extract_body_data_with_tables(self):
         xml = etree.fromstring(
             '<article>'
@@ -1573,6 +1655,30 @@ def test_extract_trans_abstract_data_title_with_inline_markup(self):
         result = xml_pipe.extract_trans_abstract_data(xml)
         self.assertEqual(result[0]['title'], 'Resumo*')
 
+    def test_extract_trans_abstract_data_structured_with_sections(self):
+        # Regression for issue #1332: same bug as extract_abstract_data,
+        # a structured trans-abstract's <p> nested in <sec> was invisible
+        # to a plain findall('p'), so the translated abstract's content
+        # came out empty (e.g. a10.xml's RESUMO in the real test corpus).
+        xml = etree.fromstring(
+            '<article>'
+            '<trans-abstract xml:lang="pt">'
+            '<title>Resumo</title>'
+            '<sec><title>Contexto:</title><p>Texto de contexto.</p></sec>'
+            '<sec><title>Métodos:</title><p>Texto de métodos.</p></sec>'
+            '</trans-abstract>'
+            '</article>'
+        )
+        expected = [
+            {
+                'lang': 'pt',
+                'title': 'Resumo',
+                'content': 'Contexto: Texto de contexto. Métodos: Texto de métodos.',
+            }
+        ]
+        result = xml_pipe.extract_trans_abstract_data(xml)
+        self.assertEqual(result, expected)
+
 
 class TestExtractFigureData(unittest.TestCase):
     """Tests for extract_figure_data's graphic href resolution, including