diff --git a/citeforge/bibtex_utils.py b/citeforge/bibtex_utils.py index 4c057d60..89ac8a16 100644 --- a/citeforge/bibtex_utils.py +++ b/citeforge/bibtex_utils.py @@ -10,15 +10,13 @@ import html import re -import threading import unicodedata from collections.abc import Sequence from functools import lru_cache from typing import Any, TypeAlias import bibtexparser -from bibtexparser.bibdatabase import BibDatabase, UndefinedString -from bibtexparser.bparser import BibTexParser +from bibtexparser.model import DuplicateBlockKeyBlock, DuplicateFieldKeyBlock, Entry, ParsingFailedBlock from .cache import response_cache from .config import ( @@ -76,8 +74,6 @@ # round-trips through the strict parser, which collapses it on read. _FIELD_WHITESPACE_RE = re.compile(r"[ \r\n\t]+") -_PARSER_LOCAL = threading.local() - _ParsedBibtex: TypeAlias = tuple[str, str, tuple[tuple[str, str], ...]] _CORPUS_ENTRY_TYPES = frozenset({"article", "book", "incollection", "inproceedings", "misc", "phdthesis"}) _CORPUS_FIELDS = frozenset( @@ -113,32 +109,169 @@ ) -class _StrictCorpusParser(BibTexParser): - """Use bibtexparser's grammar while preserving duplicate-field rejection.""" +# The entry types and month macros classic BibTeX predefines. Other entry +# types are dropped rather than parsed, as the reference BibTeX styles do. +_STANDARD_ENTRY_TYPES = frozenset( + { + "article", + "book", + "booklet", + "conference", + "inbook", + "incollection", + "inproceedings", + "manual", + "mastersthesis", + "misc", + "phdthesis", + "proceedings", + "techreport", + "unpublished", + } +) +_MONTH_STRINGS: dict[str, str] = { + "jan": "January", + "feb": "February", + "mar": "March", + "apr": "April", + "may": "May", + "jun": "June", + "jul": "July", + "aug": "August", + "sep": "September", + "oct": "October", + "nov": "November", + "dec": "December", +} - def _init_expressions(self) -> None: - super()._init_expressions() - def reject_duplicate_fields(_source: str, _location: int, tokens: Any) -> dict[str, Any]: - pairs = list(tokens.get("Fields")) - names = [str(name).casefold() for name, _value in pairs] - if len(names) != len(set(names)): - raise ValueError("committed BibTeX contains a duplicate field") - return dict(reversed(pairs)) +# Stands in for a brace-shielded quote while bibtexparser splits the text. +_SHIELDED_QUOTE = "\ue000" - seen: set[int] = set() - pending = [self._expr.entry] - while pending: - expression = pending.pop() - if id(expression) in seen: + +def _shield_braced_quotes(text: str) -> str: + """Hide each `"` nested in braces inside a quoted value from the splitter. + + In BibTeX a brace group shields a quote, so `title = "A {"} b"` holds one + value. bibtexparser 2 honours only the literal `{"}` and otherwise ends the + value at the first quote, so every shielded quote is swapped for + `_SHIELDED_QUOTE` before splitting and restored in `_resolve_value`. + """ + if _SHIELDED_QUOTE in text: + raise ValueError("BibTeX contains a reserved private-use character") + out = list(text) + index = 0 + while (index := text.find("@", index)) >= 0: + match = re.match(r"@\s*(\w+)\s*([{(])", text[index:]) + if match is None or match.group(1).lower() == "comment": + index += 1 + continue + closer = "}" if match.group(2) == "{" else ")" + depth = 0 + in_quotes = False + quote_depth = 0 + index += match.end() + while index < len(text): + char = text[index] + if in_quotes: + if char == "{": + quote_depth += 1 + elif char == "}" and quote_depth: + quote_depth -= 1 + elif char == '"': + if quote_depth: + out[index] = _SHIELDED_QUOTE + else: + in_quotes = False + elif char == '"' and depth == 0: + in_quotes, quote_depth = True, 0 + elif char == "{": + depth += 1 + elif char == "}" and depth: + depth -= 1 + elif char == closer and depth == 0: + break + index += 1 + return "".join(out) + + +def _resolve_value(raw: str, strings: dict[str, str]) -> str: + """Evaluate a raw BibTeX value: braced, quoted, integer or macro parts joined by `#`.""" + parts: list[str] = [] + depth = 0 + in_quotes = False + start = 0 + for index, char in enumerate(raw): + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + elif char == '"' and depth == 0: + in_quotes = not in_quotes + elif char == "#" and depth == 0 and not in_quotes: + parts.append(raw[start:index]) + start = index + 1 + parts.append(raw[start:]) + resolved = [] + for part in (piece.strip() for piece in parts): + if len(part) >= 2 and (part[0], part[-1]) in {("{", "}"), ('"', '"')}: + resolved.append(part[1:-1]) + elif part.isdigit(): + resolved.append(part) + elif part.lower() in strings: + resolved.append(strings[part.lower()]) + else: + raise ValueError(f"BibTeX value references an undefined string: {part!r}") + return "".join(resolved).replace(_SHIELDED_QUOTE, '"') + + +def load_bibtex_entries(text: str, *, month_strings: bool, strict_fields: bool) -> tuple[list[dict[str, str]], bool]: + """Parse BibTeX into entry dicts carrying lowercase ``ENTRYTYPE``, ``ID`` and field names. + + Macros and `#` concatenation are resolved, non-standard entry types are + dropped, and a repeated field name (compared case-insensitively) keeps its + first value, or raises when ``strict_fields`` is set. Returns the entries + and whether the text also held comments, preambles, string definitions or + other unconsumed text. Raises ``ValueError`` on malformed input. + """ + library = bibtexparser.parse_string(_shield_braced_quotes(text), parse_stack=[]) + strings = dict(_MONTH_STRINGS) if month_strings else {} + for definition in library.strings: + strings[definition.key.lower()] = _resolve_value(definition.value, strings) + blocks: list[Entry] = [] + for parsed in library.blocks: + # Repeated field names and repeated citation keys are policy decisions + # made below, so their entries are recovered; any other failure is not. + block = ( + parsed.ignore_error_block + if isinstance(parsed, (DuplicateFieldKeyBlock, DuplicateBlockKeyBlock)) + else parsed + ) + if isinstance(block, Entry): + blocks.append(block) + elif isinstance(parsed, ParsingFailedBlock): + raise ValueError("BibTeX is malformed") + entries: list[dict[str, str]] = [] + for block in blocks: + entry_type = block.entry_type.lower() + key = block.key.strip() + if not key or any(character.isspace() for character in key): + raise ValueError("BibTeX citation key is blank or contains whitespace") + if entry_type not in _STANDARD_ENTRY_TYPES: + continue + entry: dict[str, str] = {} + for field in block.fields: + name = field.key.lower() + if name in entry: + if strict_fields: + raise ValueError("committed BibTeX contains a duplicate field") continue - seen.add(id(expression)) - if getattr(expression, "resultsName", None) == "Fields": - expression.set_parse_action(reject_duplicate_fields) - pending.extend(getattr(expression, "exprs", ())) - child = getattr(expression, "expr", None) - if child is not None: - pending.append(child) + entry[name] = _resolve_value(str(field.value), strings) + entry["ENTRYTYPE"] = entry_type + entry["ID"] = key + entries.append(entry) + unconsumed = bool(library.comments or library.preambles or library.strings) + return entries, unconsumed def make_bibkey(title: str, authors: Sequence[str], year: int, fallback: str = "entry") -> str: @@ -180,26 +313,12 @@ def build_minimal_bibtex(title: str, authors: list[str], year: int, keyhint: str return "\n".join(lines) + "\n" -def _parser_for_thread() -> BibTexParser: - """Return one prepared parser per worker thread.""" - parser = getattr(_PARSER_LOCAL, "parser", None) - if parser is None: - parser = BibTexParser() - parser.expect_multiple_parse = True - _PARSER_LOCAL.parser = parser - return parser - - @lru_cache(maxsize=BIBTEX_PARSE_CACHE_SIZE) def _parse_bibtex_immutable(bibtex: str) -> _ParsedBibtex | None: """Parse into an immutable value safe to share through the LRU cache.""" - parser = _parser_for_thread() - parser.bib_database = BibDatabase() - if parser.common_strings: - parser.bib_database.load_common_strings() try: - entries = bibtexparser.loads(bibtex, parser=parser).entries - except (TypeError, UndefinedString, ValueError): + entries, _unconsumed = load_bibtex_entries(bibtex, month_strings=True, strict_fields=False) + except (TypeError, ValueError): entries = [] if not entries: return None @@ -234,21 +353,19 @@ def parse_strict_bibtex_document(content: bytes) -> dict[str, Any]: raise ValueError("committed BibTeX must not be empty") if any(unicodedata.category(character) == "Cc" and character not in "\r\n\t" for character in text): raise ValueError("committed BibTeX contains control characters") - parser = _StrictCorpusParser(common_strings=False) - parser.expect_multiple_parse = True try: - database = bibtexparser.loads(text, parser=parser) + entries, unconsumed = load_bibtex_entries(text, month_strings=False, strict_fields=True) except ValueError as exc: if "duplicate field" in str(exc): raise raise ValueError("committed BibTeX is malformed") from exc - except (TypeError, UndefinedString) as exc: + except TypeError as exc: raise ValueError("committed BibTeX is malformed") from exc - if database.comments or database.preambles or database.strings: + if unconsumed: raise ValueError("committed BibTeX contains directives or unconsumed text") - if len(database.entries) != 1: + if len(entries) != 1: raise ValueError("committed BibTeX requires exactly one entry") - raw = dict(database.entries[0]) + raw: dict[str, Any] = dict(entries[0]) entry_type = raw.pop("ENTRYTYPE", None) key = raw.pop("ID", None) if not isinstance(entry_type, str) or entry_type.casefold() not in _CORPUS_ENTRY_TYPES: diff --git a/citeforge/refresh/decoders.py b/citeforge/refresh/decoders.py index 2dc14718..3d9a2a35 100644 --- a/citeforge/refresh/decoders.py +++ b/citeforge/refresh/decoders.py @@ -12,12 +12,9 @@ from urllib.parse import urlsplit from xml.etree.ElementTree import Element -import bibtexparser -from bibtexparser.bibdatabase import UndefinedString -from bibtexparser.bparser import BibTexParser from defusedxml.ElementTree import fromstring as safe_xml_fromstring -from ..bibtex_utils import parse_bibtex_to_dict +from ..bibtex_utils import load_bibtex_entries, parse_bibtex_to_dict from ..id_utils import find_arxiv_in_text, find_doi_in_text, normalize_doi from .capabilities import ResponseMediaType from .transport import RawProviderResponse, SchemaChangedError @@ -90,10 +87,7 @@ def decode(raw: RawProviderResponse, _context: Mapping[str, object]) -> DecodedR if ( not isinstance(item, dict) or any(not item.get(name) for name in required_fields) - or any( - not isinstance(item.get(name), str) or not str(item[name]).strip() - for name in string_fields - ) + or any(not isinstance(item.get(name), str) or not str(item[name]).strip() for name in string_fields) or (record_validator is not None and not record_validator(item)) ): raise SchemaChangedError("provider record lacks required reducer evidence") @@ -604,15 +598,12 @@ def _doi_bibtex(raw: RawProviderResponse, context: Mapping[str, object]) -> Deco text = raw.body.decode("utf-8") except UnicodeDecodeError as exc: raise ValueError("DOI BibTeX is not UTF-8") from exc - parser = BibTexParser(common_strings=False) - parser.expect_multiple_parse = True try: - database = bibtexparser.loads(text, parser=parser) - except (TypeError, UndefinedString, ValueError) as exc: + entries, unconsumed = load_bibtex_entries(text, month_strings=False, strict_fields=False) + except (TypeError, ValueError) as exc: raise ValueError("DOI BibTeX is malformed") from exc - if database.comments or database.preambles or database.strings: + if unconsumed: raise ValueError("DOI BibTeX contains unparsed text or directives") - entries = database.entries if len(entries) != 1: raise SchemaChangedError("DOI BibTeX must contain exactly one entry") normalized = parse_bibtex_to_dict(text) diff --git a/pyproject.toml b/pyproject.toml index 40c4579f..52336d88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires-python = ">=3.10" readme = "README.md" license = "AGPL-3.0-or-later" dependencies = [ - "bibtexparser==1.4.4", + "bibtexparser==2.0.0", "pylatexenc==2.11", "requests>=2.32.0", "rapidfuzz>=3.0.0", diff --git a/requirements-build.in b/requirements-build.in index 054b1c1e..8879d2f4 100644 --- a/requirements-build.in +++ b/requirements-build.in @@ -1,4 +1,4 @@ # CI build backend input. Keep its minimum aligned with pyproject.toml. setuptools>=77.0 # CI uses this exact compiler to verify all generated locks. -uv==0.12.5 +uv==0.12.13 diff --git a/requirements-build.lock b/requirements-build.lock index 15acbfe6..7eb00e1e 100644 --- a/requirements-build.lock +++ b/requirements-build.lock @@ -4,24 +4,24 @@ setuptools==84.0.0 \ --hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670 \ --hash=sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73 # via -r requirements-build.in -uv==0.12.5 \ - --hash=sha256:1a06c8bc4d43b5f6c1e3f2ae3d0f6455b07515f762516f95e52e6c0cbccedf15 \ - --hash=sha256:2bd62134e56af35b9cf017aaf8ae41a605d6501dd49afc35b70b544a45dd8354 \ - --hash=sha256:2d65b7b3bc3fd28678f62aa7fb5d90f106ad9782c1354af60b6cecdf9ea9ecd9 \ - --hash=sha256:39bb102766c95571781a7b4c611675ea213e08df5c680f3936279b3c0d1f6c3c \ - --hash=sha256:3e195ccf1ed60c8bb24a6447ce306441a4181d54b602407e09bc56e963911c15 \ - --hash=sha256:442a21d181faae21742aaaf6d2091a0d27755d3eac344061a9a00c90169b7524 \ - --hash=sha256:455c3e57602e2141e66e2f0bf685898c9c5e5a70377d14c9a71554a3baf3ddbf \ - --hash=sha256:568485b44e848eb3693f85d6b00299ccd8fc4d26902030dbf24f549c276db9ca \ - --hash=sha256:58abfb0f658b39a834307a11223bc170294ea214263b4c99ecc7663720d43544 \ - --hash=sha256:6ad2c455f1fe4d2962f6fd7ccb3b1f61c61856681c9d99f40e170b2074353fa3 \ - --hash=sha256:712624b62e25c84e5a10fc6aa144d8a81b685fdc067a54a7ca4367d75d2cf791 \ - --hash=sha256:7817f8e957960f9ddc452ea353f283c0d6393e2e31b400276485adced5b1f371 \ - --hash=sha256:a05b497c2a948c8600f4c831a89852b4d2514b7f561074225cc9edd0cc4811e2 \ - --hash=sha256:bd08c82831b0033330f8eeeb0d90f938a4d999f25569bee68a975c736142d795 \ - --hash=sha256:bea86f27a027e0e3af908db4bdd4f1ceef3ca2bd47673b5ccca7f550e325b1b4 \ - --hash=sha256:d87156bc174d94fae890bb7a261e2867140abb9fe1e9de81a5295e582fb9d0f5 \ - --hash=sha256:dc14e4f81a99b585a891350c60d1ff4557d54cb3c3c81fa45fd4e0dd512ba752 \ - --hash=sha256:edd9ff6154b891146a342c143cd29b330ad97ac6a4b20ff4a99a20a4da84ceca \ - --hash=sha256:f9656ac7a00fd4314980fb0f790df1c1f3fa9cbcf9af9c6f611b19448b9da687 +uv==0.12.13 \ + --hash=sha256:15f368be55e87109391a31c50846e81086b9792ccb43a1731774e961ea0641b7 \ + --hash=sha256:19264bcb664138bd94f66a39d6d1fe7bc4614f94deea98a995f88d779db0c4ba \ + --hash=sha256:263700a9b02d12bc8f1322b250f959435c4077fe3c16ff588e68672f3eddbd07 \ + --hash=sha256:3efae0a910a25e18732f38ac1ef7b02e23f47b7fe67735039921fceff01d518c \ + --hash=sha256:5312891392eb5b72eaca9b880e85fe3e706a2b11342e59f7a5cf0304d14d5cc5 \ + --hash=sha256:6e6b9b2464aa917699fd6206932fca89483e45663ce438dac4e5bca5342e9bd0 \ + --hash=sha256:7aafa9fad6d8a2f0eec5dafdc6279633fb8fd5642c0934d4d73696c4edb9379e \ + --hash=sha256:80c05ac556fec1504466c4cdcd135eb8f99e358ce3f17da336de696852791551 \ + --hash=sha256:891aff3e186cd5844102750372304e9ca20edffc50cb39a14cd047ea3416de75 \ + --hash=sha256:93efdd41ccce008368ba6677ac9f788203b3c95cb2424657c3079b2f1d0022ee \ + --hash=sha256:959d330f99a653d115b440c28fb2abc3e720ca10e9f0131d97a48bfd5e59b4d2 \ + --hash=sha256:970ef5e688bce5936172bec9b2c73876fc1996d4e0318a26562310af92b0a75d \ + --hash=sha256:9ecd65bb4fbae642418ecc2f92d0811143eeaa96fcf96a7f18b74c475a7601c5 \ + --hash=sha256:ac7f135750b7419283029edb93453b4d41c7d7ff90f9b3860a6e474ddf98f016 \ + --hash=sha256:ad61c396a57caac8b351429294e44ee52a1ecb1629b95d6898bd7c63ed3c13f3 \ + --hash=sha256:b9854ef55f665401f95237e1db55162dd9d7a4a46929326e6f52a5388ca57842 \ + --hash=sha256:b9c703a4aaee1288f33e2ce8b39a0245f432b57bd5d769073756b68e9ca6bafe \ + --hash=sha256:bc83ca045c140fcc5d010d451def731be1d225c472f7e4d474610f9e8deb88db \ + --hash=sha256:f86e5f02883c2e7a21bf522f4aa520c20d4f779d9a3368fbd6cdfe4a8f9549b5 # via -r requirements-build.in diff --git a/requirements-dev.lock b/requirements-dev.lock index 50e784bb..37b7ec2e 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -36,8 +36,9 @@ ast-serialize==0.6.0 \ --hash=sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad \ --hash=sha256:f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554 # via mypy -bibtexparser==1.4.4 \ - --hash=sha256:093b6c824f7a71d3a748867c4057b71f77c55b8dbc07efc993b781771520d8fb +bibtexparser==2.0.0 \ + --hash=sha256:7f1e1d989282836a0ba39caf88ecc8982dfd924a4f49de58167d7effe99768e8 \ + --hash=sha256:e5e6df506d53e354874240c7f7fbd98a36db4b725e139f30e175f710a72a68b7 # via citeforge (pyproject.toml) certifi==2026.6.17 \ --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ @@ -606,11 +607,9 @@ pygments==2.20.0 \ pylatexenc==2.11 \ --hash=sha256:305a072a99ce736246049c9da05841b9d718c0f7ea8888f5f596cf15cb621053 \ --hash=sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0 - # via citeforge (pyproject.toml) -pyparsing==3.3.2 \ - --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ - --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc - # via bibtexparser + # via + # citeforge (pyproject.toml) + # bibtexparser pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c diff --git a/requirements.lock b/requirements.lock index 5896c9c7..2296d0e1 100644 --- a/requirements.lock +++ b/requirements.lock @@ -1,7 +1,8 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml --universal --python-version 3.10 --generate-hashes -o requirements.lock -bibtexparser==1.4.4 \ - --hash=sha256:093b6c824f7a71d3a748867c4057b71f77c55b8dbc07efc993b781771520d8fb +bibtexparser==2.0.0 \ + --hash=sha256:7f1e1d989282836a0ba39caf88ecc8982dfd924a4f49de58167d7effe99768e8 \ + --hash=sha256:e5e6df506d53e354874240c7f7fbd98a36db4b725e139f30e175f710a72a68b7 # via citeforge (pyproject.toml) certifi==2026.6.17 \ --hash=sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432 \ @@ -303,11 +304,9 @@ pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementatio pylatexenc==2.11 \ --hash=sha256:305a072a99ce736246049c9da05841b9d718c0f7ea8888f5f596cf15cb621053 \ --hash=sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0 - # via citeforge (pyproject.toml) -pyparsing==3.3.2 \ - --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ - --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc - # via bibtexparser + # via + # citeforge (pyproject.toml) + # bibtexparser rapidfuzz==3.14.5 \ --hash=sha256:0084b687b02b4e569b46d8d6d4ad25659528e6081cd6d067ca453a69035f07e4 \ --hash=sha256:01550fe5f60fd176aa66b7611289d46dc4aa4b1b904874c7b6d1d54e581c5ec1 \