From d1856a14fea36a52b0d93e8c91f1f94b97eba7ff Mon Sep 17 00:00:00 2001 From: Tobias Kuhn Date: Mon, 6 Jul 2026 14:55:19 +0200 Subject: [PATCH] Add drift fingerprint + incremental mint/supersede publishing Publishing was append-only: mint_publish skipped any term already in the id-map keyed solely on its old id, so a change to an already-published term -- whether to its content or to a batch-level wrapper (template, part-of, nanopub-type, license) -- was never detected or re-issued. Add a semantic drift fingerprint and an incremental publisher that acts on it: - fingerprint.py: a key-independent SHA-256 over the identity-defining inputs only (URDNA2015-canonicalized assertion + suggester/derived_from + license/introduces/nanopub-type/template), computed on the placeholder form so the thing URI never feeds its own hash. Excludes the build timestamp, signature and blank-node/order noise (no-op tier) and the signing key/toolchain (build-provenance tier), so a key rotation cannot read as content drift. - idmap.py: fourth `fingerprint` column; reads legacy 3-column files. - incremental.py: publish_incremental() -- per term, mint (new), skip (fingerprint unchanged), or supersede (drifted) against the existing fixed thing URI so the term keeps its identity across versions. Legacy rows without a fingerprint are backfilled as a baseline, not mass-superseded. - cli/mint_publish.py: build a matching SupersessionBuilder, drive publish_incremental, write minted and superseding nanopubs by their own artifact code, and persist fingerprints in the id-map. Co-Authored-By: Claude Opus 4.8 --- src/pubmate/__init__.py | 15 ++++ src/pubmate/cli/mint_publish.py | 55 ++++++++---- src/pubmate/fingerprint.py | 149 +++++++++++++++++++++++++++++++ src/pubmate/idmap.py | 47 +++++++--- src/pubmate/incremental.py | 153 ++++++++++++++++++++++++++++++++ tests/test_cli_mint_publish.py | 61 ++++++++++++- tests/test_fingerprint.py | 143 +++++++++++++++++++++++++++++ tests/test_idmap.py | 18 +++- tests/test_incremental.py | 101 +++++++++++++++++++++ 9 files changed, 708 insertions(+), 34 deletions(-) create mode 100644 src/pubmate/fingerprint.py create mode 100644 src/pubmate/incremental.py create mode 100644 tests/test_fingerprint.py create mode 100644 tests/test_incremental.py diff --git a/src/pubmate/__init__.py b/src/pubmate/__init__.py index 814aa98..e713377 100644 --- a/src/pubmate/__init__.py +++ b/src/pubmate/__init__.py @@ -1,5 +1,13 @@ from pubmate.defining import DEFAULT_LICENSE, DefiningNanopubBuilder +from pubmate.fingerprint import ( + FP_SCHEME, + IdentityFields, + canonical_assertion, + fingerprint_term, + identity_fields, +) from pubmate.idmap import IdMap, IdMapEntry +from pubmate.incremental import IncrementalResult, publish_incremental from pubmate.introduction import build_introduction from pubmate.migrate import MigrationResult, MintedSupersession, migrate_terms from pubmate.mint import IdentifierGenerator @@ -11,10 +19,13 @@ __all__ = [ "DEFAULT_LICENSE", + "FP_SCHEME", "DefiningNanopubBuilder", "IdMap", "IdMapEntry", "IdentifierGenerator", + "IdentityFields", + "IncrementalResult", "MintBatch", "MintedTerm", "MigrationResult", @@ -25,8 +36,12 @@ "SupersessionBuilder", "TermInput", "build_introduction", + "canonical_assertion", + "fingerprint_term", + "identity_fields", "migrate_terms", "order_terms", + "publish_incremental", "referenced_terms", "serialize_nanopub", "sign_and_publish", diff --git a/src/pubmate/cli/mint_publish.py b/src/pubmate/cli/mint_publish.py index 9470386..2265a5d 100644 --- a/src/pubmate/cli/mint_publish.py +++ b/src/pubmate/cli/mint_publish.py @@ -7,7 +7,9 @@ from pubmate.cli._signing import resolve_signing from pubmate.defining import DefiningNanopubBuilder from pubmate.idmap import IdMap +from pubmate.incremental import publish_incremental from pubmate.minting import SequentialMinter, term_input_from_assertion +from pubmate.supersede import SupersessionBuilder from pubmate.utils import serialize_nanopub logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -55,14 +57,18 @@ def cli( dry_run: bool, pattern: str, ) -> None: - """Sequentially mint defining nanopubs from per-term assertions and publish them. - - Each assertion is re-keyed onto the artifact-code placeholder, signed (which - lands the code on the term's thing URI), and -- unless --dry-run -- published. - Minted nanopubs are written to --output-dir as .trig and the - old_id -> thing_uri/np_uri mapping is written/merged into --id-map-file. - - Inter-term links (forward refs/cycles) are intentionally left to a later + """Incrementally mint/supersede defining nanopubs from per-term assertions. + + Each assertion is re-keyed onto the artifact-code placeholder. Per term, this + compares its identity fingerprint against the one recorded in --id-map-file: + a new term is minted, an unchanged term is skipped, and a *drifted* term + (content or wrapper changed) is superseded -- re-stated against its existing + thing URI in a nanopub that supersedes the recorded one, keeping the term's + identity. Published nanopubs (unless --dry-run) are written to --output-dir as + .trig and the updated old_id -> thing_uri/np_uri/fingerprint + map is written to --id-map-file. + + Inter-term links (forward refs/cycles) are intentionally left to the migration superseding pass (see the migration tooling); this mints the assertions as given. """ @@ -82,6 +88,10 @@ def cli( namespace, profile=signing.profile, test_server=signing.test_server, nanopub_types=nanopub_types, template=template, ) + supersession_builder = SupersessionBuilder( + profile=signing.profile, test_server=signing.test_server, + license=builder.license, nanopub_types=nanopub_types, template=template, + ) files = sorted(assertion_folder.glob(pattern)) if not files: @@ -105,26 +115,33 @@ def cli( existing = IdMap.from_tsv(id_map_file.read_text(encoding="utf-8")) if id_map_file and id_map_file.exists() else IdMap() minter = SequentialMinter(builder, default_suggester_orcid=default_suggester) - batch = minter.mint_all( + result = publish_incremental( terms, + minter=minter, + supersession_builder=supersession_builder, + existing=existing, dry_run=dry_run, - already_minted=existing.np_uri_map, ) - # Write each nanopub as .trig (the thing/np code under scheme A). + # Write each minted/superseding nanopub as .trig (its own code: + # for a defining nanopub that equals the thing code, for a supersession its own). output_dir.mkdir(parents=True, exist_ok=True) - for minted in batch.terms: - code = minted.thing_uri.removeprefix(namespace) - (output_dir / f"{code}.trig").write_text(serialize_nanopub(minted.nanopub), encoding="utf-8") + published = [(m.np_uri, m.nanopub) for m in result.minted.terms] + published += [(s.np_uri, s.nanopub) for s in result.superseded] + for np_uri, np in published: + code = np_uri.rsplit("/", 1)[-1] + (output_dir / f"{code}.trig").write_text(serialize_nanopub(np), encoding="utf-8") if id_map_file is not None: - merged = IdMap(list(existing)) - merged.merge(IdMap.from_batch(batch), overwrite=True) id_map_file.parent.mkdir(parents=True, exist_ok=True) - merged.write_tsv(id_map_file) - logger.info("Wrote id-map (%d entries) -> %s", len(merged), id_map_file) + result.id_map.write_tsv(id_map_file) + logger.info("Wrote id-map (%d entries) -> %s", len(result.id_map), id_map_file) - logger.info("Minted %d new term(s)%s -> %s", len(batch.terms), " (dry-run)" if dry_run else "", output_dir) + logger.info( + "Minted %d, superseded %d, skipped %d term(s)%s -> %s", + len(result.minted.terms), len(result.superseded), len(result.skipped), + " (dry-run)" if dry_run else "", output_dir, + ) if __name__ == "__main__": diff --git a/src/pubmate/fingerprint.py b/src/pubmate/fingerprint.py new file mode 100644 index 0000000..681c07f --- /dev/null +++ b/src/pubmate/fingerprint.py @@ -0,0 +1,149 @@ +"""Semantic drift fingerprint for defining nanopublications. + +A defining nanopub's trusty artifact code is a hash of the *whole* nanopub, so it +cannot answer "does the already-published version need re-issuing?": it also moves +with the build timestamp, the signing key and the toolchain, and it only exists +*after* signing. This module computes a stable, key-independent fingerprint over +just the **identity-defining** inputs, so a caller can tell an unchanged term from +one whose content or wrapper has drifted -- and, on drift, supersede rather than +mint a fresh (differently-identified) nanopub. + +Three tiers, and where each goes: + +* **No-op** -- build timestamp, signature, blank-node ids, triple order, + serialization prefixes. Excluded: the assertion is URDNA2015-canonicalized + (:func:`canonical_assertion`), and nothing time/signature-derived enters. +* **Identity-defining** -- the assertion itself (which already carries + ``dcterms:isPartOf`` when minted with ``part_of``), the suggester and + ``derived_from`` provenance, and the pubinfo the vocabulary commits to: + ``dct:license``, ``npx:introduces``, ``npx:hasNanopubType``, + ``nt:wasCreatedFromTemplate``. This is the fingerprint domain. +* **Build-provenance** -- signing key, ``nanopub``/pubmate version, trusty + algorithm. Deliberately **not** hashed here (a key rotation must not read as + content drift); record it beside the fingerprint and gate re-issue on policy. + +The fingerprint is computed on the *placeholder* form of the assertion (subject = +``namespace + ~~~ARTIFACTCODE~~~``), i.e. the graph the builder holds before +signing, so the thing URI -- which contains the code, itself a hash of the whole +nanopub -- never feeds its own fingerprint. + +Keep :func:`identity_fields` in lockstep with +:meth:`~pubmate.defining.DefiningNanopubBuilder.build`: if a future change adds an +identity-bearing pubinfo triple there, mirror it here and bump :data:`FP_SCHEME`. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import Any, Optional + +import rdflib +from rdflib.compare import to_canonical_graph + +from pubmate._nanopub_build import UNSET as _UNSET +from pubmate.defining import DefiningNanopubBuilder +from pubmate.minting import TermInput + +#: Fingerprint scheme tag, embedded in every digest. Bump when the domain or the +#: canonicalization changes, so digests from an older scheme are recomputed +#: rather than silently compared across incompatible definitions. +FP_SCHEME = "pubmate-fp-1" + + +def canonical_assertion(graph: rdflib.Graph) -> str: + """Canonicalize ``graph`` (URDNA2015) to sorted N-Triples. + + Stable across blank-node labels, triple order and serialization prefixes -- + the no-op tier -- so cosmetic RDF churn does not move the fingerprint. + """ + canonical = to_canonical_graph(graph) + lines = canonical.serialize(format="nt").splitlines() + return "\n".join(sorted(line for line in lines if line.strip())) + + +@dataclass(frozen=True) +class IdentityFields: + """The identity-defining inputs a defining nanopub commits to. + + A change to any field means the published nanopub is materially stale and + should be re-issued by *superseding*. The signing key and toolchain are + intentionally absent (see the module docstring).""" + + assertion: str + suggester: str + derived_from: str + license: str + introduces: str + nanopub_types: tuple[str, ...] + template: str + + def digest(self) -> str: + """Hex SHA-256 over the canonical JSON of these fields plus the scheme.""" + payload = { + "scheme": FP_SCHEME, + "assertion": self.assertion, + "suggester": self.suggester, + "derived_from": self.derived_from, + "license": self.license, + "introduces": self.introduces, + "nanopub_types": list(self.nanopub_types), + "template": self.template, + } + blob = json.dumps(payload, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def identity_fields( + term: TermInput, + builder: DefiningNanopubBuilder, + *, + default_suggester: Optional[str] = None, + license: Any = _UNSET, + introduces: Any = _UNSET, +) -> IdentityFields: + """Extract ``term``'s identity-defining fields as ``builder`` would emit them. + + Mirrors :meth:`DefiningNanopubBuilder.build` and + :meth:`SequentialMinter.mint` defaulting so the fingerprint tracks exactly + what gets signed: ``license`` falls back to the builder's license, + ``introduces`` to the placeholder thing URI, and the suggester to + ``default_suggester`` when the term carries none. ``part_of`` is already in + ``term.assertion`` (added by ``term_input_from_assertion``), so it is covered + by the assertion hash rather than a field here. + """ + effective_license = builder.license if license is _UNSET else license + effective_introduces = builder.thing_uri if introduces is _UNSET else introduces + return IdentityFields( + assertion=canonical_assertion(term.assertion), + suggester=term.suggester_orcid or default_suggester or "", + derived_from=term.derived_from or "", + license=effective_license or "", + introduces="" if effective_introduces is None else str(effective_introduces), + nanopub_types=tuple(builder.nanopub_types), + template=builder.template or "", + ) + + +def fingerprint_term( + term: TermInput, + builder: DefiningNanopubBuilder, + *, + default_suggester: Optional[str] = None, + license: Any = _UNSET, + introduces: Any = _UNSET, +) -> str: + """Hex SHA-256 drift fingerprint for ``term`` as built by ``builder``. + + Convenience wrapper over :func:`identity_fields`; the same defaulting rules + apply. Two terms share a fingerprint iff they would sign to nanopubs that are + identical in every identity-defining respect (ignoring timestamp, signature, + key and blank-node/serialization noise).""" + return identity_fields( + term, + builder, + default_suggester=default_suggester, + license=license, + introduces=introduces, + ).digest() diff --git a/src/pubmate/idmap.py b/src/pubmate/idmap.py index 9055bb4..9df215a 100644 --- a/src/pubmate/idmap.py +++ b/src/pubmate/idmap.py @@ -2,12 +2,16 @@ Records, per term, the mapping from its old/local identifier to the new nanopub-based identifiers minted for it: the term's thing URI (its trusty -artifact-code URI) and the URI of its defining nanopub. +artifact-code URI) and the URI of its defining nanopub, plus an optional +drift ``fingerprint`` (see :mod:`pubmate.fingerprint`) of the identity-defining +inputs the published nanopub was built from -- so a later run can tell an +unchanged term from one that has drifted and needs superseding. The map is meant to be kept permanently and grown incrementally, so old identifiers stay resolvable and re-runs can append without losing prior entries. It round-trips to a tab-separated file (a superset of a simple redirect table) -and to JSON. +and to JSON. The TSV gained a fourth ``fingerprint`` column; 3-column files +written by older versions still read (their fingerprint is empty). """ from __future__ import annotations @@ -19,16 +23,20 @@ from pubmate.minting import MintBatch -_TSV_HEADER = ("old_id", "thing_uri", "np_uri") +_TSV_HEADER = ("old_id", "thing_uri", "np_uri", "fingerprint") @dataclass(frozen=True) class IdMapEntry: - """One term's old identifier and its new nanopub-based identifiers.""" + """One term's old identifier and its new nanopub-based identifiers. + + ``fingerprint`` is the drift fingerprint of the identity-defining inputs the + nanopub was built from (empty when unknown, e.g. legacy 3-column rows).""" old_id: str thing_uri: str np_uri: str + fingerprint: str = "" class IdMap: @@ -60,13 +68,21 @@ def merge(self, other: "IdMap", *, overwrite: bool = False) -> None: self.add(entry, overwrite=overwrite) @classmethod - def from_batch(cls, batch: MintBatch) -> "IdMap": + def from_batch(cls, batch: MintBatch, *, fingerprints: Optional[Dict[str, str]] = None) -> "IdMap": """Build a map from a :class:`~pubmate.minting.MintBatch`. - The minter's ``term_id`` is used as the old identifier. + The minter's ``term_id`` is used as the old identifier. ``fingerprints``, + if given, supplies each term's drift fingerprint keyed by ``term_id`` + (missing terms get an empty fingerprint). """ + fingerprints = fingerprints or {} return cls( - IdMapEntry(old_id=t.term_id, thing_uri=t.thing_uri, np_uri=t.np_uri) + IdMapEntry( + old_id=t.term_id, + thing_uri=t.thing_uri, + np_uri=t.np_uri, + fingerprint=fingerprints.get(t.term_id, ""), + ) for t in batch.terms ) @@ -94,6 +110,11 @@ def np_uri_map(self) -> Dict[str, str]: """``old_id -> nanopub URI``.""" return {e.old_id: e.np_uri for e in self} + @property + def fingerprint_map(self) -> Dict[str, str]: + """``old_id -> drift fingerprint`` (empty string when unknown).""" + return {e.old_id: e.fingerprint for e in self} + def _sorted(self) -> List[IdMapEntry]: return sorted(self._entries.values(), key=lambda e: e.old_id) @@ -101,7 +122,7 @@ def _sorted(self) -> List[IdMapEntry]: def to_tsv(self) -> str: lines = ["\t".join(_TSV_HEADER)] - lines += ["\t".join((e.old_id, e.thing_uri, e.np_uri)) for e in self._sorted()] + lines += ["\t".join((e.old_id, e.thing_uri, e.np_uri, e.fingerprint)) for e in self._sorted()] return "\n".join(lines) + "\n" @classmethod @@ -110,10 +131,12 @@ def from_tsv(cls, text: str) -> "IdMap": lines = [ln for ln in text.splitlines() if ln.strip()] for line in lines: fields = line.split("\t") - if tuple(fields) == _TSV_HEADER: - continue - if len(fields) != 3: - raise ValueError(f"expected 3 tab-separated fields, got {len(fields)}: {line!r}") + if fields[0] == _TSV_HEADER[0]: + continue # header row (3- or 4-column) + if len(fields) == 3: # legacy row, no fingerprint + fields = (*fields, "") + if len(fields) != 4: + raise ValueError(f"expected 3 or 4 tab-separated fields, got {len(fields)}: {line!r}") id_map.add(IdMapEntry(*fields)) return id_map diff --git a/src/pubmate/incremental.py b/src/pubmate/incremental.py new file mode 100644 index 0000000..e00f2f0 --- /dev/null +++ b/src/pubmate/incremental.py @@ -0,0 +1,153 @@ +"""Incremental mint/supersede publishing with drift detection. + +Re-runnable publishing of defining nanopubs. For each term, compare its current +identity fingerprint (:mod:`pubmate.fingerprint`) against the one recorded in the +id-map and take one of three actions: + +* **new** -- term absent from the id-map: mint a defining nanopub, record its + thing/nanopub URIs and fingerprint. +* **unchanged** -- fingerprint matches the recorded one: skip (carry the id-map + entry forward untouched). +* **drifted** -- fingerprint differs: *supersede*. Re-state the changed assertion + against the term's **existing fixed thing URI** (so the term keeps its identity) + in a nanopub that ``npx:supersedes`` the recorded one, then repoint the id-map's + ``np_uri``/``fingerprint`` at the new version. The thing URI is unchanged -- + only the defining/superseding nanopub URI advances, chained by supersession. + +Legacy id-map rows carry no fingerprint. Rather than mass-supersede on the first +fingerprinted run, such a term is treated as unchanged and its current fingerprint +is *backfilled* as the baseline (with a warning) -- adopting "what is published +now" as the reference. A genuine wrapper change made before fingerprints existed +must therefore be forced by hand; it cannot be inferred without a prior baseline. + +I/O (reading assertions, writing trig/id-map) is left to the caller/CLI, mirroring +:func:`pubmate.migrate.migrate_terms`. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import List, Optional, Sequence + +import rdflib + +from pubmate._nanopub_build import preferred_label +from pubmate.fingerprint import fingerprint_term +from pubmate.idmap import IdMap, IdMapEntry +from pubmate.migrate import MintedSupersession +from pubmate.minting import MintBatch, SequentialMinter, TermInput +from pubmate.rdf2nanopub import sign_and_publish +from pubmate.supersede import SupersessionBuilder + +logger = logging.getLogger(__name__) + + +@dataclass +class IncrementalResult: + """The outcome of an incremental publish run.""" + + minted: MintBatch = field(default_factory=MintBatch) + superseded: List[MintedSupersession] = field(default_factory=list) + #: term_ids that were unchanged and skipped (includes backfilled legacy rows). + skipped: List[str] = field(default_factory=list) + id_map: IdMap = field(default_factory=IdMap) + + +def _rekey_to_fixed( + assertion: rdflib.Graph, *, placeholder: rdflib.URIRef, fixed: rdflib.URIRef +) -> rdflib.Graph: + """Rewrite the placeholder thing URI (subject and self-references) to the + term's already-minted fixed URI, so a superseding nanopub keeps its identity.""" + out = rdflib.Graph() + for s, p, o in assertion: + out.add((fixed if s == placeholder else s, p, fixed if o == placeholder else o)) + return out + + +def publish_incremental( + terms: Sequence[TermInput], + *, + minter: SequentialMinter, + supersession_builder: SupersessionBuilder, + existing: Optional[IdMap] = None, + dry_run: bool = True, +) -> IncrementalResult: + """Mint new terms, skip unchanged ones, and supersede drifted ones. + + Args: + terms: the terms to publish (assertions keyed on the builder's placeholder + thing URI, e.g. from ``term_input_from_assertion``). + minter: a :class:`~pubmate.minting.SequentialMinter` carrying the defining + builder + signing profile. + supersession_builder: a :class:`~pubmate.supersede.SupersessionBuilder` + configured with the *same* signing profile/wrapper, used for drifted + terms. + existing: id-map from previous runs (fingerprints included where known). + dry_run: sign only (offline), do not publish. + + Returns an :class:`IncrementalResult` whose ``id_map`` is the complete, + updated map (existing entries carried forward, plus new/superseded/backfilled). + """ + result = IncrementalResult(id_map=IdMap(list(existing or []))) + placeholder = minter.builder.thing_uri + default_suggester = minter.default_suggester_orcid + + for term in terms: + current_fp = fingerprint_term(term, minter.builder, default_suggester=default_suggester) + + if term.term_id not in result.id_map: + minted = minter.mint(term, dry_run=dry_run) + result.id_map.add( + IdMapEntry(term.term_id, minted.thing_uri, minted.np_uri, current_fp), + overwrite=True, + ) + result.minted.terms.append(minted) + continue + + entry = result.id_map[term.term_id] + + if entry.fingerprint == "": + logger.warning( + "Backfilling baseline fingerprint for legacy term (no prior fingerprint " + "to compare against; adopting current build as baseline): %s", + term.term_id, + ) + result.id_map.add( + IdMapEntry(term.term_id, entry.thing_uri, entry.np_uri, current_fp), + overwrite=True, + ) + result.skipped.append(term.term_id) + continue + + if entry.fingerprint == current_fp: + logger.info("Skipping unchanged term: %s", term.term_id) + result.skipped.append(term.term_id) + continue + + # Drift: supersede against the existing fixed thing URI. + fixed = rdflib.URIRef(entry.thing_uri) + full = _rekey_to_fixed(term.assertion, placeholder=placeholder, fixed=fixed) + sup_np = supersession_builder.build( + full, + supersedes_np_uri=entry.np_uri, + label=term.label or preferred_label(full, fixed), + suggester_orcid=term.suggester_orcid or default_suggester, + derived_from=term.derived_from, + ) + sup_uri = sign_and_publish(sup_np, dry_run=dry_run) + logger.info("Superseded drifted term %s (%s) -> %s", term.term_id, entry.np_uri, sup_uri) + result.id_map.add( + IdMapEntry(term.term_id, entry.thing_uri, sup_uri, current_fp), + overwrite=True, + ) + result.superseded.append( + MintedSupersession( + term_id=term.term_id, + supersedes_np_uri=entry.np_uri, + np_uri=sup_uri, + nanopub=sup_np, + ) + ) + + return result diff --git a/tests/test_cli_mint_publish.py b/tests/test_cli_mint_publish.py index de2425e..43a3d0c 100644 --- a/tests/test_cli_mint_publish.py +++ b/tests/test_cli_mint_publish.py @@ -131,5 +131,64 @@ def test_mint_publish_skips_already_minted(tmp_path) -> None: ) assert result.exit_code == 0, result.output - # Nothing newly minted, so no .trig written. + # Legacy 3-column entry has no fingerprint: backfilled, not re-issued. assert list(out.glob("*.trig")) == [] + parsed = IdMap.from_tsv(idmap.read_text(encoding="utf-8")) + assert parsed[OLD_ID].fingerprint != "" + + +def test_mint_publish_writes_fingerprint_and_reruns_idempotently(tmp_path) -> None: + assertions = tmp_path / "assertions" + assertions.mkdir() + _assertion_graph(assertions / "caffeine.ttl") + out = tmp_path / "published" + idmap = tmp_path / "id-map.tsv" + args = ["-a", str(assertions), "--output-dir", str(out), "--id-map-file", str(idmap), "--dry-run"] + + first = CliRunner().invoke(cli, args) + assert first.exit_code == 0, first.output + fp1 = IdMap.from_tsv(idmap.read_text(encoding="utf-8"))[OLD_ID].fingerprint + assert fp1 != "" + + # Second run: unchanged term is skipped (no new trig), fingerprint stable. + for trig in out.glob("*.trig"): + trig.unlink() + second = CliRunner().invoke(cli, args) + assert second.exit_code == 0, second.output + assert list(out.glob("*.trig")) == [] + assert IdMap.from_tsv(idmap.read_text(encoding="utf-8"))[OLD_ID].fingerprint == fp1 + + +def test_mint_publish_supersedes_a_drifted_term(tmp_path) -> None: + NPX = rdflib.Namespace("http://purl.org/nanopub/x/") + assertions = tmp_path / "assertions" + assertions.mkdir() + _assertion_graph(assertions / "caffeine.ttl") + out = tmp_path / "published" + idmap = tmp_path / "id-map.tsv" + args = ["-a", str(assertions), "--output-dir", str(out), "--id-map-file", str(idmap), "--dry-run"] + + assert CliRunner().invoke(cli, args).exit_code == 0 + before = IdMap.from_tsv(idmap.read_text(encoding="utf-8"))[OLD_ID] + for trig in out.glob("*.trig"): + trig.unlink() + + # Edit the term's label and re-run: the drift must be superseded. + g = _assertion_graph() + s = rdflib.URIRef(OLD_ID) + g.remove((s, RDFS.label, rdflib.Literal("Caffeine"))) + g.add((s, RDFS.label, rdflib.Literal("Caffeine (edited)"))) + g.serialize(destination=assertions / "caffeine.ttl", format="turtle") + + assert CliRunner().invoke(cli, args).exit_code == 0 + trigs = list(out.glob("*.trig")) + assert len(trigs) == 1 # one superseding nanopub written + np = rdflib.Dataset() + np.parse(trigs[0], format="trig") + supersedes = {str(o) for _s, _p, o, _g in np.quads((None, NPX.supersedes, None, None))} + assert before.np_uri in supersedes + + after = IdMap.from_tsv(idmap.read_text(encoding="utf-8"))[OLD_ID] + assert after.thing_uri == before.thing_uri + assert after.np_uri != before.np_uri + assert after.fingerprint != before.fingerprint diff --git a/tests/test_fingerprint.py b/tests/test_fingerprint.py new file mode 100644 index 0000000..eb4ecc8 --- /dev/null +++ b/tests/test_fingerprint.py @@ -0,0 +1,143 @@ +import rdflib + +from pubmate.defining import DefiningNanopubBuilder +from pubmate.fingerprint import ( + canonical_assertion, + fingerprint_term, + identity_fields, +) +from pubmate.minting import TermInput, term_input_from_assertion + +NAMESPACE = "https://w3id.org/peh/biochementities/" +TEMPLATE = "https://w3id.org/np/RAtemplate" +NANOPUB_TYPE = "https://w3id.org/peh/terms/BioChemEntity" +SUGGESTER = "https://orcid.org/0000-0002-1825-0097" + +# A term assertion with a blank node (a context-alias) so canonicalization has +# something non-trivial to normalize. +TERM_TTL = """ +@prefix rdfs: . +@prefix schema1: . +@prefix pehterms: . + a ; + rdfs:label "Polychlorinated biphenyl 187" ; + schema1:alternateName "PCB 187" ; + pehterms:hasContextAlias [ a pehterms:ContextAlias ; + schema1:alternateName "pcb187" ; + schema1:identifier "short_name" ] . +""" + + +def _builder(**kwargs) -> DefiningNanopubBuilder: + kwargs.setdefault("nanopub_types", [NANOPUB_TYPE]) + kwargs.setdefault("template", TEMPLATE) + return DefiningNanopubBuilder(NAMESPACE, **kwargs) + + +def _term(ttl: str = TERM_TTL, *, part_of=None, default_suggester=None) -> TermInput: + graph = rdflib.Graph() + graph.parse(data=ttl, format="turtle") + builder = _builder() + return term_input_from_assertion( + graph, + namespace=NAMESPACE, + thing_uri=builder.thing_uri, + part_of=part_of, + default_suggester=default_suggester, + ) + + +def test_fingerprint_is_deterministic(): + builder = _builder() + term = _term() + assert fingerprint_term(term, builder) == fingerprint_term(term, builder) + + +def test_digest_is_hex_sha256(): + digest = fingerprint_term(_term(), _builder()) + assert len(digest) == 64 + int(digest, 16) # raises if not hex + + +def test_scheme_participates_in_digest(monkeypatch): + # The scheme tag is inside the hashed payload: changing it moves the digest, + # so digests are never silently compared across incompatible schemes. + fields = identity_fields(_term(), _builder()) + before = fields.digest() + monkeypatch.setattr("pubmate.fingerprint.FP_SCHEME", "pubmate-fp-test") + assert fields.digest() != before + + +def test_cosmetic_reserialization_does_not_change_fingerprint(): + # Round-trip through N-Triples (relabels blank nodes, reorders triples): a + # no-op in the identity domain. + builder = _builder() + base = _term() + churned = rdflib.Graph() + churned.parse(data=base.assertion.serialize(format="nt"), format="nt") + reserialized = TermInput( + term_id=base.term_id, + assertion=churned, + suggester_orcid=base.suggester_orcid, + label=base.label, + ) + assert canonical_assertion(base.assertion) == canonical_assertion(churned) + assert fingerprint_term(base, builder) == fingerprint_term(reserialized, builder) + + +def test_assertion_change_moves_fingerprint(): + builder = _builder() + edited = TERM_TTL.replace("Polychlorinated biphenyl 187", "Polychlorinated biphenyl 187 (edited)") + assert fingerprint_term(_term(), builder) != fingerprint_term(_term(edited), builder) + + +def test_part_of_change_moves_fingerprint(): + builder = _builder() + a = fingerprint_term(_term(part_of="https://w3id.org/spaces/biochementity/r/vocabulary"), builder) + b = fingerprint_term(_term(part_of="https://w3id.org/spaces/biochementity/r/vocabulary-v2"), builder) + assert a != b + + +def test_template_change_moves_fingerprint(): + term = _term() + a = fingerprint_term(term, _builder(template=TEMPLATE)) + b = fingerprint_term(term, _builder(template="https://w3id.org/np/RAother")) + assert a != b + + +def test_nanopub_type_change_moves_fingerprint(): + term = _term() + a = fingerprint_term(term, _builder(nanopub_types=[NANOPUB_TYPE])) + b = fingerprint_term(term, _builder(nanopub_types=[NANOPUB_TYPE, "https://w3id.org/peh/terms/Extra"])) + assert a != b + + +def test_license_change_moves_fingerprint(): + term = _term() + a = fingerprint_term(term, _builder(license="https://creativecommons.org/licenses/by/4.0/")) + b = fingerprint_term(term, _builder(license="https://creativecommons.org/publicdomain/zero/1.0/")) + assert a != b + + +def test_suggester_change_moves_fingerprint(): + builder = _builder() + a = fingerprint_term(_term(default_suggester=SUGGESTER), builder) + b = fingerprint_term(_term(default_suggester="https://orcid.org/0000-0001-0000-0000"), builder) + assert a != b + + +def test_default_suggester_fallback_matches_resolved_term_suggester(): + # A term with no suggester + a batch default must fingerprint the same as a + # term that already carries that suggester. + builder = _builder() + via_default = fingerprint_term(_term(), builder, default_suggester=SUGGESTER) + via_term = fingerprint_term(_term(default_suggester=SUGGESTER), builder) + assert via_default == via_term + + +def test_namespace_change_moves_fingerprint(): + # The namespace rides in via the placeholder subject / introduces URI. + term = _term() + a = fingerprint_term(term, DefiningNanopubBuilder(NAMESPACE)) + b = fingerprint_term(term, DefiningNanopubBuilder("https://example.org/other/")) + assert a != b diff --git a/tests/test_idmap.py b/tests/test_idmap.py index 3752f0c..6e4b468 100644 --- a/tests/test_idmap.py +++ b/tests/test_idmap.py @@ -41,19 +41,33 @@ def test_merge_preserves_existing_and_adds_new(): def test_tsv_roundtrip_and_header(): id_map = IdMap([_entry("b"), _entry("a")]) tsv = id_map.to_tsv() - assert tsv.splitlines()[0] == "old_id\tthing_uri\tnp_uri" + assert tsv.splitlines()[0] == "old_id\tthing_uri\tnp_uri\tfingerprint" # entries are sorted by old_id assert tsv.splitlines()[1].startswith("a\t") assert IdMap.from_tsv(tsv).thing_uri_map == id_map.thing_uri_map +def test_tsv_roundtrips_fingerprint(): + id_map = IdMap([IdMapEntry("a", "https://example.org/terms/RAa", "https://w3id.org/np/RAa", "deadbeef")]) + restored = IdMap.from_tsv(id_map.to_tsv()) + assert restored["a"].fingerprint == "deadbeef" + assert restored.fingerprint_map == {"a": "deadbeef"} + + +def test_from_tsv_reads_legacy_three_column_rows(): + legacy = "old_id\tthing_uri\tnp_uri\nalpha\thttps://example.org/terms/RAa\thttps://w3id.org/np/RAa\n" + id_map = IdMap.from_tsv(legacy) + assert id_map["alpha"].np_uri == "https://w3id.org/np/RAa" + assert id_map["alpha"].fingerprint == "" + + def test_json_roundtrip(): id_map = IdMap([_entry("a"), _entry("b", "https://example.org/terms/RAb", "https://w3id.org/np/RAb")]) assert {e.old_id for e in IdMap.from_json(id_map.to_json())} == {"a", "b"} def test_from_tsv_rejects_malformed_rows(): - with pytest.raises(ValueError, match="expected 3 tab-separated fields"): + with pytest.raises(ValueError, match="expected 3 or 4 tab-separated fields"): IdMap.from_tsv("old_id\tthing_uri\tnp_uri\nalpha\tonly-two-fields\n") diff --git a/tests/test_incremental.py b/tests/test_incremental.py new file mode 100644 index 0000000..db16828 --- /dev/null +++ b/tests/test_incremental.py @@ -0,0 +1,101 @@ +import rdflib +from rdflib import Literal +from rdflib.namespace import RDF, RDFS + +from nanopub.namespaces import NPX +from pubmate.defining import DefiningNanopubBuilder +from pubmate.idmap import IdMap, IdMapEntry +from pubmate.incremental import publish_incremental +from pubmate.minting import SequentialMinter, TermInput +from pubmate.supersede import SupersessionBuilder + +NAMESPACE = "https://example.org/terms/" + + +def _minter() -> SequentialMinter: + return SequentialMinter(DefiningNanopubBuilder(NAMESPACE)) + + +def _sup() -> SupersessionBuilder: + return SupersessionBuilder() + + +def _term(term_id: str, label: str) -> TermInput: + builder = DefiningNanopubBuilder(NAMESPACE) + assertion = builder.make_assertion([(RDF.type, RDFS.Class), (RDFS.label, Literal(label))]) + return TermInput(term_id=term_id, assertion=assertion, label=label) + + +def _run(terms, existing=None): + return publish_incremental( + terms, minter=_minter(), supersession_builder=_sup(), + existing=existing or IdMap(), dry_run=True, + ) + + +def test_new_term_is_minted_and_records_fingerprint(): + result = _run([_term("alpha", "Alpha")]) + assert [m.term_id for m in result.minted.terms] == ["alpha"] + assert result.superseded == [] + entry = result.id_map["alpha"] + assert entry.thing_uri.startswith(f"{NAMESPACE}RA") + assert entry.np_uri.startswith("https://w3id.org/np/RA") + assert entry.fingerprint != "" + + +def test_unchanged_term_is_skipped(): + first = _run([_term("alpha", "Alpha")]) + again = _run([_term("alpha", "Alpha")], existing=first.id_map) + assert again.minted.terms == [] + assert again.superseded == [] + assert again.skipped == ["alpha"] + # id-map entry carried forward untouched. + assert again.id_map["alpha"] == first.id_map["alpha"] + + +def test_drifted_term_is_superseded_keeping_thing_uri(): + first = _run([_term("alpha", "Alpha")]) + before = first.id_map["alpha"] + + drifted = _run([_term("alpha", "Alpha (edited)")], existing=first.id_map) + + assert drifted.minted.terms == [] + assert [s.term_id for s in drifted.superseded] == ["alpha"] + sup = drifted.superseded[0] + # The new nanopub supersedes the recorded one... + assert sup.supersedes_np_uri == before.np_uri + assert (None, NPX.supersedes, rdflib.URIRef(before.np_uri)) in sup.nanopub.pubinfo + # ...and is written against the term's fixed, unchanged thing URI. + assert rdflib.URIRef(before.thing_uri) in set(sup.nanopub.assertion.subjects()) + + after = drifted.id_map["alpha"] + assert after.thing_uri == before.thing_uri # identity preserved + assert after.np_uri == sup.np_uri != before.np_uri # version advanced + assert after.fingerprint != before.fingerprint # fingerprint updated + + +def test_legacy_entry_without_fingerprint_is_backfilled_not_superseded(): + legacy = IdMap([IdMapEntry("alpha", f"{NAMESPACE}RAseed", "https://w3id.org/np/RAseed", "")]) + result = _run([_term("alpha", "Alpha")], existing=legacy) + assert result.minted.terms == [] + assert result.superseded == [] + assert result.skipped == ["alpha"] + entry = result.id_map["alpha"] + # URIs untouched (no re-issue), fingerprint adopted as baseline. + assert entry.thing_uri == f"{NAMESPACE}RAseed" + assert entry.np_uri == "https://w3id.org/np/RAseed" + assert entry.fingerprint != "" + + +def test_mixed_batch_mints_skips_and_supersedes(): + first = _run([_term("keep", "Keep"), _term("edit", "Edit")]) + batch = [ + _term("keep", "Keep"), # unchanged + _term("edit", "Edit (edited)"), # drifted + _term("new", "New"), # new + ] + result = _run(batch, existing=first.id_map) + assert [m.term_id for m in result.minted.terms] == ["new"] + assert [s.term_id for s in result.superseded] == ["edit"] + assert result.skipped == ["keep"] + assert set(result.id_map.fingerprint_map) == {"keep", "edit", "new"}