diff --git a/README.md b/README.md index 06cb513..e9dc9dc 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,9 @@ positive) regardless of how the source expressed direction. `fitid`, `account_id`, and currency. - De-duplicates transactions with deterministic heuristics: - Bank-assigned `FITID` (from OFX) is the authoritative key when present. - - Otherwise a content hash over (date, signed amount, currency, canonical - description), while preserving legitimately-repeated same-day charges. + - Otherwise a content hash over (account ID, date, signed amount, currency, + canonical description), while preserving legitimately-repeated same-day + charges. - Merges multiple statements (e.g. overlapping monthly exports) into one de-duplicated list. - Ships a CLI and a small Python API. diff --git a/statement_normalizer/dedup.py b/statement_normalizer/dedup.py index 654fcbf..90229ec 100644 --- a/statement_normalizer/dedup.py +++ b/statement_normalizer/dedup.py @@ -10,11 +10,12 @@ * If two transactions share a non-empty ``fitid`` (and account), they are the same transaction. FITID is bank-assigned and authoritative. -* Otherwise fall back to a content hash over (date, signed amount, currency, - canonical description). To avoid collapsing genuinely-repeated charges (e.g. - two identical $4.75 coffees on the same day), identical content rows are only - treated as duplicates beyond the count seen on a *single* source statement — - i.e. we keep the max multiplicity observed within any one input, not the sum. +* Otherwise fall back to a content hash over (account, date, signed amount, + currency, canonical description). To avoid collapsing genuinely-repeated + charges (e.g. two identical $4.75 coffees on the same day), identical content + rows are only treated as duplicates beyond the count seen on a *single* source + statement — i.e. we keep the max multiplicity observed within any one input, + not the sum. The first occurrence is kept; order is otherwise preserved (stable). """ diff --git a/statement_normalizer/schema.py b/statement_normalizer/schema.py index 7808f9f..863387b 100644 --- a/statement_normalizer/schema.py +++ b/statement_normalizer/schema.py @@ -120,18 +120,19 @@ def canonical_description(self) -> str: def content_hash(self) -> str: """Stable hash over the identity-bearing fields. - Two rows with the same date, signed amount, currency and canonical - description are considered the same transaction when no FITID is - available. Used by the dedup heuristics. + Two rows from the same account with the same date, signed amount, + currency and canonical description are considered the same transaction + when no FITID is available. Used by the dedup heuristics. """ - key = "|".join( - [ - self.date.isoformat(), - f"{self.amount:.2f}", - self.currency, - self.canonical_description, - ] - ) + fields = [ + self.date.isoformat(), + f"{self.amount:.2f}", + self.currency, + self.canonical_description, + ] + if self.account_id: + fields.insert(0, self.account_id) + key = "|".join(fields) return hashlib.sha256(key.encode("utf-8")).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/tests/test_dedup.py b/tests/test_dedup.py index d1dc286..586ae16 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -25,12 +25,21 @@ def test_fitid_dedup(): def test_content_dedup_collapses_without_multiplicity(): - a = _txn(5, "-4.75", "COFFEE ROASTERS") - b = _txn(5, "-4.75", "Coffee Roasters") # same after canonicalization + a = _txn(5, "-4.75", "COFFEE ROASTERS", acct="CHECKING") + b = _txn(5, "-4.75", "Coffee Roasters", acct="CHECKING") out = dedup_transactions([a, b]) assert len(out) == 1 +def test_content_dedup_preserves_transactions_from_different_accounts(): + a = _txn(5, "-4.75", "COFFEE ROASTERS", acct="CHECKING") + b = _txn(5, "-4.75", "Coffee Roasters", acct="SAVINGS") + + out = dedup_transactions([a, b]) + + assert out == [a, b] + + def test_content_dedup_preserves_legit_repeats_within_source(): # Two identical coffees on the same day within ONE statement are legitimate. a = _txn(5, "-4.75", "COFFEE")