From 9406fb3b5b2d1ace6d9bccea20a777e8c862194a Mon Sep 17 00:00:00 2001 From: Changwan Seo Date: Thu, 10 Sep 2026 17:13:00 +0900 Subject: [PATCH 1/3] Fix KeyError crash when dropping empty-after-trim sequences validate_alignments removes sequences that are left with no residues after trimming, and logged each removal by looking the hash up in dict_hash_id. That dictionary is initialized empty and never populated, so the lookup always raised KeyError and killed the run at the very end of alignment validation. The branch only became reachable in 6d151a8, which fixed the membership test that had silently guarded it (a hash string was compared against a list of Funinfo objects, so it was never true). Take the id from the Funinfo being removed instead of any dictionary, and add regression tests covering the removal, the warning text, the rewritten trimmed alignment, and the untouched-alignment case. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EZTJs2CAFcHw9Ye4k7p9Mx --- funvip/src/dataset.py | 5 +- tests/test_dataset.py | 139 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 tests/test_dataset.py diff --git a/funvip/src/dataset.py b/funvip/src/dataset.py index fc55aca..2e3bbea 100644 --- a/funvip/src/dataset.py +++ b/funvip/src/dataset.py @@ -614,14 +614,15 @@ def validate_alignments(self, path, opt): _lst = getattr( self.dict_dataset[group][gene], _attr ) - if any(x.hash == _hash for x in _lst): + _removed = [x for x in _lst if x.hash == _hash] + if _removed: setattr( self.dict_dataset[group][gene], _attr, [x for x in _lst if x.hash != _hash], ) logging.warning( - f"{self.dict_hash_id[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" + f"{_removed[0].id} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" ) # Finally, check again if the datasets meet criteria diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 0000000..fbad2aa --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,139 @@ +"""Tests for funvip.src.dataset.FunVIP_var.validate_alignments cleanup steps.""" +import logging +import os + +import pytest +from Bio import SeqIO + +from funvip.src.dataset import Dataset, FunVIP_var +from funvip.src.validate_input import Funinfo + + +class _Method: + tcs = False + + +class _Opt: + runname = "test" + terminate = False + verbose = 2 + thread = 1 + method = _Method() + + +class _Path: + def __init__(self, out_alignment): + self.out_alignment = out_alignment + + +def _FI(_id, _hash, datatype, group="G"): + FI = Funinfo() + FI.original_id = _id + FI.id = _id + FI.hash = _hash + FI.genus = "Genus" + FI.ori_species = "species" + FI.datatype = datatype + FI.group = group + FI.adjusted_group = group + FI.seq = {"gene1": "ATGCAT"} + return FI + + +def _write_alignment(out_alignment, records): + os.makedirs(f"{out_alignment}/hash", exist_ok=True) + os.makedirs(f"{out_alignment}/failed", exist_ok=True) + with open(f"{out_alignment}/test_trimmed_G_gene1.fasta", "w") as fp: + for _hash, seq in records: + fp.write(f">{_hash}\n{seq}\n") + + +def _make_V(records): + """Five FIs (3 db, 1 query, 1 outgroup) in one group/gene dataset.""" + db = [_FI("DB1", "HS1HE", "db"), _FI("DB2", "HS2HE", "db"), _FI("DB3", "HS3HE", "db")] + qr = [_FI("QR_empty", "HS64628HE", "query")] + og = [_FI("OG1", "HS4HE", "db")] + + V = FunVIP_var() + V.list_FI = db + qr + og + V.dict_hash_FI = {FI.hash: FI for FI in V.list_FI} + V.dict_dataset = { + "G": {"gene1": Dataset(gene="gene1", group="G", list_qr=qr, list_db=db, list_og=og)} + } + return V + + +@pytest.fixture +def aligned(tmp_path): + """One sequence is all gaps after trimming; the rest fully overlap.""" + out_alignment = str(tmp_path) + _write_alignment( + out_alignment, + [ + ("HS1HE", "ATGCAT"), + ("HS2HE", "ATGCAT"), + ("HS3HE", "ATGCAT"), + ("HS64628HE", "------"), + ("HS4HE", "ATGCAT"), + ], + ) + return _make_V([]), _Path(out_alignment), _Opt() + + +def test_empty_after_trim_sequence_is_removed_without_crashing(aligned, caplog): + # Regression: the removal branch used to look up the never-populated + # dict_hash_id and die with KeyError: 'HS64628HE'. + V, path, opt = aligned + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=path, opt=opt) + + dataset = V.dict_dataset["G"]["gene1"] + assert [FI.hash for FI in dataset.list_qr_FI] == [] + assert [FI.hash for FI in dataset.list_db_FI] == ["HS1HE", "HS2HE", "HS3HE"] + assert [FI.hash for FI in dataset.list_og_FI] == ["HS4HE"] + + +def test_removal_warning_reports_the_readable_id_not_the_hash(aligned, caplog): + V, path, opt = aligned + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=path, opt=opt) + + removals = [ + rec.message for rec in caplog.records if "removed from dataset" in rec.message + ] + assert removals == ["QR_empty removed from dataset G gene1. Please check the alignment and see the region is correct"] + + +def test_trimmed_alignment_is_rewritten_without_the_empty_sequence(aligned): + V, path, opt = aligned + V.validate_alignments(path=path, opt=opt) + + seq_list = list( + SeqIO.parse(f"{path.out_alignment}/test_trimmed_G_gene1.fasta", "fasta") + ) + assert [seq.id for seq in seq_list] == ["HS1HE", "HS2HE", "HS3HE", "HS4HE"] + + +def test_intact_alignment_keeps_every_sequence(tmp_path, caplog): + out_alignment = str(tmp_path) + _write_alignment( + out_alignment, + [ + ("HS1HE", "ATGCAT"), + ("HS2HE", "ATGCAT"), + ("HS3HE", "ATGCAT"), + ("HS64628HE", "ATGCAT"), + ("HS4HE", "ATGCAT"), + ], + ) + V = _make_V([]) + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=_Path(out_alignment), opt=_Opt()) + + dataset = V.dict_dataset["G"]["gene1"] + assert len(dataset.list_qr_FI) == 1 + assert len(dataset.list_db_FI) == 3 + assert len(dataset.list_og_FI) == 1 + assert not [ + rec.message for rec in caplog.records if "removed from dataset" in rec.message + ] From f80294bebf273333362ada5480a7df2ecb614def Mon Sep 17 00:00:00 2001 From: Changwan Seo Date: Thu, 10 Sep 2026 17:13:09 +0900 Subject: [PATCH 2/3] Decode hashes as UTF-8 regardless of the platform locale hasher.decode opened both the hashed input and the decoded output with the platform default encoding. On a cp949 (Korean) Windows console, any non-ASCII character in a sequence name, such as an accented author name, raised UnicodeEncodeError while writing, so decoding trees, SVGs and alignments crashed the run. CI never saw it because Linux and macOS default to UTF-8. Pin both ends to UTF-8, and read and write the test fixtures the same way so the round-trip test exercises the real contract on any locale. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EZTJs2CAFcHw9Ye4k7p9Mx --- funvip/src/hasher.py | 4 ++-- tests/test_hasher.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/funvip/src/hasher.py b/funvip/src/hasher.py index 50d58c7..d104cb4 100644 --- a/funvip/src/hasher.py +++ b/funvip/src/hasher.py @@ -61,7 +61,7 @@ def encode(funinfo_list: list, newick: bool = False) -> dict: def decode( hash_dict: dict, file: str, out: str, newick: bool = True, svg: bool = False ) -> None: - with open(file, "rt") as fp: + with open(file, "rt", encoding="utf-8") as fp: content = fp.read() if newick and svg: @@ -81,7 +81,7 @@ def decode( pattern = re.compile(r"HS\d+HE") decoded_content = pattern.sub(lambda m: lookup.get(m.group(0), m.group(0)), content) - with open(out, "w") as fw: + with open(out, "w", encoding="utf-8") as fw: fw.write(decoded_content) diff --git a/tests/test_hasher.py b/tests/test_hasher.py index 128899a..a805fa7 100644 --- a/tests/test_hasher.py +++ b/tests/test_hasher.py @@ -47,18 +47,18 @@ def _reference_decode(hash_dict, content, newick=True, svg=False): @pytest.mark.parametrize("newick,svg", [(True, True), (True, False), (False, True), (False, False)]) def test_decode_matches_reference(tmp_path, newick, svg): infile = tmp_path / "in.txt" - infile.write_text(CONTENT) + infile.write_text(CONTENT, encoding="utf-8") out = tmp_path / "out.txt" hasher.decode(HASH_DICT, str(infile), str(out), newick=newick, svg=svg) - assert out.read_text() == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg) + assert out.read_text(encoding="utf-8") == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg) def test_decode_leaves_unknown_hash(tmp_path): infile = tmp_path / "in.txt" - infile.write_text("keep HS999HE unchanged\n") + infile.write_text("keep HS999HE unchanged\n", encoding="utf-8") out = tmp_path / "out.txt" hasher.decode({"HS0HE": "x"}, str(infile), str(out), newick=False) - assert out.read_text() == "keep HS999HE unchanged\n" + assert out.read_text(encoding="utf-8") == "keep HS999HE unchanged\n" def test_decode_df_exact_cells_only(): From be856ac28bdc7c688159f7db47b37ab6fb3a1c16 Mon Sep 17 00:00:00 2001 From: Changwan Seo Date: Thu, 10 Sep 2026 17:13:09 +0900 Subject: [PATCH 3/3] Prepare 1.0.1: version bump and CHANGELOG Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01EZTJs2CAFcHw9Ye4k7p9Mx --- CHANGELOG.md | 15 +++++++++++++++ pyproject.toml | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84afb55..8835df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to FunVIP are documented here. This project adheres to [Semantic Versioning](https://semver.org/). +## [1.0.1] - 2026-09-10 + +### Fixed +- **Crash while cleaning up empty alignments.** A sequence left with no residues + after trimming was dropped from its dataset with a warning that looked its id up + in `dict_hash_id`, a dictionary that is never populated, so the run died with + `KeyError: 'HSHE'` instead of continuing. The warning now reads the id from the + removed sequence itself. +- **UnicodeEncodeError when decoding hashes on a non-UTF-8 Windows locale.** + `hasher.decode` read and wrote with the platform default encoding, so on a cp949 + (Korean) console any non-ASCII character in a sequence name (an accented author + name, for instance) aborted the decode of trees, SVGs and alignments. Both ends + are now explicitly UTF-8. + ## [1.0.0] - 2026-07-09 First stable release. The pipeline was ported from ete3 to ete4, made @@ -68,4 +82,5 @@ so rebuild the conda environment instead of `pip install --upgrade`. See the fitted regression line. - Removed confirmed-dead / broken code paths. +[1.0.1]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.1 [1.0.0]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.0 diff --git a/pyproject.toml b/pyproject.toml index fade60b..b5a0ba2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "FunVIP" -version = "1.0.0" +version = "1.0.1" description = "Fungal Validation & Identification Pipeline" authors = [{name = "Changwan Seo", email = "wan101010@snu.ac.kr"}] urls = { "Homepage" = "https://github.com/Changwanseo/FunVIP" }