diff --git a/CHANGELOG.md b/CHANGELOG.md index 84afb55..8835df0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to FunVIP are documented here. This project adheres to [Semantic Versioning](https://semver.org/). +## [1.0.1] - 2026-09-10 + +### Fixed +- **Crash while cleaning up empty alignments.** A sequence left with no residues + after trimming was dropped from its dataset with a warning that looked its id up + in `dict_hash_id`, a dictionary that is never populated, so the run died with + `KeyError: 'HSHE'` instead of continuing. The warning now reads the id from the + removed sequence itself. +- **UnicodeEncodeError when decoding hashes on a non-UTF-8 Windows locale.** + `hasher.decode` read and wrote with the platform default encoding, so on a cp949 + (Korean) console any non-ASCII character in a sequence name (an accented author + name, for instance) aborted the decode of trees, SVGs and alignments. Both ends + are now explicitly UTF-8. + ## [1.0.0] - 2026-07-09 First stable release. The pipeline was ported from ete3 to ete4, made @@ -68,4 +82,5 @@ so rebuild the conda environment instead of `pip install --upgrade`. See the fitted regression line. - Removed confirmed-dead / broken code paths. +[1.0.1]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.1 [1.0.0]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.0 diff --git a/funvip/src/dataset.py b/funvip/src/dataset.py index fc55aca..2e3bbea 100644 --- a/funvip/src/dataset.py +++ b/funvip/src/dataset.py @@ -614,14 +614,15 @@ def validate_alignments(self, path, opt): _lst = getattr( self.dict_dataset[group][gene], _attr ) - if any(x.hash == _hash for x in _lst): + _removed = [x for x in _lst if x.hash == _hash] + if _removed: setattr( self.dict_dataset[group][gene], _attr, [x for x in _lst if x.hash != _hash], ) logging.warning( - f"{self.dict_hash_id[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" + f"{_removed[0].id} removed from dataset {group} {gene}. Please check the alignment and see the region is correct" ) # Finally, check again if the datasets meet criteria diff --git a/funvip/src/hasher.py b/funvip/src/hasher.py index 50d58c7..d104cb4 100644 --- a/funvip/src/hasher.py +++ b/funvip/src/hasher.py @@ -61,7 +61,7 @@ def encode(funinfo_list: list, newick: bool = False) -> dict: def decode( hash_dict: dict, file: str, out: str, newick: bool = True, svg: bool = False ) -> None: - with open(file, "rt") as fp: + with open(file, "rt", encoding="utf-8") as fp: content = fp.read() if newick and svg: @@ -81,7 +81,7 @@ def decode( pattern = re.compile(r"HS\d+HE") decoded_content = pattern.sub(lambda m: lookup.get(m.group(0), m.group(0)), content) - with open(out, "w") as fw: + with open(out, "w", encoding="utf-8") as fw: fw.write(decoded_content) diff --git a/pyproject.toml b/pyproject.toml index fade60b..b5a0ba2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "FunVIP" -version = "1.0.0" +version = "1.0.1" description = "Fungal Validation & Identification Pipeline" authors = [{name = "Changwan Seo", email = "wan101010@snu.ac.kr"}] urls = { "Homepage" = "https://github.com/Changwanseo/FunVIP" } diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 0000000..fbad2aa --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,139 @@ +"""Tests for funvip.src.dataset.FunVIP_var.validate_alignments cleanup steps.""" +import logging +import os + +import pytest +from Bio import SeqIO + +from funvip.src.dataset import Dataset, FunVIP_var +from funvip.src.validate_input import Funinfo + + +class _Method: + tcs = False + + +class _Opt: + runname = "test" + terminate = False + verbose = 2 + thread = 1 + method = _Method() + + +class _Path: + def __init__(self, out_alignment): + self.out_alignment = out_alignment + + +def _FI(_id, _hash, datatype, group="G"): + FI = Funinfo() + FI.original_id = _id + FI.id = _id + FI.hash = _hash + FI.genus = "Genus" + FI.ori_species = "species" + FI.datatype = datatype + FI.group = group + FI.adjusted_group = group + FI.seq = {"gene1": "ATGCAT"} + return FI + + +def _write_alignment(out_alignment, records): + os.makedirs(f"{out_alignment}/hash", exist_ok=True) + os.makedirs(f"{out_alignment}/failed", exist_ok=True) + with open(f"{out_alignment}/test_trimmed_G_gene1.fasta", "w") as fp: + for _hash, seq in records: + fp.write(f">{_hash}\n{seq}\n") + + +def _make_V(records): + """Five FIs (3 db, 1 query, 1 outgroup) in one group/gene dataset.""" + db = [_FI("DB1", "HS1HE", "db"), _FI("DB2", "HS2HE", "db"), _FI("DB3", "HS3HE", "db")] + qr = [_FI("QR_empty", "HS64628HE", "query")] + og = [_FI("OG1", "HS4HE", "db")] + + V = FunVIP_var() + V.list_FI = db + qr + og + V.dict_hash_FI = {FI.hash: FI for FI in V.list_FI} + V.dict_dataset = { + "G": {"gene1": Dataset(gene="gene1", group="G", list_qr=qr, list_db=db, list_og=og)} + } + return V + + +@pytest.fixture +def aligned(tmp_path): + """One sequence is all gaps after trimming; the rest fully overlap.""" + out_alignment = str(tmp_path) + _write_alignment( + out_alignment, + [ + ("HS1HE", "ATGCAT"), + ("HS2HE", "ATGCAT"), + ("HS3HE", "ATGCAT"), + ("HS64628HE", "------"), + ("HS4HE", "ATGCAT"), + ], + ) + return _make_V([]), _Path(out_alignment), _Opt() + + +def test_empty_after_trim_sequence_is_removed_without_crashing(aligned, caplog): + # Regression: the removal branch used to look up the never-populated + # dict_hash_id and die with KeyError: 'HS64628HE'. + V, path, opt = aligned + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=path, opt=opt) + + dataset = V.dict_dataset["G"]["gene1"] + assert [FI.hash for FI in dataset.list_qr_FI] == [] + assert [FI.hash for FI in dataset.list_db_FI] == ["HS1HE", "HS2HE", "HS3HE"] + assert [FI.hash for FI in dataset.list_og_FI] == ["HS4HE"] + + +def test_removal_warning_reports_the_readable_id_not_the_hash(aligned, caplog): + V, path, opt = aligned + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=path, opt=opt) + + removals = [ + rec.message for rec in caplog.records if "removed from dataset" in rec.message + ] + assert removals == ["QR_empty removed from dataset G gene1. Please check the alignment and see the region is correct"] + + +def test_trimmed_alignment_is_rewritten_without_the_empty_sequence(aligned): + V, path, opt = aligned + V.validate_alignments(path=path, opt=opt) + + seq_list = list( + SeqIO.parse(f"{path.out_alignment}/test_trimmed_G_gene1.fasta", "fasta") + ) + assert [seq.id for seq in seq_list] == ["HS1HE", "HS2HE", "HS3HE", "HS4HE"] + + +def test_intact_alignment_keeps_every_sequence(tmp_path, caplog): + out_alignment = str(tmp_path) + _write_alignment( + out_alignment, + [ + ("HS1HE", "ATGCAT"), + ("HS2HE", "ATGCAT"), + ("HS3HE", "ATGCAT"), + ("HS64628HE", "ATGCAT"), + ("HS4HE", "ATGCAT"), + ], + ) + V = _make_V([]) + with caplog.at_level(logging.WARNING): + V.validate_alignments(path=_Path(out_alignment), opt=_Opt()) + + dataset = V.dict_dataset["G"]["gene1"] + assert len(dataset.list_qr_FI) == 1 + assert len(dataset.list_db_FI) == 3 + assert len(dataset.list_og_FI) == 1 + assert not [ + rec.message for rec in caplog.records if "removed from dataset" in rec.message + ] diff --git a/tests/test_hasher.py b/tests/test_hasher.py index 128899a..a805fa7 100644 --- a/tests/test_hasher.py +++ b/tests/test_hasher.py @@ -47,18 +47,18 @@ def _reference_decode(hash_dict, content, newick=True, svg=False): @pytest.mark.parametrize("newick,svg", [(True, True), (True, False), (False, True), (False, False)]) def test_decode_matches_reference(tmp_path, newick, svg): infile = tmp_path / "in.txt" - infile.write_text(CONTENT) + infile.write_text(CONTENT, encoding="utf-8") out = tmp_path / "out.txt" hasher.decode(HASH_DICT, str(infile), str(out), newick=newick, svg=svg) - assert out.read_text() == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg) + assert out.read_text(encoding="utf-8") == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg) def test_decode_leaves_unknown_hash(tmp_path): infile = tmp_path / "in.txt" - infile.write_text("keep HS999HE unchanged\n") + infile.write_text("keep HS999HE unchanged\n", encoding="utf-8") out = tmp_path / "out.txt" hasher.decode({"HS0HE": "x"}, str(infile), str(out), newick=False) - assert out.read_text() == "keep HS999HE unchanged\n" + assert out.read_text(encoding="utf-8") == "keep HS999HE unchanged\n" def test_decode_df_exact_cells_only():