Development

CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to FunVIP are documented here. This project adheres to
 [Semantic Versioning](https://semver.org/).
 
+## [1.0.1] - 2026-09-10
+
+### Fixed
+- **Crash while cleaning up empty alignments.** A sequence left with no residues
+  after trimming was dropped from its dataset with a warning that looked its id up
+  in `dict_hash_id`, a dictionary that is never populated, so the run died with
+  `KeyError: 'HS<n>HE'` instead of continuing. The warning now reads the id from the
+  removed sequence itself.
+- **UnicodeEncodeError when decoding hashes on a non-UTF-8 Windows locale.**
+  `hasher.decode` read and wrote with the platform default encoding, so on a cp949
+  (Korean) console any non-ASCII character in a sequence name (an accented author
+  name, for instance) aborted the decode of trees, SVGs and alignments. Both ends
+  are now explicitly UTF-8.
+
 ## [1.0.0] - 2026-07-09
 
 First stable release. The pipeline was ported from ete3 to ete4, made
@@ -68,4 +82,5 @@ so rebuild the conda environment instead of `pip install --upgrade`. See
   the fitted regression line.
 - Removed confirmed-dead / broken code paths.
 
+[1.0.1]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.1
 [1.0.0]: https://github.com/Changwanseo/FunVIP/releases/tag/v1.0.0

funvip/src/dataset.py

@@ -614,14 +614,15 @@ def validate_alignments(self, path, opt):
                                 _lst = getattr(
                                     self.dict_dataset[group][gene], _attr
                                 )
-                                if any(x.hash == _hash for x in _lst):
+                                _removed = [x for x in _lst if x.hash == _hash]
+                                if _removed:
                                     setattr(
                                         self.dict_dataset[group][gene],
                                         _attr,
                                         [x for x in _lst if x.hash != _hash],
                                     )
                                     logging.warning(
-                                        f"{self.dict_hash_id[_hash]} removed from dataset {group} {gene}. Please check the alignment and see the region is correct"
+                                        f"{_removed[0].id} removed from dataset {group} {gene}. Please check the alignment and see the region is correct"
                                     )
 
         # Finally, check again if the datasets meet criteria

funvip/src/hasher.py

@@ -61,7 +61,7 @@ def encode(funinfo_list: list, newick: bool = False) -> dict:
 def decode(
     hash_dict: dict, file: str, out: str, newick: bool = True, svg: bool = False
 ) -> None:
-    with open(file, "rt") as fp:
+    with open(file, "rt", encoding="utf-8") as fp:
         content = fp.read()
 
     if newick and svg:
@@ -81,7 +81,7 @@ def decode(
     pattern = re.compile(r"HS\d+HE")
     decoded_content = pattern.sub(lambda m: lookup.get(m.group(0), m.group(0)), content)
 
-    with open(out, "w") as fw:
+    with open(out, "w", encoding="utf-8") as fw:
         fw.write(decoded_content)
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "FunVIP"
-version = "1.0.0"
+version = "1.0.1"
 description = "Fungal Validation & Identification Pipeline"
 authors = [{name = "Changwan Seo", email = "wan101010@snu.ac.kr"}]
 urls = { "Homepage" = "https://github.com/Changwanseo/FunVIP" }

tests/test_dataset.py

@@ -0,0 +1,139 @@
+"""Tests for funvip.src.dataset.FunVIP_var.validate_alignments cleanup steps."""
+import logging
+import os
+
+import pytest
+from Bio import SeqIO
+
+from funvip.src.dataset import Dataset, FunVIP_var
+from funvip.src.validate_input import Funinfo
+
+
+class _Method:
+    tcs = False
+
+
+class _Opt:
+    runname = "test"
+    terminate = False
+    verbose = 2
+    thread = 1
+    method = _Method()
+
+
+class _Path:
+    def __init__(self, out_alignment):
+        self.out_alignment = out_alignment
+
+
+def _FI(_id, _hash, datatype, group="G"):
+    FI = Funinfo()
+    FI.original_id = _id
+    FI.id = _id
+    FI.hash = _hash
+    FI.genus = "Genus"
+    FI.ori_species = "species"
+    FI.datatype = datatype
+    FI.group = group
+    FI.adjusted_group = group
+    FI.seq = {"gene1": "ATGCAT"}
+    return FI
+
+
+def _write_alignment(out_alignment, records):
+    os.makedirs(f"{out_alignment}/hash", exist_ok=True)
+    os.makedirs(f"{out_alignment}/failed", exist_ok=True)
+    with open(f"{out_alignment}/test_trimmed_G_gene1.fasta", "w") as fp:
+        for _hash, seq in records:
+            fp.write(f">{_hash}\n{seq}\n")
+
+
+def _make_V(records):
+    """Five FIs (3 db, 1 query, 1 outgroup) in one group/gene dataset."""
+    db = [_FI("DB1", "HS1HE", "db"), _FI("DB2", "HS2HE", "db"), _FI("DB3", "HS3HE", "db")]
+    qr = [_FI("QR_empty", "HS64628HE", "query")]
+    og = [_FI("OG1", "HS4HE", "db")]
+
+    V = FunVIP_var()
+    V.list_FI = db + qr + og
+    V.dict_hash_FI = {FI.hash: FI for FI in V.list_FI}
+    V.dict_dataset = {
+        "G": {"gene1": Dataset(gene="gene1", group="G", list_qr=qr, list_db=db, list_og=og)}
+    }
+    return V
+
+
+@pytest.fixture
+def aligned(tmp_path):
+    """One sequence is all gaps after trimming; the rest fully overlap."""
+    out_alignment = str(tmp_path)
+    _write_alignment(
+        out_alignment,
+        [
+            ("HS1HE", "ATGCAT"),
+            ("HS2HE", "ATGCAT"),
+            ("HS3HE", "ATGCAT"),
+            ("HS64628HE", "------"),
+            ("HS4HE", "ATGCAT"),
+        ],
+    )
+    return _make_V([]), _Path(out_alignment), _Opt()
+
+
+def test_empty_after_trim_sequence_is_removed_without_crashing(aligned, caplog):
+    # Regression: the removal branch used to look up the never-populated
+    # dict_hash_id and die with KeyError: 'HS64628HE'.
+    V, path, opt = aligned
+    with caplog.at_level(logging.WARNING):
+        V.validate_alignments(path=path, opt=opt)
+
+    dataset = V.dict_dataset["G"]["gene1"]
+    assert [FI.hash for FI in dataset.list_qr_FI] == []
+    assert [FI.hash for FI in dataset.list_db_FI] == ["HS1HE", "HS2HE", "HS3HE"]
+    assert [FI.hash for FI in dataset.list_og_FI] == ["HS4HE"]
+
+
+def test_removal_warning_reports_the_readable_id_not_the_hash(aligned, caplog):
+    V, path, opt = aligned
+    with caplog.at_level(logging.WARNING):
+        V.validate_alignments(path=path, opt=opt)
+
+    removals = [
+        rec.message for rec in caplog.records if "removed from dataset" in rec.message
+    ]
+    assert removals == ["QR_empty removed from dataset G gene1. Please check the alignment and see the region is correct"]
+
+
+def test_trimmed_alignment_is_rewritten_without_the_empty_sequence(aligned):
+    V, path, opt = aligned
+    V.validate_alignments(path=path, opt=opt)
+
+    seq_list = list(
+        SeqIO.parse(f"{path.out_alignment}/test_trimmed_G_gene1.fasta", "fasta")
+    )
+    assert [seq.id for seq in seq_list] == ["HS1HE", "HS2HE", "HS3HE", "HS4HE"]
+
+
+def test_intact_alignment_keeps_every_sequence(tmp_path, caplog):
+    out_alignment = str(tmp_path)
+    _write_alignment(
+        out_alignment,
+        [
+            ("HS1HE", "ATGCAT"),
+            ("HS2HE", "ATGCAT"),
+            ("HS3HE", "ATGCAT"),
+            ("HS64628HE", "ATGCAT"),
+            ("HS4HE", "ATGCAT"),
+        ],
+    )
+    V = _make_V([])
+    with caplog.at_level(logging.WARNING):
+        V.validate_alignments(path=_Path(out_alignment), opt=_Opt())
+
+    dataset = V.dict_dataset["G"]["gene1"]
+    assert len(dataset.list_qr_FI) == 1
+    assert len(dataset.list_db_FI) == 3
+    assert len(dataset.list_og_FI) == 1
+    assert not [
+        rec.message for rec in caplog.records if "removed from dataset" in rec.message
+    ]

tests/test_hasher.py

@@ -47,18 +47,18 @@ def _reference_decode(hash_dict, content, newick=True, svg=False):
 @pytest.mark.parametrize("newick,svg", [(True, True), (True, False), (False, True), (False, False)])
 def test_decode_matches_reference(tmp_path, newick, svg):
     infile = tmp_path / "in.txt"
-    infile.write_text(CONTENT)
+    infile.write_text(CONTENT, encoding="utf-8")
     out = tmp_path / "out.txt"
     hasher.decode(HASH_DICT, str(infile), str(out), newick=newick, svg=svg)
-    assert out.read_text() == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg)
+    assert out.read_text(encoding="utf-8") == _reference_decode(HASH_DICT, CONTENT, newick=newick, svg=svg)
 
 
 def test_decode_leaves_unknown_hash(tmp_path):
     infile = tmp_path / "in.txt"
-    infile.write_text("keep HS999HE unchanged\n")
+    infile.write_text("keep HS999HE unchanged\n", encoding="utf-8")
     out = tmp_path / "out.txt"
     hasher.decode({"HS0HE": "x"}, str(infile), str(out), newick=False)
-    assert out.read_text() == "keep HS999HE unchanged\n"
+    assert out.read_text(encoding="utf-8") == "keep HS999HE unchanged\n"
 
 
 def test_decode_df_exact_cells_only():