fix(shadow-ai): close SQLite handles and read UTF-8 explicitly

shadow-ai/README.md

@@ -55,7 +55,7 @@ JSON objects, passed as a list or read from a JSONL file (one object per line):
 
 These are enriched records: whatever produces them must supply a trustworthy `agent_id`. cMCP audit entries carry `session_id` instead of `agent_id` and name their timestamp `timestamp_utc`. `ShadowAIScanner` still does not map session identity or read `timestamp_utc`; use the separate adapter below for identity attribution. Neither class infers identity from a tool catalog. The adapter addresses the remaining identity-mapping work in [integrations#215](https://github.com/agentrust-io/integrations/issues/215).
 
-JSONL files are read with Python's default text encoding (`Path.read_text()`). Lines are split only on `\n`, `\r\n` and `\r`; blank lines are skipped but still counted, so reported line numbers are physical line numbers. A line that is not valid JSON raises `json.JSONDecodeError`, and a record that is not a JSON object raises `AttributeError`; both stop the scan, from every method.
+Catalogs and JSONL files are read as UTF-8, independently of the platform's default text encoding. Lines are split only on `\n`, `\r\n` and `\r`; blank lines are skipped but still counted, so reported line numbers are physical line numbers. A line that is not valid JSON raises `json.JSONDecodeError`, and a record that is not a JSON object raises `AttributeError`; both stop the scan, from every method.
 
 ## Install
 

shadow-ai/src/shadow_ai_discovery.py

@@ -145,7 +145,7 @@ def __init__(self, catalog_path: str | Path) -> None:
         self._load_catalog(Path(catalog_path))
 
     def _load_catalog(self, path: Path) -> None:
-        raw: Any = json.loads(path.read_text())
+        raw: Any = json.loads(path.read_text(encoding="utf-8"))
         # catalog.json: {"agents": [{"id": "...", "tools": ["tool1", ...]}]}
         # also accept flat {"agent-id": ["tool1", ...]} map
         if isinstance(raw, dict) and "agents" in raw:
@@ -201,7 +201,7 @@ def _read_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]:
         # read_text() applies universal newlines, so "\n" is the only line break
         # left. str.splitlines() would also split on characters such as U+2028
         # that JSON allows raw inside strings, breaking valid records apart.
-        for number, line in enumerate(path.read_text().split("\n"), start=1):
+        for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1):
             line = line.strip()
             if not line:
                 continue

shadow-ai/tests/cmcp_producer.py

@@ -5,6 +5,7 @@
 import sqlite3
 import sys
 import tempfile
+from contextlib import closing
 from importlib.metadata import version
 from pathlib import Path
 
@@ -31,7 +32,7 @@ def write_audit(path: Path) -> list[dict]:
                     chain.append("tool_call", call_id=session + "/2", tool_name="delete_all", policy_decision="deny")
         finally:
             store.close()
-        with sqlite3.connect(database) as connection:
+        with closing(sqlite3.connect(database)) as connection:
             payloads = [row[0] for row in connection.execute("SELECT payload FROM audit_entries ORDER BY rowid")]
     path.write_text("\n".join(payloads) + "\n", encoding="utf-8")
     return [json.loads(payload) for payload in payloads]

shadow-ai/tests/test_cmcp_interop.py

@@ -2,6 +2,7 @@
 import hashlib
 import json
 import os
+import sqlite3
 import sys
 from importlib.metadata import version
 from pathlib import Path
@@ -18,6 +19,27 @@
 from shadow_ai_discovery import IdentityUnavailableError, ShadowAIScanner
 
 
+def test_producer_closes_sqlite_connections_before_return(tmp_path, monkeypatch):
+    connections = []
+    real_connect = sqlite3.connect
+
+    def connect(*args, **kwargs):
+        connection = real_connect(*args, **kwargs)
+        connections.append(connection)
+        return connection
+
+    monkeypatch.setattr(sqlite3, "connect", connect)
+    try:
+        assert len(write_audit(tmp_path / "audit.jsonl")) == 7
+        assert connections
+        for connection in connections:
+            with pytest.raises(sqlite3.ProgrammingError, match="closed"):
+                connection.execute("SELECT 1")
+    finally:
+        for connection in connections:
+            connection.close()
+
+
 @pytest.fixture
 def registry(tmp_path):
     path = tmp_path / "agents.json"

shadow-ai/tests/test_cmcp_shadow_ai.py

@@ -177,7 +177,7 @@ def test_non_tool_calls_and_timestamp_contract_are_unchanged(registry):
 
 def test_jsonl_unicode_separator_is_not_a_physical_line_break(registry, tmp_path):
     path = tmp_path / "audit.jsonl"
-    path.write_text(json.dumps(record(note="a\u2028b"), ensure_ascii=False))
+    path.write_text(json.dumps(record(note="a\u2028b"), ensure_ascii=False), encoding="utf-8")
     assert adapter(registry).scan_audit_log_report(path).complete
 
 

shadow-ai/tests/test_shadow_ai_discovery.py

@@ -407,6 +407,28 @@ def test_u2028_inside_json_string_does_not_split_line(scanner, tmp_path, monkeyp
 
 # ── 14. JSON serialization ────────────────────────────────────────────────────
 
+def test_utf8_catalog_and_jsonl_ignore_cp1252_default(tmp_path, monkeypatch):
+    catalog = tmp_path / "agents.json"
+    log = tmp_path / "audit.jsonl"
+    agent = "caf\u00e9-agent"
+    catalog.write_bytes(json.dumps({agent: ["read"]}, ensure_ascii=False).encode("utf-8"))
+    log.write_bytes((json.dumps({
+        "agent_id": agent, "tool_name": "delete", "note": "a\u2028b",
+    }, ensure_ascii=False) + "\n").encode("utf-8"))
+    real_open = Path.open
+
+    def cp1252_open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
+        if "b" not in mode and encoding in (None, "locale"):
+            encoding = "cp1252"
+        return real_open(self, mode, buffering, encoding, errors, newline)
+
+    monkeypatch.setattr(Path, "open", cp1252_open)
+    report = ShadowAIScanner(catalog).scan_audit_log_report(log)
+    assert report.complete
+    assert [(event.agent_id, event.reason) for event in report.findings] == [
+        (agent, "undeclared_tool"),
+    ]
+
 def test_discovery_event_to_dict_exact():
     e = DiscoveryEvent("My Agent/v2 (prod)", "bad_tool", "2026-06-25T00:00:00Z", "unregistered_agent")
     assert e.to_dict() == {