diff --git a/shadow-ai/README.md b/shadow-ai/README.md index 60c28d3..0c9f53d 100644 --- a/shadow-ai/README.md +++ b/shadow-ai/README.md @@ -55,7 +55,7 @@ JSON objects, passed as a list or read from a JSONL file (one object per line): These are enriched records: whatever produces them must supply a trustworthy `agent_id`. cMCP audit entries carry `session_id` instead of `agent_id` and name their timestamp `timestamp_utc`. `ShadowAIScanner` still does not map session identity or read `timestamp_utc`; use the separate adapter below for identity attribution. Neither class infers identity from a tool catalog. The adapter addresses the remaining identity-mapping work in [integrations#215](https://github.com/agentrust-io/integrations/issues/215). -JSONL files are read with Python's default text encoding (`Path.read_text()`). Lines are split only on `\n`, `\r\n` and `\r`; blank lines are skipped but still counted, so reported line numbers are physical line numbers. A line that is not valid JSON raises `json.JSONDecodeError`, and a record that is not a JSON object raises `AttributeError`; both stop the scan, from every method. +Catalogs and JSONL files are read as UTF-8, independently of the platform's default text encoding. Lines are split only on `\n`, `\r\n` and `\r`; blank lines are skipped but still counted, so reported line numbers are physical line numbers. A line that is not valid JSON raises `json.JSONDecodeError`, and a record that is not a JSON object raises `AttributeError`; both stop the scan, from every method. ## Install diff --git a/shadow-ai/src/shadow_ai_discovery.py b/shadow-ai/src/shadow_ai_discovery.py index eaac14e..8668a1a 100644 --- a/shadow-ai/src/shadow_ai_discovery.py +++ b/shadow-ai/src/shadow_ai_discovery.py @@ -145,7 +145,7 @@ def __init__(self, catalog_path: str | Path) -> None: self._load_catalog(Path(catalog_path)) def _load_catalog(self, path: Path) -> None: - raw: Any = json.loads(path.read_text()) + raw: Any = json.loads(path.read_text(encoding="utf-8")) # catalog.json: {"agents": [{"id": "...", "tools": ["tool1", ...]}]} # also accept flat {"agent-id": ["tool1", ...]} map if isinstance(raw, dict) and "agents" in raw: @@ -201,7 +201,7 @@ def _read_jsonl(path: Path) -> Iterator[tuple[int, dict[str, Any]]]: # read_text() applies universal newlines, so "\n" is the only line break # left. str.splitlines() would also split on characters such as U+2028 # that JSON allows raw inside strings, breaking valid records apart. - for number, line in enumerate(path.read_text().split("\n"), start=1): + for number, line in enumerate(path.read_text(encoding="utf-8").split("\n"), start=1): line = line.strip() if not line: continue diff --git a/shadow-ai/tests/cmcp_producer.py b/shadow-ai/tests/cmcp_producer.py index 8088560..69b7187 100644 --- a/shadow-ai/tests/cmcp_producer.py +++ b/shadow-ai/tests/cmcp_producer.py @@ -5,6 +5,7 @@ import sqlite3 import sys import tempfile +from contextlib import closing from importlib.metadata import version from pathlib import Path @@ -31,7 +32,7 @@ def write_audit(path: Path) -> list[dict]: chain.append("tool_call", call_id=session + "/2", tool_name="delete_all", policy_decision="deny") finally: store.close() - with sqlite3.connect(database) as connection: + with closing(sqlite3.connect(database)) as connection: payloads = [row[0] for row in connection.execute("SELECT payload FROM audit_entries ORDER BY rowid")] path.write_text("\n".join(payloads) + "\n", encoding="utf-8") return [json.loads(payload) for payload in payloads] diff --git a/shadow-ai/tests/test_cmcp_interop.py b/shadow-ai/tests/test_cmcp_interop.py index e3ba41a..2152c88 100644 --- a/shadow-ai/tests/test_cmcp_interop.py +++ b/shadow-ai/tests/test_cmcp_interop.py @@ -2,6 +2,7 @@ import hashlib import json import os +import sqlite3 import sys from importlib.metadata import version from pathlib import Path @@ -18,6 +19,27 @@ from shadow_ai_discovery import IdentityUnavailableError, ShadowAIScanner +def test_producer_closes_sqlite_connections_before_return(tmp_path, monkeypatch): + connections = [] + real_connect = sqlite3.connect + + def connect(*args, **kwargs): + connection = real_connect(*args, **kwargs) + connections.append(connection) + return connection + + monkeypatch.setattr(sqlite3, "connect", connect) + try: + assert len(write_audit(tmp_path / "audit.jsonl")) == 7 + assert connections + for connection in connections: + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + finally: + for connection in connections: + connection.close() + + @pytest.fixture def registry(tmp_path): path = tmp_path / "agents.json" diff --git a/shadow-ai/tests/test_cmcp_shadow_ai.py b/shadow-ai/tests/test_cmcp_shadow_ai.py index f3ad9f9..0596a93 100644 --- a/shadow-ai/tests/test_cmcp_shadow_ai.py +++ b/shadow-ai/tests/test_cmcp_shadow_ai.py @@ -177,7 +177,7 @@ def test_non_tool_calls_and_timestamp_contract_are_unchanged(registry): def test_jsonl_unicode_separator_is_not_a_physical_line_break(registry, tmp_path): path = tmp_path / "audit.jsonl" - path.write_text(json.dumps(record(note="a\u2028b"), ensure_ascii=False)) + path.write_text(json.dumps(record(note="a\u2028b"), ensure_ascii=False), encoding="utf-8") assert adapter(registry).scan_audit_log_report(path).complete diff --git a/shadow-ai/tests/test_shadow_ai_discovery.py b/shadow-ai/tests/test_shadow_ai_discovery.py index 47b965d..4d46cbf 100644 --- a/shadow-ai/tests/test_shadow_ai_discovery.py +++ b/shadow-ai/tests/test_shadow_ai_discovery.py @@ -407,6 +407,28 @@ def test_u2028_inside_json_string_does_not_split_line(scanner, tmp_path, monkeyp # ── 14. JSON serialization ──────────────────────────────────────────────────── +def test_utf8_catalog_and_jsonl_ignore_cp1252_default(tmp_path, monkeypatch): + catalog = tmp_path / "agents.json" + log = tmp_path / "audit.jsonl" + agent = "caf\u00e9-agent" + catalog.write_bytes(json.dumps({agent: ["read"]}, ensure_ascii=False).encode("utf-8")) + log.write_bytes((json.dumps({ + "agent_id": agent, "tool_name": "delete", "note": "a\u2028b", + }, ensure_ascii=False) + "\n").encode("utf-8")) + real_open = Path.open + + def cp1252_open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None): + if "b" not in mode and encoding in (None, "locale"): + encoding = "cp1252" + return real_open(self, mode, buffering, encoding, errors, newline) + + monkeypatch.setattr(Path, "open", cp1252_open) + report = ShadowAIScanner(catalog).scan_audit_log_report(log) + assert report.complete + assert [(event.agent_id, event.reason) for event in report.findings] == [ + (agent, "undeclared_tool"), + ] + def test_discovery_event_to_dict_exact(): e = DiscoveryEvent("My Agent/v2 (prod)", "bad_tool", "2026-06-25T00:00:00Z", "unregistered_agent") assert e.to_dict() == {