From 9ff75ef55461d2a79d606d11ad8c028668c86243 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:12:27 +0800 Subject: [PATCH 01/11] test: capture official Codex transport against loopback fixtures --- .github/scripts/capture_codex_transport.py | 389 ++++++++++++++++++ .github/workflows/codex-transport-samples.yml | 55 +++ 2 files changed, 444 insertions(+) create mode 100644 .github/scripts/capture_codex_transport.py create mode 100644 .github/workflows/codex-transport-samples.yml diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py new file mode 100644 index 000000000000..65f7a34b9b92 --- /dev/null +++ b/.github/scripts/capture_codex_transport.py @@ -0,0 +1,389 @@ +"""Capture a fixed official Codex binary against loopback only; no real API calls. + +The capture includes complete ClientHello parameters and decoded HTTP/2 headers, +not just a JA3 digest. Certificates, keys and the isolated CODEX_HOME are scratch. +""" +import argparse +import base64 +import hashlib +import json +import os +import pathlib +import platform +import queue +import socket +import ssl +import struct +import subprocess +import tempfile +import threading +import time +from datetime import datetime, timedelta, timezone + +import h2.config +import h2.connection +import h2.events +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID + +VERSION = "0.162.1" +SOURCE = "092d3acd6bec3e3a14bdc7e7a2810ab628ab759d" + + +def u16(data): + return list(struct.unpack(">" + "H" * (len(data) // 2), data)) + + +def client_hello(raw): + # A ClientHello may span TLS records. Preserve handshake bytes only. + data, offset = b"", 0 + while offset + 5 <= len(raw): + size = int.from_bytes(raw[offset + 3:offset + 5], "big") + if offset + 5 + size > len(raw): + return None + if raw[offset] == 22: + data += raw[offset + 5:offset + 5 + size] + offset += 5 + size + if len(data) >= 4 and len(data) >= 4 + int.from_bytes(data[1:4], "big"): + break + if len(data) < 4 or data[0] != 1 or len(data) < 4 + int.from_bytes(data[1:4], "big"): + return None + body = data[4:4 + int.from_bytes(data[1:4], "big")] + pos = 34 + session_length = body[pos] + pos += 1 + session_length + length = int.from_bytes(body[pos:pos + 2], "big") + ciphers = u16(body[pos + 2:pos + 2 + length]) + pos += 2 + length + compression = list(body[pos + 1:pos + 1 + body[pos]]) + pos += 1 + body[pos] + end = pos + 2 + int.from_bytes(body[pos:pos + 2], "big") + pos += 2 + result = dict(legacy_version=int.from_bytes(body[:2], "big"), cipher_suites=ciphers, + compression_methods=compression, session_id_length=session_length, + extensions=[], extension_data={}, alpn=[], groups=[], key_share_groups=[], + signature_algorithms=[], versions=[], psk_modes=[]) + while pos + 4 <= end: + kind, length = struct.unpack(">HH", body[pos:pos + 4]) + value = body[pos + 4:pos + 4 + length] + pos += 4 + length + result["extensions"].append(kind) + if kind not in (0, 21, 35, 41, 51): + result["extension_data"][str(kind)] = value.hex() + if kind in (10, 13): + result["groups" if kind == 10 else "signature_algorithms"] = u16(value[2:]) + elif kind == 43: + result["versions"] = u16(value[1:]) + elif kind == 45: + result["psk_modes"] = list(value[1:]) + elif kind == 16: + p = 2 + while p < len(value): + n = value[p] + result["alpn"].append(value[p + 1:p + 1 + n].decode()) + p += 1 + n + elif kind == 51: + p = 2 + while p + 4 <= len(value): + group, size = struct.unpack(">HH", value[p:p + 4]) + result["key_share_groups"].append(group) + p += 4 + size + return result + + +def events(number, prewarm=False): + rid = f"resp_local_capture_{number}" + output = [] if prewarm else [{"type": "message", "id": f"msg_{number}", "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}]}] + values = [{"type": "response.created", "response": {"id": rid}}] + if output: + values.append({"type": "response.output_item.done", "item": output[0]}) + values.append({"type": "response.completed", "response": {"id": rid, "status": "completed", + "output": output, "usage": {"input_tokens": 1, "output_tokens": 1, + "total_tokens": 2}}}) + return values + + +def recv_exact(conn, size): + out = b"" + while len(out) < size: + part = conn.recv(size - len(out)) + if not part: + raise EOFError() + out += part + return out + + +def ws_send(conn, value, opcode=1): + raw = json.dumps(value).encode() if opcode == 1 else value + header = bytes([0x80 | opcode]) + header += bytes([len(raw)]) if len(raw) < 126 else b"\x7e" + struct.pack(">H", len(raw)) + conn.sendall(header + raw) + + +class Capture: + def __init__(self, directory): + self.records, self.errors = [], [] + self.lock = threading.Lock() + self.context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Codex loopback capture")]) + now = datetime.now(timezone.utc) + cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key()) + .serial_number(x509.random_serial_number()).not_valid_before(now - timedelta(minutes=5)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) + .sign(key, hashes.SHA256())) + self.ca = directory / "ca.pem" + self.ca.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + leaf_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + leaf = (x509.CertificateBuilder().subject_name(leaf_name).issuer_name(name).public_key(key.public_key()) + .serial_number(x509.random_serial_number()).not_valid_before(now - timedelta(minutes=5)) + .not_valid_after(now + timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) + .sign(key, hashes.SHA256())) + leaf_path = directory / "server.pem" + leaf_path.write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) + private = directory / "key.pem" + private.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + self.context.load_cert_chain(leaf_path, private) + self.context.set_alpn_protocols(["h2", "http/1.1"]) + self.listener = socket.socket() + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen() + self.port = self.listener.getsockname()[1] + threading.Thread(target=self.accept, daemon=True).start() + + def accept(self): + while True: + try: + conn, _ = self.listener.accept() + except OSError: + return + threading.Thread(target=self.handle, args=(conn,), daemon=True).start() + + def handle(self, raw): + record = None + try: + raw.settimeout(20) + deadline = time.monotonic() + 15 + hello = None + while time.monotonic() < deadline: + hello = client_hello(raw.recv(65535, socket.MSG_PEEK)) + if hello: + break + time.sleep(0.01) + if not hello: + raise ValueError("incomplete ClientHello") + record = {"client_hello": hello, "requests": []} + with self.lock: + self.records.append(record) + conn = self.context.wrap_socket(raw, server_side=True) + record.update(negotiated_alpn=conn.selected_alpn_protocol(), tls_version=conn.version(), + cipher=conn.cipher()[0], session_reused=conn.session_reused) + with conn: + if conn.selected_alpn_protocol() == "h2": + self.http2(conn, record) + else: + self.http1(conn, record) + except (EOFError, ConnectionResetError, BrokenPipeError): + pass + except Exception as exc: + # Local dummy traffic only. Keep failures distinguishable from captures. + with self.lock: + self.errors.append(type(exc).__name__ + ": " + str(exc)) + finally: + raw.close() + + def http1(self, conn, record): + data = b"" + while b"\r\n\r\n" not in data: + data += recv_exact(conn, 1) + lines = data.decode("latin1").split("\r\n") + headers = [line.split(":", 1) for line in lines[1:] if ":" in line] + headers = [(k, v.strip()) for k, v in headers] + lower = {k.lower(): v for k, v in headers} + request = {"request_line": lines[0], "headers": headers} + record["requests"].append(request) + if lower.get("upgrade", "").lower() == "websocket": + accept = base64.b64encode(hashlib.sha1((lower["sec-websocket-key"] + + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode()).digest()) + conn.sendall(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + accept + b"\r\n\r\n") + request["frames"] = [] + while True: + a, z = recv_exact(conn, 2) + size = z & 127 + if size == 126: + size = int.from_bytes(recv_exact(conn, 2), "big") + elif size == 127: + size = int.from_bytes(recv_exact(conn, 8), "big") + mask = recv_exact(conn, 4) if z & 128 else b"" + payload = recv_exact(conn, size) + if mask: + payload = bytes(c ^ mask[i % 4] for i, c in enumerate(payload)) + op = a & 15 + if op == 8: + return + if op == 9: + ws_send(conn, payload, 10) + elif op == 1: + value = json.loads(payload) + request["frames"].append({"opcode": op, "masked": bool(mask), "keys": sorted(value)}) + for event in events(len(request["frames"]), value.get("generate") is False): + ws_send(conn, event) + return + if "content-length" in lower: + recv_exact(conn, int(lower["content-length"])) + body = b"".join(("event: " + value["type"] + "\ndata: " + json.dumps(value) + "\n\n").encode() + for value in events(len(self.records))) + if lines[0].startswith("GET "): + body = b'{"models":[]}' + conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: " + + str(len(body)).encode() + b"\r\n\r\n" + body) + + def http2(self, conn, record): + h = h2.connection.H2Connection(config=h2.config.H2Configuration(client_side=False, header_encoding="utf-8")) + h.initiate_connection() + conn.sendall(h.data_to_send()) + first = True + while True: + data = conn.recv(65535) + if not data: + return + if first: + record["http2_initial_bytes"] = data.hex() + first = False + for event in h.receive_data(data): + if isinstance(event, h2.events.RequestReceived): + record["requests"].append({"headers": event.headers}) + elif isinstance(event, h2.events.DataReceived): + h.acknowledge_received_data(event.flow_controlled_length, event.stream_id) + elif isinstance(event, h2.events.StreamEnded): + body = b"".join(("event: " + v["type"] + "\ndata: " + json.dumps(v) + "\n\n").encode() + for v in events(len(self.records))) + h.send_headers(event.stream_id, [(":status", "200"), ("content-type", "text/event-stream")]) + h.send_data(event.stream_id, body, end_stream=True) + h.close_connection(last_stream_id=event.stream_id) + conn.sendall(h.data_to_send()) + return + conn.sendall(h.data_to_send()) + + +def drive(binary, capture, scratch, mode): + home = scratch / ("home-" + mode) + home.mkdir() + workspace = scratch / "workspace" + workspace.mkdir(exist_ok=True) + env = dict(os.environ) + for key in ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy", + "OPENAI_API_KEY", "CODEX_API_KEY", "ACCESS_TOKEN"]: + env.pop(key, None) + env.update(CODEX_HOME=str(home), CODEX_CA_CERTIFICATE=str(capture.ca), + SSL_CERT_FILE=str(capture.ca), CODEX_PROFILE_TEST_KEY="local-dummy-key", NO_PROXY="localhost,127.0.0.1") + provider = dict(name="Loopback only", base_url=f"https://localhost:{capture.port}/v1", + env_key="CODEX_PROFILE_TEST_KEY", wire_api="responses", requires_openai_auth=False, + supports_websockets=mode == "ws", request_max_retries=0, stream_max_retries=0) + def toml(value): + return str(value).lower() if isinstance(value, bool) else json.dumps(value) + config = "model_providers.capture={" + ",".join(k + "=" + toml(v) for k, v in provider.items()) + "}" + args = [str(binary), "app-server", "--listen", "stdio://", "-c", 'model_provider="capture"', + "-c", config, "-c", 'model="gpt-5.4"', "-c", 'approval_policy="never"', + "-c", 'sandbox_mode="read-only"', "-c", 'features.responses_websockets_v2=' + str(mode == "ws").lower(), + "-c", 'analytics.enabled=false'] + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=workspace, env=env, text=True, encoding="utf-8") + incoming, errors = queue.Queue(), [] + def reader(): + for line in process.stdout: + try: + incoming.put(json.loads(line)) + except json.JSONDecodeError: + pass + threading.Thread(target=reader, daemon=True).start() + threading.Thread(target=lambda: errors.extend(process.stderr.read().splitlines()), daemon=True).start() + def send(value): + process.stdin.write(json.dumps(value) + "\n") + process.stdin.flush() + def until(predicate): + deadline = time.monotonic() + 45 + observed = [] + while time.monotonic() < deadline: + try: + value = incoming.get(timeout=1) + except queue.Empty: + if process.poll() is not None: + break + continue + if "error" in value: + raise RuntimeError(json.dumps(value["error"])) + observed.append(value) + if predicate(value): + return value + raise RuntimeError("local app-server did not complete: " + json.dumps(observed[-5:]) + "\n" + "\n".join(errors[-8:])) + try: + send({"id": 1, "method": "initialize", "params": {"clientInfo": {"name": "codex-tui", "title": "Capture", "version": VERSION}}}) + until(lambda v: v.get("id") == 1) + send({"method": "initialized"}) + send({"id": 2, "method": "thread/start", "params": {"model": "gpt-5.4", "cwd": str(workspace)}}) + thread = until(lambda v: v.get("id") == 2)["result"]["thread"]["id"] + for i in range(2): + send({"id": 3 + i, "method": "turn/start", "params": {"threadId": thread, + "input": [{"type": "text", "text": "Reply ok from the loopback fixture."}]}}) + result = until(lambda v: v.get("method") == "turn/completed") + if result["params"]["turn"]["status"] != "completed": + raise RuntimeError("local fixture turn failed: " + json.dumps(result)) + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True, type=pathlib.Path) + parser.add_argument("--output", required=True, type=pathlib.Path) + args = parser.parse_args() + binary = args.binary.resolve() + version = subprocess.check_output([str(binary), "--version"], text=True).strip() + if version != "codex-cli " + VERSION: + raise SystemExit("unexpected official binary version: " + version) + with tempfile.TemporaryDirectory(prefix="codex-transport-capture-") as temp: + scratch = pathlib.Path(temp) + capture = Capture(scratch) + modes = {} + for mode in ["http", "ws"]: + print("capturing " + mode, flush=True) + start = len(capture.records) + try: + drive(binary, capture, scratch, mode) + except Exception: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps({"incomplete": True, "mode": mode, "captures": capture.records, + "errors": capture.errors}, indent=2), encoding="utf-8") + raise + modes[mode] = capture.records[start:] + if not modes[mode] or not any(row["requests"] for row in modes[mode]): + raise RuntimeError("no completed " + mode + " capture") + if mode == "ws" and not any("frames" in req for row in modes[mode] for req in row["requests"]): + raise RuntimeError("official client did not use WebSocket") + capture.listener.close() + result = dict(schema=1, version=VERSION, source_commit=SOURCE, binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(), + system=platform.system(), release=platform.release(), machine=platform.machine(), + trust_mode="isolated CODEX_CA_CERTIFICATE", real_model_requests=0, captures=modes, + errors=capture.errors) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"output": str(args.output), "system": result["system"], + "connections": {k: len(v) for k, v in modes.items()}, "errors": capture.errors})) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/codex-transport-samples.yml b/.github/workflows/codex-transport-samples.yml new file mode 100644 index 000000000000..08ca7bb30e42 --- /dev/null +++ b/.github/workflows/codex-transport-samples.yml @@ -0,0 +1,55 @@ +name: Codex transport samples +on: + push: + branches: [codex/fixed-device-conversations] + paths: + - .github/scripts/capture_codex_transport.py + - .github/workflows/codex-transport-samples.yml + workflow_dispatch: +permissions: + contents: read +jobs: + capture: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + digest: 86f268d81b898f3e144c5ecff2ad2fda2c5802e042d6195b72538fe0fa125057 + - runner: macos-15 + target: aarch64-apple-darwin + digest: b293a0fcdd871900a44c0a4f40a1aa401b31e02854f790ebdb8bd5d1177fd1fe + - runner: macos-15-intel + target: x86_64-apple-darwin + digest: d68e39d8f62639f33f7d00656554cb9c3ef7809e664a3827e62ab3ace3c21b21 + - runner: windows-2025 + target: x86_64-pc-windows-msvc.exe + digest: a97c427a033b08d58902af3e52e8586ad2013267212112d306100223e2b12ec7 + runs-on: ${{ matrix.runner }} + timeout-minutes: 12 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - run: python -m pip install cryptography==48.0.1 h2==4.3.0 + - name: Download fixed official binary and verify archive + shell: bash + env: + TARGET: ${{ matrix.target }} + EXPECTED: ${{ matrix.digest }} + run: | + curl --fail --location --retry 2 "https://github.com/openai/codex/releases/download/rust-v0.162.1/codex-${TARGET}.tar.gz" -o codex.tar.gz + python -c 'import hashlib,os,pathlib; assert hashlib.sha256(pathlib.Path("codex.tar.gz").read_bytes()).hexdigest()==os.environ["EXPECTED"]' + mkdir official + tar -xzf codex.tar.gz -C official + - name: Capture loopback HTTP and WS only + shell: bash + env: + TARGET: ${{ matrix.target }} + run: python .github/scripts/capture_codex_transport.py --binary "official/codex-${TARGET}" --output "capture-${TARGET}.json" + - uses: actions/upload-artifact@v6 + with: + name: codex-transport-${{ matrix.target }} + path: capture-*.json From 1343c880a4b93092350fdfeb442193da9491b0f0 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:13:24 +0800 Subject: [PATCH 02/11] test: capture reconnects and HTTP2 startup without inherited client settings --- .github/scripts/capture_codex_transport.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py index 65f7a34b9b92..ddef878ae306 100644 --- a/.github/scripts/capture_codex_transport.py +++ b/.github/scripts/capture_codex_transport.py @@ -237,6 +237,9 @@ def http1(self, conn, record): request["frames"].append({"opcode": op, "masked": bool(mask), "keys": sorted(value)}) for event in events(len(request["frames"]), value.get("generate") is False): ws_send(conn, event) + if value.get("generate") is not False: + ws_send(conn, b"\x03\xe8", 8) + return return if "content-length" in lower: recv_exact(conn, int(lower["content-length"])) @@ -251,14 +254,14 @@ def http2(self, conn, record): h = h2.connection.H2Connection(config=h2.config.H2Configuration(client_side=False, header_encoding="utf-8")) h.initiate_connection() conn.sendall(h.data_to_send()) - first = True + initial = bytearray() while True: data = conn.recv(65535) if not data: return - if first: - record["http2_initial_bytes"] = data.hex() - first = False + if not record["requests"]: + initial.extend(data) + record["http2_initial_bytes"] = initial.hex() for event in h.receive_data(data): if isinstance(event, h2.events.RequestReceived): record["requests"].append({"headers": event.headers}) @@ -281,6 +284,9 @@ def drive(binary, capture, scratch, mode): workspace = scratch / "workspace" workspace.mkdir(exist_ok=True) env = dict(os.environ) + for key in list(env): + if key.startswith(("CODEX_", "OPENAI_")): + env.pop(key, None) for key in ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy", "OPENAI_API_KEY", "CODEX_API_KEY", "ACCESS_TOKEN"]: env.pop(key, None) From 79885172689a3fbe0b4f419063c80be431fd73ee Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:16:22 +0800 Subject: [PATCH 03/11] fix: isolate native capture lifecycle and retain failed capture evidence --- .github/scripts/capture_codex_transport.py | 33 +++++++++++++------ .github/workflows/codex-transport-samples.yml | 1 + 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py index ddef878ae306..c1ab7bea7fe7 100644 --- a/.github/scripts/capture_codex_transport.py +++ b/.github/scripts/capture_codex_transport.py @@ -9,6 +9,7 @@ import json import os import pathlib +import ipaddress import platform import queue import socket @@ -135,7 +136,7 @@ def __init__(self, directory): .serial_number(x509.random_serial_number()).not_valid_before(now - timedelta(minutes=5)) .not_valid_after(now + timedelta(days=1)) .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) - .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False) .sign(key, hashes.SHA256())) self.ca = directory / "ca.pem" self.ca.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) @@ -145,7 +146,7 @@ def __init__(self, directory): .not_valid_after(now + timedelta(days=1)) .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) .add_extension(x509.ExtendedKeyUsage([x509.oid.ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) - .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False) .sign(key, hashes.SHA256())) leaf_path = directory / "server.pem" leaf_path.write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) @@ -158,12 +159,22 @@ def __init__(self, directory): self.listener.bind(("127.0.0.1", 0)) self.listener.listen() self.port = self.listener.getsockname()[1] - threading.Thread(target=self.accept, daemon=True).start() + self.listeners = [self.listener] + try: + ipv6 = socket.socket(socket.AF_INET6) + ipv6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) + ipv6.bind(("::1", self.port)) + ipv6.listen() + self.listeners.append(ipv6) + except OSError: + pass + for listener in self.listeners: + threading.Thread(target=self.accept, args=(listener,), daemon=True).start() - def accept(self): + def accept(self, listener): while True: try: - conn, _ = self.listener.accept() + conn, _ = listener.accept() except OSError: return threading.Thread(target=self.handle, args=(conn,), daemon=True).start() @@ -301,7 +312,7 @@ def toml(value): args = [str(binary), "app-server", "--listen", "stdio://", "-c", 'model_provider="capture"', "-c", config, "-c", 'model="gpt-5.4"', "-c", 'approval_policy="never"', "-c", 'sandbox_mode="read-only"', "-c", 'features.responses_websockets_v2=' + str(mode == "ws").lower(), - "-c", 'analytics.enabled=false'] + "-c", 'analytics.enabled=false', "-c", 'features.plugins=false'] process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workspace, env=env, text=True, encoding="utf-8") incoming, errors = queue.Queue(), [] @@ -345,11 +356,12 @@ def until(predicate): if result["params"]["turn"]["status"] != "completed": raise RuntimeError("local fixture turn failed: " + json.dumps(result)) finally: - process.terminate() + process.stdin.close() try: process.wait(timeout=5) except subprocess.TimeoutExpired: - process.kill() + process.terminate() + process.wait(timeout=5) def main(): @@ -361,7 +373,7 @@ def main(): version = subprocess.check_output([str(binary), "--version"], text=True).strip() if version != "codex-cli " + VERSION: raise SystemExit("unexpected official binary version: " + version) - with tempfile.TemporaryDirectory(prefix="codex-transport-capture-") as temp: + with tempfile.TemporaryDirectory(prefix="codex-transport-capture-", ignore_cleanup_errors=True) as temp: scratch = pathlib.Path(temp) capture = Capture(scratch) modes = {} @@ -380,7 +392,8 @@ def main(): raise RuntimeError("no completed " + mode + " capture") if mode == "ws" and not any("frames" in req for row in modes[mode] for req in row["requests"]): raise RuntimeError("official client did not use WebSocket") - capture.listener.close() + for listener in capture.listeners: + listener.close() result = dict(schema=1, version=VERSION, source_commit=SOURCE, binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(), system=platform.system(), release=platform.release(), machine=platform.machine(), trust_mode="isolated CODEX_CA_CERTIFICATE", real_model_requests=0, captures=modes, diff --git a/.github/workflows/codex-transport-samples.yml b/.github/workflows/codex-transport-samples.yml index 08ca7bb30e42..60ccfc2e2cee 100644 --- a/.github/workflows/codex-transport-samples.yml +++ b/.github/workflows/codex-transport-samples.yml @@ -50,6 +50,7 @@ jobs: TARGET: ${{ matrix.target }} run: python .github/scripts/capture_codex_transport.py --binary "official/codex-${TARGET}" --output "capture-${TARGET}.json" - uses: actions/upload-artifact@v6 + if: always() with: name: codex-transport-${{ matrix.target }} path: capture-*.json From 50977362e3863ac5d737c4c5bbfc7abde8b1f113 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:23:47 +0800 Subject: [PATCH 04/11] test: diagnose remaining Linux capture without rerunning verified platforms --- .github/scripts/capture_codex_transport.py | 8 ++++++-- .github/workflows/codex-transport-samples.yml | 9 --------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py index c1ab7bea7fe7..9678eae51ad0 100644 --- a/.github/scripts/capture_codex_transport.py +++ b/.github/scripts/capture_codex_transport.py @@ -303,6 +303,7 @@ def drive(binary, capture, scratch, mode): env.pop(key, None) env.update(CODEX_HOME=str(home), CODEX_CA_CERTIFICATE=str(capture.ca), SSL_CERT_FILE=str(capture.ca), CODEX_PROFILE_TEST_KEY="local-dummy-key", NO_PROXY="localhost,127.0.0.1") + env["RUST_LOG"] = "codex_http_client=debug,reqwest=debug,hyper_util=debug" provider = dict(name="Loopback only", base_url=f"https://localhost:{capture.port}/v1", env_key="CODEX_PROFILE_TEST_KEY", wire_api="responses", requires_openai_auth=False, supports_websockets=mode == "ws", request_max_retries=0, stream_max_retries=0) @@ -323,7 +324,10 @@ def reader(): except json.JSONDecodeError: pass threading.Thread(target=reader, daemon=True).start() - threading.Thread(target=lambda: errors.extend(process.stderr.read().splitlines()), daemon=True).start() + def read_errors(): + for line in process.stderr: + errors.append(line.rstrip()) + threading.Thread(target=read_errors, daemon=True).start() def send(value): process.stdin.write(json.dumps(value) + "\n") process.stdin.flush() @@ -354,7 +358,7 @@ def until(predicate): "input": [{"type": "text", "text": "Reply ok from the loopback fixture."}]}}) result = until(lambda v: v.get("method") == "turn/completed") if result["params"]["turn"]["status"] != "completed": - raise RuntimeError("local fixture turn failed: " + json.dumps(result)) + raise RuntimeError("local fixture turn failed: " + json.dumps(result) + "\n" + "\n".join(errors[-15:])) finally: process.stdin.close() try: diff --git a/.github/workflows/codex-transport-samples.yml b/.github/workflows/codex-transport-samples.yml index 60ccfc2e2cee..98cdf2ac417d 100644 --- a/.github/workflows/codex-transport-samples.yml +++ b/.github/workflows/codex-transport-samples.yml @@ -17,15 +17,6 @@ jobs: - runner: ubuntu-24.04 target: x86_64-unknown-linux-musl digest: 86f268d81b898f3e144c5ecff2ad2fda2c5802e042d6195b72538fe0fa125057 - - runner: macos-15 - target: aarch64-apple-darwin - digest: b293a0fcdd871900a44c0a4f40a1aa401b31e02854f790ebdb8bd5d1177fd1fe - - runner: macos-15-intel - target: x86_64-apple-darwin - digest: d68e39d8f62639f33f7d00656554cb9c3ef7809e664a3827e62ab3ace3c21b21 - - runner: windows-2025 - target: x86_64-pc-windows-msvc.exe - digest: a97c427a033b08d58902af3e52e8586ad2013267212112d306100223e2b12ec7 runs-on: ${{ matrix.runner }} timeout-minutes: 12 steps: From 2f60ee85707c89abd64fb48c710b035cfba8259d Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:27:54 +0800 Subject: [PATCH 05/11] fix: keep capture HTTP2 sessions alive through completed turns --- .github/scripts/capture_codex_transport.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py index 9678eae51ad0..3ae476e8b061 100644 --- a/.github/scripts/capture_codex_transport.py +++ b/.github/scripts/capture_codex_transport.py @@ -250,6 +250,13 @@ def http1(self, conn, record): ws_send(conn, event) if value.get("generate") is not False: ws_send(conn, b"\x03\xe8", 8) + # Let the peer consume the completed response and close + # frame before the TLS socket is destroyed. + try: + conn.settimeout(2) + conn.unwrap().close() + except (OSError, TimeoutError): + pass return return if "content-length" in lower: @@ -283,9 +290,9 @@ def http2(self, conn, record): for v in events(len(self.records))) h.send_headers(event.stream_id, [(":status", "200"), ("content-type", "text/event-stream")]) h.send_data(event.stream_id, body, end_stream=True) - h.close_connection(last_stream_id=event.stream_id) conn.sendall(h.data_to_send()) - return + # Keep the H2 session alive. Closing immediately after + # END_STREAM can reset unread control frames on Linux. conn.sendall(h.data_to_send()) From 74422d06965610b056d9611ab3b04a9335818fbb Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 11:51:55 +0800 Subject: [PATCH 06/11] test: capture HTTP1 negotiation for every supported Codex build --- .github/scripts/capture_codex_transport.py | 8 +++++++- .github/workflows/codex-transport-samples.yml | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/scripts/capture_codex_transport.py b/.github/scripts/capture_codex_transport.py index 3ae476e8b061..a1334acf2f3d 100644 --- a/.github/scripts/capture_codex_transport.py +++ b/.github/scripts/capture_codex_transport.py @@ -267,6 +267,11 @@ def http1(self, conn, record): body = b'{"models":[]}' conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\nContent-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body) + try: + conn.settimeout(2) + conn.unwrap().close() + except (OSError, TimeoutError): + pass def http2(self, conn, record): h = h2.connection.H2Connection(config=h2.config.H2Configuration(client_side=False, header_encoding="utf-8")) @@ -388,7 +393,8 @@ def main(): scratch = pathlib.Path(temp) capture = Capture(scratch) modes = {} - for mode in ["http", "ws"]: + for mode in ["http", "http1", "ws"]: + capture.context.set_alpn_protocols(["http/1.1"] if mode=="http1" else ["h2","http/1.1"]) print("capturing " + mode, flush=True) start = len(capture.records) try: diff --git a/.github/workflows/codex-transport-samples.yml b/.github/workflows/codex-transport-samples.yml index 98cdf2ac417d..60ccfc2e2cee 100644 --- a/.github/workflows/codex-transport-samples.yml +++ b/.github/workflows/codex-transport-samples.yml @@ -17,6 +17,15 @@ jobs: - runner: ubuntu-24.04 target: x86_64-unknown-linux-musl digest: 86f268d81b898f3e144c5ecff2ad2fda2c5802e042d6195b72538fe0fa125057 + - runner: macos-15 + target: aarch64-apple-darwin + digest: b293a0fcdd871900a44c0a4f40a1aa401b31e02854f790ebdb8bd5d1177fd1fe + - runner: macos-15-intel + target: x86_64-apple-darwin + digest: d68e39d8f62639f33f7d00656554cb9c3ef7809e664a3827e62ab3ace3c21b21 + - runner: windows-2025 + target: x86_64-pc-windows-msvc.exe + digest: a97c427a033b08d58902af3e52e8586ad2013267212112d306100223e2b12ec7 runs-on: ${{ matrix.runner }} timeout-minutes: 12 steps: From 23e52e01be53cd926fc4435fba72d59dc5ccf316 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 12:52:05 +0800 Subject: [PATCH 07/11] fix: preserve Codex device conversations with verified strict transport --- .downstream/native-domains.md | 77 ++- .github/scripts/verify_codex_transport.py | 83 +++ .github/workflows/codex-transport-samples.yml | 9 +- .github/workflows/downstream-verify.yml | 5 + THIRD_PARTY_NOTICES.md | 7 + backend/cmd/codex-transport-probe/main.go | 114 ++++ backend/internal/pkg/codextransport/http1.go | 163 +++++ .../internal/pkg/codextransport/http1_test.go | 52 ++ .../internal/pkg/codextransport/profile.go | 147 +++++ .../internal/pkg/codextransport/proxy_test.go | 236 +++++++ .../pkg/codextransport/reference.json | 407 ++++++++++++ .../internal/pkg/codextransport/transport.go | 204 ++++++ .../pkg/codextransport/transport_test.go | 110 ++++ backend/internal/pkg/codexws/LICENSE.txt | 13 + backend/internal/pkg/codexws/UPSTREAM.md | 8 + backend/internal/pkg/codexws/accept.go | 378 +++++++++++ backend/internal/pkg/codexws/close.go | 335 ++++++++++ .../pkg/codexws/codex_compress_test.go | 53 ++ backend/internal/pkg/codexws/compress.go | 249 ++++++++ backend/internal/pkg/codexws/conn.go | 306 +++++++++ backend/internal/pkg/codexws/dial.go | 360 +++++++++++ backend/internal/pkg/codexws/doc.go | 33 + backend/internal/pkg/codexws/errors.go | 8 + backend/internal/pkg/codexws/frame.go | 173 +++++ backend/internal/pkg/codexws/hijack.go | 33 + .../pkg/codexws/internal/bpool/bpool.go | 25 + .../pkg/codexws/internal/errd/wrap.go | 14 + .../pkg/codexws/internal/util/util.go | 15 + .../pkg/codexws/internal/wsjs/wsjs_js.go | 169 +++++ .../internal/pkg/codexws/internal/xsync/go.go | 26 + backend/internal/pkg/codexws/mask.go | 128 ++++ backend/internal/pkg/codexws/mask_amd64.s | 127 ++++ backend/internal/pkg/codexws/mask_arm64.s | 72 +++ backend/internal/pkg/codexws/mask_asm.go | 26 + backend/internal/pkg/codexws/mask_go.go | 7 + backend/internal/pkg/codexws/netconn.go | 233 +++++++ backend/internal/pkg/codexws/netconn_js.go | 11 + backend/internal/pkg/codexws/netconn_notjs.go | 19 + backend/internal/pkg/codexws/read.go | 540 ++++++++++++++++ backend/internal/pkg/codexws/stringer.go | 91 +++ backend/internal/pkg/codexws/write.go | 389 ++++++++++++ backend/internal/pkg/codexws/ws_js.go | 598 ++++++++++++++++++ backend/internal/pkg/codexws/wsjson/wsjson.go | 68 ++ backend/internal/repository/http_upstream.go | 8 +- .../repository/http_upstream_codex.go | 51 ++ .../repository/openai_oauth_service.go | 24 +- .../internal/repository/req_client_pool.go | 29 +- .../internal/service/account_test_service.go | 74 ++- .../service/codex_fingerprint_account_view.go | 19 +- .../codex_fingerprint_account_view_test.go | 21 +- .../service/codex_fingerprint_policy_test.go | 31 +- .../internal/service/openai_alpha_search.go | 4 +- .../service/openai_codex_account_identity.go | 94 ++- .../openai_codex_conversations_test.go | 203 ++++++ .../service/openai_codex_fingerprint.go | 269 +++----- .../service/openai_codex_fingerprint_test.go | 95 +-- .../service/openai_codex_identity_snapshot.go | 69 +- .../internal/service/openai_codex_metadata.go | 174 +++++ .../service/openai_codex_models_service.go | 14 + .../openai_codex_request_compression.go | 15 +- .../service/openai_codex_test_identity.go | 106 ++++ .../service/openai_codex_transport.go | 89 +++ .../openai_gateway_chat_completions.go | 4 +- .../service/openai_gateway_forward.go | 44 +- .../service/openai_gateway_messages.go | 4 +- .../service/openai_gateway_passthrough.go | 6 +- .../service/openai_gateway_service.go | 10 +- .../internal/service/openai_oauth_service.go | 9 + .../service/openai_plugin_transport.go | 21 +- .../openai_upstream_transport_error.go | 5 + backend/internal/service/openai_ws_client.go | 4 + .../service/openai_ws_codex_transport.go | 123 ++++ .../internal/service/openai_ws_forwarder.go | 4 + .../service/openai_ws_forwarder_ingress.go | 15 +- .../service/openai_ws_forwarder_payload.go | 2 +- .../service/openai_ws_forwarder_v2.go | 7 +- backend/internal/service/openai_ws_pool.go | 18 +- .../openai_ws_v2_passthrough_adapter.go | 10 +- .../service/setting_codex_fingerprint.go | 20 +- .../service/setting_gateway_runtime.go | 4 + backend/internal/service/setting_update.go | 4 + backend/internal/service/upstream_models.go | 5 + frontend/src/api/admin/codexFingerprint.ts | 5 + .../admin/CodexFingerprintIdentity.vue | 25 +- .../src/views/admin/CodexFingerprintView.vue | 10 +- .../__tests__/CodexFingerprintView.spec.ts | 36 +- 86 files changed, 7455 insertions(+), 450 deletions(-) create mode 100644 .github/scripts/verify_codex_transport.py create mode 100644 backend/cmd/codex-transport-probe/main.go create mode 100644 backend/internal/pkg/codextransport/http1.go create mode 100644 backend/internal/pkg/codextransport/http1_test.go create mode 100644 backend/internal/pkg/codextransport/profile.go create mode 100644 backend/internal/pkg/codextransport/proxy_test.go create mode 100644 backend/internal/pkg/codextransport/reference.json create mode 100644 backend/internal/pkg/codextransport/transport.go create mode 100644 backend/internal/pkg/codextransport/transport_test.go create mode 100644 backend/internal/pkg/codexws/LICENSE.txt create mode 100644 backend/internal/pkg/codexws/UPSTREAM.md create mode 100644 backend/internal/pkg/codexws/accept.go create mode 100644 backend/internal/pkg/codexws/close.go create mode 100644 backend/internal/pkg/codexws/codex_compress_test.go create mode 100644 backend/internal/pkg/codexws/compress.go create mode 100644 backend/internal/pkg/codexws/conn.go create mode 100644 backend/internal/pkg/codexws/dial.go create mode 100644 backend/internal/pkg/codexws/doc.go create mode 100644 backend/internal/pkg/codexws/errors.go create mode 100644 backend/internal/pkg/codexws/frame.go create mode 100644 backend/internal/pkg/codexws/hijack.go create mode 100644 backend/internal/pkg/codexws/internal/bpool/bpool.go create mode 100644 backend/internal/pkg/codexws/internal/errd/wrap.go create mode 100644 backend/internal/pkg/codexws/internal/util/util.go create mode 100644 backend/internal/pkg/codexws/internal/wsjs/wsjs_js.go create mode 100644 backend/internal/pkg/codexws/internal/xsync/go.go create mode 100644 backend/internal/pkg/codexws/mask.go create mode 100644 backend/internal/pkg/codexws/mask_amd64.s create mode 100644 backend/internal/pkg/codexws/mask_arm64.s create mode 100644 backend/internal/pkg/codexws/mask_asm.go create mode 100644 backend/internal/pkg/codexws/mask_go.go create mode 100644 backend/internal/pkg/codexws/netconn.go create mode 100644 backend/internal/pkg/codexws/netconn_js.go create mode 100644 backend/internal/pkg/codexws/netconn_notjs.go create mode 100644 backend/internal/pkg/codexws/read.go create mode 100644 backend/internal/pkg/codexws/stringer.go create mode 100644 backend/internal/pkg/codexws/write.go create mode 100644 backend/internal/pkg/codexws/ws_js.go create mode 100644 backend/internal/pkg/codexws/wsjson/wsjson.go create mode 100644 backend/internal/repository/http_upstream_codex.go create mode 100644 backend/internal/service/openai_codex_conversations_test.go create mode 100644 backend/internal/service/openai_codex_metadata.go create mode 100644 backend/internal/service/openai_codex_test_identity.go create mode 100644 backend/internal/service/openai_codex_transport.go create mode 100644 backend/internal/service/openai_ws_codex_transport.go diff --git a/.downstream/native-domains.md b/.downstream/native-domains.md index 94a181cc81da..99e44cf1d381 100644 --- a/.downstream/native-domains.md +++ b/.downstream/native-domains.md @@ -50,28 +50,61 @@ General account edits preserve the latest stored mode when it is omitted. Existing seeds and persisted TUI identities are retained, and new accounts still default to `device`. -Enabled simulation applies a fixed account TUI identity to HTTP, passthrough, -WS, account tests and credential refresh. Disabled simulation preserves caller -identity and identifiers, filling only missing protocol identity headers. -Mode `off` preserves device/session identifiers while simulation stays enabled; -`device` changes only device identifiers; `session` keeps separate real threads -and maps their parent references; `full` combines threads and removes self-parent -references. Real turn IDs, parent/root turn IDs, window indices and timestamps -are retained. Sandbox `none`, `external`, permission modes and unknown tags are -preserved; known platform backends are aligned without inventing a sandbox for -an absent tag. Explicit custom cache keys are not rewritten. - -The built-in protocol/version baseline is official Codex `rust-v0.161.0`. -The existing six-hour stable-version sync and manual-version precedence remain. -Successful saves publish an immutable policy; failed saves retain the old one. -Each attempt shares it across headers/body. WS reuse includes identity and -policy revision. Setting changes let the current turn finish, then request a -retryable reconnect before another turn is sent through an old handshake. -Account mode changes invalidate only that account's next handshake. - -Account connection tests exclude and reject `ultra` across every capability -source before dispatch; stale UI selections reset to default. Codex client -orchestration catalogs retain their existing capabilities. +Enabled simulation uses one fixed credential-source TUI device with independent +real conversations. Stored `session` / `full` values and API inputs remain +compatible but have effective mode `device`; the UI offers only `off` and +fixed-device/multiple-conversation mode. Their stored raw values are not migrated. +Every enabled mode scopes session/thread/parent/window identities one-to-one by +credential source and API key, preserving UUIDv7 timestamps and real root/child/fork +relationships. Explicit custom cache keys remain untouched; default session cache +keys follow the same mapping. Real turns, timestamps, permission and sandbox +metadata are retained. Calls without a client conversation get one synthetic UUIDv7 +root per logical request (shared by its attempts), never a permanent account root. + +The valid embedded body turn metadata is canonical; flat fields and bounded +HTTP/WS headers are compatibility projections. Malformed core metadata returns a +request-scoped 400. Unknown metadata, including large JSON numbers, is retained. +Original field conflicts are recorded when canonical projection resolves them. +Ordinary/compaction account tests use the same projection and synthetic root +lifecycle, including credential shadows; wire status distinguishes configured +policy, actually sent identity and negotiated TLS/ALPN. + +The verified client baseline is official Codex `rust-v0.162.1`, commit +`092d3acd6bec3e3a14bdc7e7a2810ab628ab759d`. The transport reference in +`backend/internal/pkg/codextransport/reference.json` records fixed official-release archive +digests, binary/capture digests and loopback sample runs for Windows x86_64, +Linux x86_64 and macOS arm64/x86_64. These captures use an isolated custom CA +configuration; verification is for that official-client transport variant, not +an assertion that every native trust-store configuration is identical. TLS +extensions are permuted as observed. HTTP/2 settings/flow/pseudo-header order, +HTTP/1 header order/spelling and WebSocket offers are verified against actual +wire captures. Native reconnect samples did not resume TLS sessions; the current +profile therefore uses fresh TLS handshakes and does not share session tickets. + +Strict transport applies to enabled simulation: an unavailable profile or failed +transport does not fall back to the generic Go client or replay an uncertain POST. +Existing explicit upstream protocol-error recovery remains separate. Third-party +plugins that own transmission cannot be used under strict simulation without a +matching attested profile; their configuration is retained and a clear error is +returned. API-key/other-provider paths and master-off behavior retain their clients. +Codex HTTP, WS, refresh and model-list clients isolate their pools by credential +source, proxy and profile. The Codex-only WebSocket adapter retains upstream +coder/websocket v1.8.14 and its license, adding actual bounded deflate-window support; +other services use the original dependency. + +Version discovery still runs every six hours. Automatic effective identity advances +only within the shipped verified profile set; newly discovered unsupported releases +are visible as pending validation. Saving an unsupported manual version returns an +error. Settings reads add verified/pending/strict-profile details without changing +the four-field write payload. Existing device seeds and persisted hardware identities +are retained; their supported platform/architecture classes match the verified set. + +One immutable policy/identity selection is shared by each attempt's headers, body +and transport. New requests freeze the compression choice too. WS pool compatibility +includes actual header and body device/session/thread/window identity even for +`device` / `off`, plus client and policy revision. Changed identity is applied after +the current turn by requiring a new handshake, never by reusing an old conversation's +handshake. Account mode edits invalidate only that account's subsequent connections. ## Independent Pelican tests diff --git a/.github/scripts/verify_codex_transport.py b/.github/scripts/verify_codex_transport.py new file mode 100644 index 000000000000..37092a9f8b07 --- /dev/null +++ b/.github/scripts/verify_codex_transport.py @@ -0,0 +1,83 @@ +"""Compare the implementation's actual wire fields with native capture data.""" +import argparse +import json +import pathlib +import struct +import subprocess +import tempfile +from capture_codex_transport import Capture + + +def settings(row): + raw = bytes.fromhex(row.get("http2_initial_bytes", "")) + if raw.startswith(b"PRI "): + raw = raw[24:] + values, flow = [], 0 + while len(raw) >= 9: + length = int.from_bytes(raw[:3], "big") + data = raw[9:9 + length] + if len(data) < length: + break + if raw[3] == 4 and data: + values = [{"id": k, "value": v} for k, v in + [struct.unpack(">HI", data[i:i + 6]) for i in range(0, len(data), 6)]] + if raw[3] == 8: + flow = int.from_bytes(data, "big") + raw = raw[9 + length:] + return values, flow + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--probe", type=pathlib.Path, required=True) + parser.add_argument("--reference", type=pathlib.Path, required=True) + parser.add_argument("--output", type=pathlib.Path, required=True) + args = parser.parse_args() + reference = json.loads(args.reference.read_text()) + results = {} + with tempfile.TemporaryDirectory(prefix="codex-transport-verify-") as temp: + capture = Capture(pathlib.Path(temp)) + for mode in ["http", "http1", "ws"]: + capture.context.set_alpn_protocols(["http/1.1"] if mode=="http1" else ["h2","http/1.1"]) + start = len(capture.records) + subprocess.run([str(args.probe.resolve()), "--url", f"https://localhost:{capture.port}/v1/responses", + "--ca", str(capture.ca), "--mode", mode], check=True) + rows = capture.records[start:] + if not rows or not rows[0]["requests"]: + raise AssertionError("no wire capture for " + mode) + expected = reference["profiles"][mode] + for row in rows: + actual = row["client_hello"] + for key in ["cipher_suites", "groups", "key_share_groups", "signature_algorithms", "versions", "psk_modes", "alpn", "session_id_length"]: + if actual[key] != expected["tls"][key]: + raise AssertionError(f"{mode} TLS {key}: {actual[key]} != {expected['tls'][key]}") + if sorted(actual["extensions"]) != sorted(expected["tls"]["extensions"]): + raise AssertionError(f"{mode} TLS extensions: {actual['extensions']}") + if actual["extension_data"] != expected["tls"]["extension_data"]: + raise AssertionError(f"{mode} TLS extension payloads differ") + if mode == "http": + actual_settings, flow = settings(rows[0]) + if actual_settings != expected["settings"] or flow != expected["connection_flow"]: + raise AssertionError(f"HTTP2 startup differs: {actual_settings}, flow={flow}") + elif mode=="ws": + headers = {k.lower(): v for k, v in rows[0]["requests"][0]["headers"]} + if headers.get("sec-websocket-extensions") != "permessage-deflate; client_max_window_bits": + raise AssertionError("WebSocket compression offer differs: " + str(headers.get("sec-websocket-extensions"))) + names = [k.lower() for k, _ in rows[0]["requests"][0]["headers"]] + expected_names = expected["pseudo_header_order"] + expected["header_order"] + if names != expected_names: + raise AssertionError(f"{mode} header ordering differs: {names}") + if mode in ("ws", "http1"): + for key, _ in rows[0]["requests"][0]["headers"]: + if key != expected["header_case"].get(key.lower(), key): + raise AssertionError("WebSocket header spelling differs: " + key) + results[mode] = rows + for listener in capture.listeners: + listener.close() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps({"matched": True, "captures": results, "errors": capture.errors}, indent=2) + "\n") + print("official TLS, HTTP2 startup and WebSocket offer matched") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/codex-transport-samples.yml b/.github/workflows/codex-transport-samples.yml index 60ccfc2e2cee..fa766b500283 100644 --- a/.github/workflows/codex-transport-samples.yml +++ b/.github/workflows/codex-transport-samples.yml @@ -1,10 +1,5 @@ name: Codex transport samples on: - push: - branches: [codex/fixed-device-conversations] - paths: - - .github/scripts/capture_codex_transport.py - - .github/workflows/codex-transport-samples.yml workflow_dispatch: permissions: contents: read @@ -30,7 +25,7 @@ jobs: timeout-minutes: 12 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: '3.13' - run: python -m pip install cryptography==48.0.1 h2==4.3.0 @@ -49,7 +44,7 @@ jobs: env: TARGET: ${{ matrix.target }} run: python .github/scripts/capture_codex_transport.py --binary "official/codex-${TARGET}" --output "capture-${TARGET}.json" - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 if: always() with: name: codex-transport-${{ matrix.target }} diff --git a/.github/workflows/downstream-verify.yml b/.github/workflows/downstream-verify.yml index c372917a1e53..3da5d34ceda8 100644 --- a/.github/workflows/downstream-verify.yml +++ b/.github/workflows/downstream-verify.yml @@ -29,6 +29,11 @@ jobs: check-latest: false cache: true cache-dependency-path: backend/go.sum + - name: Verified Codex transport wire contract + run: | + python3 -m pip install cryptography==48.0.1 h2==4.3.0 + go -C backend build -o "$RUNNER_TEMP/codex-transport-probe" ./cmd/codex-transport-probe + python3 .github/scripts/verify_codex_transport.py --probe "$RUNNER_TEMP/codex-transport-probe" --reference backend/internal/pkg/codextransport/reference.json --output "$RUNNER_TEMP/codex-wire.json" - name: Release metadata contracts run: | python3 .github/scripts/test_release_preflight.py diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 28c89f2df8b7..64613953ff0c 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -75,3 +75,10 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## Codex WebSocket transport adapter + +`backend/internal/pkg/codexws` is derived from `github.com/coder/websocket` v1.8.14 +under the MIT license; its original license is retained in `LICENSE.txt`. +Local changes add opt-in client compression-window negotiation using the already +locked klauspost/compress dependency. Other service paths retain the original module. diff --git a/backend/cmd/codex-transport-probe/main.go b/backend/cmd/codex-transport-probe/main.go new file mode 100644 index 000000000000..20ebe8c98b7a --- /dev/null +++ b/backend/cmd/codex-transport-probe/main.go @@ -0,0 +1,114 @@ +// codex-transport-probe verifies the transport on a loopback fixture only. +package main + +import ( + "bytes" + "context" + "crypto/x509" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + websocket "github.com/Wei-Shaw/sub2api/internal/pkg/codexws" +) + +func run() error { + target := flag.String("url", "", "loopback capture URL") + ca := flag.String("ca", "", "fixture CA") + ua := flag.String("ua", "codex-tui/0.162.1 (Windows 10.0.26100; x86_64) WindowsTerminal (codex-tui; 0.162.1)", "fixed identity") + mode := flag.String("mode", "http", "http or ws") + flag.Parse() + u, err := url.Parse(*target) + if err != nil { + return err + } + if u.Hostname() != "localhost" && u.Hostname() != "127.0.0.1" && u.Hostname() != "::1" { + return fmt.Errorf("probe requires a loopback endpoint") + } + certificate, err := os.ReadFile(*ca) + if err != nil { + return err + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(certificate) { + return fmt.Errorf("invalid fixture CA") + } + binding, err := codextransport.Select(*ua, "local-fixture", *mode) + if err != nil { + return err + } + rt, err := codextransport.New(binding, codextransport.Options{RootCAs: roots, MaxIdleConns: 2, MaxIdleConnsPerHost: 2, + MaxConnsPerHost: 2, IdleConnTimeout: time.Second, ResponseHeaderTimeout: 5 * time.Second}) + if err != nil { + return err + } + defer rt.CloseIdleConnections() + client := &http.Client{Transport: rt} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + headers := http.Header{} + for k, v := range map[string]string{"User-Agent": *ua, "Originator": "codex-tui", "Authorization": "Bearer local-dummy-key", + "X-Codex-Beta-Features": "remote_compaction_v2", "X-Codex-Window-Id": "root:0", "X-Codex-Turn-Metadata": "{}", + "X-Client-Request-Id": "root", "Session-Id": "root", "Thread-Id": "root", "Content-Type": "application/json", "Accept": "text/event-stream"} { + headers.Set(k, v) + } + if *mode == "ws" { + u.Scheme = "wss" + headers.Del("Content-Type") + headers.Del("Accept") + headers.Set("OpenAI-Beta", "responses_websockets=2026-02-06") + conn, _, err := websocket.Dial(ctx, u.String(), &websocket.DialOptions{HTTPClient: client, HTTPHeader: headers, ClientMaxWindowBits: true, CompressionMode: websocket.CompressionContextTakeover}) + if err != nil { + return err + } + defer conn.CloseNow() + if err = conn.Write(ctx, websocket.MessageText, []byte(`{"type":"response.create","model":"local-fixture","input":[]}`)); err != nil { + return err + } + for { + _, data, err := conn.Read(ctx) + if err != nil { + return err + } + if strings.Contains(string(data), `"response.completed"`) { + return nil + } + } + } + for range 2 { + request, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), bytes.NewBufferString(`{"model":"local-fixture","input":[]}`)) + if err != nil { + return err + } + request.Header = headers.Clone() + resp, err := client.Do(request) + if err != nil { + return err + } + _, readErr := io.Copy(io.Discard, resp.Body) + closeErr := resp.Body.Close() + if readErr != nil { + return readErr + } + if closeErr != nil { + return closeErr + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fixture returned %d", resp.StatusCode) + } + } + return nil +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/backend/internal/pkg/codextransport/http1.go b/backend/internal/pkg/codextransport/http1.go new file mode 100644 index 000000000000..8dbab70ad7cc --- /dev/null +++ b/backend/internal/pkg/codextransport/http1.go @@ -0,0 +1,163 @@ +package codextransport + +import ( + "bytes" + "fmt" + "net" + "strconv" + "strings" +) + +// http1WireConn only changes header-name spelling. Request bodies, chunk +// boundaries, trailers and upgraded WebSocket frames remain byte-for-byte intact. +// req's HTTP/1 writer owns header ordering, including Host and Content-Length. +type http1WireConn struct { + net.Conn + spelling map[string]string + upgrade bool + header bytes.Buffer + bodyRemaining int64 + chunked bool + chunkPhase byte + chunkLine []byte +} + +func (c *http1WireConn) Write(input []byte) (int, error) { + if c.upgrade { + return c.Conn.Write(input) + } + p := input + for len(p) > 0 { + if !c.chunked && c.bodyRemaining > 0 { + n := int64(len(p)) + if n > c.bodyRemaining { + n = c.bodyRemaining + } + if _, err := c.Conn.Write(p[:n]); err != nil { + _ = c.Conn.Close() + return 0, err + } + c.bodyRemaining -= n + p = p[n:] + continue + } + if c.chunked { + n, err := c.consumeChunk(p) + if err != nil { + _ = c.Conn.Close() + return 0, err + } + if _, err = c.Conn.Write(p[:n]); err != nil { + _ = c.Conn.Close() + return 0, err + } + p = p[n:] + continue + } + // Headers may be split across writes. Bound only the header allocation, + // retaining net/http's existing default header-size boundary. + c.header.WriteByte(p[0]) + p = p[1:] + if c.header.Len() > 1<<20 { + _ = c.Conn.Close() + return 0, fmt.Errorf("Codex HTTP1 headers exceed 1 MiB") + } + if !bytes.HasSuffix(c.header.Bytes(), []byte("\r\n\r\n")) { + continue + } + raw := c.header.Bytes() + lines := bytes.Split(raw, []byte("\r\n")) + for i := 1; i < len(lines); i++ { + colon := bytes.IndexByte(lines[i], ':') + if colon < 0 { + continue + } + name := strings.ToLower(string(lines[i][:colon])) + value := strings.TrimSpace(string(lines[i][colon+1:])) + if spelling := c.spelling[name]; spelling != "" { + copy(lines[i][:colon], spelling) + } + switch name { + case "content-length": + length, err := strconv.ParseInt(value, 10, 64) + if err != nil || length < 0 { + _ = c.Conn.Close() + return 0, fmt.Errorf("invalid HTTP1 Content-Length") + } + c.bodyRemaining = length + case "transfer-encoding": + c.chunked = strings.Contains(strings.ToLower(value), "chunked") + c.chunkPhase = 's' + case "upgrade": + c.upgrade = strings.EqualFold(value, "websocket") + } + } + if _, err := c.Conn.Write(raw); err != nil { + _ = c.Conn.Close() + return 0, err + } + c.header.Reset() + if c.upgrade && len(p) > 0 { + if _, err := c.Conn.Write(p); err != nil { + return 0, err + } + p = nil + } + } + return len(input), nil +} + +func (c *http1WireConn) consumeChunk(p []byte) (int, error) { + n := 0 + for n < len(p) && c.chunked { + if c.chunkPhase == 'd' { + take := int64(len(p) - n) + if take > c.bodyRemaining { + take = c.bodyRemaining + } + n += int(take) + c.bodyRemaining -= take + if c.bodyRemaining == 0 { + c.chunkPhase = 'r' + } + continue + } + c.chunkLine = append(c.chunkLine, p[n]) + n++ + if len(c.chunkLine) > 1<<20 { + return 0, fmt.Errorf("HTTP1 chunk control line too long") + } + if !bytes.HasSuffix(c.chunkLine, []byte("\r\n")) { + continue + } + line := strings.TrimSuffix(string(c.chunkLine), "\r\n") + c.chunkLine = c.chunkLine[:0] + switch c.chunkPhase { + case 's': + if i := strings.IndexByte(line, ';'); i >= 0 { + line = line[:i] + } + size, err := strconv.ParseInt(line, 16, 64) + if err != nil || size < 0 { + return 0, fmt.Errorf("invalid HTTP1 chunk size") + } + if size == 0 { + c.chunkPhase = 't' + } else { + c.bodyRemaining = size + c.chunkPhase = 'd' + } + case 'r': + if line != "" { + return 0, fmt.Errorf("invalid chunk delimiter") + } + c.chunkPhase = 's' + case 't': + if line == "" { + c.chunked = false + c.chunkPhase = 0 + } + } + } + return n, nil +} diff --git a/backend/internal/pkg/codextransport/http1_test.go b/backend/internal/pkg/codextransport/http1_test.go new file mode 100644 index 000000000000..058e1e693ae3 --- /dev/null +++ b/backend/internal/pkg/codextransport/http1_test.go @@ -0,0 +1,52 @@ +package codextransport + +import ( + "bytes" + "net" + "testing" + "time" +) + +type memoryConn struct{ bytes.Buffer } + +func (*memoryConn) Close() error { return nil } +func (*memoryConn) LocalAddr() net.Addr { return nil } +func (*memoryConn) RemoteAddr() net.Addr { return nil } +func (*memoryConn) SetDeadline(time.Time) error { return nil } +func (*memoryConn) SetReadDeadline(time.Time) error { return nil } +func (*memoryConn) SetWriteDeadline(time.Time) error { return nil } + +func TestHTTP1HeaderSpellingPreservesOpaqueBodiesAndReuse(t *testing.T) { + for _, chunked := range []bool{false, true} { + t.Run(map[bool]string{false: "content-length", true: "chunked"}[chunked], func(t *testing.T) { + body := "Host: should-stay\r\nUser-Agent: body-text" + framing := "Content-Length: 39\r\n" + if chunked { + framing = "Transfer-Encoding: chunked\r\n" + body = "4\r\nHost\r\n6\r\n: body\r\n0\r\nUser-Agent: trailer\r\n\r\n" + } + first := "POST / HTTP/1.1\r\nUser-Agent: test\r\nHost: localhost\r\n" + framing + "\r\n" + body + second := "GET /two HTTP/1.1\r\nHost: localhost\r\n\r\n" + wire := &memoryConn{} + conn := &http1WireConn{Conn: wire, spelling: map[string]string{"host": "host", "user-agent": "user-agent", "content-length": "content-length"}} + input := first + second + for start := 0; start < len(input); { + end := start + 3 + if end > len(input) { + end = len(input) + } + if n, err := conn.Write([]byte(input[start:end])); err != nil || n != end-start { + t.Fatalf("write %d: %d %v", start, n, err) + } + start = end + } + expected := "POST / HTTP/1.1\r\nuser-agent: test\r\nhost: localhost\r\n" + framing + "\r\n" + body + "GET /two HTTP/1.1\r\nhost: localhost\r\n\r\n" + if !chunked { + expected = string(bytes.ReplaceAll([]byte(expected), []byte("Content-Length:"), []byte("content-length:"))) + } + if wire.String() != expected { + t.Fatalf("body/trailer or reused headers changed:\n%q\nwant:\n%q", wire.String(), expected) + } + }) + } +} diff --git a/backend/internal/pkg/codextransport/profile.go b/backend/internal/pkg/codextransport/profile.go new file mode 100644 index 000000000000..b6577116eea7 --- /dev/null +++ b/backend/internal/pkg/codextransport/profile.go @@ -0,0 +1,147 @@ +// Package codextransport implements the verified official Codex transport. +// Application identity, TLS and HTTP settings are selected before transmission; +// unsupported profiles never silently fall back to another transport. +package codextransport + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/json" + "errors" + "fmt" + "regexp" + "strings" +) + +//go:embed reference.json +var referenceJSON []byte + +type hello struct { + CipherSuites []uint16 `json:"cipher_suites"` + Extensions []uint16 `json:"extensions"` + ExtensionData map[string]string `json:"extension_data"` + ALPN []string `json:"alpn"` + Groups []uint16 `json:"groups"` + KeyShareGroups []uint16 `json:"key_share_groups"` + Signatures []uint16 `json:"signature_algorithms"` + Versions []uint16 `json:"versions"` + PSKModes []uint8 `json:"psk_modes"` +} + +type wireProfile struct { + HeaderCase map[string]string `json:"header_case"` + TLS hello `json:"tls"` + HeaderOrder []string `json:"header_order"` + PseudoHeaderOrder []string `json:"pseudo_header_order"` + Settings []struct { + ID uint16 `json:"id"` + Value uint32 `json:"value"` + } `json:"settings"` + ConnectionFlow uint32 `json:"connection_flow"` +} + +type reference struct { + Version string `json:"version"` + SourceCommit string `json:"source_commit"` + Builds []struct { + System string `json:"system"` + Machine string `json:"machine"` + BinarySHA256 string `json:"binary_sha256"` + } `json:"builds"` + Profiles map[string]wireProfile `json:"profiles"` +} + +var verified = func() reference { + var value reference + if err := json.Unmarshal(referenceJSON, &value); err != nil { + panic(err) + } + return value +}() + +func LatestVersion() string { return verified.Version } +func SupportsVersion(version string) bool { return version == verified.Version } + +type Error struct{ Reason string } + +func (e *Error) Error() string { return "Codex strict transport: " + e.Reason } + +// Failure marks an uncertain transport outcome, including errors after a body +// may have been written. Protocol retry code must not turn it into an HTTP replay. +type Failure struct{ Err error } + +func (e *Failure) Error() string { return "Codex strict transport failed: " + e.Err.Error() } +func (e *Failure) Unwrap() error { return e.Err } +func IsFailure(err error) bool { var failure *Failure; return errors.As(err, &failure) } + +type Binding struct { + ReferenceConfiguration string `json:"reference_configuration"` + RequestZstd bool `json:"-"` + ProfileID string `json:"profile_id"` + Version string `json:"version"` + SourceCommit string `json:"source_commit"` + System string `json:"system"` + Arch string `json:"arch"` + Strict bool `json:"strict"` + Validated bool `json:"validated"` + Scope string `json:"-"` + Plane string `json:"plane"` +} + +var userAgent = regexp.MustCompile(`^[^/]+/([0-9A-Za-z.\-]+) \((Mac OS|Windows|Ubuntu) [^;]+; (arm64|x86_64)\)`) + +func Select(ua, scope, plane string) (Binding, error) { + m := userAgent.FindStringSubmatch(strings.TrimSpace(ua)) + if len(m) == 0 { + return Binding{}, &Error{"User-Agent has no supported fixed device profile"} + } + if !SupportsVersion(m[1]) { + return Binding{}, &Error{fmt.Sprintf("version %s has no verified profile", m[1])} + } + system := map[string]string{"Mac OS": "Darwin", "Windows": "Windows", "Ubuntu": "Linux"}[m[2]] + arch := m[3] + matched := false + for _, build := range verified.Builds { + machine := strings.ToLower(build.Machine) + if machine == "amd64" { + machine = "x86_64" + } + if machine == "aarch64" { + machine = "arm64" + } + if build.System == system && machine == arch { + matched = true + break + } + } + if !matched { + return Binding{}, &Error{fmt.Sprintf("%s/%s has no verified official sample", system, arch)} + } + if plane != "ws" { + plane = "http" + } + return Binding{ProfileID: "codex-rust-" + verified.Version + "-r1", Version: verified.Version, SourceCommit: verified.SourceCommit, + System: system, Arch: arch, Strict: true, Validated: true, Scope: scope, Plane: plane, ReferenceConfiguration: "official binary with isolated custom CA"}, nil +} + +func (b Binding) Key() string { + sum := sha256.Sum256([]byte(b.ProfileID + "\x00" + b.System + "\x00" + b.Arch + "\x00" + b.Scope + "\x00" + b.Plane)) + return fmt.Sprintf("%x", sum[:16]) +} + +type contextKey struct{} + +func WithBinding(ctx context.Context, b Binding) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, contextKey{}, b) +} +func FromContext(ctx context.Context) (Binding, bool) { + if ctx == nil { + return Binding{}, false + } + b, ok := ctx.Value(contextKey{}).(Binding) + return b, ok +} diff --git a/backend/internal/pkg/codextransport/proxy_test.go b/backend/internal/pkg/codextransport/proxy_test.go new file mode 100644 index 000000000000..3d1d26d8f87b --- /dev/null +++ b/backend/internal/pkg/codextransport/proxy_test.go @@ -0,0 +1,236 @@ +package codextransport + +import ( + "bufio" + "context" + "crypto/x509" + "encoding/binary" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +func relayFixture(a, b net.Conn) { + done := make(chan struct{}) + go func() { _, _ = io.Copy(a, b); _ = a.Close(); _ = b.Close(); close(done) }() + _, _ = io.Copy(b, a) + _ = a.Close() + _ = b.Close() + <-done +} + +func TestStrictCodexProxyVariantsPreserveAuthenticationAndStreaming(t *testing.T) { + var posts atomic.Int32 + target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + posts.Add(1) + if r.Header.Get("Proxy-Authorization") != "" { + t.Error("proxy credential reached target") + } + data, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + _, _ = w.Write(data) + })) + defer target.Close() + targetURL, _ := url.Parse(target.URL) + for _, kind := range []string{"http", "https", "socks5"} { + t.Run(kind, func(t *testing.T) { + roots := x509.NewCertPool() + roots.AddCert(target.Certificate()) + var proxyURL *url.URL + if kind == "socks5" { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go func() { + client, err := listener.Accept() + if err != nil { + return + } + defer client.Close() + _ = client.SetDeadline(time.Now().Add(5 * time.Second)) + head := make([]byte, 2) + if _, err = io.ReadFull(client, head); err != nil { + return + } + methods := make([]byte, int(head[1])) + if _, err = io.ReadFull(client, methods); err != nil { + return + } + _, _ = client.Write([]byte{5, 2}) + if _, err = io.ReadFull(client, head); err != nil { + return + } + user := make([]byte, int(head[1])) + _, _ = io.ReadFull(client, user) + one := make([]byte, 1) + _, _ = io.ReadFull(client, one) + password := make([]byte, int(one[0])) + _, _ = io.ReadFull(client, password) + if string(user) != "fixture" || string(password) != "pass" { + t.Error("SOCKS authentication mismatch") + return + } + _, _ = client.Write([]byte{1, 0}) + request := make([]byte, 4) + if _, err = io.ReadFull(client, request); err != nil { + return + } + size := 4 + if request[3] == 3 { + _, _ = io.ReadFull(client, one) + size = int(one[0]) + } else if request[3] == 4 { + size = 16 + } + address := make([]byte, size+2) + if _, err = io.ReadFull(client, address); err != nil { + return + } + _, expectedPort, _ := net.SplitHostPort(targetURL.Host) + port, _ := strconv.Atoi(expectedPort) + if int(binary.BigEndian.Uint16(address[size:])) != port { + t.Error("SOCKS target mismatch") + return + } + upstream, err := net.DialTimeout("tcp", targetURL.Host, time.Second) + if err != nil { + return + } + _, _ = client.Write([]byte{5, 0, 0, 1, 127, 0, 0, 1, 0, 0}) + _ = client.SetDeadline(time.Time{}) + relayFixture(client, upstream) + }() + proxyURL, _ = url.Parse("socks5://fixture:pass@" + listener.Addr().String()) + } else { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect || r.Host != targetURL.Host { + http.Error(w, "invalid fixture tunnel", 400) + return + } + if r.Header.Get("Proxy-Authorization") == "" { + t.Error("missing proxy authentication") + } + upstream, err := net.DialTimeout("tcp", targetURL.Host, time.Second) + if err != nil { + http.Error(w, "tunnel failed", 502) + return + } + client, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + _ = upstream.Close() + return + } + _, _ = buffer.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n") + _ = buffer.Flush() + relayFixture(client, upstream) + }) + var proxy *httptest.Server + if kind == "https" { + proxy = httptest.NewTLSServer(handler) + roots.AddCert(proxy.Certificate()) + } else { + proxy = httptest.NewServer(handler) + } + defer proxy.Close() + proxyURL, _ = url.Parse(proxy.URL) + proxyURL.User = url.UserPassword("fixture", "pass") + } + rt, err := New(testBinding(t), Options{ProxyURL: proxyURL, RootCAs: roots, MaxIdleConns: 2, MaxIdleConnsPerHost: 2, IdleConnTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer rt.CloseIdleConnections() + body := strings.Repeat("opaque proxy body ", 2048) + request, _ := http.NewRequest(http.MethodPost, target.URL, strings.NewReader(body)) + response, err := (&http.Client{Transport: rt, Timeout: 5 * time.Second}).Do(request) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || string(data) != body { + t.Fatalf("proxy body changed: %v", err) + } + }) + } + if posts.Load() != 3 { + t.Fatalf("unexpected physical POST count: %d", posts.Load()) + } +} + +func TestStrictCodexDoesNotReplaySentPOSTOrCanceledCONNECT(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + posts.Add(1) + _, _ = io.Copy(io.Discard, r.Body) + conn, _, err := w.(http.Hijacker).Hijack() + if err == nil { + _ = conn.Close() + } + })) + defer server.Close() + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + rt, err := New(testBinding(t), Options{RootCAs: roots}) + if err != nil { + t.Fatal(err) + } + defer rt.CloseIdleConnections() + request, _ := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("body already delivered")) + request.Header.Set("Idempotency-Key", "fixture-id") + if response, err := (&http.Client{Transport: rt, Timeout: time.Second}).Do(request); err == nil { + _ = response.Body.Close() + t.Fatal("expected an uncertain response failure") + } else if !IsFailure(err) { + t.Fatalf("failure lost its no-replay classification: %v", err) + } + if posts.Load() != 1 { + t.Fatalf("sent POST was replayed %d times", posts.Load()) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = http.ReadRequest(bufio.NewReader(conn)) + _, _ = io.Copy(io.Discard, conn) + }() + proxyURL, _ := url.Parse("http://" + listener.Addr().String()) + blocked, err := New(testBinding(t), Options{ProxyURL: proxyURL, RootCAs: roots}) + if err != nil { + t.Fatal(err) + } + defer blocked.CloseIdleConnections() + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + request, _ = http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader("never sent")) + start := time.Now() + if response, err := (&http.Client{Transport: blocked}).Do(request); err == nil { + _ = response.Body.Close() + t.Fatal("canceled CONNECT succeeded") + } + if time.Since(start) > time.Second { + t.Fatal("CONNECT ignored cancellation") + } + if posts.Load() != 1 { + t.Fatal("canceled tunnel sent an upstream request") + } +} diff --git a/backend/internal/pkg/codextransport/reference.json b/backend/internal/pkg/codextransport/reference.json new file mode 100644 index 000000000000..a30da6f1802e --- /dev/null +++ b/backend/internal/pkg/codextransport/reference.json @@ -0,0 +1,407 @@ +{ + "version": "0.162.1", + "source_commit": "092d3acd6bec3e3a14bdc7e7a2810ab628ab759d", + "trust_mode": "isolated CODEX_CA_CERTIFICATE", + "builds": [ + { + "system": "Darwin", + "machine": "arm64", + "release": "24.6.0", + "binary_sha256": "74c6a6d263f40d65f99c27a815275f5b4ed0bcc2baecc396b4cf3255498bdfc0", + "capture_sha256": "1305a24b6bf1f4ae581e8758a45ad0458b72e68b96698ad3248d03d2a5a53063", + "run": 38678469179 + }, + { + "system": "Darwin", + "machine": "x86_64", + "release": "24.6.0", + "binary_sha256": "f86b8cfd3de94dc70763d251da3ff292e94f4a53ad41b81faab3619a7bcb626c", + "capture_sha256": "9a277bb573e779c7e56b648f1d2557e79d03661a7430a5039787c0f4240377f6", + "run": 38678469179 + }, + { + "system": "Windows", + "machine": "AMD64", + "release": "2025Server", + "binary_sha256": "dd13bdb162a184fa04039488b234a8258ef9b76f84e926c505d5742b19249a6a", + "capture_sha256": "ee28fc637855987c22cef3f66a39e40c61b1aef1112bfb0b6d5271eac34b9e9a", + "run": 38678469179 + }, + { + "system": "Linux", + "machine": "x86_64", + "release": "6.17.0-1022-azure", + "binary_sha256": "628e35487d888a28f0427fc37501da529dd3d81fcbad7a2c3421c1e8670d92e1", + "capture_sha256": "fac35060f0b97a3b2fd8b98a0c7e753f2ffd2784dc0b36545e139ab90d374766", + "run": 38678469179 + } + ], + "profiles": { + "http": { + "tls": { + "legacy_version": 771, + "cipher_suites": [ + 4866, + 4865, + 4867, + 49196, + 49195, + 52393, + 49200, + 49199, + 52392, + 255 + ], + "compression_methods": [ + 0 + ], + "session_id_length": 32, + "extensions": [ + 23, + 0, + 45, + 11, + 5, + 35, + 13, + 16, + 51, + 43, + 10 + ], + "extension_data": { + "23": "", + "45": "0101", + "11": "0100", + "5": "0100000000", + "13": "001a0503040306030807080608050804060105010401090409050906", + "16": "000c02683208687474702f312e31", + "43": "0403040303", + "10": "000811ec001d00170018" + }, + "alpn": [ + "h2", + "http/1.1" + ], + "groups": [ + 4588, + 29, + 23, + 24 + ], + "key_share_groups": [ + 4588, + 29 + ], + "signature_algorithms": [ + 1283, + 1027, + 1539, + 2055, + 2054, + 2053, + 2052, + 1537, + 1281, + 1025, + 2308, + 2309, + 2310 + ], + "versions": [ + 772, + 771 + ], + "psk_modes": [ + 1 + ] + }, + "header_order": [ + "x-codex-beta-features", + "x-codex-window-id", + "x-codex-turn-metadata", + "x-client-request-id", + "session-id", + "thread-id", + "accept", + "content-type", + "authorization", + "originator", + "user-agent", + "content-length" + ], + "header_case": { + "x-codex-beta-features": "x-codex-beta-features", + "x-codex-window-id": "x-codex-window-id", + "x-codex-turn-metadata": "x-codex-turn-metadata", + "x-client-request-id": "x-client-request-id", + "session-id": "session-id", + "thread-id": "thread-id", + "accept": "accept", + "content-type": "content-type", + "authorization": "authorization", + "originator": "originator", + "user-agent": "user-agent", + "content-length": "content-length" + }, + "pseudo_header_order": [ + ":method", + ":scheme", + ":authority", + ":path" + ], + "settings": [ + { + "id": 2, + "value": 0 + }, + { + "id": 4, + "value": 2097152 + }, + { + "id": 5, + "value": 16384 + }, + { + "id": 6, + "value": 16384 + } + ], + "connection_flow": 5177345, + "extension_order": "randomized" + }, + "ws": { + "tls": { + "legacy_version": 771, + "cipher_suites": [ + 4866, + 4865, + 4867, + 49196, + 49195, + 52393, + 49200, + 49199, + 52392, + 255 + ], + "compression_methods": [ + 0 + ], + "session_id_length": 32, + "extensions": [ + 35, + 23, + 0, + 13, + 5, + 11, + 51, + 43, + 45, + 10 + ], + "extension_data": { + "23": "", + "13": "001a0503040306030807080608050804060105010401090409050906", + "5": "0100000000", + "11": "0100", + "43": "0403040303", + "45": "0101", + "10": "000811ec001d00170018" + }, + "alpn": [], + "groups": [ + 4588, + 29, + 23, + 24 + ], + "key_share_groups": [ + 4588, + 29 + ], + "signature_algorithms": [ + 1283, + 1027, + 1539, + 2055, + 2054, + 2053, + 2052, + 1537, + 1281, + 1025, + 2308, + 2309, + 2310 + ], + "versions": [ + 772, + 771 + ], + "psk_modes": [ + 1 + ] + }, + "header_order": [ + "host", + "connection", + "upgrade", + "sec-websocket-version", + "sec-websocket-key", + "authorization", + "user-agent", + "originator", + "openai-beta", + "x-codex-turn-metadata", + "x-codex-beta-features", + "x-client-request-id", + "session-id", + "thread-id", + "x-codex-window-id", + "sec-websocket-extensions" + ], + "header_case": { + "host": "Host", + "connection": "Connection", + "upgrade": "Upgrade", + "sec-websocket-version": "Sec-WebSocket-Version", + "sec-websocket-key": "Sec-WebSocket-Key", + "authorization": "authorization", + "user-agent": "user-agent", + "originator": "originator", + "openai-beta": "openai-beta", + "x-codex-turn-metadata": "x-codex-turn-metadata", + "x-codex-beta-features": "x-codex-beta-features", + "x-client-request-id": "x-client-request-id", + "session-id": "session-id", + "thread-id": "thread-id", + "x-codex-window-id": "x-codex-window-id", + "sec-websocket-extensions": "sec-websocket-extensions" + }, + "pseudo_header_order": [], + "settings": [], + "connection_flow": 0, + "extension_order": "randomized" + }, + "http1": { + "tls": { + "legacy_version": 771, + "cipher_suites": [ + 4866, + 4865, + 4867, + 49196, + 49195, + 52393, + 49200, + 49199, + 52392, + 255 + ], + "compression_methods": [ + 0 + ], + "session_id_length": 32, + "extensions": [ + 10, + 0, + 13, + 16, + 11, + 5, + 35, + 23, + 45, + 43, + 51 + ], + "extension_data": { + "10": "000811ec001d00170018", + "13": "001a0503040306030807080608050804060105010401090409050906", + "16": "000c02683208687474702f312e31", + "11": "0100", + "5": "0100000000", + "23": "", + "45": "0101", + "43": "0403040303" + }, + "alpn": [ + "h2", + "http/1.1" + ], + "groups": [ + 4588, + 29, + 23, + 24 + ], + "key_share_groups": [ + 4588, + 29 + ], + "signature_algorithms": [ + 1283, + 1027, + 1539, + 2055, + 2054, + 2053, + 2052, + 1537, + 1281, + 1025, + 2308, + 2309, + 2310 + ], + "versions": [ + 772, + 771 + ], + "psk_modes": [ + 1 + ] + }, + "header_order": [ + "x-codex-beta-features", + "x-codex-window-id", + "x-codex-turn-metadata", + "x-client-request-id", + "session-id", + "thread-id", + "accept", + "content-type", + "authorization", + "originator", + "user-agent", + "host", + "content-length" + ], + "header_case": { + "x-codex-beta-features": "x-codex-beta-features", + "x-codex-window-id": "x-codex-window-id", + "x-codex-turn-metadata": "x-codex-turn-metadata", + "x-client-request-id": "x-client-request-id", + "session-id": "session-id", + "thread-id": "thread-id", + "accept": "accept", + "content-type": "content-type", + "authorization": "authorization", + "originator": "originator", + "user-agent": "user-agent", + "host": "host", + "content-length": "content-length" + }, + "pseudo_header_order": [], + "settings": [], + "connection_flow": 0, + "extension_order": "randomized" + } + }, + "real_model_requests": 0, + "resumption_observed": false, + "resumption_policy": "fresh handshake; official reconnect captures did not resume TLS sessions" +} diff --git a/backend/internal/pkg/codextransport/transport.go b/backend/internal/pkg/codextransport/transport.go new file mode 100644 index 000000000000..ee57bfad3e74 --- /dev/null +++ b/backend/internal/pkg/codextransport/transport.go @@ -0,0 +1,204 @@ +package codextransport + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/imroc/req/v3" + reqhttp2 "github.com/imroc/req/v3/http2" + utls "github.com/refraction-networking/utls" +) + +type Options struct { + ProxyURL *url.URL + RootCAs *x509.CertPool + MaxIdleConns int + MaxIdleConnsPerHost int + MaxConnsPerHost int + IdleConnTimeout time.Duration + ResponseHeaderTimeout time.Duration + DialContext func(context.Context, string, string) (net.Conn, error) +} + +// New returns a streaming RoundTripper. It does not use req's high-level retry +// or response-decoding layer. TLS state is private to this binding and pool. +func New(binding Binding, opts Options) (*req.Transport, error) { + if !binding.Validated || !SupportsVersion(binding.Version) { + return nil, &Error{"profile has not been verified"} + } + profile, ok := verified.Profiles[binding.Plane] + if !ok { + return nil, &Error{"missing protocol profile"} + } + client := req.C().DisableCompression().DisableAutoDecode().SetTimeout(0) + order := profile.HeaderOrder + if binding.Plane == "http" { + if h1, ok := verified.Profiles["http1"]; ok { + order = h1.HeaderOrder + } + } + client.SetCommonHeaderOrder(order...) + if len(profile.PseudoHeaderOrder) > 0 { + client.SetCommonPseudoHeaderOder(profile.PseudoHeaderOrder...) + } + settings := make([]reqhttp2.Setting, 0, len(profile.Settings)) + for _, s := range profile.Settings { + settings = append(settings, reqhttp2.Setting{ID: reqhttp2.SettingID(s.ID), Val: s.Value}) + } + if len(settings) > 0 { + client.SetHTTP2SettingsFrame(settings...).SetHTTP2ConnectionFlow(profile.ConnectionFlow).SetHTTP2MaxHeaderListSize(16384) + } + transport := client.GetTransport() + transport.Proxy = nil + if opts.ProxyURL != nil { + transport.Proxy = http.ProxyURL(opts.ProxyURL) + } + transport.MaxIdleConns = opts.MaxIdleConns + transport.MaxIdleConnsPerHost = opts.MaxIdleConnsPerHost + transport.MaxConnsPerHost = opts.MaxConnsPerHost + transport.IdleConnTimeout = opts.IdleConnTimeout + transport.ResponseHeaderTimeout = opts.ResponseHeaderTimeout + transport.TLSHandshakeTimeout = 10 * time.Second + transport.DisableCompression = true + transport.DialContext = opts.DialContext + if transport.DialContext == nil { + transport.DialContext = (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext + } + if opts.ProxyURL != nil { + dial := transport.DialContext + transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + if err != nil { + return nil, err + } + if err = conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + _ = conn.Close() + return nil, err + } + return conn, nil + } + } + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: opts.RootCAs} + transport.SetTLSHandshake(func(ctx context.Context, addr string, plain net.Conn) (net.Conn, *tls.ConnectionState, error) { + host := addr + if parsed, _, err := net.SplitHostPort(addr); err == nil { + host = parsed + } + host = strings.Trim(host, "[]") + var err error + config := &utls.Config{ServerName: host, RootCAs: opts.RootCAs, MinVersion: utls.VersionTLS12, MaxVersion: utls.VersionTLS13} + conn := utls.UClient(plain, config, utls.HelloCustom) + if err = conn.ApplyPreset(clientHello(profile.TLS)); err != nil { + _ = plain.Close() + return nil, nil, fmt.Errorf("apply Codex TLS profile: %w", err) + } + handshake, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if err = conn.HandshakeContext(handshake); err != nil { + _ = plain.Close() + return nil, nil, fmt.Errorf("Codex TLS handshake: %w", err) + } + if err = conn.SetDeadline(time.Time{}); err != nil { + _ = conn.Close() + return nil, nil, err + } + cs := conn.ConnectionState() + state := &tls.ConnectionState{Version: cs.Version, HandshakeComplete: cs.HandshakeComplete, DidResume: cs.DidResume, + CipherSuite: cs.CipherSuite, NegotiatedProtocol: cs.NegotiatedProtocol, NegotiatedProtocolIsMutual: cs.NegotiatedProtocolIsMutual, + ServerName: cs.ServerName, PeerCertificates: cs.PeerCertificates, VerifiedChains: cs.VerifiedChains, + SignedCertificateTimestamps: cs.SignedCertificateTimestamps, OCSPResponse: cs.OCSPResponse} + verifiedConn := &verifiedTLSConn{UConn: conn, state: *state} + if cs.NegotiatedProtocol != "h2" { + spelling := profile.HeaderCase + if binding.Plane == "http" { + spelling = verified.Profiles["http1"].HeaderCase + } + return &http1WireConn{Conn: verifiedConn, spelling: spelling}, state, nil + } + return verifiedConn, state, nil + }) + // Own the request copy: internal ordering keys and transport retry decisions + // must not mutate the caller's frozen headers or replay a posted JSON body. + transport.WrapRoundTripFunc(func(next http.RoundTripper) req.HttpRoundTripFunc { + return func(request *http.Request) (*http.Response, error) { + wire := request.Clone(request.Context()) + wire.GetBody = nil + response, err := next.RoundTrip(wire) + if err != nil { + return response, &Failure{Err: err} + } + return response, nil + } + }) + return transport, nil +} + +func clientHello(h hello) *utls.ClientHelloSpec { + extensions := make([]utls.TLSExtension, 0, len(h.Extensions)) + for _, kind := range h.Extensions { + var ext utls.TLSExtension + switch kind { + case 0: + ext = &utls.SNIExtension{} + case 5: + ext = &utls.StatusRequestExtension{} + case 10: + groups := make([]utls.CurveID, len(h.Groups)) + for i, g := range h.Groups { + groups[i] = utls.CurveID(g) + } + ext = &utls.SupportedCurvesExtension{Curves: groups} + case 11: + ext = &utls.SupportedPointsExtension{SupportedPoints: []uint8{0}} + case 13: + signatures := make([]utls.SignatureScheme, len(h.Signatures)) + for i, s := range h.Signatures { + signatures[i] = utls.SignatureScheme(s) + } + ext = &utls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: signatures} + case 16: + ext = &utls.ALPNExtension{AlpnProtocols: append([]string(nil), h.ALPN...)} + case 23: + ext = &utls.ExtendedMasterSecretExtension{} + case 35: + ext = &utls.SessionTicketExtension{} + case 43: + ext = &utls.SupportedVersionsExtension{Versions: append([]uint16(nil), h.Versions...)} + case 45: + ext = &utls.PSKKeyExchangeModesExtension{Modes: append([]uint8(nil), h.PSKModes...)} + case 51: + shares := make([]utls.KeyShare, len(h.KeyShareGroups)) + for i, g := range h.KeyShareGroups { + shares[i] = utls.KeyShare{Group: utls.CurveID(g)} + } + ext = &utls.KeyShareExtension{KeyShares: shares} + default: + data, err := hex.DecodeString(h.ExtensionData[fmt.Sprint(kind)]) + if err != nil { + panic("invalid embedded Codex extension") + } + ext = &utls.GenericExtension{Id: kind, Data: data} + } + extensions = append(extensions, ext) + } + // Official captures permute their extension order on fresh handshakes. + extensions = utls.ShuffleChromeTLSExtensions(extensions) + return &utls.ClientHelloSpec{TLSVersMin: utls.VersionTLS12, TLSVersMax: utls.VersionTLS13, + CipherSuites: append([]uint16(nil), h.CipherSuites...), CompressionMethods: []uint8{0}, Extensions: extensions} +} + +// req/h2 requires the standard ConnectionState signature on custom TLS connections. +type verifiedTLSConn struct { + *utls.UConn + state tls.ConnectionState +} + +func (c *verifiedTLSConn) ConnectionState() tls.ConnectionState { return c.state } diff --git a/backend/internal/pkg/codextransport/transport_test.go b/backend/internal/pkg/codextransport/transport_test.go new file mode 100644 index 000000000000..2474bc0c0d8a --- /dev/null +++ b/backend/internal/pkg/codextransport/transport_test.go @@ -0,0 +1,110 @@ +package codextransport + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func testBinding(t *testing.T) Binding { + t.Helper() + binding, err := Select("codex-tui/0.162.1 (Windows 10.0.26100; x86_64) WindowsTerminal", "test-account", "http") + if err != nil { + t.Fatal(err) + } + return binding +} + +func TestStrictCodexTLSValidationAndHTTP2Streaming(t *testing.T) { + var calls atomic.Int32 + data := bytes.Repeat([]byte("streaming-window"), 200000) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.ProtoMajor != 2 { + t.Errorf("expected H2, got %s", r.Proto) + } + if r.Header.Get("Accept-Encoding") != "" { + t.Errorf("unsolicited encoding %s", r.Header.Get("Accept-Encoding")) + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(data) + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + for _, trusted := range []bool{false, true} { + pool := x509.NewCertPool() + if trusted { + pool = roots + } + rt, err := New(testBinding(t), Options{RootCAs: pool, MaxIdleConns: 2, MaxIdleConnsPerHost: 2, IdleConnTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Transport: rt, Timeout: 5 * time.Second} + request, _ := http.NewRequest(http.MethodPost, server.URL, strings.NewReader(`{"input":[]}`)) + resp, err := client.Do(request) + if !trusted { + if err == nil { + _ = resp.Body.Close() + t.Fatal("untrusted certificate accepted") + } + if calls.Load() != 0 { + t.Fatal("posted a body after certificate failure") + } + } else { + if err != nil { + t.Fatal(err) + } + got, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil || !bytes.Equal(got, data) { + t.Fatalf("flow control/streaming: length=%d error=%v", len(got), readErr) + } + if resp.TLS == nil || resp.TLS.Version != tls.VersionTLS13 { + t.Fatal("missing real TLS observation") + } + } + rt.CloseIdleConnections() + } + if calls.Load() != 1 { + t.Fatalf("physical POSTs: %d", calls.Load()) + } +} + +func TestStrictCodexUnsupportedAndCanceledRequests(t *testing.T) { + if _, err := Select("codex-tui/999.0.0 (Windows 10.0.26100; x86_64) WindowsTerminal", "test", "http"); err == nil { + t.Fatal("accepted unverified version") + } + if _, err := New(Binding{Version: LatestVersion()}, Options{}); err == nil { + t.Fatal("accepted unverified binding") + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) })) + defer server.Close() + rt, err := New(testBinding(t), Options{}) + if err != nil { + t.Fatal(err) + } + defer rt.CloseIdleConnections() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + request, _ := http.NewRequestWithContext(ctx, http.MethodPost, server.URL, strings.NewReader("one logical post")) + if response, err := (&http.Client{Transport: rt}).Do(request); err == nil { + _ = response.Body.Close() + t.Fatal("canceled request sent") + } + if calls.Load() != 0 { + t.Fatal("sent a canceled POST") + } +} diff --git a/backend/internal/pkg/codexws/LICENSE.txt b/backend/internal/pkg/codexws/LICENSE.txt new file mode 100644 index 000000000000..7e79329fc1e2 --- /dev/null +++ b/backend/internal/pkg/codexws/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright (c) 2025 Coder + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/backend/internal/pkg/codexws/UPSTREAM.md b/backend/internal/pkg/codexws/UPSTREAM.md new file mode 100644 index 000000000000..e178f88724f7 --- /dev/null +++ b/backend/internal/pkg/codexws/UPSTREAM.md @@ -0,0 +1,8 @@ +Codex-only adapter of github.com/coder/websocket v1.8.14, copied from the locked Go module. + +Original license: LICENSE.txt. Other providers retain the original dependency. + +Local changes: opt-in client_max_window_bits offer and validation; bounded-window +compression with the already locked klauspost/compress implementation; narrow +window writers never enter the default 32 KiB writer pool. No protocol parser +or connection-lifecycle behavior is otherwise changed. diff --git a/backend/internal/pkg/codexws/accept.go b/backend/internal/pkg/codexws/accept.go new file mode 100644 index 000000000000..bea93fc2b5a5 --- /dev/null +++ b/backend/internal/pkg/codexws/accept.go @@ -0,0 +1,378 @@ +//go:build !js + +package websocket + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/textproto" + "net/url" + "path" + "strings" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" +) + +// AcceptOptions represents Accept's options. +type AcceptOptions struct { + // Subprotocols lists the WebSocket subprotocols that Accept will negotiate with the client. + // The empty subprotocol will always be negotiated as per RFC 6455. If you would like to + // reject it, close the connection when c.Subprotocol() == "". + Subprotocols []string + + // InsecureSkipVerify is used to disable Accept's origin verification behaviour. + // + // You probably want to use OriginPatterns instead. + InsecureSkipVerify bool + + // OriginPatterns lists the host patterns for authorized origins. + // The request host is always authorized. + // Use this to enable cross origin WebSockets. + // + // i.e javascript running on example.com wants to access a WebSocket server at chat.example.com. + // In such a case, example.com is the origin and chat.example.com is the request host. + // One would set this field to []string{"example.com"} to authorize example.com to connect. + // + // Each pattern is matched case insensitively with path.Match (see + // https://golang.org/pkg/path/#Match). By default, it is matched + // against the request origin host. If the pattern contains a URI + // scheme ("://"), it will be matched against "scheme://host". + // + // Please ensure you understand the ramifications of enabling this. + // If used incorrectly your WebSocket server will be open to CSRF attacks. + // + // Do not use * as a pattern to allow any origin, prefer to use InsecureSkipVerify instead + // to bring attention to the danger of such a setting. + OriginPatterns []string + + // CompressionMode controls the compression mode. + // Defaults to CompressionDisabled. + // + // See docs on CompressionMode for details. + CompressionMode CompressionMode + + // CompressionThreshold controls the minimum size of a message before compression is applied. + // + // Defaults to 512 bytes for CompressionNoContextTakeover and 128 bytes + // for CompressionContextTakeover. + CompressionThreshold int + + // OnPingReceived is an optional callback invoked synchronously when a ping frame is received. + // + // The payload contains the application data of the ping frame. + // If the callback returns false, the subsequent pong frame will not be sent. + // To avoid blocking, any expensive processing should be performed asynchronously using a goroutine. + OnPingReceived func(ctx context.Context, payload []byte) bool + + // OnPongReceived is an optional callback invoked synchronously when a pong frame is received. + // + // The payload contains the application data of the pong frame. + // To avoid blocking, any expensive processing should be performed asynchronously using a goroutine. + // + // Unlike OnPingReceived, this callback does not return a value because a pong frame + // is a response to a ping and does not trigger any further frame transmission. + OnPongReceived func(ctx context.Context, payload []byte) +} + +func (opts *AcceptOptions) cloneWithDefaults() *AcceptOptions { + var o AcceptOptions + if opts != nil { + o = *opts + } + return &o +} + +// Accept accepts a WebSocket handshake from a client and upgrades the +// the connection to a WebSocket. +// +// Accept will not allow cross origin requests by default. +// See the InsecureSkipVerify and OriginPatterns options to allow cross origin requests. +// +// Accept will write a response to w on all errors. +// +// Note that using the http.Request Context after Accept returns may lead to +// unexpected behavior (see http.Hijacker). +func Accept(w http.ResponseWriter, r *http.Request, opts *AcceptOptions) (*Conn, error) { + return accept(w, r, opts) +} + +func accept(w http.ResponseWriter, r *http.Request, opts *AcceptOptions) (_ *Conn, err error) { + defer errd.Wrap(&err, "failed to accept WebSocket connection") + + errCode, err := verifyClientRequest(w, r) + if err != nil { + http.Error(w, err.Error(), errCode) + return nil, err + } + + opts = opts.cloneWithDefaults() + if !opts.InsecureSkipVerify { + err = authenticateOrigin(r, opts.OriginPatterns) + if err != nil { + if errors.Is(err, path.ErrBadPattern) { + log.Printf("websocket: %v", err) + err = errors.New(http.StatusText(http.StatusForbidden)) + } + http.Error(w, err.Error(), http.StatusForbidden) + return nil, err + } + } + + hj, ok := hijacker(w) + if !ok { + err = errors.New("http.ResponseWriter does not implement http.Hijacker") + http.Error(w, http.StatusText(http.StatusNotImplemented), http.StatusNotImplemented) + return nil, err + } + + w.Header().Set("Upgrade", "websocket") + w.Header().Set("Connection", "Upgrade") + + key := r.Header.Get("Sec-WebSocket-Key") + w.Header().Set("Sec-WebSocket-Accept", secWebSocketAccept(key)) + + subproto := selectSubprotocol(r, opts.Subprotocols) + if subproto != "" { + w.Header().Set("Sec-WebSocket-Protocol", subproto) + } + + copts, ok := selectDeflate(websocketExtensions(r.Header), opts.CompressionMode) + if ok { + w.Header().Set("Sec-WebSocket-Extensions", copts.String()) + } + + w.WriteHeader(http.StatusSwitchingProtocols) + // See https://github.com/nhooyr/websocket/issues/166 + if ginWriter, ok := w.(interface { + WriteHeaderNow() + }); ok { + ginWriter.WriteHeaderNow() + } + + netConn, brw, err := hj.Hijack() + if err != nil { + err = fmt.Errorf("failed to hijack connection: %w", err) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return nil, err + } + + // https://github.com/golang/go/issues/32314 + b, _ := brw.Reader.Peek(brw.Reader.Buffered()) + brw.Reader.Reset(io.MultiReader(bytes.NewReader(b), netConn)) + + return newConn(connConfig{ + subprotocol: w.Header().Get("Sec-WebSocket-Protocol"), + rwc: netConn, + client: false, + copts: copts, + flateThreshold: opts.CompressionThreshold, + onPingReceived: opts.OnPingReceived, + onPongReceived: opts.OnPongReceived, + + br: brw.Reader, + bw: brw.Writer, + }), nil +} + +func verifyClientRequest(w http.ResponseWriter, r *http.Request) (errCode int, _ error) { + if !r.ProtoAtLeast(1, 1) { + return http.StatusUpgradeRequired, fmt.Errorf("WebSocket protocol violation: handshake request must be at least HTTP/1.1: %q", r.Proto) + } + + if !headerContainsTokenIgnoreCase(r.Header, "Connection", "Upgrade") { + w.Header().Set("Connection", "Upgrade") + w.Header().Set("Upgrade", "websocket") + return http.StatusUpgradeRequired, fmt.Errorf("WebSocket protocol violation: Connection header %q does not contain Upgrade", r.Header.Get("Connection")) + } + + if !headerContainsTokenIgnoreCase(r.Header, "Upgrade", "websocket") { + w.Header().Set("Connection", "Upgrade") + w.Header().Set("Upgrade", "websocket") + return http.StatusUpgradeRequired, fmt.Errorf("WebSocket protocol violation: Upgrade header %q does not contain websocket", r.Header.Get("Upgrade")) + } + + if r.Method != "GET" { + return http.StatusMethodNotAllowed, fmt.Errorf("WebSocket protocol violation: handshake request method is not GET but %q", r.Method) + } + + if r.Header.Get("Sec-WebSocket-Version") != "13" { + w.Header().Set("Sec-WebSocket-Version", "13") + return http.StatusBadRequest, fmt.Errorf("unsupported WebSocket protocol version (only 13 is supported): %q", r.Header.Get("Sec-WebSocket-Version")) + } + + websocketSecKeys := r.Header.Values("Sec-WebSocket-Key") + if len(websocketSecKeys) == 0 { + return http.StatusBadRequest, errors.New("WebSocket protocol violation: missing Sec-WebSocket-Key") + } + + if len(websocketSecKeys) > 1 { + return http.StatusBadRequest, errors.New("WebSocket protocol violation: multiple Sec-WebSocket-Key headers") + } + + // The RFC states to remove any leading or trailing whitespace. + websocketSecKey := strings.TrimSpace(websocketSecKeys[0]) + if v, err := base64.StdEncoding.DecodeString(websocketSecKey); err != nil || len(v) != 16 { + return http.StatusBadRequest, fmt.Errorf("WebSocket protocol violation: invalid Sec-WebSocket-Key %q, must be a 16 byte base64 encoded string", websocketSecKey) + } + + return 0, nil +} + +func authenticateOrigin(r *http.Request, originHosts []string) error { + origin := r.Header.Get("Origin") + if origin == "" { + return nil + } + + u, err := url.Parse(origin) + if err != nil { + return fmt.Errorf("failed to parse Origin header %q: %w", origin, err) + } + + if strings.EqualFold(r.Host, u.Host) { + return nil + } + + for _, hostPattern := range originHosts { + target := u.Host + if strings.Contains(hostPattern, "://") { + target = u.Scheme + "://" + u.Host + } + matched, err := match(hostPattern, target) + if err != nil { + return fmt.Errorf("failed to parse path pattern %q: %w", hostPattern, err) + } + if matched { + return nil + } + } + if u.Host == "" { + return fmt.Errorf("request Origin %q is not a valid URL with a host", origin) + } + return fmt.Errorf("request Origin %q is not authorized for Host %q", u.Host, r.Host) +} + +func match(pattern, s string) (bool, error) { + return path.Match(strings.ToLower(pattern), strings.ToLower(s)) +} + +func selectSubprotocol(r *http.Request, subprotocols []string) string { + cps := headerTokens(r.Header, "Sec-WebSocket-Protocol") + for _, sp := range subprotocols { + for _, cp := range cps { + if strings.EqualFold(sp, cp) { + return cp + } + } + } + return "" +} + +func selectDeflate(extensions []websocketExtension, mode CompressionMode) (*compressionOptions, bool) { + if mode == CompressionDisabled { + return nil, false + } + for _, ext := range extensions { + switch ext.name { + // We used to implement x-webkit-deflate-frame too for Safari but Safari has bugs... + // See https://github.com/nhooyr/websocket/issues/218 + case "permessage-deflate": + copts, ok := acceptDeflate(ext, mode) + if ok { + return copts, true + } + } + } + return nil, false +} + +func acceptDeflate(ext websocketExtension, mode CompressionMode) (*compressionOptions, bool) { + copts := mode.opts() + for _, p := range ext.params { + switch p { + case "client_no_context_takeover": + copts.clientNoContextTakeover = true + continue + case "server_no_context_takeover": + copts.serverNoContextTakeover = true + continue + case "client_max_window_bits", + "server_max_window_bits=15": + continue + } + + if strings.HasPrefix(p, "client_max_window_bits=") { + // We can't adjust the deflate window, but decoding with a larger window is acceptable. + continue + } + return nil, false + } + return copts, true +} + +func headerContainsTokenIgnoreCase(h http.Header, key, token string) bool { + for _, t := range headerTokens(h, key) { + if strings.EqualFold(t, token) { + return true + } + } + return false +} + +type websocketExtension struct { + name string + params []string +} + +func websocketExtensions(h http.Header) []websocketExtension { + var exts []websocketExtension + extStrs := headerTokens(h, "Sec-WebSocket-Extensions") + for _, extStr := range extStrs { + if extStr == "" { + continue + } + + vals := strings.Split(extStr, ";") + for i := range vals { + vals[i] = strings.TrimSpace(vals[i]) + } + + e := websocketExtension{ + name: vals[0], + params: vals[1:], + } + + exts = append(exts, e) + } + return exts +} + +func headerTokens(h http.Header, key string) []string { + key = textproto.CanonicalMIMEHeaderKey(key) + var tokens []string + for _, v := range h[key] { + v = strings.TrimSpace(v) + for _, t := range strings.Split(v, ",") { + t = strings.TrimSpace(t) + tokens = append(tokens, t) + } + } + return tokens +} + +var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") + +func secWebSocketAccept(secWebSocketKey string) string { + h := sha1.New() + h.Write([]byte(secWebSocketKey)) + h.Write(keyGUID) + + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} diff --git a/backend/internal/pkg/codexws/close.go b/backend/internal/pkg/codexws/close.go new file mode 100644 index 000000000000..4bff517a8831 --- /dev/null +++ b/backend/internal/pkg/codexws/close.go @@ -0,0 +1,335 @@ +//go:build !js + +package websocket + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "net" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" +) + +// StatusCode represents a WebSocket status code. +// https://tools.ietf.org/html/rfc6455#section-7.4 +type StatusCode int + +// https://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number +// +// These are only the status codes defined by the protocol. +// +// You can define custom codes in the 3000-4999 range. +// The 3000-3999 range is reserved for use by libraries, frameworks and applications. +// The 4000-4999 range is reserved for private use. +const ( + StatusNormalClosure StatusCode = 1000 + StatusGoingAway StatusCode = 1001 + StatusProtocolError StatusCode = 1002 + StatusUnsupportedData StatusCode = 1003 + + // 1004 is reserved and so unexported. + statusReserved StatusCode = 1004 + + // StatusNoStatusRcvd cannot be sent in a close message. + // It is reserved for when a close message is received without + // a status code. + StatusNoStatusRcvd StatusCode = 1005 + + // StatusAbnormalClosure is exported for use only with Wasm. + // In non Wasm Go, the returned error will indicate whether the + // connection was closed abnormally. + StatusAbnormalClosure StatusCode = 1006 + + StatusInvalidFramePayloadData StatusCode = 1007 + StatusPolicyViolation StatusCode = 1008 + StatusMessageTooBig StatusCode = 1009 + StatusMandatoryExtension StatusCode = 1010 + StatusInternalError StatusCode = 1011 + StatusServiceRestart StatusCode = 1012 + StatusTryAgainLater StatusCode = 1013 + StatusBadGateway StatusCode = 1014 + + // StatusTLSHandshake is only exported for use with Wasm. + // In non Wasm Go, the returned error will indicate whether there was + // a TLS handshake failure. + StatusTLSHandshake StatusCode = 1015 +) + +// CloseError is returned when the connection is closed with a status and reason. +// +// Use Go 1.13's errors.As to check for this error. +// Also see the CloseStatus helper. +type CloseError struct { + Code StatusCode + Reason string +} + +func (ce CloseError) Error() string { + return fmt.Sprintf("status = %v and reason = %q", ce.Code, ce.Reason) +} + +// CloseStatus is a convenience wrapper around Go 1.13's errors.As to grab +// the status code from a CloseError. +// +// -1 will be returned if the passed error is nil or not a CloseError. +func CloseStatus(err error) StatusCode { + var ce CloseError + if errors.As(err, &ce) { + return ce.Code + } + return -1 +} + +// Close performs the WebSocket close handshake with the given status code and reason. +// +// It will write a WebSocket close frame with a timeout of 5s and then wait 5s for +// the peer to send a close frame. +// All data messages received from the peer during the close handshake will be discarded. +// +// The connection can only be closed once. Additional calls to Close +// are no-ops. +// +// The maximum length of reason must be 125 bytes. Avoid sending a dynamic reason. +// +// Close will unblock all goroutines interacting with the connection once +// complete. +func (c *Conn) Close(code StatusCode, reason string) (err error) { + defer errd.Wrap(&err, "failed to close WebSocket") + + if c.casClosing() { + err = c.waitGoroutines() + if err != nil { + return err + } + return net.ErrClosed + } + defer func() { + if errors.Is(err, net.ErrClosed) { + err = nil + } + }() + + err = c.closeHandshake(code, reason) + + err2 := c.close() + if err == nil && err2 != nil { + err = err2 + } + + err2 = c.waitGoroutines() + if err == nil && err2 != nil { + err = err2 + } + + return err +} + +// CloseNow closes the WebSocket connection without attempting a close handshake. +// Use when you do not want the overhead of the close handshake. +func (c *Conn) CloseNow() (err error) { + defer errd.Wrap(&err, "failed to immediately close WebSocket") + + if c.casClosing() { + err = c.waitGoroutines() + if err != nil { + return err + } + return net.ErrClosed + } + defer func() { + if errors.Is(err, net.ErrClosed) { + err = nil + } + }() + + err = c.close() + + err2 := c.waitGoroutines() + if err == nil && err2 != nil { + err = err2 + } + return err +} + +func (c *Conn) closeHandshake(code StatusCode, reason string) error { + err := c.writeClose(code, reason) + if err != nil { + return err + } + + err = c.waitCloseHandshake() + if CloseStatus(err) != code { + return err + } + return nil +} + +func (c *Conn) writeClose(code StatusCode, reason string) error { + ce := CloseError{ + Code: code, + Reason: reason, + } + + var p []byte + var err error + if ce.Code != StatusNoStatusRcvd { + p, err = ce.bytes() + if err != nil { + return err + } + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err = c.writeControl(ctx, opClose, p) + // If the connection closed as we're writing we ignore the error as we might + // have written the close frame, the peer responded and then someone else read it + // and closed the connection. + if err != nil && !errors.Is(err, net.ErrClosed) { + return err + } + return nil +} + +func (c *Conn) waitCloseHandshake() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err := c.readMu.lock(ctx) + if err != nil { + return err + } + defer c.readMu.unlock() + + for i := int64(0); i < c.msgReader.payloadLength; i++ { + _, err := c.br.ReadByte() + if err != nil { + return err + } + } + + for { + h, err := c.readLoop(ctx) + if err != nil { + return err + } + + for i := int64(0); i < h.payloadLength; i++ { + _, err := c.br.ReadByte() + if err != nil { + return err + } + } + } +} + +func (c *Conn) waitGoroutines() error { + t := time.NewTimer(time.Second * 15) + defer t.Stop() + + c.closeReadMu.Lock() + closeRead := c.closeReadCtx != nil + c.closeReadMu.Unlock() + if closeRead { + select { + case <-c.closeReadDone: + case <-t.C: + return errors.New("failed to wait for close read goroutine to exit") + } + } + + select { + case <-c.closed: + case <-t.C: + return errors.New("failed to wait for connection to be closed") + } + + return nil +} + +func parseClosePayload(p []byte) (CloseError, error) { + if len(p) == 0 { + return CloseError{ + Code: StatusNoStatusRcvd, + }, nil + } + + if len(p) < 2 { + return CloseError{}, fmt.Errorf("close payload %q too small, cannot even contain the 2 byte status code", p) + } + + ce := CloseError{ + Code: StatusCode(binary.BigEndian.Uint16(p)), + Reason: string(p[2:]), + } + + if !validWireCloseCode(ce.Code) { + return CloseError{}, fmt.Errorf("invalid status code %v", ce.Code) + } + + return ce, nil +} + +// See http://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number +// and https://tools.ietf.org/html/rfc6455#section-7.4.1 +func validWireCloseCode(code StatusCode) bool { + switch code { + case statusReserved, StatusNoStatusRcvd, StatusAbnormalClosure, StatusTLSHandshake: + return false + } + + if code >= StatusNormalClosure && code <= StatusBadGateway { + return true + } + if code >= 3000 && code <= 4999 { + return true + } + + return false +} + +func (ce CloseError) bytes() ([]byte, error) { + p, err := ce.bytesErr() + if err != nil { + err = fmt.Errorf("failed to marshal close frame: %w", err) + ce = CloseError{ + Code: StatusInternalError, + } + p, _ = ce.bytesErr() + } + return p, err +} + +const maxCloseReason = maxControlPayload - 2 + +func (ce CloseError) bytesErr() ([]byte, error) { + if len(ce.Reason) > maxCloseReason { + return nil, fmt.Errorf("reason string max is %v but got %q with length %v", maxCloseReason, ce.Reason, len(ce.Reason)) + } + + if !validWireCloseCode(ce.Code) { + return nil, fmt.Errorf("status code %v cannot be set", ce.Code) + } + + buf := make([]byte, 2+len(ce.Reason)) + binary.BigEndian.PutUint16(buf, uint16(ce.Code)) + copy(buf[2:], ce.Reason) + return buf, nil +} + +func (c *Conn) casClosing() bool { + return c.closing.Swap(true) +} + +func (c *Conn) isClosed() bool { + select { + case <-c.closed: + return true + default: + return false + } +} diff --git a/backend/internal/pkg/codexws/codex_compress_test.go b/backend/internal/pkg/codexws/codex_compress_test.go new file mode 100644 index 000000000000..1a7cb30a948d --- /dev/null +++ b/backend/internal/pkg/codexws/codex_compress_test.go @@ -0,0 +1,53 @@ +package websocket + +import ( + "bytes" + "io" + "net/http" + "strconv" + "testing" + + "github.com/klauspost/compress/flate" +) + +func TestCodexBoundedWindowNegotiationAndCompression(t *testing.T) { + for _, bits := range []int{8, 10, 15} { + t.Run(strconv.Itoa(bits), func(t *testing.T) { + offer := &compressionOptions{offerClientWindow: true} + if offer.String() != "permessage-deflate; client_max_window_bits" { + t.Fatal(offer.String()) + } + options, err := verifyServerExtensions(offer, http.Header{"Sec-Websocket-Extensions": {"permessage-deflate; client_max_window_bits=" + strconv.Itoa(bits)}}) + if err != nil || options.clientWindowBits != bits { + t.Fatalf("negotiation: %+v %v", options, err) + } + input := bytes.Repeat([]byte("bounded Codex websocket window "), 4096) + var wire bytes.Buffer + writer := getFlateWriter(&wire, bits) + if _, err = writer.Write(input); err != nil { + t.Fatal(err) + } + if err = writer.Flush(); err != nil { + t.Fatal(err) + } + // RFC 7692 stream tail, followed by the final empty DEFLATE block. + wire.Write([]byte{1, 0, 0, 255, 255}) + reader := flate.NewReader(bytes.NewReader(wire.Bytes())) + decoded, err := io.ReadAll(reader) + _ = reader.Close() + if err != nil || !bytes.Equal(input, decoded) { + t.Fatalf("compressed round trip: %v", err) + } + putFlateWriter(writer, bits) + }) + } + for _, value := range []string{"7", "16", "invalid"} { + _, err := verifyServerExtensions(&compressionOptions{offerClientWindow: true}, http.Header{"Sec-Websocket-Extensions": {"permessage-deflate; client_max_window_bits=" + value}}) + if err == nil { + t.Fatalf("accepted invalid window %s", value) + } + } + if _, err := verifyServerExtensions(&compressionOptions{}, http.Header{"Sec-Websocket-Extensions": {"permessage-deflate; client_max_window_bits=10"}}); err == nil { + t.Fatal("accepted an unoffered window parameter") + } +} diff --git a/backend/internal/pkg/codexws/compress.go b/backend/internal/pkg/codexws/compress.go new file mode 100644 index 000000000000..29db77f467e1 --- /dev/null +++ b/backend/internal/pkg/codexws/compress.go @@ -0,0 +1,249 @@ +//go:build !js + +package websocket + +import ( + "github.com/klauspost/compress/flate" + "io" + "sync" +) + +// CompressionMode represents the modes available to the permessage-deflate extension. +// See https://tools.ietf.org/html/rfc7692 +// +// Works in all modern browsers except Safari which does not implement the permessage-deflate extension. +// +// Compression is only used if the peer supports the mode selected. +type CompressionMode int + +const ( + // CompressionDisabled disables the negotiation of the permessage-deflate extension. + // + // This is the default. Do not enable compression without benchmarking for your particular use case first. + CompressionDisabled CompressionMode = iota + + // CompressionContextTakeover compresses each message greater than 128 bytes reusing the 32 KB sliding window from + // previous messages. i.e compression context across messages is preserved. + // + // As most WebSocket protocols are text based and repetitive, this compression mode can be very efficient. + // + // The memory overhead is a fixed 32 KB sliding window, a fixed 1.2 MB flate.Writer and a sync.Pool of 40 KB flate.Reader's + // that are used when reading and then returned. + // + // Thus, it uses more memory than CompressionNoContextTakeover but compresses more efficiently. + // + // If the peer does not support CompressionContextTakeover then we will fall back to CompressionNoContextTakeover. + CompressionContextTakeover + + // CompressionNoContextTakeover compresses each message greater than 512 bytes. Each message is compressed with + // a new 1.2 MB flate.Writer pulled from a sync.Pool. Each message is read with a 40 KB flate.Reader pulled from + // a sync.Pool. + // + // This means less efficient compression as the sliding window from previous messages will not be used but the + // memory overhead will be lower as there will be no fixed cost for the flate.Writer nor the 32 KB sliding window. + // Especially if the connections are long lived and seldom written to. + // + // Thus, it uses less memory than CompressionContextTakeover but compresses less efficiently. + // + // If the peer does not support CompressionNoContextTakeover then we will fall back to CompressionDisabled. + CompressionNoContextTakeover +) + +func (m CompressionMode) opts() *compressionOptions { + return &compressionOptions{ + clientNoContextTakeover: m == CompressionNoContextTakeover, + serverNoContextTakeover: m == CompressionNoContextTakeover, + } +} + +type compressionOptions struct { + offerClientWindow bool + clientWindowBits int + clientNoContextTakeover bool + serverNoContextTakeover bool +} + +func (copts *compressionOptions) String() string { + s := "permessage-deflate" + if copts.offerClientWindow { + s += "; client_max_window_bits" + } + if copts.clientNoContextTakeover { + s += "; client_no_context_takeover" + } + if copts.serverNoContextTakeover { + s += "; server_no_context_takeover" + } + return s +} + +// These bytes are required to get flate.Reader to return. +// They are removed when sending to avoid the overhead as +// WebSocket framing tell's when the message has ended but then +// we need to add them back otherwise flate.Reader keeps +// trying to read more bytes. +const deflateMessageTail = "\x00\x00\xff\xff" + +type trimLastFourBytesWriter struct { + w io.Writer + tail []byte +} + +func (tw *trimLastFourBytesWriter) reset() { + if tw != nil && tw.tail != nil { + tw.tail = tw.tail[:0] + } +} + +func (tw *trimLastFourBytesWriter) Write(p []byte) (int, error) { + if tw.tail == nil { + tw.tail = make([]byte, 0, 4) + } + + extra := len(tw.tail) + len(p) - 4 + + if extra <= 0 { + tw.tail = append(tw.tail, p...) + return len(p), nil + } + + // Now we need to write as many extra bytes as we can from the previous tail. + if extra > len(tw.tail) { + extra = len(tw.tail) + } + if extra > 0 { + _, err := tw.w.Write(tw.tail[:extra]) + if err != nil { + return 0, err + } + + // Shift remaining bytes in tail over. + n := copy(tw.tail, tw.tail[extra:]) + tw.tail = tw.tail[:n] + } + + // If p is less than or equal to 4 bytes, + // all of it is is part of the tail. + if len(p) <= 4 { + tw.tail = append(tw.tail, p...) + return len(p), nil + } + + // Otherwise, only the last 4 bytes are. + tw.tail = append(tw.tail, p[len(p)-4:]...) + + p = p[:len(p)-4] + n, err := tw.w.Write(p) + return n + 4, err +} + +var flateReaderPool sync.Pool + +func getFlateReader(r io.Reader, dict []byte) io.Reader { + fr, ok := flateReaderPool.Get().(io.Reader) + if !ok { + return flate.NewReaderDict(r, dict) + } + fr.(flate.Resetter).Reset(r, dict) + return fr +} + +func putFlateReader(fr io.Reader) { + flateReaderPool.Put(fr) +} + +var flateWriterPool sync.Pool + +func getFlateWriter(w io.Writer, bits int) *flate.Writer { + if bits > 0 && bits < 15 { + fw, err := flate.NewWriterWindow(w, 1< 0 && bits < 15 { + return + } + flateWriterPool.Put(w) +} + +type slidingWindow struct { + buf []byte +} + +var ( + swPoolMu sync.RWMutex + swPool = map[int]*sync.Pool{} +) + +func slidingWindowPool(n int) *sync.Pool { + swPoolMu.RLock() + p, ok := swPool[n] + swPoolMu.RUnlock() + if ok { + return p + } + + p = &sync.Pool{} + + swPoolMu.Lock() + swPool[n] = p + swPoolMu.Unlock() + + return p +} + +func (sw *slidingWindow) init(n int) { + if sw.buf != nil { + return + } + + if n == 0 { + n = 32768 + } + + p := slidingWindowPool(n) + sw2, ok := p.Get().(*slidingWindow) + if ok { + *sw = *sw2 + } else { + sw.buf = make([]byte, 0, n) + } +} + +func (sw *slidingWindow) close() { + sw.buf = sw.buf[:0] + swPoolMu.Lock() + swPool[cap(sw.buf)].Put(sw) + swPoolMu.Unlock() +} + +func (sw *slidingWindow) write(p []byte) { + if len(p) >= cap(sw.buf) { + sw.buf = sw.buf[:cap(sw.buf)] + p = p[len(p)-cap(sw.buf):] + copy(sw.buf, p) + return + } + + left := cap(sw.buf) - len(sw.buf) + if left < len(p) { + // We need to shift spaceNeeded bytes from the end to make room for p at the end. + spaceNeeded := len(p) - left + copy(sw.buf, sw.buf[spaceNeeded:]) + sw.buf = sw.buf[:len(sw.buf)-spaceNeeded] + } + + sw.buf = append(sw.buf, p...) +} diff --git a/backend/internal/pkg/codexws/conn.go b/backend/internal/pkg/codexws/conn.go new file mode 100644 index 000000000000..092348712d8a --- /dev/null +++ b/backend/internal/pkg/codexws/conn.go @@ -0,0 +1,306 @@ +//go:build !js + +package websocket + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "runtime" + "strconv" + "sync" + "sync/atomic" +) + +// MessageType represents the type of a WebSocket message. +// See https://tools.ietf.org/html/rfc6455#section-5.6 +type MessageType int + +// MessageType constants. +const ( + // MessageText is for UTF-8 encoded text messages like JSON. + MessageText MessageType = iota + 1 + // MessageBinary is for binary messages like protobufs. + MessageBinary +) + +// Conn represents a WebSocket connection. +// All methods may be called concurrently except for Reader and Read. +// +// You must always read from the connection. Otherwise control +// frames will not be handled. See Reader and CloseRead. +// +// Be sure to call Close on the connection when you +// are finished with it to release associated resources. +// +// On any error from any method, the connection is closed +// with an appropriate reason. +// +// This applies to context expirations as well unfortunately. +// See https://github.com/nhooyr/websocket/issues/242#issuecomment-633182220 +type Conn struct { + noCopy noCopy + + subprotocol string + rwc io.ReadWriteCloser + client bool + copts *compressionOptions + flateThreshold int + br *bufio.Reader + bw *bufio.Writer + + readTimeoutStop atomic.Pointer[func() bool] + writeTimeoutStop atomic.Pointer[func() bool] + + // Read state. + readMu *mu + readHeaderBuf [8]byte + readControlBuf [maxControlPayload]byte + msgReader *msgReader + + // Write state. + msgWriter *msgWriter + writeFrameMu *mu + writeBuf []byte + writeHeaderBuf [8]byte + writeHeader header + + // Close handshake state. + closeStateMu sync.RWMutex + closeReceivedErr error + closeSentErr error + + // CloseRead state. + closeReadMu sync.Mutex + closeReadCtx context.Context + closeReadDone chan struct{} + + closing atomic.Bool + closeMu sync.Mutex // Protects following. + closed chan struct{} + + pingCounter atomic.Int64 + activePingsMu sync.Mutex + activePings map[string]chan<- struct{} + onPingReceived func(context.Context, []byte) bool + onPongReceived func(context.Context, []byte) +} + +type connConfig struct { + subprotocol string + rwc io.ReadWriteCloser + client bool + copts *compressionOptions + flateThreshold int + onPingReceived func(context.Context, []byte) bool + onPongReceived func(context.Context, []byte) + + br *bufio.Reader + bw *bufio.Writer +} + +func newConn(cfg connConfig) *Conn { + c := &Conn{ + subprotocol: cfg.subprotocol, + rwc: cfg.rwc, + client: cfg.client, + copts: cfg.copts, + flateThreshold: cfg.flateThreshold, + + br: cfg.br, + bw: cfg.bw, + + closed: make(chan struct{}), + activePings: make(map[string]chan<- struct{}), + onPingReceived: cfg.onPingReceived, + onPongReceived: cfg.onPongReceived, + } + + c.readMu = newMu(c) + c.writeFrameMu = newMu(c) + + c.msgReader = newMsgReader(c) + + c.msgWriter = newMsgWriter(c) + if c.client { + c.writeBuf = extractBufioWriterBuf(c.bw, c.rwc) + } + + if c.flate() && c.flateThreshold == 0 { + c.flateThreshold = 128 + if !c.msgWriter.flateContextTakeover() { + c.flateThreshold = 512 + } + } + + runtime.SetFinalizer(c, func(c *Conn) { + c.close() + }) + + return c +} + +// Subprotocol returns the negotiated subprotocol. +// An empty string means the default protocol. +func (c *Conn) Subprotocol() string { + return c.subprotocol +} + +func (c *Conn) close() error { + c.closeMu.Lock() + defer c.closeMu.Unlock() + + if c.isClosed() { + return net.ErrClosed + } + runtime.SetFinalizer(c, nil) + close(c.closed) + + // Have to close after c.closed is closed to ensure any goroutine that wakes up + // from the connection being closed also sees that c.closed is closed and returns + // closeErr. + err := c.rwc.Close() + // With the close of rwc, these become safe to close. + c.msgWriter.close() + c.msgReader.close() + return err +} + +func (c *Conn) setupWriteTimeout(ctx context.Context) { + stop := context.AfterFunc(ctx, func() { + c.clearWriteTimeout() + c.close() + }) + swapTimeoutStop(&c.writeTimeoutStop, &stop) +} + +func (c *Conn) clearWriteTimeout() { + swapTimeoutStop(&c.writeTimeoutStop, nil) +} + +func (c *Conn) setupReadTimeout(ctx context.Context) { + stop := context.AfterFunc(ctx, func() { + c.clearReadTimeout() + c.close() + }) + swapTimeoutStop(&c.readTimeoutStop, &stop) +} + +func (c *Conn) clearReadTimeout() { + swapTimeoutStop(&c.readTimeoutStop, nil) +} + +func swapTimeoutStop(p *atomic.Pointer[func() bool], newStop *func() bool) { + oldStop := p.Swap(newStop) + if oldStop != nil { + (*oldStop)() + } +} + +func (c *Conn) flate() bool { + return c.copts != nil +} + +// Ping sends a ping to the peer and waits for a pong. +// Use this to measure latency or ensure the peer is responsive. +// Ping must be called concurrently with Reader as it does +// not read from the connection but instead waits for a Reader call +// to read the pong. +// +// TCP Keepalives should suffice for most use cases. +func (c *Conn) Ping(ctx context.Context) error { + p := c.pingCounter.Add(1) + + err := c.ping(ctx, strconv.FormatInt(p, 10)) + if err != nil { + return fmt.Errorf("failed to ping: %w", err) + } + return nil +} + +func (c *Conn) ping(ctx context.Context, p string) error { + pong := make(chan struct{}, 1) + + c.activePingsMu.Lock() + c.activePings[p] = pong + c.activePingsMu.Unlock() + + defer func() { + c.activePingsMu.Lock() + delete(c.activePings, p) + c.activePingsMu.Unlock() + }() + + err := c.writeControl(ctx, opPing, []byte(p)) + if err != nil { + return err + } + + select { + case <-c.closed: + return net.ErrClosed + case <-ctx.Done(): + return fmt.Errorf("failed to wait for pong: %w", ctx.Err()) + case <-pong: + return nil + } +} + +type mu struct { + c *Conn + ch chan struct{} +} + +func newMu(c *Conn) *mu { + return &mu{ + c: c, + ch: make(chan struct{}, 1), + } +} + +func (m *mu) forceLock() { + m.ch <- struct{}{} +} + +func (m *mu) tryLock() bool { + select { + case m.ch <- struct{}{}: + return true + default: + return false + } +} + +func (m *mu) lock(ctx context.Context) error { + select { + case <-m.c.closed: + return net.ErrClosed + case <-ctx.Done(): + return fmt.Errorf("failed to acquire lock: %w", ctx.Err()) + case m.ch <- struct{}{}: + // To make sure the connection is certainly alive. + // As it's possible the send on m.ch was selected + // over the receive on closed. + select { + case <-m.c.closed: + // Make sure to release. + m.unlock() + return net.ErrClosed + default: + } + return nil + } +} + +func (m *mu) unlock() { + select { + case <-m.ch: + default: + } +} + +type noCopy struct{} + +func (*noCopy) Lock() {} diff --git a/backend/internal/pkg/codexws/dial.go b/backend/internal/pkg/codexws/dial.go new file mode 100644 index 000000000000..f76b62f76e1d --- /dev/null +++ b/backend/internal/pkg/codexws/dial.go @@ -0,0 +1,360 @@ +//go:build !js + +package websocket + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" +) + +// DialOptions represents Dial's options. +type DialOptions struct { + // HTTPClient is used for the connection. + // Its Transport must return writable bodies for WebSocket handshakes. + // http.Transport does beginning with Go 1.12. + HTTPClient *http.Client + + // HTTPHeader specifies the HTTP headers included in the handshake request. + HTTPHeader http.Header + + // Host optionally overrides the Host HTTP header to send. If empty, the value + // of URL.Host will be used. + Host string + + // Subprotocols lists the WebSocket subprotocols to negotiate with the server. + Subprotocols []string + + // CompressionMode controls the compression mode. + // Defaults to CompressionDisabled. + // + // See docs on CompressionMode for details. + CompressionMode CompressionMode + + // ClientMaxWindowBits advertises the official Codex bounded compression offer. + ClientMaxWindowBits bool + + // CompressionThreshold controls the minimum size of a message before compression is applied. + // + // Defaults to 512 bytes for CompressionNoContextTakeover and 128 bytes + // for CompressionContextTakeover. + CompressionThreshold int + + // OnPingReceived is an optional callback invoked synchronously when a ping frame is received. + // + // The payload contains the application data of the ping frame. + // If the callback returns false, the subsequent pong frame will not be sent. + // To avoid blocking, any expensive processing should be performed asynchronously using a goroutine. + OnPingReceived func(ctx context.Context, payload []byte) bool + + // OnPongReceived is an optional callback invoked synchronously when a pong frame is received. + // + // The payload contains the application data of the pong frame. + // To avoid blocking, any expensive processing should be performed asynchronously using a goroutine. + // + // Unlike OnPingReceived, this callback does not return a value because a pong frame + // is a response to a ping and does not trigger any further frame transmission. + OnPongReceived func(ctx context.Context, payload []byte) +} + +func (opts *DialOptions) cloneWithDefaults(ctx context.Context) (context.Context, context.CancelFunc, *DialOptions) { + var cancel context.CancelFunc + + var o DialOptions + if opts != nil { + o = *opts + } + if o.HTTPClient == nil { + o.HTTPClient = http.DefaultClient + } + if o.HTTPClient.Timeout > 0 { + ctx, cancel = context.WithTimeout(ctx, o.HTTPClient.Timeout) + + newClient := *o.HTTPClient + newClient.Timeout = 0 + o.HTTPClient = &newClient + } + if o.HTTPHeader == nil { + o.HTTPHeader = http.Header{} + } + newClient := *o.HTTPClient + oldCheckRedirect := o.HTTPClient.CheckRedirect + newClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + switch req.URL.Scheme { + case "ws": + req.URL.Scheme = "http" + case "wss": + req.URL.Scheme = "https" + } + if oldCheckRedirect != nil { + return oldCheckRedirect(req, via) + } + return nil + } + o.HTTPClient = &newClient + + return ctx, cancel, &o +} + +// Dial performs a WebSocket handshake on url. +// +// The response is the WebSocket handshake response from the server. +// You never need to close resp.Body yourself. +// +// If an error occurs, the returned response may be non nil. +// However, you can only read the first 1024 bytes of the body. +// +// This function requires at least Go 1.12 as it uses a new feature +// in net/http to perform WebSocket handshakes. +// See docs on the HTTPClient option and https://github.com/golang/go/issues/26937#issuecomment-415855861 +// +// URLs with http/https schemes will work and are interpreted as ws/wss. +func Dial(ctx context.Context, u string, opts *DialOptions) (*Conn, *http.Response, error) { + return dial(ctx, u, opts, nil) +} + +func dial(ctx context.Context, urls string, opts *DialOptions, rand io.Reader) (_ *Conn, _ *http.Response, err error) { + defer errd.Wrap(&err, "failed to WebSocket dial") + + var cancel context.CancelFunc + ctx, cancel, opts = opts.cloneWithDefaults(ctx) + if cancel != nil { + defer cancel() + } + + secWebSocketKey, err := secWebSocketKey(rand) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate Sec-WebSocket-Key: %w", err) + } + + var copts *compressionOptions + if opts.CompressionMode != CompressionDisabled { + copts = opts.CompressionMode.opts() + copts.offerClientWindow = opts.ClientMaxWindowBits + } + + resp, err := handshakeRequest(ctx, urls, opts, copts, secWebSocketKey) + if err != nil { + return nil, resp, err + } + respBody := resp.Body + resp.Body = nil + defer func() { + if err != nil { + // We read a bit of the body for easier debugging. + r := io.LimitReader(respBody, 1024) + + timer := time.AfterFunc(time.Second*3, func() { + respBody.Close() + }) + defer timer.Stop() + + b, _ := io.ReadAll(r) + respBody.Close() + resp.Body = io.NopCloser(bytes.NewReader(b)) + } + }() + + copts, err = verifyServerResponse(opts, copts, secWebSocketKey, resp) + if err != nil { + return nil, resp, err + } + + rwc, ok := respBody.(io.ReadWriteCloser) + if !ok { + return nil, resp, fmt.Errorf("response body is not a io.ReadWriteCloser: %T", respBody) + } + + return newConn(connConfig{ + subprotocol: resp.Header.Get("Sec-WebSocket-Protocol"), + rwc: rwc, + client: true, + copts: copts, + flateThreshold: opts.CompressionThreshold, + onPingReceived: opts.OnPingReceived, + onPongReceived: opts.OnPongReceived, + br: getBufioReader(rwc), + bw: getBufioWriter(rwc), + }), resp, nil +} + +func handshakeRequest(ctx context.Context, urls string, opts *DialOptions, copts *compressionOptions, secWebSocketKey string) (*http.Response, error) { + u, err := url.Parse(urls) + if err != nil { + return nil, fmt.Errorf("failed to parse url: %w", err) + } + + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + case "http", "https": + default: + return nil, fmt.Errorf("unexpected url scheme: %q", u.Scheme) + } + + req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create new http request: %w", err) + } + if len(opts.Host) > 0 { + req.Host = opts.Host + } + req.Header = opts.HTTPHeader.Clone() + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") + req.Header.Set("Sec-WebSocket-Version", "13") + req.Header.Set("Sec-WebSocket-Key", secWebSocketKey) + if len(opts.Subprotocols) > 0 { + req.Header.Set("Sec-WebSocket-Protocol", strings.Join(opts.Subprotocols, ",")) + } + if copts != nil { + req.Header.Set("Sec-WebSocket-Extensions", copts.String()) + } + + resp, err := opts.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send handshake request: %w", err) + } + return resp, nil +} + +func secWebSocketKey(rr io.Reader) (string, error) { + if rr == nil { + rr = rand.Reader + } + b := make([]byte, 16) + _, err := io.ReadFull(rr, b) + if err != nil { + return "", fmt.Errorf("failed to read random data from rand.Reader: %w", err) + } + return base64.StdEncoding.EncodeToString(b), nil +} + +func verifyServerResponse(opts *DialOptions, copts *compressionOptions, secWebSocketKey string, resp *http.Response) (*compressionOptions, error) { + if resp.StatusCode != http.StatusSwitchingProtocols { + return nil, fmt.Errorf("expected handshake response status code %v but got %v", http.StatusSwitchingProtocols, resp.StatusCode) + } + + if !headerContainsTokenIgnoreCase(resp.Header, "Connection", "Upgrade") { + return nil, fmt.Errorf("WebSocket protocol violation: Connection header %q does not contain Upgrade", resp.Header.Get("Connection")) + } + + if !headerContainsTokenIgnoreCase(resp.Header, "Upgrade", "WebSocket") { + return nil, fmt.Errorf("WebSocket protocol violation: Upgrade header %q does not contain websocket", resp.Header.Get("Upgrade")) + } + + if resp.Header.Get("Sec-WebSocket-Accept") != secWebSocketAccept(secWebSocketKey) { + return nil, fmt.Errorf("WebSocket protocol violation: invalid Sec-WebSocket-Accept %q, key %q", + resp.Header.Get("Sec-WebSocket-Accept"), + secWebSocketKey, + ) + } + + err := verifySubprotocol(opts.Subprotocols, resp) + if err != nil { + return nil, err + } + + return verifyServerExtensions(copts, resp.Header) +} + +func verifySubprotocol(subprotos []string, resp *http.Response) error { + proto := resp.Header.Get("Sec-WebSocket-Protocol") + if proto == "" { + return nil + } + + for _, sp2 := range subprotos { + if strings.EqualFold(sp2, proto) { + return nil + } + } + + return fmt.Errorf("WebSocket protocol violation: unexpected Sec-WebSocket-Protocol from server: %q", proto) +} + +func verifyServerExtensions(copts *compressionOptions, h http.Header) (*compressionOptions, error) { + exts := websocketExtensions(h) + if len(exts) == 0 { + return nil, nil + } + + ext := exts[0] + if ext.name != "permessage-deflate" || len(exts) > 1 || copts == nil { + return nil, fmt.Errorf("WebSocket protcol violation: unsupported extensions from server: %+v", exts[1:]) + } + + _copts := *copts + copts = &_copts + + for _, p := range ext.params { + if strings.HasPrefix(p, "client_max_window_bits=") && copts.offerClientWindow { + bits, err := strconv.Atoi(strings.TrimPrefix(p, "client_max_window_bits=")) + if err != nil || bits < 8 || bits > 15 { + return nil, fmt.Errorf("invalid client_max_window_bits: %q", p) + } + copts.clientWindowBits = bits + continue + } + switch p { + case "client_no_context_takeover": + copts.clientNoContextTakeover = true + continue + case "server_no_context_takeover": + copts.serverNoContextTakeover = true + continue + } + if strings.HasPrefix(p, "server_max_window_bits=") { + // We can't adjust the deflate window, but decoding with a larger window is acceptable. + continue + } + + return nil, fmt.Errorf("unsupported permessage-deflate parameter: %q", p) + } + + return copts, nil +} + +var bufioReaderPool sync.Pool + +func getBufioReader(r io.Reader) *bufio.Reader { + br, ok := bufioReaderPool.Get().(*bufio.Reader) + if !ok { + return bufio.NewReader(r) + } + br.Reset(r) + return br +} + +func putBufioReader(br *bufio.Reader) { + bufioReaderPool.Put(br) +} + +var bufioWriterPool sync.Pool + +func getBufioWriter(w io.Writer) *bufio.Writer { + bw, ok := bufioWriterPool.Get().(*bufio.Writer) + if !ok { + return bufio.NewWriter(w) + } + bw.Reset(w) + return bw +} + +func putBufioWriter(bw *bufio.Writer) { + bufioWriterPool.Put(bw) +} diff --git a/backend/internal/pkg/codexws/doc.go b/backend/internal/pkg/codexws/doc.go new file mode 100644 index 000000000000..b459b60bafd6 --- /dev/null +++ b/backend/internal/pkg/codexws/doc.go @@ -0,0 +1,33 @@ +//go:build !js + +// Package websocket implements the RFC 6455 WebSocket protocol. +// +// https://tools.ietf.org/html/rfc6455 +// +// Use Dial to dial a WebSocket server. +// +// Use Accept to accept a WebSocket client. +// +// Conn represents the resulting WebSocket connection. +// +// The examples are the best way to understand how to correctly use the library. +// +// The wsjson subpackage contain helpers for JSON and protobuf messages. +// +// More documentation at https://github.com/Wei-Shaw/sub2api/internal/pkg/codexws. +// +// # Wasm +// +// The client side supports compiling to Wasm. +// It wraps the WebSocket browser API. +// +// See https://developer.mozilla.org/en-US/docs/Web/API/WebSocket +// +// Some important caveats to be aware of: +// +// - Accept always errors out +// - Conn.Ping is no-op +// - Conn.CloseNow is Close(StatusGoingAway, "") +// - HTTPClient, HTTPHeader and CompressionMode in DialOptions are no-op +// - *http.Response from Dial is &http.Response{} with a 101 status code on success +package websocket // import "github.com/Wei-Shaw/sub2api/internal/pkg/codexws" diff --git a/backend/internal/pkg/codexws/errors.go b/backend/internal/pkg/codexws/errors.go new file mode 100644 index 000000000000..bf4fc2b01ece --- /dev/null +++ b/backend/internal/pkg/codexws/errors.go @@ -0,0 +1,8 @@ +package websocket + +import ( + "errors" +) + +// ErrMessageTooBig is returned when a message exceeds the read limit. +var ErrMessageTooBig = errors.New("websocket: message too big") diff --git a/backend/internal/pkg/codexws/frame.go b/backend/internal/pkg/codexws/frame.go new file mode 100644 index 000000000000..3dd7a2ca7478 --- /dev/null +++ b/backend/internal/pkg/codexws/frame.go @@ -0,0 +1,173 @@ +//go:build !js + +package websocket + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" +) + +// opcode represents a WebSocket opcode. +type opcode int + +// https://tools.ietf.org/html/rfc6455#section-11.8. +const ( + opContinuation opcode = iota + opText + opBinary + // 3 - 7 are reserved for further non-control frames. + _ + _ + _ + _ + _ + opClose + opPing + opPong + // 11-16 are reserved for further control frames. +) + +// header represents a WebSocket frame header. +// See https://tools.ietf.org/html/rfc6455#section-5.2. +type header struct { + fin bool + rsv1 bool + rsv2 bool + rsv3 bool + opcode opcode + + payloadLength int64 + + masked bool + maskKey uint32 +} + +// readFrameHeader reads a header from the reader. +// See https://tools.ietf.org/html/rfc6455#section-5.2. +func readFrameHeader(r *bufio.Reader, readBuf []byte) (h header, err error) { + defer errd.Wrap(&err, "failed to read frame header") + + b, err := r.ReadByte() + if err != nil { + return header{}, err + } + + h.fin = b&(1<<7) != 0 + h.rsv1 = b&(1<<6) != 0 + h.rsv2 = b&(1<<5) != 0 + h.rsv3 = b&(1<<4) != 0 + + h.opcode = opcode(b & 0xf) + + b, err = r.ReadByte() + if err != nil { + return header{}, err + } + + h.masked = b&(1<<7) != 0 + + payloadLength := b &^ (1 << 7) + switch { + case payloadLength < 126: + h.payloadLength = int64(payloadLength) + case payloadLength == 126: + _, err = io.ReadFull(r, readBuf[:2]) + h.payloadLength = int64(binary.BigEndian.Uint16(readBuf)) + case payloadLength == 127: + _, err = io.ReadFull(r, readBuf) + h.payloadLength = int64(binary.BigEndian.Uint64(readBuf)) + } + if err != nil { + return header{}, err + } + + if h.payloadLength < 0 { + return header{}, fmt.Errorf("received negative payload length: %v", h.payloadLength) + } + + if h.masked { + _, err = io.ReadFull(r, readBuf[:4]) + if err != nil { + return header{}, err + } + h.maskKey = binary.LittleEndian.Uint32(readBuf) + } + + return h, nil +} + +// maxControlPayload is the maximum length of a control frame payload. +// See https://tools.ietf.org/html/rfc6455#section-5.5. +const maxControlPayload = 125 + +// writeFrameHeader writes the bytes of the header to w. +// See https://tools.ietf.org/html/rfc6455#section-5.2 +func writeFrameHeader(h header, w *bufio.Writer, buf []byte) (err error) { + defer errd.Wrap(&err, "failed to write frame header") + + var b byte + if h.fin { + b |= 1 << 7 + } + if h.rsv1 { + b |= 1 << 6 + } + if h.rsv2 { + b |= 1 << 5 + } + if h.rsv3 { + b |= 1 << 4 + } + + b |= byte(h.opcode) + + err = w.WriteByte(b) + if err != nil { + return err + } + + lengthByte := byte(0) + if h.masked { + lengthByte |= 1 << 7 + } + + switch { + case h.payloadLength > math.MaxUint16: + lengthByte |= 127 + case h.payloadLength > 125: + lengthByte |= 126 + case h.payloadLength >= 0: + lengthByte |= byte(h.payloadLength) + } + err = w.WriteByte(lengthByte) + if err != nil { + return err + } + + switch { + case h.payloadLength > math.MaxUint16: + binary.BigEndian.PutUint64(buf, uint64(h.payloadLength)) + _, err = w.Write(buf) + case h.payloadLength > 125: + binary.BigEndian.PutUint16(buf, uint16(h.payloadLength)) + _, err = w.Write(buf[:2]) + } + if err != nil { + return err + } + + if h.masked { + binary.LittleEndian.PutUint32(buf, h.maskKey) + _, err = w.Write(buf[:4]) + if err != nil { + return err + } + } + + return nil +} diff --git a/backend/internal/pkg/codexws/hijack.go b/backend/internal/pkg/codexws/hijack.go new file mode 100644 index 000000000000..9cce45cad66d --- /dev/null +++ b/backend/internal/pkg/codexws/hijack.go @@ -0,0 +1,33 @@ +//go:build !js + +package websocket + +import ( + "net/http" +) + +type rwUnwrapper interface { + Unwrap() http.ResponseWriter +} + +// hijacker returns the Hijacker interface of the http.ResponseWriter. +// It follows the Unwrap method of the http.ResponseWriter if available, +// matching the behavior of http.ResponseController. If the Hijacker +// interface is not found, it returns false. +// +// Since the http.ResponseController is not available in Go 1.19, and +// does not support checking the presence of the Hijacker interface, +// this function is used to provide a consistent way to check for the +// Hijacker interface across Go versions. +func hijacker(rw http.ResponseWriter) (http.Hijacker, bool) { + for { + switch t := rw.(type) { + case http.Hijacker: + return t, true + case rwUnwrapper: + rw = t.Unwrap() + default: + return nil, false + } + } +} diff --git a/backend/internal/pkg/codexws/internal/bpool/bpool.go b/backend/internal/pkg/codexws/internal/bpool/bpool.go new file mode 100644 index 000000000000..12cf577a9f7a --- /dev/null +++ b/backend/internal/pkg/codexws/internal/bpool/bpool.go @@ -0,0 +1,25 @@ +package bpool + +import ( + "bytes" + "sync" +) + +var bpool = sync.Pool{ + New: func() any { + return &bytes.Buffer{} + }, +} + +// Get returns a buffer from the pool or creates a new one if +// the pool is empty. +func Get() *bytes.Buffer { + b := bpool.Get() + return b.(*bytes.Buffer) +} + +// Put returns a buffer into the pool. +func Put(b *bytes.Buffer) { + b.Reset() + bpool.Put(b) +} diff --git a/backend/internal/pkg/codexws/internal/errd/wrap.go b/backend/internal/pkg/codexws/internal/errd/wrap.go new file mode 100644 index 000000000000..c80d0a653f84 --- /dev/null +++ b/backend/internal/pkg/codexws/internal/errd/wrap.go @@ -0,0 +1,14 @@ +package errd + +import ( + "fmt" +) + +// Wrap wraps err with fmt.Errorf if err is non nil. +// Intended for use with defer and a named error return. +// Inspired by https://github.com/golang/go/issues/32676. +func Wrap(err *error, f string, v ...any) { + if *err != nil { + *err = fmt.Errorf(f+": %w", append(v, *err)...) + } +} diff --git a/backend/internal/pkg/codexws/internal/util/util.go b/backend/internal/pkg/codexws/internal/util/util.go new file mode 100644 index 000000000000..aa2107031323 --- /dev/null +++ b/backend/internal/pkg/codexws/internal/util/util.go @@ -0,0 +1,15 @@ +package util + +// WriterFunc is used to implement one off io.Writers. +type WriterFunc func(p []byte) (int, error) + +func (f WriterFunc) Write(p []byte) (int, error) { + return f(p) +} + +// ReaderFunc is used to implement one off io.Readers. +type ReaderFunc func(p []byte) (int, error) + +func (f ReaderFunc) Read(p []byte) (int, error) { + return f(p) +} diff --git a/backend/internal/pkg/codexws/internal/wsjs/wsjs_js.go b/backend/internal/pkg/codexws/internal/wsjs/wsjs_js.go new file mode 100644 index 000000000000..45ecf49d382d --- /dev/null +++ b/backend/internal/pkg/codexws/internal/wsjs/wsjs_js.go @@ -0,0 +1,169 @@ +//go:build js +// +build js + +// Package wsjs implements typed access to the browser javascript WebSocket API. +// +// https://developer.mozilla.org/en-US/docs/Web/API/WebSocket +package wsjs + +import ( + "syscall/js" +) + +func handleJSError(err *error, onErr func()) { + r := recover() + + if jsErr, ok := r.(js.Error); ok { + *err = jsErr + + if onErr != nil { + onErr() + } + return + } + + if r != nil { + panic(r) + } +} + +// New is a wrapper around the javascript WebSocket constructor. +func New(url string, protocols []string) (c WebSocket, err error) { + defer handleJSError(&err, func() { + c = WebSocket{} + }) + + jsProtocols := make([]any, len(protocols)) + for i, p := range protocols { + jsProtocols[i] = p + } + + c = WebSocket{ + v: js.Global().Get("WebSocket").New(url, jsProtocols), + } + + c.setBinaryType("arraybuffer") + + return c, nil +} + +// WebSocket is a wrapper around a javascript WebSocket object. +type WebSocket struct { + v js.Value +} + +func (c WebSocket) setBinaryType(typ string) { + c.v.Set("binaryType", string(typ)) +} + +func (c WebSocket) addEventListener(eventType string, fn func(e js.Value)) func() { + f := js.FuncOf(func(this js.Value, args []js.Value) any { + fn(args[0]) + return nil + }) + c.v.Call("addEventListener", eventType, f) + + return func() { + c.v.Call("removeEventListener", eventType, f) + f.Release() + } +} + +// CloseEvent is the type passed to a WebSocket close handler. +type CloseEvent struct { + Code uint16 + Reason string + WasClean bool +} + +// OnClose registers a function to be called when the WebSocket is closed. +func (c WebSocket) OnClose(fn func(CloseEvent)) (remove func()) { + return c.addEventListener("close", func(e js.Value) { + ce := CloseEvent{ + Code: uint16(e.Get("code").Int()), + Reason: e.Get("reason").String(), + WasClean: e.Get("wasClean").Bool(), + } + fn(ce) + }) +} + +// OnError registers a function to be called when there is an error +// with the WebSocket. +func (c WebSocket) OnError(fn func(e js.Value)) (remove func()) { + return c.addEventListener("error", fn) +} + +// MessageEvent is the type passed to a message handler. +type MessageEvent struct { + // string or []byte. + Data any + + // There are more fields to the interface but we don't use them. + // See https://developer.mozilla.org/en-US/docs/Web/API/MessageEvent +} + +// OnMessage registers a function to be called when the WebSocket receives a message. +func (c WebSocket) OnMessage(fn func(m MessageEvent)) (remove func()) { + return c.addEventListener("message", func(e js.Value) { + var data any + + arrayBuffer := e.Get("data") + if arrayBuffer.Type() == js.TypeString { + data = arrayBuffer.String() + } else { + data = extractArrayBuffer(arrayBuffer) + } + + me := MessageEvent{ + Data: data, + } + fn(me) + }) +} + +// Subprotocol returns the WebSocket subprotocol in use. +func (c WebSocket) Subprotocol() string { + return c.v.Get("protocol").String() +} + +// OnOpen registers a function to be called when the WebSocket is opened. +func (c WebSocket) OnOpen(fn func(e js.Value)) (remove func()) { + return c.addEventListener("open", fn) +} + +// Close closes the WebSocket with the given code and reason. +func (c WebSocket) Close(code int, reason string) (err error) { + defer handleJSError(&err, nil) + c.v.Call("close", code, reason) + return err +} + +// SendText sends the given string as a text message +// on the WebSocket. +func (c WebSocket) SendText(v string) (err error) { + defer handleJSError(&err, nil) + c.v.Call("send", v) + return err +} + +// SendBytes sends the given message as a binary message +// on the WebSocket. +func (c WebSocket) SendBytes(v []byte) (err error) { + defer handleJSError(&err, nil) + c.v.Call("send", uint8Array(v)) + return err +} + +func extractArrayBuffer(arrayBuffer js.Value) []byte { + uint8Array := js.Global().Get("Uint8Array").New(arrayBuffer) + dst := make([]byte, uint8Array.Length()) + js.CopyBytesToGo(dst, uint8Array) + return dst +} + +func uint8Array(src []byte) js.Value { + uint8Array := js.Global().Get("Uint8Array").New(len(src)) + js.CopyBytesToJS(uint8Array, src) + return uint8Array +} diff --git a/backend/internal/pkg/codexws/internal/xsync/go.go b/backend/internal/pkg/codexws/internal/xsync/go.go new file mode 100644 index 000000000000..5229b12a1c9a --- /dev/null +++ b/backend/internal/pkg/codexws/internal/xsync/go.go @@ -0,0 +1,26 @@ +package xsync + +import ( + "fmt" + "runtime/debug" +) + +// Go allows running a function in another goroutine +// and waiting for its error. +func Go(fn func() error) <-chan error { + errs := make(chan error, 1) + go func() { + defer func() { + r := recover() + if r != nil { + select { + case errs <- fmt.Errorf("panic in go fn: %v, %s", r, debug.Stack()): + default: + } + } + }() + errs <- fn() + }() + + return errs +} diff --git a/backend/internal/pkg/codexws/mask.go b/backend/internal/pkg/codexws/mask.go new file mode 100644 index 000000000000..7bc0c8d5f59e --- /dev/null +++ b/backend/internal/pkg/codexws/mask.go @@ -0,0 +1,128 @@ +package websocket + +import ( + "encoding/binary" + "math/bits" +) + +// maskGo applies the WebSocket masking algorithm to p +// with the given key. +// See https://tools.ietf.org/html/rfc6455#section-5.3 +// +// The returned value is the correctly rotated key to +// to continue to mask/unmask the message. +// +// It is optimized for LittleEndian and expects the key +// to be in little endian. +// +// See https://github.com/golang/go/issues/31586 +func maskGo(b []byte, key uint32) uint32 { + if len(b) >= 8 { + key64 := uint64(key)<<32 | uint64(key) + + // At some point in the future we can clean these unrolled loops up. + // See https://github.com/golang/go/issues/31586#issuecomment-487436401 + + // Then we xor until b is less than 128 bytes. + for len(b) >= 128 { + v := binary.LittleEndian.Uint64(b) + binary.LittleEndian.PutUint64(b, v^key64) + v = binary.LittleEndian.Uint64(b[8:16]) + binary.LittleEndian.PutUint64(b[8:16], v^key64) + v = binary.LittleEndian.Uint64(b[16:24]) + binary.LittleEndian.PutUint64(b[16:24], v^key64) + v = binary.LittleEndian.Uint64(b[24:32]) + binary.LittleEndian.PutUint64(b[24:32], v^key64) + v = binary.LittleEndian.Uint64(b[32:40]) + binary.LittleEndian.PutUint64(b[32:40], v^key64) + v = binary.LittleEndian.Uint64(b[40:48]) + binary.LittleEndian.PutUint64(b[40:48], v^key64) + v = binary.LittleEndian.Uint64(b[48:56]) + binary.LittleEndian.PutUint64(b[48:56], v^key64) + v = binary.LittleEndian.Uint64(b[56:64]) + binary.LittleEndian.PutUint64(b[56:64], v^key64) + v = binary.LittleEndian.Uint64(b[64:72]) + binary.LittleEndian.PutUint64(b[64:72], v^key64) + v = binary.LittleEndian.Uint64(b[72:80]) + binary.LittleEndian.PutUint64(b[72:80], v^key64) + v = binary.LittleEndian.Uint64(b[80:88]) + binary.LittleEndian.PutUint64(b[80:88], v^key64) + v = binary.LittleEndian.Uint64(b[88:96]) + binary.LittleEndian.PutUint64(b[88:96], v^key64) + v = binary.LittleEndian.Uint64(b[96:104]) + binary.LittleEndian.PutUint64(b[96:104], v^key64) + v = binary.LittleEndian.Uint64(b[104:112]) + binary.LittleEndian.PutUint64(b[104:112], v^key64) + v = binary.LittleEndian.Uint64(b[112:120]) + binary.LittleEndian.PutUint64(b[112:120], v^key64) + v = binary.LittleEndian.Uint64(b[120:128]) + binary.LittleEndian.PutUint64(b[120:128], v^key64) + b = b[128:] + } + + // Then we xor until b is less than 64 bytes. + for len(b) >= 64 { + v := binary.LittleEndian.Uint64(b) + binary.LittleEndian.PutUint64(b, v^key64) + v = binary.LittleEndian.Uint64(b[8:16]) + binary.LittleEndian.PutUint64(b[8:16], v^key64) + v = binary.LittleEndian.Uint64(b[16:24]) + binary.LittleEndian.PutUint64(b[16:24], v^key64) + v = binary.LittleEndian.Uint64(b[24:32]) + binary.LittleEndian.PutUint64(b[24:32], v^key64) + v = binary.LittleEndian.Uint64(b[32:40]) + binary.LittleEndian.PutUint64(b[32:40], v^key64) + v = binary.LittleEndian.Uint64(b[40:48]) + binary.LittleEndian.PutUint64(b[40:48], v^key64) + v = binary.LittleEndian.Uint64(b[48:56]) + binary.LittleEndian.PutUint64(b[48:56], v^key64) + v = binary.LittleEndian.Uint64(b[56:64]) + binary.LittleEndian.PutUint64(b[56:64], v^key64) + b = b[64:] + } + + // Then we xor until b is less than 32 bytes. + for len(b) >= 32 { + v := binary.LittleEndian.Uint64(b) + binary.LittleEndian.PutUint64(b, v^key64) + v = binary.LittleEndian.Uint64(b[8:16]) + binary.LittleEndian.PutUint64(b[8:16], v^key64) + v = binary.LittleEndian.Uint64(b[16:24]) + binary.LittleEndian.PutUint64(b[16:24], v^key64) + v = binary.LittleEndian.Uint64(b[24:32]) + binary.LittleEndian.PutUint64(b[24:32], v^key64) + b = b[32:] + } + + // Then we xor until b is less than 16 bytes. + for len(b) >= 16 { + v := binary.LittleEndian.Uint64(b) + binary.LittleEndian.PutUint64(b, v^key64) + v = binary.LittleEndian.Uint64(b[8:16]) + binary.LittleEndian.PutUint64(b[8:16], v^key64) + b = b[16:] + } + + // Then we xor until b is less than 8 bytes. + for len(b) >= 8 { + v := binary.LittleEndian.Uint64(b) + binary.LittleEndian.PutUint64(b, v^key64) + b = b[8:] + } + } + + // Then we xor until b is less than 4 bytes. + for len(b) >= 4 { + v := binary.LittleEndian.Uint32(b) + binary.LittleEndian.PutUint32(b, v^key) + b = b[4:] + } + + // xor remaining bytes. + for i := range b { + b[i] ^= byte(key) + key = bits.RotateLeft32(key, -8) + } + + return key +} diff --git a/backend/internal/pkg/codexws/mask_amd64.s b/backend/internal/pkg/codexws/mask_amd64.s new file mode 100644 index 000000000000..bd42be31f10c --- /dev/null +++ b/backend/internal/pkg/codexws/mask_amd64.s @@ -0,0 +1,127 @@ +#include "textflag.h" + +// func maskAsm(b *byte, len int, key uint32) +TEXT ·maskAsm(SB), NOSPLIT, $0-28 + // AX = b + // CX = len (left length) + // SI = key (uint32) + // DI = uint64(SI) | uint64(SI)<<32 + MOVQ b+0(FP), AX + MOVQ len+8(FP), CX + MOVL key+16(FP), SI + + // calculate the DI + // DI = SI<<32 | SI + MOVL SI, DI + MOVQ DI, DX + SHLQ $32, DI + ORQ DX, DI + + CMPQ CX, $15 + JLE less_than_16 + CMPQ CX, $63 + JLE less_than_64 + CMPQ CX, $128 + JLE sse + TESTQ $31, AX + JNZ unaligned + +unaligned_loop_1byte: + XORB SI, (AX) + INCQ AX + DECQ CX + ROLL $24, SI + TESTQ $7, AX + JNZ unaligned_loop_1byte + + // calculate DI again since SI was modified + // DI = SI<<32 | SI + MOVL SI, DI + MOVQ DI, DX + SHLQ $32, DI + ORQ DX, DI + + TESTQ $31, AX + JZ sse + +unaligned: + TESTQ $7, AX // AND $7 & len, if not zero jump to loop_1b. + JNZ unaligned_loop_1byte + +unaligned_loop: + // we don't need to check the CX since we know it's above 128 + XORQ DI, (AX) + ADDQ $8, AX + SUBQ $8, CX + TESTQ $31, AX + JNZ unaligned_loop + JMP sse + +sse: + CMPQ CX, $0x40 + JL less_than_64 + MOVQ DI, X0 + PUNPCKLQDQ X0, X0 + +sse_loop: + MOVOU 0*16(AX), X1 + MOVOU 1*16(AX), X2 + MOVOU 2*16(AX), X3 + MOVOU 3*16(AX), X4 + PXOR X0, X1 + PXOR X0, X2 + PXOR X0, X3 + PXOR X0, X4 + MOVOU X1, 0*16(AX) + MOVOU X2, 1*16(AX) + MOVOU X3, 2*16(AX) + MOVOU X4, 3*16(AX) + ADDQ $0x40, AX + SUBQ $0x40, CX + CMPQ CX, $0x40 + JAE sse_loop + +less_than_64: + TESTQ $32, CX + JZ less_than_32 + XORQ DI, (AX) + XORQ DI, 8(AX) + XORQ DI, 16(AX) + XORQ DI, 24(AX) + ADDQ $32, AX + +less_than_32: + TESTQ $16, CX + JZ less_than_16 + XORQ DI, (AX) + XORQ DI, 8(AX) + ADDQ $16, AX + +less_than_16: + TESTQ $8, CX + JZ less_than_8 + XORQ DI, (AX) + ADDQ $8, AX + +less_than_8: + TESTQ $4, CX + JZ less_than_4 + XORL SI, (AX) + ADDQ $4, AX + +less_than_4: + TESTQ $2, CX + JZ less_than_2 + XORW SI, (AX) + ROLL $16, SI + ADDQ $2, AX + +less_than_2: + TESTQ $1, CX + JZ done + XORB SI, (AX) + ROLL $24, SI + +done: + MOVL SI, ret+24(FP) + RET diff --git a/backend/internal/pkg/codexws/mask_arm64.s b/backend/internal/pkg/codexws/mask_arm64.s new file mode 100644 index 000000000000..e494b43ab91f --- /dev/null +++ b/backend/internal/pkg/codexws/mask_arm64.s @@ -0,0 +1,72 @@ +#include "textflag.h" + +// func maskAsm(b *byte, len int, key uint32) +TEXT ·maskAsm(SB), NOSPLIT, $0-28 + // R0 = b + // R1 = len + // R3 = key (uint32) + // R2 = uint64(key)<<32 | uint64(key) + MOVD b_ptr+0(FP), R0 + MOVD b_len+8(FP), R1 + MOVWU key+16(FP), R3 + MOVD R3, R2 + ORR R2<<32, R2, R2 + VDUP R2, V0.D2 + CMP $64, R1 + BLT less_than_64 + +loop_64: + VLD1 (R0), [V1.B16, V2.B16, V3.B16, V4.B16] + VEOR V1.B16, V0.B16, V1.B16 + VEOR V2.B16, V0.B16, V2.B16 + VEOR V3.B16, V0.B16, V3.B16 + VEOR V4.B16, V0.B16, V4.B16 + VST1.P [V1.B16, V2.B16, V3.B16, V4.B16], 64(R0) + SUBS $64, R1 + CMP $64, R1 + BGE loop_64 + +less_than_64: + CBZ R1, end + TBZ $5, R1, less_than_32 + VLD1 (R0), [V1.B16, V2.B16] + VEOR V1.B16, V0.B16, V1.B16 + VEOR V2.B16, V0.B16, V2.B16 + VST1.P [V1.B16, V2.B16], 32(R0) + +less_than_32: + TBZ $4, R1, less_than_16 + LDP (R0), (R11, R12) + EOR R11, R2, R11 + EOR R12, R2, R12 + STP.P (R11, R12), 16(R0) + +less_than_16: + TBZ $3, R1, less_than_8 + MOVD (R0), R11 + EOR R2, R11, R11 + MOVD.P R11, 8(R0) + +less_than_8: + TBZ $2, R1, less_than_4 + MOVWU (R0), R11 + EORW R2, R11, R11 + MOVWU.P R11, 4(R0) + +less_than_4: + TBZ $1, R1, less_than_2 + MOVHU (R0), R11 + EORW R3, R11, R11 + MOVHU.P R11, 2(R0) + RORW $16, R3 + +less_than_2: + TBZ $0, R1, end + MOVBU (R0), R11 + EORW R3, R11, R11 + MOVBU.P R11, 1(R0) + RORW $8, R3 + +end: + MOVWU R3, ret+24(FP) + RET diff --git a/backend/internal/pkg/codexws/mask_asm.go b/backend/internal/pkg/codexws/mask_asm.go new file mode 100644 index 000000000000..f9484b5bc097 --- /dev/null +++ b/backend/internal/pkg/codexws/mask_asm.go @@ -0,0 +1,26 @@ +//go:build amd64 || arm64 + +package websocket + +func mask(b []byte, key uint32) uint32 { + // TODO: Will enable in v1.9.0. + return maskGo(b, key) + /* + if len(b) > 0 { + return maskAsm(&b[0], len(b), key) + } + return key + */ +} + +// @nhooyr: I am not confident that the amd64 or the arm64 implementations of this +// function are perfect. There are almost certainly missing optimizations or +// opportunities for simplification. I'm confident there are no bugs though. +// For example, the arm64 implementation doesn't align memory like the amd64. +// Or the amd64 implementation could use AVX512 instead of just AVX2. +// The AVX2 code I had to disable anyway as it wasn't performing as expected. +// See https://github.com/nhooyr/websocket/pull/326#issuecomment-1771138049 +// +//go:noescape +//lint:ignore U1000 disabled till v1.9.0 +func maskAsm(b *byte, len int, key uint32) uint32 diff --git a/backend/internal/pkg/codexws/mask_go.go b/backend/internal/pkg/codexws/mask_go.go new file mode 100644 index 000000000000..b29435e956a5 --- /dev/null +++ b/backend/internal/pkg/codexws/mask_go.go @@ -0,0 +1,7 @@ +//go:build !amd64 && !arm64 && !js + +package websocket + +func mask(b []byte, key uint32) uint32 { + return maskGo(b, key) +} diff --git a/backend/internal/pkg/codexws/netconn.go b/backend/internal/pkg/codexws/netconn.go new file mode 100644 index 000000000000..1f73b04be6c7 --- /dev/null +++ b/backend/internal/pkg/codexws/netconn.go @@ -0,0 +1,233 @@ +package websocket + +import ( + "context" + "fmt" + "io" + "math" + "net" + "sync/atomic" + "time" +) + +// NetConn converts a *websocket.Conn into a net.Conn. +// +// It's for tunneling arbitrary protocols over WebSockets. +// Few users of the library will need this but it's tricky to implement +// correctly and so provided in the library. +// See https://github.com/nhooyr/websocket/issues/100. +// +// Every Write to the net.Conn will correspond to a message write of +// the given type on *websocket.Conn. +// +// The passed ctx bounds the lifetime of the net.Conn. If cancelled, +// all reads and writes on the net.Conn will be cancelled. +// +// If a message is read that is not of the correct type, the connection +// will be closed with StatusUnsupportedData and an error will be returned. +// +// Close will close the *websocket.Conn with StatusNormalClosure. +// +// When a deadline is hit and there is an active read or write goroutine, the +// connection will be closed. This is different from most net.Conn implementations +// where only the reading/writing goroutines are interrupted but the connection +// is kept alive. +// +// The Addr methods will return the real addresses for connections obtained +// from websocket.Accept. But for connections obtained from websocket.Dial, a mock net.Addr +// will be returned that gives "websocket" for Network() and "websocket/unknown-addr" for +// String(). This is because websocket.Dial only exposes a io.ReadWriteCloser instead of the +// full net.Conn to us. +// +// When running as WASM, the Addr methods will always return the mock address described above. +// +// A received StatusNormalClosure or StatusGoingAway close frame will be translated to +// io.EOF when reading. +// +// Furthermore, the ReadLimit is set to -1 to disable it. +func NetConn(ctx context.Context, c *Conn, msgType MessageType) net.Conn { + c.SetReadLimit(-1) + + nc := &netConn{ + c: c, + msgType: msgType, + readMu: newMu(c), + writeMu: newMu(c), + } + + nc.writeCtx, nc.writeCancel = context.WithCancel(ctx) + nc.readCtx, nc.readCancel = context.WithCancel(ctx) + + nc.writeTimer = time.AfterFunc(math.MaxInt64, func() { + if !nc.writeMu.tryLock() { + // If the lock cannot be acquired, then there is an + // active write goroutine and so we should cancel the context. + nc.writeCancel() + return + } + defer nc.writeMu.unlock() + + // Prevents future writes from writing until the deadline is reset. + nc.writeExpired.Store(1) + }) + if !nc.writeTimer.Stop() { + <-nc.writeTimer.C + } + + nc.readTimer = time.AfterFunc(math.MaxInt64, func() { + if !nc.readMu.tryLock() { + // If the lock cannot be acquired, then there is an + // active read goroutine and so we should cancel the context. + nc.readCancel() + return + } + defer nc.readMu.unlock() + + // Prevents future reads from reading until the deadline is reset. + nc.readExpired.Store(1) + }) + if !nc.readTimer.Stop() { + <-nc.readTimer.C + } + + return nc +} + +type netConn struct { + c *Conn + msgType MessageType + + writeTimer *time.Timer + writeMu *mu + writeExpired atomic.Int64 + writeCtx context.Context + writeCancel context.CancelFunc + + readTimer *time.Timer + readMu *mu + readExpired atomic.Int64 + readCtx context.Context + readCancel context.CancelFunc + readEOFed bool + reader io.Reader +} + +var _ net.Conn = &netConn{} + +func (nc *netConn) Close() error { + nc.writeTimer.Stop() + nc.writeCancel() + nc.readTimer.Stop() + nc.readCancel() + return nc.c.Close(StatusNormalClosure, "") +} + +func (nc *netConn) Write(p []byte) (int, error) { + nc.writeMu.forceLock() + defer nc.writeMu.unlock() + + if nc.writeExpired.Load() == 1 { + return 0, fmt.Errorf("failed to write: %w", context.DeadlineExceeded) + } + + err := nc.c.Write(nc.writeCtx, nc.msgType, p) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (nc *netConn) Read(p []byte) (int, error) { + nc.readMu.forceLock() + defer nc.readMu.unlock() + + for { + n, err := nc.read(p) + if err != nil { + return n, err + } + if n == 0 { + continue + } + return n, nil + } +} + +func (nc *netConn) read(p []byte) (int, error) { + if nc.readExpired.Load() == 1 { + return 0, fmt.Errorf("failed to read: %w", context.DeadlineExceeded) + } + + if nc.readEOFed { + return 0, io.EOF + } + + if nc.reader == nil { + typ, r, err := nc.c.Reader(nc.readCtx) + if err != nil { + switch CloseStatus(err) { + case StatusNormalClosure, StatusGoingAway: + nc.readEOFed = true + return 0, io.EOF + } + return 0, err + } + if typ != nc.msgType { + err := fmt.Errorf("unexpected frame type read (expected %v): %v", nc.msgType, typ) + nc.c.Close(StatusUnsupportedData, err.Error()) + return 0, err + } + nc.reader = r + } + + n, err := nc.reader.Read(p) + if err == io.EOF { + nc.reader = nil + err = nil + } + return n, err +} + +type websocketAddr struct{} + +func (a websocketAddr) Network() string { + return "websocket" +} + +func (a websocketAddr) String() string { + return "websocket/unknown-addr" +} + +func (nc *netConn) SetDeadline(t time.Time) error { + nc.SetWriteDeadline(t) + nc.SetReadDeadline(t) + return nil +} + +func (nc *netConn) SetWriteDeadline(t time.Time) error { + nc.writeExpired.Store(0) + if t.IsZero() { + nc.writeTimer.Stop() + } else { + dur := time.Until(t) + if dur <= 0 { + dur = 1 + } + nc.writeTimer.Reset(dur) + } + return nil +} + +func (nc *netConn) SetReadDeadline(t time.Time) error { + nc.readExpired.Store(0) + if t.IsZero() { + nc.readTimer.Stop() + } else { + dur := time.Until(t) + if dur <= 0 { + dur = 1 + } + nc.readTimer.Reset(dur) + } + return nil +} diff --git a/backend/internal/pkg/codexws/netconn_js.go b/backend/internal/pkg/codexws/netconn_js.go new file mode 100644 index 000000000000..ccc8c89fb2dc --- /dev/null +++ b/backend/internal/pkg/codexws/netconn_js.go @@ -0,0 +1,11 @@ +package websocket + +import "net" + +func (nc *netConn) RemoteAddr() net.Addr { + return websocketAddr{} +} + +func (nc *netConn) LocalAddr() net.Addr { + return websocketAddr{} +} diff --git a/backend/internal/pkg/codexws/netconn_notjs.go b/backend/internal/pkg/codexws/netconn_notjs.go new file mode 100644 index 000000000000..cab763492894 --- /dev/null +++ b/backend/internal/pkg/codexws/netconn_notjs.go @@ -0,0 +1,19 @@ +//go:build !js + +package websocket + +import "net" + +func (nc *netConn) RemoteAddr() net.Addr { + if unc, ok := nc.c.rwc.(net.Conn); ok { + return unc.RemoteAddr() + } + return websocketAddr{} +} + +func (nc *netConn) LocalAddr() net.Addr { + if unc, ok := nc.c.rwc.(net.Conn); ok { + return unc.LocalAddr() + } + return websocketAddr{} +} diff --git a/backend/internal/pkg/codexws/read.go b/backend/internal/pkg/codexws/read.go new file mode 100644 index 000000000000..782881113b52 --- /dev/null +++ b/backend/internal/pkg/codexws/read.go @@ -0,0 +1,540 @@ +//go:build !js + +package websocket + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "strings" + "sync/atomic" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/util" +) + +// Reader reads from the connection until there is a WebSocket +// data message to be read. It will handle ping, pong and close frames as appropriate. +// +// It returns the type of the message and an io.Reader to read it. +// The passed context will also bound the reader. +// Ensure you read to EOF otherwise the connection will hang. +// +// Call CloseRead if you do not expect any data messages from the peer. +// +// Only one Reader may be open at a time. +// +// If you need a separate timeout on the Reader call and the Read itself, +// use time.AfterFunc to cancel the context passed in. +// See https://github.com/nhooyr/websocket/issues/87#issue-451703332 +// Most users should not need this. +func (c *Conn) Reader(ctx context.Context) (MessageType, io.Reader, error) { + return c.reader(ctx) +} + +// Read is a convenience method around Reader to read a single message +// from the connection. +func (c *Conn) Read(ctx context.Context) (MessageType, []byte, error) { + typ, r, err := c.Reader(ctx) + if err != nil { + return 0, nil, err + } + + b, err := io.ReadAll(r) + return typ, b, err +} + +// CloseRead starts a goroutine to read from the connection until it is closed +// or a data message is received. +// +// Once CloseRead is called you cannot read any messages from the connection. +// The returned context will be cancelled when the connection is closed. +// +// If a data message is received, the connection will be closed with StatusPolicyViolation. +// +// Call CloseRead when you do not expect to read any more messages. +// Since it actively reads from the connection, it will ensure that ping, pong and close +// frames are responded to. This means c.Ping and c.Close will still work as expected. +// +// This function is idempotent. +func (c *Conn) CloseRead(ctx context.Context) context.Context { + c.closeReadMu.Lock() + ctx2 := c.closeReadCtx + if ctx2 != nil { + c.closeReadMu.Unlock() + return ctx2 + } + ctx, cancel := context.WithCancel(ctx) + c.closeReadCtx = ctx + c.closeReadDone = make(chan struct{}) + c.closeReadMu.Unlock() + + go func() { + defer close(c.closeReadDone) + defer cancel() + defer c.close() + _, _, err := c.Reader(ctx) + if err == nil { + c.Close(StatusPolicyViolation, "unexpected data message") + } + }() + return ctx +} + +// SetReadLimit sets the max number of bytes to read for a single message. +// It applies to the Reader and Read methods. +// +// By default, the connection has a message read limit of 32768 bytes. +// +// When the limit is hit, reads return an error wrapping ErrMessageTooBig and +// the connection is closed with StatusMessageTooBig. +// +// Set to -1 to disable. +func (c *Conn) SetReadLimit(n int64) { + if n >= 0 { + // We read one more byte than the limit in case + // there is a fin frame that needs to be read. + n++ + } + + c.msgReader.limitReader.limit.Store(n) +} + +const defaultReadLimit = 32768 + +func newMsgReader(c *Conn) *msgReader { + mr := &msgReader{ + c: c, + fin: true, + } + mr.readFunc = mr.read + + mr.limitReader = newLimitReader(c, mr.readFunc, defaultReadLimit+1) + return mr +} + +func (mr *msgReader) resetFlate() { + if mr.flateContextTakeover() { + if mr.dict == nil { + mr.dict = &slidingWindow{} + } + mr.dict.init(32768) + } + if mr.flateBufio == nil { + mr.flateBufio = getBufioReader(mr.readFunc) + } + + if mr.flateContextTakeover() { + mr.flateReader = getFlateReader(mr.flateBufio, mr.dict.buf) + } else { + mr.flateReader = getFlateReader(mr.flateBufio, nil) + } + mr.limitReader.r = mr.flateReader + mr.flateTail.Reset(deflateMessageTail) +} + +func (mr *msgReader) putFlateReader() { + if mr.flateReader != nil { + putFlateReader(mr.flateReader) + mr.flateReader = nil + } +} + +func (mr *msgReader) close() { + mr.c.readMu.forceLock() + mr.putFlateReader() + if mr.dict != nil { + mr.dict.close() + mr.dict = nil + } + if mr.flateBufio != nil { + putBufioReader(mr.flateBufio) + } + + if mr.c.client { + putBufioReader(mr.c.br) + mr.c.br = nil + } +} + +func (mr *msgReader) flateContextTakeover() bool { + if mr.c.client { + return !mr.c.copts.serverNoContextTakeover + } + return !mr.c.copts.clientNoContextTakeover +} + +func (c *Conn) readRSV1Illegal(h header) bool { + // If compression is disabled, rsv1 is illegal. + if !c.flate() { + return true + } + // rsv1 is only allowed on data frames beginning messages. + if h.opcode != opText && h.opcode != opBinary { + return true + } + return false +} + +func (c *Conn) readLoop(ctx context.Context) (header, error) { + for { + h, err := c.readFrameHeader(ctx) + if err != nil { + return header{}, err + } + + if h.rsv1 && c.readRSV1Illegal(h) || h.rsv2 || h.rsv3 { + err := fmt.Errorf("received header with unexpected rsv bits set: %v:%v:%v", h.rsv1, h.rsv2, h.rsv3) + c.writeError(StatusProtocolError, err) + return header{}, err + } + + if !c.client && !h.masked { + return header{}, errors.New("received unmasked frame from client") + } + + switch h.opcode { + case opClose, opPing, opPong: + err = c.handleControl(ctx, h) + if err != nil { + // Pass through CloseErrors when receiving a close frame. + if h.opcode == opClose && CloseStatus(err) != -1 { + return header{}, err + } + return header{}, fmt.Errorf("failed to handle control frame %v: %w", h.opcode, err) + } + case opContinuation, opText, opBinary: + return h, nil + default: + err := fmt.Errorf("received unknown opcode %v", h.opcode) + c.writeError(StatusProtocolError, err) + return header{}, err + } + } +} + +// prepareRead sets the readTimeout context and returns a done function +// to be called after the read is done. It also returns an error if the +// connection is closed. The reference to the error is used to assign +// an error depending on if the connection closed or the context timed +// out during use. Typically, the referenced error is a named return +// variable of the function calling this method. +func (c *Conn) prepareRead(ctx context.Context, err *error) (func(), error) { + select { + case <-c.closed: + return nil, net.ErrClosed + default: + } + c.setupReadTimeout(ctx) + + done := func() { + c.clearReadTimeout() + select { + case <-c.closed: + if *err != nil { + *err = net.ErrClosed + } + default: + } + if *err != nil && ctx.Err() != nil { + *err = ctx.Err() + } + } + + c.closeStateMu.Lock() + closeReceivedErr := c.closeReceivedErr + c.closeStateMu.Unlock() + if closeReceivedErr != nil { + defer done() + return nil, closeReceivedErr + } + + return done, nil +} + +func (c *Conn) readFrameHeader(ctx context.Context) (_ header, err error) { + readDone, err := c.prepareRead(ctx, &err) + if err != nil { + return header{}, err + } + defer readDone() + + h, err := readFrameHeader(c.br, c.readHeaderBuf[:]) + if err != nil { + return header{}, err + } + + return h, nil +} + +func (c *Conn) readFramePayload(ctx context.Context, p []byte) (_ int, err error) { + readDone, err := c.prepareRead(ctx, &err) + if err != nil { + return 0, err + } + defer readDone() + + n, err := io.ReadFull(c.br, p) + if err != nil { + return n, fmt.Errorf("failed to read frame payload: %w", err) + } + + return n, nil +} + +func (c *Conn) handleControl(ctx context.Context, h header) (err error) { + if h.payloadLength < 0 || h.payloadLength > maxControlPayload { + err := fmt.Errorf("received control frame payload with invalid length: %d", h.payloadLength) + c.writeError(StatusProtocolError, err) + return err + } + + if !h.fin { + err := errors.New("received fragmented control frame") + c.writeError(StatusProtocolError, err) + return err + } + + ctx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + b := c.readControlBuf[:h.payloadLength] + _, err = c.readFramePayload(ctx, b) + if err != nil { + return err + } + + if h.masked { + mask(b, h.maskKey) + } + + switch h.opcode { + case opPing: + if c.onPingReceived != nil { + if !c.onPingReceived(ctx, b) { + return nil + } + } + return c.writeControl(ctx, opPong, b) + case opPong: + if c.onPongReceived != nil { + c.onPongReceived(ctx, b) + } + c.activePingsMu.Lock() + pong, ok := c.activePings[string(b)] + c.activePingsMu.Unlock() + if ok { + select { + case pong <- struct{}{}: + default: + } + } + return nil + } + + // opClose + + ce, err := parseClosePayload(b) + if err != nil { + err = fmt.Errorf("received invalid close payload: %w", err) + c.writeError(StatusProtocolError, err) + return err + } + + err = fmt.Errorf("received close frame: %w", ce) + c.closeStateMu.Lock() + c.closeReceivedErr = err + closeSent := c.closeSentErr != nil + c.closeStateMu.Unlock() + + // Only unlock readMu if this connection is being closed becaue + // c.close will try to acquire the readMu lock. We unlock for + // writeClose as well because it may also call c.close. + if !closeSent { + c.readMu.unlock() + _ = c.writeClose(ce.Code, ce.Reason) + } + if !c.casClosing() { + c.readMu.unlock() + _ = c.close() + } + return err +} + +func (c *Conn) reader(ctx context.Context) (_ MessageType, _ io.Reader, err error) { + defer errd.Wrap(&err, "failed to get reader") + + err = c.readMu.lock(ctx) + if err != nil { + return 0, nil, err + } + defer c.readMu.unlock() + + if !c.msgReader.fin { + return 0, nil, errors.New("previous message not read to completion") + } + + h, err := c.readLoop(ctx) + if err != nil { + return 0, nil, err + } + + if h.opcode == opContinuation { + err := errors.New("received continuation frame without text or binary frame") + c.writeError(StatusProtocolError, err) + return 0, nil, err + } + + c.msgReader.reset(ctx, h) + + return MessageType(h.opcode), c.msgReader, nil +} + +type msgReader struct { + c *Conn + + ctx context.Context + flate bool + flateReader io.Reader + flateBufio *bufio.Reader + flateTail strings.Reader + limitReader *limitReader + dict *slidingWindow + + fin bool + payloadLength int64 + maskKey uint32 + + // util.ReaderFunc(mr.Read) to avoid continuous allocations. + readFunc util.ReaderFunc +} + +func (mr *msgReader) reset(ctx context.Context, h header) { + mr.ctx = ctx + mr.flate = h.rsv1 + mr.limitReader.reset(mr.readFunc) + + if mr.flate { + mr.resetFlate() + } + + mr.setFrame(h) +} + +func (mr *msgReader) setFrame(h header) { + mr.fin = h.fin + mr.payloadLength = h.payloadLength + mr.maskKey = h.maskKey +} + +func (mr *msgReader) Read(p []byte) (n int, err error) { + err = mr.c.readMu.lock(mr.ctx) + if err != nil { + return 0, fmt.Errorf("failed to read: %w", err) + } + defer mr.c.readMu.unlock() + + n, err = mr.limitReader.Read(p) + if mr.flate && mr.flateContextTakeover() { + p = p[:n] + mr.dict.write(p) + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) && mr.fin && mr.flate { + mr.putFlateReader() + return n, io.EOF + } + if err != nil { + return n, fmt.Errorf("failed to read: %w", err) + } + return n, nil +} + +func (mr *msgReader) read(p []byte) (int, error) { + for { + if mr.payloadLength == 0 { + if mr.fin { + if mr.flate { + return mr.flateTail.Read(p) + } + return 0, io.EOF + } + + h, err := mr.c.readLoop(mr.ctx) + if err != nil { + return 0, err + } + if h.opcode != opContinuation { + err := errors.New("received new data message without finishing the previous message") + mr.c.writeError(StatusProtocolError, err) + return 0, err + } + mr.setFrame(h) + + continue + } + + if int64(len(p)) > mr.payloadLength { + p = p[:mr.payloadLength] + } + + n, err := mr.c.readFramePayload(mr.ctx, p) + if err != nil { + return n, err + } + + mr.payloadLength -= int64(n) + + if !mr.c.client { + mr.maskKey = mask(p, mr.maskKey) + } + + return n, nil + } +} + +type limitReader struct { + c *Conn + r io.Reader + limit atomic.Int64 + n int64 +} + +func newLimitReader(c *Conn, r io.Reader, limit int64) *limitReader { + lr := &limitReader{ + c: c, + } + lr.limit.Store(limit) + lr.reset(r) + return lr +} + +func (lr *limitReader) reset(r io.Reader) { + lr.n = lr.limit.Load() + lr.r = r +} + +func (lr *limitReader) Read(p []byte) (int, error) { + if lr.n < 0 { + return lr.r.Read(p) + } + + if lr.n == 0 { + reason := fmt.Errorf("read limited at %d bytes", lr.limit.Load()) + lr.c.writeError(StatusMessageTooBig, reason) + return 0, fmt.Errorf("%w: %v", ErrMessageTooBig, reason) + } + + if int64(len(p)) > lr.n { + p = p[:lr.n] + } + n, err := lr.r.Read(p) + lr.n -= int64(n) + if lr.n < 0 { + lr.n = 0 + } + return n, err +} diff --git a/backend/internal/pkg/codexws/stringer.go b/backend/internal/pkg/codexws/stringer.go new file mode 100644 index 000000000000..5a66ba290762 --- /dev/null +++ b/backend/internal/pkg/codexws/stringer.go @@ -0,0 +1,91 @@ +// Code generated by "stringer -type=opcode,MessageType,StatusCode -output=stringer.go"; DO NOT EDIT. + +package websocket + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[opContinuation-0] + _ = x[opText-1] + _ = x[opBinary-2] + _ = x[opClose-8] + _ = x[opPing-9] + _ = x[opPong-10] +} + +const ( + _opcode_name_0 = "opContinuationopTextopBinary" + _opcode_name_1 = "opCloseopPingopPong" +) + +var ( + _opcode_index_0 = [...]uint8{0, 14, 20, 28} + _opcode_index_1 = [...]uint8{0, 7, 13, 19} +) + +func (i opcode) String() string { + switch { + case 0 <= i && i <= 2: + return _opcode_name_0[_opcode_index_0[i]:_opcode_index_0[i+1]] + case 8 <= i && i <= 10: + i -= 8 + return _opcode_name_1[_opcode_index_1[i]:_opcode_index_1[i+1]] + default: + return "opcode(" + strconv.FormatInt(int64(i), 10) + ")" + } +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[MessageText-1] + _ = x[MessageBinary-2] +} + +const _MessageType_name = "MessageTextMessageBinary" + +var _MessageType_index = [...]uint8{0, 11, 24} + +func (i MessageType) String() string { + i -= 1 + if i < 0 || i >= MessageType(len(_MessageType_index)-1) { + return "MessageType(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _MessageType_name[_MessageType_index[i]:_MessageType_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[StatusNormalClosure-1000] + _ = x[StatusGoingAway-1001] + _ = x[StatusProtocolError-1002] + _ = x[StatusUnsupportedData-1003] + _ = x[statusReserved-1004] + _ = x[StatusNoStatusRcvd-1005] + _ = x[StatusAbnormalClosure-1006] + _ = x[StatusInvalidFramePayloadData-1007] + _ = x[StatusPolicyViolation-1008] + _ = x[StatusMessageTooBig-1009] + _ = x[StatusMandatoryExtension-1010] + _ = x[StatusInternalError-1011] + _ = x[StatusServiceRestart-1012] + _ = x[StatusTryAgainLater-1013] + _ = x[StatusBadGateway-1014] + _ = x[StatusTLSHandshake-1015] +} + +const _StatusCode_name = "StatusNormalClosureStatusGoingAwayStatusProtocolErrorStatusUnsupportedDatastatusReservedStatusNoStatusRcvdStatusAbnormalClosureStatusInvalidFramePayloadDataStatusPolicyViolationStatusMessageTooBigStatusMandatoryExtensionStatusInternalErrorStatusServiceRestartStatusTryAgainLaterStatusBadGatewayStatusTLSHandshake" + +var _StatusCode_index = [...]uint16{0, 19, 34, 53, 74, 88, 106, 127, 156, 177, 196, 220, 239, 259, 278, 294, 312} + +func (i StatusCode) String() string { + i -= 1000 + if i < 0 || i >= StatusCode(len(_StatusCode_index)-1) { + return "StatusCode(" + strconv.FormatInt(int64(i+1000), 10) + ")" + } + return _StatusCode_name[_StatusCode_index[i]:_StatusCode_index[i+1]] +} diff --git a/backend/internal/pkg/codexws/write.go b/backend/internal/pkg/codexws/write.go new file mode 100644 index 000000000000..d6c05d66d98a --- /dev/null +++ b/backend/internal/pkg/codexws/write.go @@ -0,0 +1,389 @@ +//go:build !js + +package websocket + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "github.com/klauspost/compress/flate" + "io" + "net" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/util" +) + +// Writer returns a writer bounded by the context that will write +// a WebSocket message of type dataType to the connection. +// +// You must close the writer once you have written the entire message. +// +// Only one writer can be open at a time, multiple calls will block until the previous writer +// is closed. +func (c *Conn) Writer(ctx context.Context, typ MessageType) (io.WriteCloser, error) { + w, err := c.writer(ctx, typ) + if err != nil { + return nil, fmt.Errorf("failed to get writer: %w", err) + } + return w, nil +} + +// Write writes a message to the connection. +// +// See the Writer method if you want to stream a message. +// +// If compression is disabled or the compression threshold is not met, then it +// will write the message in a single frame. +func (c *Conn) Write(ctx context.Context, typ MessageType, p []byte) error { + _, err := c.write(ctx, typ, p) + if err != nil { + return fmt.Errorf("failed to write msg: %w", err) + } + return nil +} + +type msgWriter struct { + c *Conn + + mu *mu + writeMu *mu + closed bool + + ctx context.Context + opcode opcode + flate bool + + trimWriter *trimLastFourBytesWriter + flateWriter *flate.Writer + flateWindowBits int +} + +func newMsgWriter(c *Conn) *msgWriter { + mw := &msgWriter{ + c: c, + mu: newMu(c), + writeMu: newMu(c), + } + return mw +} + +func (mw *msgWriter) ensureFlate() { + if mw.trimWriter == nil { + mw.trimWriter = &trimLastFourBytesWriter{ + w: util.WriterFunc(mw.write), + } + } + + if mw.flateWriter == nil { + mw.flateWindowBits = 15 + if mw.c.client && mw.c.copts.clientWindowBits > 0 { + mw.flateWindowBits = mw.c.copts.clientWindowBits + } + mw.flateWriter = getFlateWriter(mw.trimWriter, mw.flateWindowBits) + } + mw.flate = true +} + +func (mw *msgWriter) flateContextTakeover() bool { + if mw.c.client { + return !mw.c.copts.clientNoContextTakeover + } + return !mw.c.copts.serverNoContextTakeover +} + +func (c *Conn) writer(ctx context.Context, typ MessageType) (io.WriteCloser, error) { + err := c.msgWriter.reset(ctx, typ) + if err != nil { + return nil, err + } + return c.msgWriter, nil +} + +func (c *Conn) write(ctx context.Context, typ MessageType, p []byte) (int, error) { + mw, err := c.writer(ctx, typ) + if err != nil { + return 0, err + } + + if !c.flate() { + defer c.msgWriter.mu.unlock() + return c.writeFrame(ctx, true, false, c.msgWriter.opcode, p) + } + + n, err := mw.Write(p) + if err != nil { + return n, err + } + + err = mw.Close() + return n, err +} + +func (mw *msgWriter) reset(ctx context.Context, typ MessageType) error { + err := mw.mu.lock(ctx) + if err != nil { + return err + } + + mw.ctx = ctx + mw.opcode = opcode(typ) + mw.flate = false + mw.closed = false + + mw.trimWriter.reset() + + return nil +} + +func (mw *msgWriter) putFlateWriter() { + if mw.flateWriter != nil { + putFlateWriter(mw.flateWriter, mw.flateWindowBits) + mw.flateWriter = nil + } +} + +// Write writes the given bytes to the WebSocket connection. +func (mw *msgWriter) Write(p []byte) (_ int, err error) { + err = mw.writeMu.lock(mw.ctx) + if err != nil { + return 0, fmt.Errorf("failed to write: %w", err) + } + defer mw.writeMu.unlock() + + if mw.closed { + return 0, errors.New("cannot use closed writer") + } + + defer func() { + if err != nil { + err = fmt.Errorf("failed to write: %w", err) + } + }() + + if mw.c.flate() { + // Only enables flate if the length crosses the + // threshold on the first frame + if mw.opcode != opContinuation && len(p) >= mw.c.flateThreshold { + mw.ensureFlate() + } + } + + if mw.flate { + return mw.flateWriter.Write(p) + } + + return mw.write(p) +} + +func (mw *msgWriter) write(p []byte) (int, error) { + n, err := mw.c.writeFrame(mw.ctx, false, mw.flate, mw.opcode, p) + if err != nil { + return n, fmt.Errorf("failed to write data frame: %w", err) + } + mw.opcode = opContinuation + return n, nil +} + +// Close flushes the frame to the connection. +func (mw *msgWriter) Close() (err error) { + defer errd.Wrap(&err, "failed to close writer") + + err = mw.writeMu.lock(mw.ctx) + if err != nil { + return err + } + defer mw.writeMu.unlock() + + if mw.closed { + return errors.New("writer already closed") + } + mw.closed = true + + if mw.flate { + err = mw.flateWriter.Flush() + if err != nil { + return fmt.Errorf("failed to flush flate: %w", err) + } + } + + _, err = mw.c.writeFrame(mw.ctx, true, mw.flate, mw.opcode, nil) + if err != nil { + return fmt.Errorf("failed to write fin frame: %w", err) + } + + if mw.flate && !mw.flateContextTakeover() { + mw.putFlateWriter() + } + mw.mu.unlock() + return nil +} + +func (mw *msgWriter) close() { + if mw.c.client { + mw.c.writeFrameMu.forceLock() + putBufioWriter(mw.c.bw) + } + + mw.writeMu.forceLock() + mw.putFlateWriter() +} + +func (c *Conn) writeControl(ctx context.Context, opcode opcode, p []byte) error { + ctx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + + _, err := c.writeFrame(ctx, true, false, opcode, p) + if err != nil { + return fmt.Errorf("failed to write control frame %v: %w", opcode, err) + } + return nil +} + +// writeFrame handles all writes to the connection. +func (c *Conn) writeFrame(ctx context.Context, fin bool, flate bool, opcode opcode, p []byte) (_ int, err error) { + err = c.writeFrameMu.lock(ctx) + if err != nil { + return 0, err + } + defer c.writeFrameMu.unlock() + + defer func() { + if c.isClosed() && opcode == opClose { + err = nil + } + if err != nil { + if ctx.Err() != nil { + err = ctx.Err() + } else if c.isClosed() { + err = net.ErrClosed + } + err = fmt.Errorf("failed to write frame: %w", err) + } + }() + + c.closeStateMu.Lock() + closeSentErr := c.closeSentErr + c.closeStateMu.Unlock() + if closeSentErr != nil { + return 0, net.ErrClosed + } + + select { + case <-c.closed: + return 0, net.ErrClosed + default: + } + c.setupWriteTimeout(ctx) + defer c.clearWriteTimeout() + + c.writeHeader.fin = fin + c.writeHeader.opcode = opcode + c.writeHeader.payloadLength = int64(len(p)) + + if c.client { + c.writeHeader.masked = true + _, err = io.ReadFull(rand.Reader, c.writeHeaderBuf[:4]) + if err != nil { + return 0, fmt.Errorf("failed to generate masking key: %w", err) + } + c.writeHeader.maskKey = binary.LittleEndian.Uint32(c.writeHeaderBuf[:]) + } + + c.writeHeader.rsv1 = false + if flate && (opcode == opText || opcode == opBinary) { + c.writeHeader.rsv1 = true + } + + err = writeFrameHeader(c.writeHeader, c.bw, c.writeHeaderBuf[:]) + if err != nil { + return 0, err + } + + n, err := c.writeFramePayload(p) + if err != nil { + return n, err + } + + if c.writeHeader.fin { + err = c.bw.Flush() + if err != nil { + return n, fmt.Errorf("failed to flush: %w", err) + } + } + + if opcode == opClose { + c.closeStateMu.Lock() + c.closeSentErr = fmt.Errorf("sent close frame: %w", net.ErrClosed) + closeReceived := c.closeReceivedErr != nil + c.closeStateMu.Unlock() + + if closeReceived && !c.casClosing() { + c.writeFrameMu.unlock() + _ = c.close() + } + } + + return n, nil +} + +func (c *Conn) writeFramePayload(p []byte) (n int, err error) { + defer errd.Wrap(&err, "failed to write frame payload") + + if !c.writeHeader.masked { + return c.bw.Write(p) + } + + maskKey := c.writeHeader.maskKey + for len(p) > 0 { + // If the buffer is full, we need to flush. + if c.bw.Available() == 0 { + err = c.bw.Flush() + if err != nil { + return n, err + } + } + + // Start of next write in the buffer. + i := c.bw.Buffered() + + j := min(len(p), c.bw.Available()) + + _, err := c.bw.Write(p[:j]) + if err != nil { + return n, err + } + + maskKey = mask(c.writeBuf[i:c.bw.Buffered()], maskKey) + + p = p[j:] + n += j + } + + return n, nil +} + +// extractBufioWriterBuf grabs the []byte backing a *bufio.Writer +// and returns it. +func extractBufioWriterBuf(bw *bufio.Writer, w io.Writer) []byte { + var writeBuf []byte + bw.Reset(util.WriterFunc(func(p2 []byte) (int, error) { + writeBuf = p2[:cap(p2)] + return len(p2), nil + })) + + bw.WriteByte(0) + bw.Flush() + + bw.Reset(w) + + return writeBuf +} + +func (c *Conn) writeError(code StatusCode, err error) { + c.writeClose(code, err.Error()) +} diff --git a/backend/internal/pkg/codexws/ws_js.go b/backend/internal/pkg/codexws/ws_js.go new file mode 100644 index 000000000000..e4412834ba7d --- /dev/null +++ b/backend/internal/pkg/codexws/ws_js.go @@ -0,0 +1,598 @@ +package websocket // import "github.com/Wei-Shaw/sub2api/internal/pkg/codexws" + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "reflect" + "runtime" + "strings" + "sync" + "sync/atomic" + "syscall/js" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/bpool" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/wsjs" +) + +// opcode represents a WebSocket opcode. +type opcode int + +// https://tools.ietf.org/html/rfc6455#section-11.8. +const ( + opContinuation opcode = iota + opText + opBinary + // 3 - 7 are reserved for further non-control frames. + _ + _ + _ + _ + _ + opClose + opPing + opPong + // 11-16 are reserved for further control frames. +) + +// Conn provides a wrapper around the browser WebSocket API. +type Conn struct { + noCopy noCopy + ws wsjs.WebSocket + + // read limit for a message in bytes. + msgReadLimit atomic.Int64 + + closeReadMu sync.Mutex + closeReadCtx context.Context + + closingMu sync.Mutex + closeOnce sync.Once + closed chan struct{} + closeErrOnce sync.Once + closeErr error + closeWasClean bool + + releaseOnClose func() + releaseOnError func() + releaseOnMessage func() + + readSignal chan struct{} + readBufMu sync.Mutex + readBuf []wsjs.MessageEvent +} + +func (c *Conn) close(err error, wasClean bool) { + c.closeOnce.Do(func() { + runtime.SetFinalizer(c, nil) + + if !wasClean { + err = fmt.Errorf("unclean connection close: %w", err) + } + c.setCloseErr(err) + c.closeWasClean = wasClean + close(c.closed) + }) +} + +func (c *Conn) init() { + c.closed = make(chan struct{}) + c.readSignal = make(chan struct{}, 1) + + c.msgReadLimit.Store(32768) + + c.releaseOnClose = c.ws.OnClose(func(e wsjs.CloseEvent) { + err := CloseError{ + Code: StatusCode(e.Code), + Reason: e.Reason, + } + // We do not know if we sent or received this close as + // its possible the browser triggered it without us + // explicitly sending it. + c.close(err, e.WasClean) + + c.releaseOnClose() + c.releaseOnError() + c.releaseOnMessage() + }) + + c.releaseOnError = c.ws.OnError(func(v js.Value) { + c.setCloseErr(errors.New(v.Get("message").String())) + c.closeWithInternal() + }) + + c.releaseOnMessage = c.ws.OnMessage(func(e wsjs.MessageEvent) { + c.readBufMu.Lock() + defer c.readBufMu.Unlock() + + c.readBuf = append(c.readBuf, e) + + // Lets the read goroutine know there is definitely something in readBuf. + select { + case c.readSignal <- struct{}{}: + default: + } + }) + + runtime.SetFinalizer(c, func(c *Conn) { + c.setCloseErr(errors.New("connection garbage collected")) + c.closeWithInternal() + }) +} + +func (c *Conn) closeWithInternal() { + c.Close(StatusInternalError, "something went wrong") +} + +// Read attempts to read a message from the connection. +// The maximum time spent waiting is bounded by the context. +func (c *Conn) Read(ctx context.Context) (MessageType, []byte, error) { + c.closeReadMu.Lock() + closedRead := c.closeReadCtx != nil + c.closeReadMu.Unlock() + if closedRead { + return 0, nil, errors.New("WebSocket connection read closed") + } + + typ, p, err := c.read(ctx) + if err != nil { + return 0, nil, fmt.Errorf("failed to read: %w", err) + } + readLimit := c.msgReadLimit.Load() + if readLimit >= 0 && int64(len(p)) > readLimit { + reason := fmt.Errorf("read limited at %d bytes", c.msgReadLimit.Load()) + c.Close(StatusMessageTooBig, reason.Error()) + return 0, nil, fmt.Errorf("%w: %v", ErrMessageTooBig, reason) + } + return typ, p, nil +} + +func (c *Conn) read(ctx context.Context) (MessageType, []byte, error) { + select { + case <-ctx.Done(): + c.Close(StatusPolicyViolation, "read timed out") + return 0, nil, ctx.Err() + case <-c.readSignal: + case <-c.closed: + return 0, nil, net.ErrClosed + } + + c.readBufMu.Lock() + defer c.readBufMu.Unlock() + + me := c.readBuf[0] + // We copy the messages forward and decrease the size + // of the slice to avoid reallocating. + copy(c.readBuf, c.readBuf[1:]) + c.readBuf = c.readBuf[:len(c.readBuf)-1] + + if len(c.readBuf) > 0 { + // Next time we read, we'll grab the message. + select { + case c.readSignal <- struct{}{}: + default: + } + } + + switch p := me.Data.(type) { + case string: + return MessageText, []byte(p), nil + case []byte: + return MessageBinary, p, nil + default: + panic("websocket: unexpected data type from wsjs OnMessage: " + reflect.TypeOf(me.Data).String()) + } +} + +// Ping is mocked out for Wasm. +func (c *Conn) Ping(ctx context.Context) error { + return nil +} + +// Write writes a message of the given type to the connection. +// Always non blocking. +func (c *Conn) Write(ctx context.Context, typ MessageType, p []byte) error { + err := c.write(typ, p) + if err != nil { + // Have to ensure the WebSocket is closed after a write error + // to match the Go API. It can only error if the message type + // is unexpected or the passed bytes contain invalid UTF-8 for + // MessageText. + err := fmt.Errorf("failed to write: %w", err) + c.setCloseErr(err) + c.closeWithInternal() + return err + } + return nil +} + +func (c *Conn) write(typ MessageType, p []byte) error { + if c.isClosed() { + return net.ErrClosed + } + switch typ { + case MessageBinary: + return c.ws.SendBytes(p) + case MessageText: + return c.ws.SendText(string(p)) + default: + return fmt.Errorf("unexpected message type: %v", typ) + } +} + +// Close closes the WebSocket with the given code and reason. +// It will wait until the peer responds with a close frame +// or the connection is closed. +// It thus performs the full WebSocket close handshake. +func (c *Conn) Close(code StatusCode, reason string) error { + err := c.exportedClose(code, reason) + if err != nil { + return fmt.Errorf("failed to close WebSocket: %w", err) + } + return nil +} + +// CloseNow closes the WebSocket connection without attempting a close handshake. +// Use when you do not want the overhead of the close handshake. +// +// note: No different from Close(StatusGoingAway, "") in WASM as there is no way to close +// a WebSocket without the close handshake. +func (c *Conn) CloseNow() error { + return c.Close(StatusGoingAway, "") +} + +func (c *Conn) exportedClose(code StatusCode, reason string) error { + c.closingMu.Lock() + defer c.closingMu.Unlock() + + if c.isClosed() { + return net.ErrClosed + } + + ce := fmt.Errorf("sent close: %w", CloseError{ + Code: code, + Reason: reason, + }) + + c.setCloseErr(ce) + err := c.ws.Close(int(code), reason) + if err != nil { + return err + } + + <-c.closed + if !c.closeWasClean { + return c.closeErr + } + return nil +} + +// Subprotocol returns the negotiated subprotocol. +// An empty string means the default protocol. +func (c *Conn) Subprotocol() string { + return c.ws.Subprotocol() +} + +// DialOptions represents the options available to pass to Dial. +type DialOptions struct { + // Subprotocols lists the subprotocols to negotiate with the server. + Subprotocols []string +} + +// Dial creates a new WebSocket connection to the given url with the given options. +// The passed context bounds the maximum time spent waiting for the connection to open. +// The returned *http.Response is always nil or a mock. It's only in the signature +// to match the core API. +func Dial(ctx context.Context, url string, opts *DialOptions) (*Conn, *http.Response, error) { + c, resp, err := dial(ctx, url, opts) + if err != nil { + return nil, nil, fmt.Errorf("failed to WebSocket dial %q: %w", url, err) + } + return c, resp, nil +} + +func dial(ctx context.Context, url string, opts *DialOptions) (*Conn, *http.Response, error) { + if opts == nil { + opts = &DialOptions{} + } + + url = strings.Replace(url, "http://", "ws://", 1) + url = strings.Replace(url, "https://", "wss://", 1) + + ws, err := wsjs.New(url, opts.Subprotocols) + if err != nil { + return nil, nil, err + } + + c := &Conn{ + ws: ws, + } + c.init() + + opench := make(chan struct{}) + releaseOpen := ws.OnOpen(func(e js.Value) { + close(opench) + }) + defer releaseOpen() + + select { + case <-ctx.Done(): + c.Close(StatusPolicyViolation, "dial timed out") + return nil, nil, ctx.Err() + case <-opench: + return c, &http.Response{ + StatusCode: http.StatusSwitchingProtocols, + }, nil + case <-c.closed: + return nil, nil, net.ErrClosed + } +} + +// Reader attempts to read a message from the connection. +// The maximum time spent waiting is bounded by the context. +func (c *Conn) Reader(ctx context.Context) (MessageType, io.Reader, error) { + typ, p, err := c.Read(ctx) + if err != nil { + return 0, nil, err + } + return typ, bytes.NewReader(p), nil +} + +// Writer returns a writer to write a WebSocket data message to the connection. +// It buffers the entire message in memory and then sends it when the writer +// is closed. +func (c *Conn) Writer(ctx context.Context, typ MessageType) (io.WriteCloser, error) { + return &writer{ + c: c, + ctx: ctx, + typ: typ, + b: bpool.Get(), + }, nil +} + +type writer struct { + closed bool + + c *Conn + ctx context.Context + typ MessageType + + b *bytes.Buffer +} + +func (w *writer) Write(p []byte) (int, error) { + if w.closed { + return 0, errors.New("cannot write to closed writer") + } + n, err := w.b.Write(p) + if err != nil { + return n, fmt.Errorf("failed to write message: %w", err) + } + return n, nil +} + +func (w *writer) Close() error { + if w.closed { + return errors.New("cannot close closed writer") + } + w.closed = true + defer bpool.Put(w.b) + + err := w.c.Write(w.ctx, w.typ, w.b.Bytes()) + if err != nil { + return fmt.Errorf("failed to close writer: %w", err) + } + return nil +} + +// CloseRead implements *Conn.CloseRead for wasm. +func (c *Conn) CloseRead(ctx context.Context) context.Context { + c.closeReadMu.Lock() + ctx2 := c.closeReadCtx + if ctx2 != nil { + c.closeReadMu.Unlock() + return ctx2 + } + ctx, cancel := context.WithCancel(ctx) + c.closeReadCtx = ctx + c.closeReadMu.Unlock() + + go func() { + defer cancel() + defer c.CloseNow() + _, _, err := c.read(ctx) + if err != nil { + c.Close(StatusPolicyViolation, "unexpected data message") + } + }() + return ctx +} + +// SetReadLimit implements *Conn.SetReadLimit for wasm. +func (c *Conn) SetReadLimit(n int64) { + c.msgReadLimit.Store(n) +} + +func (c *Conn) setCloseErr(err error) { + c.closeErrOnce.Do(func() { + c.closeErr = fmt.Errorf("WebSocket closed: %w", err) + }) +} + +func (c *Conn) isClosed() bool { + select { + case <-c.closed: + return true + default: + return false + } +} + +// AcceptOptions represents Accept's options. +type AcceptOptions struct { + Subprotocols []string + InsecureSkipVerify bool + OriginPatterns []string + CompressionMode CompressionMode + CompressionThreshold int +} + +// Accept is stubbed out for Wasm. +func Accept(w http.ResponseWriter, r *http.Request, opts *AcceptOptions) (*Conn, error) { + return nil, errors.New("unimplemented") +} + +// StatusCode represents a WebSocket status code. +// https://tools.ietf.org/html/rfc6455#section-7.4 +type StatusCode int + +// https://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number +// +// These are only the status codes defined by the protocol. +// +// You can define custom codes in the 3000-4999 range. +// The 3000-3999 range is reserved for use by libraries, frameworks and applications. +// The 4000-4999 range is reserved for private use. +const ( + StatusNormalClosure StatusCode = 1000 + StatusGoingAway StatusCode = 1001 + StatusProtocolError StatusCode = 1002 + StatusUnsupportedData StatusCode = 1003 + + // 1004 is reserved and so unexported. + statusReserved StatusCode = 1004 + + // StatusNoStatusRcvd cannot be sent in a close message. + // It is reserved for when a close message is received without + // a status code. + StatusNoStatusRcvd StatusCode = 1005 + + // StatusAbnormalClosure is exported for use only with Wasm. + // In non Wasm Go, the returned error will indicate whether the + // connection was closed abnormally. + StatusAbnormalClosure StatusCode = 1006 + + StatusInvalidFramePayloadData StatusCode = 1007 + StatusPolicyViolation StatusCode = 1008 + StatusMessageTooBig StatusCode = 1009 + StatusMandatoryExtension StatusCode = 1010 + StatusInternalError StatusCode = 1011 + StatusServiceRestart StatusCode = 1012 + StatusTryAgainLater StatusCode = 1013 + StatusBadGateway StatusCode = 1014 + + // StatusTLSHandshake is only exported for use with Wasm. + // In non Wasm Go, the returned error will indicate whether there was + // a TLS handshake failure. + StatusTLSHandshake StatusCode = 1015 +) + +// CloseError is returned when the connection is closed with a status and reason. +// +// Use Go 1.13's errors.As to check for this error. +// Also see the CloseStatus helper. +type CloseError struct { + Code StatusCode + Reason string +} + +func (ce CloseError) Error() string { + return fmt.Sprintf("status = %v and reason = %q", ce.Code, ce.Reason) +} + +// CloseStatus is a convenience wrapper around Go 1.13's errors.As to grab +// the status code from a CloseError. +// +// -1 will be returned if the passed error is nil or not a CloseError. +func CloseStatus(err error) StatusCode { + var ce CloseError + if errors.As(err, &ce) { + return ce.Code + } + return -1 +} + +// CompressionMode represents the modes available to the deflate extension. +// See https://tools.ietf.org/html/rfc7692 +// Works in all browsers except Safari which does not implement the deflate extension. +type CompressionMode int + +const ( + // CompressionNoContextTakeover grabs a new flate.Reader and flate.Writer as needed + // for every message. This applies to both server and client side. + // + // This means less efficient compression as the sliding window from previous messages + // will not be used but the memory overhead will be lower if the connections + // are long lived and seldom used. + // + // The message will only be compressed if greater than 512 bytes. + CompressionNoContextTakeover CompressionMode = iota + + // CompressionContextTakeover uses a flate.Reader and flate.Writer per connection. + // This enables reusing the sliding window from previous messages. + // As most WebSocket protocols are repetitive, this can be very efficient. + // It carries an overhead of 8 kB for every connection compared to CompressionNoContextTakeover. + // + // If the peer negotiates NoContextTakeover on the client or server side, it will be + // used instead as this is required by the RFC. + CompressionContextTakeover + + // CompressionDisabled disables the deflate extension. + // + // Use this if you are using a predominantly binary protocol with very + // little duplication in between messages or CPU and memory are more + // important than bandwidth. + CompressionDisabled +) + +// MessageType represents the type of a WebSocket message. +// See https://tools.ietf.org/html/rfc6455#section-5.6 +type MessageType int + +// MessageType constants. +const ( + // MessageText is for UTF-8 encoded text messages like JSON. + MessageText MessageType = iota + 1 + // MessageBinary is for binary messages like protobufs. + MessageBinary +) + +type mu struct { + c *Conn + ch chan struct{} +} + +func newMu(c *Conn) *mu { + return &mu{ + c: c, + ch: make(chan struct{}, 1), + } +} + +func (m *mu) forceLock() { + m.ch <- struct{}{} +} + +func (m *mu) tryLock() bool { + select { + case m.ch <- struct{}{}: + return true + default: + return false + } +} + +func (m *mu) unlock() { + select { + case <-m.ch: + default: + } +} + +type noCopy struct{} + +func (*noCopy) Lock() {} diff --git a/backend/internal/pkg/codexws/wsjson/wsjson.go b/backend/internal/pkg/codexws/wsjson/wsjson.go new file mode 100644 index 000000000000..3ad3bf2f4308 --- /dev/null +++ b/backend/internal/pkg/codexws/wsjson/wsjson.go @@ -0,0 +1,68 @@ +// Package wsjson provides helpers for reading and writing JSON messages. +package wsjson // import "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/wsjson" + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/bpool" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/errd" + "github.com/Wei-Shaw/sub2api/internal/pkg/codexws/internal/util" +) + +// Read reads a JSON message from c into v. +// It will reuse buffers in between calls to avoid allocations. +func Read(ctx context.Context, c *websocket.Conn, v any) error { + return read(ctx, c, v) +} + +func read(ctx context.Context, c *websocket.Conn, v any) (err error) { + defer errd.Wrap(&err, "failed to read JSON message") + + _, r, err := c.Reader(ctx) + if err != nil { + return err + } + + b := bpool.Get() + defer bpool.Put(b) + + _, err = b.ReadFrom(r) + if err != nil { + return err + } + + err = json.Unmarshal(b.Bytes(), v) + if err != nil { + c.Close(websocket.StatusInvalidFramePayloadData, "failed to unmarshal JSON") + return fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + return nil +} + +// Write writes the JSON message v to c. +// It will reuse buffers in between calls to avoid allocations. +func Write(ctx context.Context, c *websocket.Conn, v any) error { + return write(ctx, c, v) +} + +func write(ctx context.Context, c *websocket.Conn, v any) (err error) { + defer errd.Wrap(&err, "failed to write JSON message") + + // json.Marshal cannot reuse buffers between calls as it has to return + // a copy of the byte slice but Encoder does as it directly writes to w. + err = json.NewEncoder(util.WriterFunc(func(p []byte) (int, error) { + err := c.Write(ctx, websocket.MessageText, p) + if err != nil { + return 0, err + } + return len(p), nil + })).Encode(v) + if err != nil { + return fmt.Errorf("failed to marshal JSON: %w", err) + } + return nil +} diff --git a/backend/internal/repository/http_upstream.go b/backend/internal/repository/http_upstream.go index f10ab2906d61..a49649c23d22 100644 --- a/backend/internal/repository/http_upstream.go +++ b/backend/internal/repository/http_upstream.go @@ -10,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "io" "log/slog" "net" @@ -218,7 +219,12 @@ func (s *httpUpstreamService) Do(req *http.Request, proxyURL string, accountID i } // 获取或创建对应的客户端，并标记请求占用 - entry, err := s.acquireClientWithProfile(proxyURL, accountID, accountConcurrency, profile) + var entry *upstreamClientEntry + if binding, ok := codextransport.FromContext(req.Context()); ok { + entry, err = s.acquireCodexClient(proxyURL, accountID, accountConcurrency, binding) + } else { + entry, err = s.acquireClientWithProfile(proxyURL, accountID, accountConcurrency, profile) + } if err != nil { return nil, err } diff --git a/backend/internal/repository/http_upstream_codex.go b/backend/internal/repository/http_upstream_codex.go new file mode 100644 index 000000000000..45f4b5d9940f --- /dev/null +++ b/backend/internal/repository/http_upstream_codex.go @@ -0,0 +1,51 @@ +package repository + +import ( + "net/http" + "sync/atomic" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + "github.com/Wei-Shaw/sub2api/internal/service" +) + +func (s *httpUpstreamService) acquireCodexClient(proxyURL string, accountID int64, concurrency int, binding codextransport.Binding) (*upstreamClientEntry, error) { + isolation := s.getIsolationMode() + proxyKey, parsed, err := normalizeProxyURL(proxyURL) + if err != nil { + return nil, err + } + settings := s.applyProfilePoolSettings(s.resolvePoolSettings(isolation, concurrency), service.HTTPUpstreamProfileOpenAI) + key := "codex:" + binding.Key() + ":" + buildCacheKey(isolation, proxyKey, accountID, "codex_verified") + poolKey := buildPoolKey(settings, "codex_verified") + ":" + binding.Key() + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now() + if entry, ok := s.clients[key]; ok { + if s.shouldReuseEntry(entry, isolation, proxyKey, poolKey) { + atomic.StoreInt64(&entry.lastUsed, now.UnixNano()) + atomic.AddInt64(&entry.inFlight, 1) + return entry, nil + } + s.removeClientLocked(key, entry) + } + if s.maxUpstreamClients() > 0 && len(s.clients) >= s.maxUpstreamClients() { + s.evictIdleLocked(now) + if len(s.clients) >= s.maxUpstreamClients() && !s.evictOldestIdleLocked() { + return nil, errUpstreamClientLimitReached + } + } + rt, err := codextransport.New(binding, codextransport.Options{ProxyURL: parsed, DialContext: newUpstreamDialer().DialContext, + MaxIdleConns: settings.maxIdleConns, MaxIdleConnsPerHost: settings.maxIdleConnsPerHost, MaxConnsPerHost: settings.maxConnsPerHost, + IdleConnTimeout: settings.idleConnTimeout, ResponseHeaderTimeout: settings.responseHeaderTimeout}) + if err != nil { + return nil, err + } + client := &http.Client{Transport: rt} + if s.shouldValidateResolvedIP() { + client.CheckRedirect = s.redirectChecker + } + entry := &upstreamClientEntry{client: client, proxyKey: proxyKey, poolKey: poolKey, protocolMode: "codex_verified", lastUsed: now.UnixNano(), inFlight: 1} + s.clients[key] = entry + return entry, nil +} diff --git a/backend/internal/repository/openai_oauth_service.go b/backend/internal/repository/openai_oauth_service.go index f7ad47e2e041..988b72a05b65 100644 --- a/backend/internal/repository/openai_oauth_service.go +++ b/backend/internal/repository/openai_oauth_service.go @@ -3,6 +3,7 @@ package repository import ( "context" "errors" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "net/http" "net/url" "strings" @@ -24,7 +25,7 @@ type openaiOAuthService struct { } func (s *openaiOAuthService) ExchangeCode(ctx context.Context, code, codeVerifier, redirectURI, proxyURL, clientID string) (*openai.TokenResponse, error) { - client, err := createOpenAIReqClient(proxyURL) + client, err := createOpenAIReqClientForContext(ctx, proxyURL) if err != nil { return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_OAUTH_CLIENT_INIT_FAILED", "create HTTP client: %v", err) } @@ -102,7 +103,7 @@ func (s *openaiOAuthService) RefreshTokenWithIdentity(ctx context.Context, refre } func (s *openaiOAuthService) refreshTokenWithIdentity(ctx context.Context, refreshToken, proxyURL, clientID, authUA, authOriginator string) (*openai.TokenResponse, error) { - client, err := createOpenAIReqClient(proxyURL) + client, err := createOpenAIReqClientForContext(ctx, proxyURL) if err != nil { return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_OAUTH_CLIENT_INIT_FAILED", "create HTTP client: %v", err) } @@ -138,10 +139,21 @@ func (s *openaiOAuthService) refreshTokenWithIdentity(ctx context.Context, refre } func createOpenAIReqClient(proxyURL string) (*req.Client, error) { - return getSharedReqClient(reqClientOptions{ - ProxyURL: proxyURL, - Timeout: 120 * time.Second, - }) + return createOpenAIReqClientForContext(context.Background(), proxyURL) +} +func createOpenAIReqClientForContext(ctx context.Context, proxyURL string) (*req.Client, error) { + opts := reqClientOptions{ProxyURL: proxyURL, Timeout: 120 * time.Second} + if binding, ok := codextransport.FromContext(ctx); ok { + opts.CodexProfile = &binding + } else if service.CodexFingerprintSimulationEnabled() { + ua, _ := service.CodexCanonicalAuthIdentity() + binding, err := codextransport.Select(ua, "initial-authorization", "http") + if err != nil { + return nil, err + } + opts.CodexProfile = &binding + } + return getSharedReqClient(opts) } func shouldReturnOpenAINoProxyHint(ctx context.Context, proxyURL string, err error) bool { diff --git a/backend/internal/repository/req_client_pool.go b/backend/internal/repository/req_client_pool.go index 26f3421ae0da..0b289b90589f 100644 --- a/backend/internal/repository/req_client_pool.go +++ b/backend/internal/repository/req_client_pool.go @@ -2,6 +2,7 @@ package repository import ( "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "net/http" "strings" "sync" @@ -15,10 +16,11 @@ import ( // reqClientOptions 定义 req 客户端的构建参数 type reqClientOptions struct { - ProxyURL string // 代理 URL（支持 http/https/socks5） - Timeout time.Duration // 请求超时时间 - Impersonate bool // 是否模拟浏览器指纹（当前为 Firefox，Chrome 伪装会被 chatgpt.com 的 Cloudflare 质询） - ForceHTTP2 bool // 是否强制使用 HTTP/2 + CodexProfile *codextransport.Binding + ProxyURL string // 代理 URL（支持 http/https/socks5） + Timeout time.Duration // 请求超时时间 + Impersonate bool // 是否模拟浏览器指纹（当前为 Firefox，Chrome 伪装会被 chatgpt.com 的 Cloudflare 质询） + ForceHTTP2 bool // 是否强制使用 HTTP/2 } // sharedReqClients 存储按配置参数缓存的 req 客户端实例 @@ -46,6 +48,19 @@ func getSharedReqClient(opts reqClientOptions) (*req.Client, error) { } client := req.C().SetTimeout(opts.Timeout) + if opts.CodexProfile != nil { + _, parsed, err := normalizeProxyURL(opts.ProxyURL) + if err != nil { + return nil, err + } + rt, err := codextransport.New(*opts.CodexProfile, codextransport.Options{ProxyURL: parsed, MaxIdleConns: 32, MaxIdleConnsPerHost: 8, IdleConnTimeout: 90 * time.Second}) + if err != nil { + return nil, err + } + client.Transport = rt + client.GetClient().Transport = rt + } + if opts.ForceHTTP2 { client = client.EnableForceHTTP2() } @@ -85,12 +100,16 @@ func instrumentReqClient(client *req.Client) *req.Client { } func buildReqClientKey(opts reqClientOptions) string { - return fmt.Sprintf("%s|%s|%t|%t", + key := fmt.Sprintf("%s|%s|%t|%t", strings.TrimSpace(opts.ProxyURL), opts.Timeout.String(), opts.Impersonate, opts.ForceHTTP2, ) + if opts.CodexProfile != nil { + key += "|codex:" + opts.CodexProfile.Key() + } + return key } // CreatePrivacyReqClient creates an HTTP client for OpenAI privacy settings API diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index 3f01791e696c..83f882733e8e 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/rand" + "crypto/tls" "encoding/base64" "encoding/binary" "encoding/hex" @@ -1004,6 +1005,14 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account } payload := createOpenAITestPayload(upstreamTestModelID, isOAuth, prompt) applyAccountTestReasoning(c, payload, false) + var identityHeaders http.Header + if isOAuth { + var err error + identityHeaders, err = projectCodexAccountTest(c, account, credentialAccount, payload) + if err != nil { + return s.sendErrorAndEnd(c, err.Error()) + } + } payloadBytes, _ := json.Marshal(payload) // Send test_start event once. A task-invalid Agent Identity response may @@ -1018,6 +1027,9 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account } req = req.WithContext(WithHTTPUpstreamProfile(req.Context(), HTTPUpstreamProfileOpenAI)) + for name, values := range identityHeaders { + req.Header[name] = values + } // Set common headers req.Header.Set("Content-Type", "application/json") if !isOAuth { @@ -1059,6 +1071,13 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account if account.ProxyID != nil && account.Proxy != nil { proxyURL = account.Proxy.URL() } + if isOAuth { + var err error + req, err = bindCodexTransport(req, credentialAccount, codexFingerprintPolicyForContext(c, account), "http") + if err != nil { + return s.sendErrorAndEnd(c, err.Error()) + } + } if isOAuth { // Report the request this test puts on the wire before it is sent. A // plugin round trip takes the plaintext request; every other send goes @@ -1070,7 +1089,8 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account return s.sendErrorAndEnd(c, fmt.Sprintf("Request failed: %s", err.Error())) } base := resolveCodexIdentitySnapshot(account, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount), codexFingerprintPolicyForContext(c, account)) - snapshot := base.withWire(transport, req.Header) + snapshot := base.withRequest(transport, req) + snapshot.WireIdentity = codexWireIdentity(payload, req.Header) s.sendEvent(c, TestEvent{Type: "status", Text: snapshot.summary(), Data: snapshot}) s.sendEvent(c, TestEvent{Type: "status", Text: "Codex wire headers", Data: accountTestWireHeaders(req.Header)}) } @@ -1079,6 +1099,10 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account if err != nil { return s.sendErrorAndEnd(c, fmt.Sprintf("Request failed: %s", err.Error())) } + if isOAuth && resp.TLS != nil { + s.sendEvent(c, TestEvent{Type: "status", Text: "Codex negotiated transport", Data: map[string]any{"tls_version": tls.VersionName(resp.TLS.Version), "alpn": resp.TLS.NegotiatedProtocol, "cipher_suite": tls.CipherSuiteName(resp.TLS.CipherSuite), "tls_session_resumed": resp.TLS.DidResume}}) + } + defer func() { _ = resp.Body.Close() }() if isOAuth && !account.IsShadow() && s.accountRepo != nil && !IsAccountObservation(ctx) { @@ -2368,7 +2392,16 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if isOAuth { testModelID = normalizeOpenAIModelForUpstream(credentialAccount, testModelID) } - payloadBytes, _ := json.Marshal(createOpenAICompactProbePayload(testModelID, isOAuth)) + payload := createOpenAICompactProbePayload(testModelID, isOAuth) + var identityHeaders http.Header + if isOAuth { + var err error + identityHeaders, err = projectCodexAccountTest(c, account, credentialAccount, payload) + if err != nil { + return s.sendErrorAndEnd(c, err.Error()) + } + } + payloadBytes, _ := json.Marshal(payload) if !agentIdentityTaskRecoveryWasTried(ctx) { s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID}) } @@ -2396,25 +2429,20 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account } else { req.Header.Set("Authorization", "Bearer "+authToken) } - applyOpenAICodexProbeHeaders(req.Header) + if !isOAuth { + applyOpenAICodexProbeHeaders(req.Header) + } + for name, values := range identityHeaders { + req.Header[name] = values + } if isOAuth { // 与真实转发一致：使用该账号的 Codex TUI 身份，账号级自定义 UA 经 ForceCodexCLI // 策略过滤后作为管理员显式配置传入（同普通 OAuth 连接测试）。 enforceCodexIdentityHeadersForAccount(req.Header, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount), codexFingerprintPolicyForContext(c, account)) } - probeSessionID := compactProbeSessionID(account.ID) - req.Header.Set("Session_ID", probeSessionID) - req.Header.Set("Conversation_ID", probeSessionID) - if isOAuth { req.Host = "chatgpt.com" setOpenAIChatGPTAccountHeaders(req.Header, credentialAccount) - // 指纹收敛：探测与真实转发走同一个 /responses 端点，身份也必须同构， - // 否则探测流量会以「缺 x-codex-installation-id + 非收敛 session」的 - // 形态暴露在上游眼里。账号关闭收敛（off）时返回 nil，探测保持原样。 - if fpIDs := resolveCodexFingerprintIDsFromRequest(account, req.Header, codexFingerprintPolicyForContext(c, account)); fpIDs != nil { - applyCodexFingerprintHeaders(req.Header, fpIDs) - } } // 账号级请求头覆写：测试请求与真实转发保持一致的最终头 @@ -2425,6 +2453,22 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account proxyURL = account.Proxy.URL() } + if isOAuth { + var err error + req, err = bindCodexTransport(req, credentialAccount, codexFingerprintPolicyForContext(c, account), "http") + if err != nil { + return s.sendErrorAndEnd(c, err.Error()) + } + req, err = prepareOpenAICodexWireRequest(req, account) + if err != nil { + return s.sendErrorAndEnd(c, err.Error()) + } + snapshot := resolveCodexIdentitySnapshot(account, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount), codexFingerprintPolicyForContext(c, account)).withRequest("http", req) + snapshot.WireIdentity = codexWireIdentity(payload, req.Header) + s.sendEvent(c, TestEvent{Type: "status", Text: snapshot.summary(), Data: snapshot}) + s.sendEvent(c, TestEvent{Type: "status", Text: "Codex wire headers", Data: accountTestWireHeaders(req.Header)}) + } + resp, err := s.doOpenAIAccountTestUpstream(req, proxyURL, account, true) if err != nil { if s.accountRepo != nil && !IsAccountObservation(ctx) { @@ -2434,6 +2478,10 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account } return s.sendErrorAndEnd(c, fmt.Sprintf("Request failed: %s", err.Error())) } + if isOAuth && resp.TLS != nil { + s.sendEvent(c, TestEvent{Type: "status", Text: "Codex negotiated transport", Data: map[string]any{"tls_version": tls.VersionName(resp.TLS.Version), "alpn": resp.TLS.NegotiatedProtocol, "cipher_suite": tls.CipherSuiteName(resp.TLS.CipherSuite), "tls_session_resumed": resp.TLS.DidResume}}) + } + defer func() { _ = resp.Body.Close() }() body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) diff --git a/backend/internal/service/codex_fingerprint_account_view.go b/backend/internal/service/codex_fingerprint_account_view.go index ea3c4a749abd..37374304a172 100644 --- a/backend/internal/service/codex_fingerprint_account_view.go +++ b/backend/internal/service/codex_fingerprint_account_view.go @@ -45,7 +45,7 @@ func DescribeCodexFingerprintAccount(routed, source *Account) CodexFingerprintAc } policy := currentCodexFingerprintPolicy() view := CodexFingerprintAccountView{ - Mode: string(routed.GetCodexFingerprintMode()), + Mode: string(codexFingerprintModeFromExtra(routed.Extra)), Identity: resolveCodexIdentitySnapshot(routed, source, codexAccountIdentityOverrideUA(source), policy), SimulationEnabled: policy.enabled, } @@ -109,18 +109,9 @@ func describeCodexFingerprintIdentifierPolicy(source *Account, effectiveMode str return result } result.InstallationID = CodexFingerprintIdentifierRule{Behavior: "fixed", Rule: "account_device", Value: codexFingerprintOptionalString(resolveConvergedInstallationID(source, seed))} - if mode == codexFingerprintDevice { - return result - } - sessionID := resolveConvergedSessionID(seed) - result.SessionID = CodexFingerprintIdentifierRule{Behavior: "fixed", Rule: "account_session", Value: &sessionID} - result.ThreadID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "mapped_thread"} - result.ParentThreadID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "mapped_parent_thread"} - result.WindowID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "mapped_window"} - if mode == codexFingerprintFull { - result.ThreadID = CodexFingerprintIdentifierRule{Behavior: "fixed", Rule: "account_thread", Value: &sessionID} - result.ParentThreadID.Rule = "remove_parent_self_reference" - result.WindowID.Rule = "merged_window" - } + result.SessionID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "scoped_real_session"} + result.ThreadID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "scoped_real_thread"} + result.ParentThreadID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "scoped_real_parent"} + result.WindowID = CodexFingerprintIdentifierRule{Behavior: "request_derived", Rule: "scoped_real_window"} return result } diff --git a/backend/internal/service/codex_fingerprint_account_view_test.go b/backend/internal/service/codex_fingerprint_account_view_test.go index d64e26a7fdc8..883da32631b1 100644 --- a/backend/internal/service/codex_fingerprint_account_view_test.go +++ b/backend/internal/service/codex_fingerprint_account_view_test.go @@ -121,20 +121,13 @@ func TestCodexFingerprintAccountViewPoliciesMatchSharedForwardingAndParentSource } require.Equal(t, ids.installationID, *view.IdentifierPolicy.InstallationID.Value) require.Equal(t, "43ef16f3-2c23-4ca0-b67d-7c3f0189aca4", *view.IdentifierPolicy.InstallationID.Value) - if mode == "device" { - require.Equal(t, "passthrough", view.IdentifierPolicy.SessionID.Behavior) - return - } - require.Equal(t, ids.sessionID, *view.IdentifierPolicy.SessionID.Value) - require.Nil(t, view.IdentifierPolicy.WindowID.Value, "the real request window index is not known by this endpoint") - if mode == "session" { - require.Equal(t, "request_derived", view.IdentifierPolicy.ThreadID.Behavior) - require.Nil(t, view.IdentifierPolicy.ThreadID.Value, "do not display the empty-request thread fallback") - require.Equal(t, "mapped_parent_thread", view.IdentifierPolicy.ParentThreadID.Rule) - } else { - require.Equal(t, ids.threadID, *view.IdentifierPolicy.ThreadID.Value) - require.Equal(t, "remove_parent_self_reference", view.IdentifierPolicy.ParentThreadID.Rule) - } + require.Equal(t, "request_derived", view.IdentifierPolicy.SessionID.Behavior) + require.Nil(t, view.IdentifierPolicy.SessionID.Value) + require.Nil(t, view.IdentifierPolicy.ThreadID.Value) + require.Nil(t, view.IdentifierPolicy.WindowID.Value) + require.Equal(t, "scoped_real_parent", view.IdentifierPolicy.ParentThreadID.Rule) + require.Equal(t, "device", view.Identity.FingerprintModeEffective) + }) } } diff --git a/backend/internal/service/codex_fingerprint_policy_test.go b/backend/internal/service/codex_fingerprint_policy_test.go index 83b12fabc0a8..52939077f9d7 100644 --- a/backend/internal/service/codex_fingerprint_policy_test.go +++ b/backend/internal/service/codex_fingerprint_policy_test.go @@ -63,7 +63,7 @@ func TestCodexFingerprintSettingsPreserveStorageAndPublishOnlyAfterSave(t *testi account := fingerprintTestAccount("session") c, _ := gin.CreateTestContext(httptest.NewRecorder()) frozen := stageCodexFingerprintPolicy(c, account) - sent := CodexFingerprintSettings{Enabled: false, ClientVersion: "0.162.0", VersionAutoSyncEnabled: false} + sent := CodexFingerprintSettings{Enabled: false, ClientVersion: "0.162.1", VersionAutoSyncEnabled: false} repo.fail = errors.New("storage unavailable") _, err := svc.UpdateCodexFingerprintSettings(context.Background(), sent) require.Error(t, err) @@ -73,7 +73,7 @@ func TestCodexFingerprintSettingsPreserveStorageAndPublishOnlyAfterSave(t *testi view, err := svc.UpdateCodexFingerprintSettings(context.Background(), sent) require.NoError(t, err) require.False(t, view.Enabled) - require.Equal(t, "0.162.0", view.EffectiveVersion) + require.Equal(t, "0.162.1", view.EffectiveVersion) require.Equal(t, "0.161.0", repo.values[SettingKeyOpenAICodexClientVersionSynced]) require.False(t, currentCodexFingerprintPolicy().enabled) require.Same(t, frozen, codexFingerprintPolicyForContext(c, account), "an active attempt keeps its policy") @@ -135,12 +135,7 @@ func TestCodexFingerprintLogicalTurnReferencesAndPermissions(t *testing.T) { require.Equal(t, headers.Get("thread-id"), embedded["thread_id"]) require.Equal(t, headers.Get("x-codex-window-id"), embedded["window_id"]) require.Contains(t, embedded["window_id"], ":8") - if mode == "full" { - require.NotContains(t, embedded, "parent_thread_id") - require.Empty(t, headers.Get("x-codex-parent-thread-id")) - } else { - require.Equal(t, ids.mapThread("parent-thread"), embedded["parent_thread_id"]) - } + require.Equal(t, "parent-thread", embedded["parent_thread_id"]) } }) } @@ -214,16 +209,16 @@ func TestCodexFingerprintAccountModeEpochAndGeneralEditPreservation(t *testing.T } func TestCodexFingerprintSandboxAlignmentPreservesActualPolicy(t *testing.T) { - for _, backend := range []string{"none", "external", "custom-sandbox", ""} { - metadata := map[string]any{"sandbox": backend, "sandbox_mode": "read-only"} + for _, metadata := range []map[string]any{ + {"sandbox": "none", "sandbox_mode": "danger-full-access"}, + {"sandbox": "external", "sandbox_mode": "external-sandbox"}, + {"sandbox": "seccomp", "sandbox_mode": "workspace-write"}, + {"sandbox": "windows_mxc", "sandbox_mode": "read-only"}, + {"sandbox": "future-backend", "sandbox_mode": "future-policy"}, + } { + before, _ := json.Marshal(metadata) require.False(t, alignCodexSandboxMetadata(metadata, "seatbelt")) - require.Equal(t, backend, metadata["sandbox"]) + after, _ := json.Marshal(metadata) + require.Equal(t, string(before), string(after)) } - metadata := map[string]any{"sandbox": "windows_mxc", "sandbox_mode": "workspace-write"} - require.False(t, alignCodexSandboxMetadata(metadata, "windows_sandbox")) - metadata = map[string]any{"sandbox": "seccomp", "sandbox_mode": "workspace-write", "thread_id": "original", "turn_id": "original-turn"} - require.True(t, alignCodexSandboxMetadata(metadata, "seatbelt")) - require.Equal(t, "workspace-write", metadata["sandbox_mode"]) - require.Equal(t, "original", metadata["thread_id"]) - require.Equal(t, "original-turn", metadata["turn_id"]) } diff --git a/backend/internal/service/openai_alpha_search.go b/backend/internal/service/openai_alpha_search.go index 81208b5a7b9e..28896bc48391 100644 --- a/backend/internal/service/openai_alpha_search.go +++ b/backend/internal/service/openai_alpha_search.go @@ -323,7 +323,7 @@ func (s *OpenAIGatewayService) buildOpenAIAlphaSearchResponsesWebSearchRequest(c // 下面的 applyCodexAccountIdentityHeaders 统一完成，与其他 Codex 路径一致。 fillCodexSessionIdentityHeaders(req.Header, generateSessionUUID(sessionID)) } - applyCodexAccountIdentityHeaders(req.Header, codexAccountIdentitySource(c, account), apiKeyID) + applyCodexAccountIdentityHeaders(req.Header, codexAccountProjectionSource(c, account), apiKeyID) enforceCodexIdentityHeadersForAccount(req.Header, codexAccountIdentitySource(c, account), s.codexIdentityOverrideUA(account)) account.ApplyHeaderOverrides(req.Header) return req, nil @@ -462,7 +462,7 @@ func (s *OpenAIGatewayService) buildOpenAIAlphaSearchRequest(ctx context.Context if s.cfg != nil && s.cfg.Gateway.ForceCodexCLI { req.Header.Set("User-Agent", canonical.userAgent) } - applyCodexAccountIdentityHeaders(req.Header, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c)) + applyCodexAccountIdentityHeaders(req.Header, codexAccountProjectionSource(c, account), getAPIKeyIDFromContext(c)) enforceCodexIdentityHeadersForAccount(req.Header, codexAccountIdentitySource(c, account), s.codexIdentityOverrideUA(account)) } diff --git a/backend/internal/service/openai_codex_account_identity.go b/backend/internal/service/openai_codex_account_identity.go index 2dea0b27d86e..891fc0d0ab1c 100644 --- a/backend/internal/service/openai_codex_account_identity.go +++ b/backend/internal/service/openai_codex_account_identity.go @@ -5,8 +5,10 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "maps" "net/http" "regexp" + "strconv" "strings" "github.com/gin-gonic/gin" @@ -28,6 +30,7 @@ const codexAccountIdentitySourceContextKey = "openai_codex_account_identity_sour // point overwrites the staged source before projecting outbound identity. func (s *OpenAIGatewayService) prepareCodexAccountIdentitySource(ctx context.Context, c *gin.Context, account *Account) (*Account, error) { stageCodexFingerprintPolicy(c, account) + recordCodexMetadataConflicts(c, nil) source := account if account != nil && account.IsShadow() { resolved, err := resolveCredentialAccount(ctx, s.accountRepo, account) @@ -175,9 +178,7 @@ func codexAccountIdentityProjectionFields(account *Account, policy *codexFingerp if account == nil || !policy.enabled || account.GetCodexFingerprintMode() == codexFingerprintOff { return nil } - if account.GetCodexFingerprintMode() == codexFingerprintDevice { - return []string{"installation_id", "x-codex-installation-id"} - } + return codexAccountIdentityFields } @@ -193,6 +194,9 @@ func applyCodexAccountIdentityFields(values map[string]any, account *Account, ap continue } next := scopeCodexAccountIdentityValue(account, apiKeyID, raw) + if name == "window_id" || name == "x-codex-window-id" { + next = scopeCodexWindowIdentity(account, apiKeyID, raw) + } if next != raw { values[name] = next changed = true @@ -207,7 +211,7 @@ func applyCodexAccountIdentityEmbeddedMetadata(values map[string]any, account *A return false } metadata := map[string]any{} - if err := json.Unmarshal([]byte(raw), &metadata); err != nil || metadata == nil { + if err := decodeCodexMetadataObject([]byte(raw), &metadata); err != nil || metadata == nil { return false } if !applyCodexAccountIdentityFields(metadata, account, apiKeyID, policies...) { @@ -228,6 +232,9 @@ func applyCodexAccountIdentityClientMetadataMap(requestBody map[string]any, acco } changed := false clientMetadata, _ := requestBody["client_metadata"].(map[string]any) + if clientMetadata != nil { + _, _ = normalizeCodexMetadataProjection(clientMetadata) + } originalSessionID := stringMetadataValue(clientMetadata, "session_id") if clientMetadata != nil { if applyCodexAccountIdentityFields(clientMetadata, account, apiKeyID, policy) { @@ -237,7 +244,7 @@ func applyCodexAccountIdentityClientMetadataMap(requestBody map[string]any, acco changed = true } } - if raw, ok := requestBody["prompt_cache_key"].(string); ok && raw != "" && raw == originalSessionID && (account.GetCodexFingerprintMode() == codexFingerprintSession || account.GetCodexFingerprintMode() == codexFingerprintFull) { + if raw, ok := requestBody["prompt_cache_key"].(string); ok && raw != "" && raw == originalSessionID && account.GetCodexFingerprintMode() != codexFingerprintOff { next := scopeCodexAccountIdentityValue(account, apiKeyID, raw) if next != raw { requestBody["prompt_cache_key"] = next @@ -251,7 +258,9 @@ func applyCodexAccountIdentityClientMetadataMap(requestBody map[string]any, acco // subobjects with gjson/sjson. The passthrough hot path never unmarshals the // potentially multi-megabyte request body. func applyCodexAccountIdentityClientMetadataRaw(body []byte, account *Account, apiKeyID int64, policies ...*codexFingerprintPolicy) ([]byte, bool, error) { - policy := selectCodexFingerprintPolicy(policies) + return applyCodexAccountIdentityRawWithDiagnostics(body, account, apiKeyID, selectCodexFingerprintPolicy(policies), nil) +} +func applyCodexAccountIdentityRawWithDiagnostics(body []byte, account *Account, apiKeyID int64, policy *codexFingerprintPolicy, diagnostic func([]string)) ([]byte, bool, error) { if len(body) == 0 || !policy.enabled || codexAccountIdentityNamespace(account) == "" { return body, false, nil } @@ -262,11 +271,24 @@ func applyCodexAccountIdentityClientMetadataRaw(body []byte, account *Account, a next := body changed := false + defaultSession := "" + cmValue := gjson.GetBytes(body, "client_metadata") + if cmValue.Exists() && !cmValue.IsObject() { + return body, false, codexMetadataClientError(fmt.Errorf("client_metadata must be an object")) + } if cm := gjson.GetBytes(body, "client_metadata"); cm.IsObject() { clientMetadata := map[string]any{} - if err := json.Unmarshal([]byte(cm.Raw), &clientMetadata); err != nil { + if err := decodeCodexMetadataObject([]byte(cm.Raw), &clientMetadata); err != nil { return body, false, fmt.Errorf("decode client_metadata for account identity: %w", err) } + conflicts, err := normalizeCodexMetadataProjection(clientMetadata) + if err != nil { + return body, false, codexMetadataClientError(err) + } + if diagnostic != nil { + diagnostic(conflicts) + } + defaultSession = stringMetadataValue(clientMetadata, "session_id") metadataChanged := applyCodexAccountIdentityFields(clientMetadata, account, apiKeyID, policy) if applyCodexAccountIdentityEmbeddedMetadata(clientMetadata, account, apiKeyID, policy) { metadataChanged = true @@ -284,7 +306,7 @@ func applyCodexAccountIdentityClientMetadataRaw(body []byte, account *Account, a changed = true } } - if promptCacheKey := gjson.GetBytes(body, "prompt_cache_key"); promptCacheKey.Type == gjson.String && promptCacheKey.String() != "" && promptCacheKey.String() == gjson.GetBytes(body, "client_metadata.session_id").String() && (account.GetCodexFingerprintMode() == codexFingerprintSession || account.GetCodexFingerprintMode() == codexFingerprintFull) { + if promptCacheKey := gjson.GetBytes(body, "prompt_cache_key"); promptCacheKey.Type == gjson.String && promptCacheKey.String() != "" && promptCacheKey.String() == defaultSession && account.GetCodexFingerprintMode() != codexFingerprintOff { raw := promptCacheKey.String() scoped := scopeCodexAccountIdentityValue(account, apiKeyID, raw) if scoped != raw { @@ -318,12 +340,16 @@ func applyCodexAccountIdentityHeaders(headers http.Header, account *Account, api } raw := strings.TrimSpace(headers.Get(name)) if raw != "" { - headers.Set(name, scopeCodexAccountIdentityValue(account, apiKeyID, raw)) + next := scopeCodexAccountIdentityValue(account, apiKeyID, raw) + if name == "window_id" || name == "x-codex-window-id" { + next = scopeCodexWindowIdentity(account, apiKeyID, raw) + } + headers.Set(name, next) } } if raw := strings.TrimSpace(headers.Get(openAIWSTurnMetadataHeader)); raw != "" { metadata := map[string]any{} - if err := json.Unmarshal([]byte(raw), &metadata); err == nil && metadata != nil && applyCodexAccountIdentityFields(metadata, account, apiKeyID, policy) { + if err := decodeCodexMetadataObject([]byte(raw), &metadata); err == nil && metadata != nil && applyCodexAccountIdentityFields(metadata, account, apiKeyID, policy) { if rebuilt, err := marshalCodexTurnMetadata(metadata); err == nil { headers.Set(openAIWSTurnMetadataHeader, string(rebuilt)) } @@ -368,3 +394,51 @@ func ensureCodexSessionIdentityHeaders(h http.Header, sessionID string) { } } } + +// The routed account owns the mode; the credential source owns identity and seed. +func codexAccountProjectionSource(c *gin.Context, routed *Account) *Account { + source := codexAccountIdentitySource(c, routed) + if source == nil || routed == nil { + return source + } + account := *source + account.Extra = maps.Clone(source.Extra) + if account.Extra == nil { + account.Extra = map[string]any{} + } + account.Extra[codexFingerprintModeExtraKey] = string(routed.GetCodexFingerprintMode()) + return &account +} +func (s *OpenAIGatewayService) scopeCodexRequestMetadataRaw(c *gin.Context, routed *Account, body []byte) ([]byte, bool, error) { + recordCodexMetadataConflicts(c, nil) + return applyCodexAccountIdentityRawWithDiagnostics(body, codexAccountProjectionSource(c, routed), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, routed), func(conflicts []string) { recordCodexMetadataConflicts(c, conflicts) }) +} + +func scopeCodexWindowIdentity(account *Account, apiKeyID int64, raw string) string { + if index := strings.LastIndexByte(raw, ':'); index > 0 && index < len(raw)-1 { + if _, err := strconv.ParseUint(raw[index+1:], 10, 64); err == nil { + return scopeCodexAccountIdentityValue(account, apiKeyID, raw[:index]) + raw[index:] + } + } + return scopeCodexAccountIdentityValue(account, apiKeyID, raw) +} + +func (s *OpenAIGatewayService) scopeCodexRequestMetadataMap(c *gin.Context, routed *Account, body map[string]any) (bool, error) { + source := codexAccountProjectionSource(c, routed) + policy := codexFingerprintPolicyForContext(c, routed) + if source == nil || !source.IsOpenAIOAuthLike() || !policy.enabled { + return false, nil + } + if value, exists := body["client_metadata"]; exists && value != nil { + metadata, ok := value.(map[string]any) + if !ok { + return false, codexMetadataClientError(fmt.Errorf("client_metadata must be an object")) + } + conflicts, err := normalizeCodexMetadataProjection(metadata) + if err != nil { + return false, codexMetadataClientError(err) + } + recordCodexMetadataConflicts(c, conflicts) + } + return applyCodexAccountIdentityClientMetadataMap(body, source, getAPIKeyIDFromContext(c), policy), nil +} diff --git a/backend/internal/service/openai_codex_conversations_test.go b/backend/internal/service/openai_codex_conversations_test.go new file mode 100644 index 000000000000..751b9f02684a --- /dev/null +++ b/backend/internal/service/openai_codex_conversations_test.go @@ -0,0 +1,203 @@ +package service + +import ( + "encoding/json" + "errors" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestCodexConversationsPreserveRootAndChildGraph(t *testing.T) { + policy := fingerprintTestPolicy(true, "0.162.1") + for _, mode := range []string{"device", "session", "full"} { + t.Run(mode, func(t *testing.T) { + account := fingerprintTestAccount(mode) + root, child, other := uuid.Must(uuid.NewV7()).String(), uuid.Must(uuid.NewV7()).String(), uuid.Must(uuid.NewV7()).String() + var installation string + for _, thread := range []string{root, child, other} { + session := root + if thread == other { + session = other + } + metadata := map[string]any{"session_id": session, "thread_id": thread, "window_id": thread + ":3", + "turn_id": "real-turn", "sandbox": "windows_mxc", "sandbox_mode": "workspace-write", "unknown_future": "preserve"} + if thread == child { + metadata["parent_thread_id"] = root + metadata["subagent_kind"] = "worker" + } + embedded, err := json.Marshal(metadata) + require.NoError(t, err) + body := map[string]any{"prompt_cache_key": session, "client_metadata": map[string]any{ + "session_id": session, "thread_id": thread, "x-codex-window-id": thread + ":3", "x-codex-turn-metadata": string(embedded)}} + headers := http.Header{"Session-Id": {session}, "Thread-Id": {thread}, "X-Codex-Window-Id": {thread + ":3"}} + applyCodexAccountIdentityClientMetadataMap(body, account, 9, policy) + applyCodexAccountIdentityHeaders(headers, account, 9, policy) + ids := resolveCodexFingerprintIDsFromRequest(account, headers, policy) + require.NotNil(t, ids) + require.True(t, applyCodexFingerprintClientMetadata(body, ids)) + require.NoError(t, ids.err) + applyCodexFingerprintHeaders(headers, ids) + flat := body["client_metadata"].(map[string]any) + canonical, _, err := canonicalCodexMetadata(flat) + require.NoError(t, err) + mappedThread := scopeCodexAccountIdentityValue(account, 9, thread) + require.Equal(t, mappedThread, headers.Get("thread-id")) + require.Equal(t, uuid.Version(7), uuid.MustParse(mappedThread).Version()) + require.Equal(t, scopeCodexAccountIdentityValue(account, 9, session), body["prompt_cache_key"]) + require.Equal(t, "windows_mxc", canonical["sandbox"]) + require.Equal(t, "preserve", canonical["unknown_future"]) + if thread == child { + require.NotEqual(t, headers.Get("session-id"), mappedThread) + require.Equal(t, scopeCodexAccountIdentityValue(account, 9, root), canonical["parent_thread_id"]) + require.Equal(t, "worker", canonical["subagent_kind"]) + } else { + require.Equal(t, headers.Get("session-id"), mappedThread) + } + if installation == "" { + installation = ids.installationID + } else { + require.Equal(t, installation, ids.installationID) + } + } + }) + } +} + +func TestCodexConversationsCanonicalPrecedenceAndInvalidMetadata(t *testing.T) { + policy := fingerprintTestPolicy(true, "0.162.1") + account := fingerprintTestAccount("device") + root := uuid.Must(uuid.NewV7()).String() + canonical, err := json.Marshal(map[string]any{"session_id": root, "thread_id": root, "window_id": root + ":0"}) + require.NoError(t, err) + flat := map[string]any{"session_id": "stale-flat", "thread_id": "stale-flat", "x-codex-turn-metadata": string(canonical)} + conflicts, err := normalizeCodexMetadataProjection(flat) + require.NoError(t, err) + require.Contains(t, conflicts, "session_id") + require.Equal(t, root, flat["session_id"]) + for _, raw := range []string{`{"client_metadata":{"x-codex-turn-metadata":"{invalid"}}`, `{"client_metadata":[]}`} { + _, _, err := applyCodexAccountIdentityClientMetadataRaw([]byte(raw), account, 1, policy) + require.Error(t, err) + var clientErr *UpstreamFailoverError + require.ErrorAs(t, err, &clientErr) + require.Equal(t, 400, clientErr.ClientStatusCode) + } +} + +func TestCodexConversationsSyntheticTestUsesOneRootAcrossRetry(t *testing.T) { + fingerprintPolicyTestState(t) + publishedCodexFingerprintPolicy.Store(fingerprintTestPolicy(true, "0.162.1")) + account := fingerprintTestAccount("full") + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/test", nil) + var first *CodexWireIdentity + for range 2 { + body := createOpenAITestPayload("gpt-5.4", true) + headers, err := projectCodexAccountTest(c, account, account, body) + require.NoError(t, err) + wire := codexWireIdentity(body, headers) + require.True(t, wire.Consistent) + require.NotEmpty(t, wire.InstallationID) + require.Equal(t, wire.SessionID, wire.ThreadID) + if first == nil { + first = wire + } else { + require.Equal(t, first, wire) + } + } +} + +func TestCodexConversationsWSChecksPreservedHandshakeIdentity(t *testing.T) { + for _, enabled := range []bool{true, false} { + for _, mode := range []string{"device", "off"} { + account := fingerprintTestAccount(mode) + policy := fingerprintTestPolicy(enabled, "0.162.1") + a := http.Header{"Session-Id": {"root-a"}, "Thread-Id": {"root-a"}, "X-Codex-Window-Id": {"root-a:0"}} + b := http.Header{"Session-Id": {"root-b"}, "Thread-Id": {"root-b"}, "X-Codex-Window-Id": {"root-b:0"}} + require.NotEqual(t, normalizeOpenAIWSHandshakeCompatibility(account, a, policy), normalizeOpenAIWSHandshakeCompatibility(account, b, policy)) + require.Equal(t, normalizeOpenAIWSHandshakeCompatibility(account, a, policy), normalizeOpenAIWSHandshakeCompatibility(account, a.Clone(), policy)) + } + } +} + +func TestCodexStrictNetworkFailureCannotEnterGatewayRetry(t *testing.T) { + original := &codextransport.Failure{Err: io.ErrUnexpectedEOF} + result := wrapOpenAIWSFallback("read_upstream", original) + var stop *UpstreamFailoverError + require.True(t, errors.As(result, &stop)) + require.Equal(t, NextAccountStop, stop.NextAccountAction) + _, retry := classifyOpenAIWSReconnectReason(result) + require.False(t, retry) + _, retry = classifyOpenAIWSReconnectReason(original) + require.False(t, retry) +} + +func TestCodexRequestWithoutIdentityGetsDistinctStableSyntheticRoot(t *testing.T) { + fingerprintPolicyTestState(t) + publishedCodexFingerprintPolicy.Store(fingerprintTestPolicy(true, "0.162.1")) + account := fingerprintTestAccount("device") + gateway := &OpenAIGatewayService{} + var previous, installation string + for range 2 { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest("POST", "/v1/responses", nil) + first := gateway.resolveStagedCodexFingerprintIDs(c, account, c.Request.Header) + again := gateway.resolveStagedCodexFingerprintIDs(c, account, c.Request.Header) + require.Equal(t, first.sessionID, again.sessionID) + require.Equal(t, first.sessionID, first.threadID) + require.Equal(t, uuid.Version(7), uuid.MustParse(first.threadID).Version()) + if previous != "" { + require.NotEqual(t, previous, first.threadID) + require.Equal(t, installation, first.installationID) + } + previous, installation = first.threadID, first.installationID + root := uuid.Must(uuid.NewV7()).String() + body := map[string]any{"prompt_cache_key": "explicit-cache", "client_metadata": map[string]any{"session_id": root}} + require.True(t, applyCodexFingerprintClientMetadata(body, first)) + require.NoError(t, first.err) + require.Equal(t, root, first.sessionID) + require.Equal(t, root, first.threadID) + require.Equal(t, "explicit-cache", body["prompt_cache_key"]) + } +} +func TestCodexRawMetadataPreservesUnknownLargeNumbers(t *testing.T) { + account := fingerprintTestAccount("device") + policy := fingerprintTestPolicy(true, "0.162.1") + body := []byte(`{"client_metadata":{"session_id":"root","unknown_counter":9007199254740993,"x-codex-turn-metadata":"{\"unknown_counter\":9007199254740993}"},"input":[]}`) + scoped, _, err := applyCodexAccountIdentityClientMetadataRaw(body, account, 7, policy) + require.NoError(t, err) + ids := resolveCodexFingerprintIDsFromRequest(account, nil, policy) + wire, _, err := applyCodexFingerprintClientMetadataRaw(scoped, ids) + require.NoError(t, err) + require.Equal(t, "9007199254740993", gjson.GetBytes(wire, "client_metadata.unknown_counter").Raw) + require.Contains(t, gjson.GetBytes(wire, "client_metadata.x-codex-turn-metadata").String(), "9007199254740993") +} + +func TestCodexLegacyWindowRetainsTheThreadMappingAndIndex(t *testing.T) { + account := fingerprintTestAccount("device") + policy := fingerprintTestPolicy(true, "0.162.1") + for _, raw := range []string{"legacy-thread", "project:legacy", uuid.NewString(), uuid.Must(uuid.NewV7()).String()} { + metadata := map[string]any{"thread_id": raw, "window_id": raw + ":19"} + require.True(t, applyCodexAccountIdentityFields(metadata, account, 9, policy)) + require.Equal(t, metadata["thread_id"].(string)+":19", metadata["window_id"]) + } +} + +func TestCodexWSLeaseCannotCrossBodyDeviceIdentity(t *testing.T) { + account := fingerprintTestAccount("off") + first := openAIWSAcquireRequest{Account: account, Headers: http.Header{"Session-Id": {"root"}, "Thread-Id": {"root"}}, FingerprintPolicy: fingerprintTestPolicy(true, "0.162.1")} + first.BodyIdentityKey = codexWSBodyIdentityKey([]byte(`{"client_metadata":{"x-codex-installation-id":"device-a","session_id":"root","thread_id":"root"}}`)) + conn := &openAIWSConn{handshakeCompatibility: normalizeOpenAIWSAcquireCompatibility(first)} + require.True(t, conn.matchesHandshakeCompatibility(normalizeOpenAIWSAcquireCompatibility(first))) + next := cloneOpenAIWSAcquireRequest(first) + next.BodyIdentityKey = codexWSBodyIdentityKey([]byte(`{"client_metadata":{"x-codex-installation-id":"device-b","session_id":"root","thread_id":"root"}}`)) + require.NotEqual(t, first.BodyIdentityKey, next.BodyIdentityKey) + require.False(t, conn.matchesHandshakeCompatibility(normalizeOpenAIWSAcquireCompatibility(next))) +} diff --git a/backend/internal/service/openai_codex_fingerprint.go b/backend/internal/service/openai_codex_fingerprint.go index ead5a63019ae..8653db638916 100644 --- a/backend/internal/service/openai_codex_fingerprint.go +++ b/backend/internal/service/openai_codex_fingerprint.go @@ -233,7 +233,11 @@ func (a *Account) GetCodexFingerprintMode() codexFingerprintMode { if a == nil || !a.IsOpenAIOAuthLike() { return codexFingerprintOff } - return codexFingerprintModeFromExtra(a.Extra) + mode := codexFingerprintModeFromExtra(a.Extra) + if mode == codexFingerprintSession || mode == codexFingerprintFull { + return codexFingerprintDevice + } + return mode } // deriveStableUUIDv4 从种子确定性派生一个 UUIDv4 格式的字符串。 @@ -301,6 +305,9 @@ type codexFingerprintIDs struct { bodyIdentityCaptured bool originalBodySessionID string originalBodySessionIDCaptured bool + metadataConflicts []string + synthetic bool + err error } // resolveCodexFingerprintIDs 按收敛模式计算出站 ID 集合。 @@ -316,47 +323,20 @@ func resolveCodexFingerprintIDs(account *Account, clientSessionID string, mode c if !ok { return nil } - - ids := &codexFingerprintIDs{ - seed: seed, - accountID: account.ID, - mode: mode, + ids := &codexFingerprintIDs{seed: seed, accountID: account.ID, mode: codexFingerprintDevice, + installationID: resolveConvergedInstallationID(account, seed), sessionID: clientSessionID, threadID: clientSessionID} + if ids.installationID == "" { + return nil } if identity, ok := account.CodexClientIdentity(); ok { ids.sandbox = identity.Sandbox } - - ids.installationID = resolveConvergedInstallationID(account, seed) - if ids.installationID == "" { - return nil - } - - switch mode { - case codexFingerprintDevice: - return ids - - case codexFingerprintSession: - ids.sessionID = resolveConvergedSessionID(seed) - ids.threadID = resolveConvergedThreadID(seed, clientSessionID) - if ids.threadID == "" { - ids.threadID = ids.sessionID - } + if ids.threadID != "" { ids.windowID = ids.threadID + ":0" - return ids - - case codexFingerprintFull: - ids.sessionID = resolveConvergedSessionID(seed) - ids.threadID = ids.sessionID - ids.windowID = ids.threadID + ":0" - return ids } - - return nil + return ids } -// extractClientSessionID 从请求头中提取客户端原始的会话标识。 -// 优先取 session-id（连字符形式，Codex CLI 标准），回退到 session_id（下划线形式）。 -// 返回的值尚未被 isolateOpenAISessionID 改写，是客户端的真实标识。 func extractClientSessionID(h http.Header) string { if v := strings.TrimSpace(h.Get("session-id")); v != "" { return v @@ -383,6 +363,14 @@ func resolveCodexFingerprintIDsFromRequest(account *Account, clientHeaders http. ids := resolveCodexFingerprintIDs(account, clientSessionID, mode) if ids != nil { ids.captureThreadAndWindow(clientHeaders.Get("thread-id"), clientHeaders.Get("x-codex-window-id")) + if raw := clientHeaders.Get("x-codex-turn-metadata"); raw != "" { + var metadata map[string]any + if decodeCodexMetadataObject([]byte(raw), &metadata) != nil || metadata == nil { + ids.err = codexMetadataClientError(fmt.Errorf("x-codex-turn-metadata must contain a JSON object")) + } else { + ids.logicalMetadata = metadata + } + } } return ids } @@ -390,50 +378,30 @@ func resolveCodexFingerprintIDsFromRequest(account *Account, clientHeaders http. // applyCodexFingerprintHeaders 按预计算的收敛 ID 改写出站 HTTP 头中的设备指纹。 // 在 buildUpstreamRequest 的白名单透传之后、enforceCodexIdentityHeaders 之前调用。 func applyCodexFingerprintHeaders(h http.Header, ids *codexFingerprintIDs) { - if h == nil || ids == nil { + if h == nil || ids == nil || ids.err != nil { return } - - // 真实 Codex 从不以请求头形式发送 installation id（只在 body client_metadata 中）， - // 收敛后的值写入 turn metadata / client_metadata，头上一律不带。 h.Del("x-codex-installation-id") - - if ids.mode == codexFingerprintDevice { - ids.rewriteHeaderTurnMetadata(h) - return - } - - // session / full 模式：改写所有相关头。真实 Codex 只发连字符形式；下划线形式 - // 只在调用方（legacy compact 桥接）已经写入时同步改写，不主动添加。 - h.Set("x-codex-window-id", ids.windowID) - h.Set("x-client-request-id", ids.threadID) - h.Set("session-id", ids.sessionID) - h.Set("thread-id", ids.threadID) - if h.Get("session_id") != "" { - h.Set("session_id", ids.sessionID) - } - - if parent := h.Get("x-codex-parent-thread-id"); parent != "" { - if ids.mode == codexFingerprintFull { - h.Del("x-codex-parent-thread-id") - } else { - h.Set("x-codex-parent-thread-id", ids.mapThread(parent)) + h.Del("session_id") + h.Del("conversation_id") + for key, value := range map[string]string{"session-id": ids.sessionID, "thread-id": ids.threadID, "x-client-request-id": ids.threadID, "x-codex-window-id": ids.windowID} { + if value != "" { + h.Set(key, value) } } + if parent := stringMetadataValue(ids.logicalMetadata, "parent_thread_id"); parent != "" { + h.Set("x-codex-parent-thread-id", parent) + } ids.rewriteHeaderTurnMetadata(h) } -// alignSandboxWithUserAgent 让 sandbox 跟随最终出站 UA 声明的系统（管理员显式 UA -// 覆盖时可能与账号身份不同）。UA 无法解析时不改写 sandbox。 func (ids *codexFingerprintIDs) alignSandboxWithUserAgent(userAgent string) { - if ids == nil { - return + // This is a descriptive device label only. Projection never changes real sandbox metadata. + if ids != nil { + ids.sandbox = codexSandboxForUserAgent(userAgent) } - ids.sandbox = codexSandboxForUserAgent(userAgent) } -// applyCodexFingerprintClientMetadata 按预计算的收敛 ID 改写请求体中的 client_metadata。 -// 使用与头改写相同的 ids 实例，确保 turn_id 等随机字段一致。 func applyCodexFingerprintClientMetadata(reqBody map[string]any, ids *codexFingerprintIDs) bool { if reqBody == nil || ids == nil { return false @@ -463,30 +431,36 @@ func applyCodexFingerprintToClientMetadataMap(existing map[string]any, ids *code if existing == nil || ids == nil { return false } - - modified := false - // Body metadata describes the current logical turn. Use it consistently in - // the outgoing body and header without manufacturing a new turn or time. ids.captureLogicalMetadata(existing) - - if ids.installationID != "" { - existing["x-codex-installation-id"] = ids.installationID - if _, exists := existing["installation_id"]; exists { - existing["installation_id"] = ids.installationID - } - modified = true + if ids.err != nil { + return false } - - if ids.mode == codexFingerprintDevice { - ids.rewriteEmbeddedTurnMetadata(existing) - return modified + existing["x-codex-installation-id"] = ids.installationID + if _, ok := existing["installation_id"]; ok { + existing["installation_id"] = ids.installationID + } + if ids.sessionID != "" { + existing["session_id"] = ids.sessionID + } + if ids.threadID != "" { + existing["thread_id"] = ids.threadID + } + if ids.windowID != "" { + existing["x-codex-window-id"] = ids.windowID + } + if _, err := normalizeCodexMetadataProjection(existing); err != nil { + ids.err = codexMetadataClientError(err) + return false + } + // The installation and graph are already final. Do not re-import stale embedded IDs. + projection := ids.headerMetadata() + for key, name := range codexMetadataFlatFields { + if value, ok := projection[name]; ok { + if _, exists := existing[key]; exists { + existing[key] = value + } + } } - - // session / full 模式 - existing["session_id"] = ids.sessionID - existing["thread_id"] = ids.threadID - existing["x-codex-window-id"] = ids.windowID - ids.rewriteThreadReferences(existing) ids.rewriteEmbeddedTurnMetadata(existing) return true } @@ -496,121 +470,41 @@ func stringMetadataValue(metadata map[string]any, key string) string { return value } -func (ids *codexFingerprintIDs) mapThread(original string) string { - if ids.mode == codexFingerprintFull { - return ids.threadID - } - return resolveConvergedThreadID(ids.seed, original) -} +func (ids *codexFingerprintIDs) mapThread(original string) string { return original } func (ids *codexFingerprintIDs) captureThreadAndWindow(thread, window string) { - if ids.mode == codexFingerprintDevice { - return + if strings.TrimSpace(thread) != "" { + ids.threadID = thread } - if ids.mode == codexFingerprintSession && strings.TrimSpace(thread) != "" { - ids.threadID = ids.mapThread(thread) + if ids.threadID == "" { + ids.threadID = ids.sessionID } - suffix := "0" - if index := strings.LastIndexByte(window, ':'); index >= 0 && index < len(window)-1 { - suffix = window[index+1:] - } else if index := strings.LastIndexByte(ids.windowID, ':'); index >= 0 { - suffix = ids.windowID[index+1:] + if window != "" { + ids.windowID = window + } else if ids.windowID == "" && ids.threadID != "" { + ids.windowID = ids.threadID + ":0" } - ids.windowID = ids.threadID + ":" + suffix } func (ids *codexFingerprintIDs) rewriteThreadReferences(metadata map[string]any) { - if ids.mode == codexFingerprintDevice { - return - } - for _, key := range []string{"parent_thread_id", "x-codex-parent-thread-id", "forked_from_thread_id"} { - if original := stringMetadataValue(metadata, key); original != "" { - if ids.mode == codexFingerprintFull { - delete(metadata, key) - if key == "forked_from_thread_id" { - delete(metadata, "forked_from_ordinal_exclusive") - } - } else { - metadata[key] = ids.mapThread(original) - } - } - } + // The account-scoping pass preserves the complete root/child/fork graph. } func (ids *codexFingerprintIDs) rewriteTurnMetadata(metadata map[string]any) { - for key, value := range ids.logicalMetadata { + for key, value := range ids.headerMetadata() { metadata[key] = value } - metadata["installation_id"] = ids.installationID - if ids.mode != codexFingerprintDevice { - metadata["session_id"] = ids.sessionID - metadata["thread_id"] = ids.threadID - metadata["window_id"] = ids.windowID - ids.rewriteThreadReferences(metadata) - } - alignCodexSandboxMetadata(metadata, ids.sandbox) } func (ids *codexFingerprintIDs) captureLogicalMetadata(clientMetadata map[string]any) { - if ids.bodyIdentityCaptured { - return - } - ids.bodyIdentityCaptured = true - embedded := map[string]any{} - if raw := stringMetadataValue(clientMetadata, "x-codex-turn-metadata"); raw != "" { - _ = json.Unmarshal([]byte(raw), &embedded) - } - if ids.logicalMetadata == nil { - ids.logicalMetadata = map[string]any{} - } - for _, key := range []string{"turn_id", "turn_started_at_unix_ms", "parent_turn_id", "root_turn_id", "window_number", "context_window_id"} { - if value, ok := embedded[key]; ok { - ids.logicalMetadata[key] = value - } - if value, ok := clientMetadata[key]; ok { - ids.logicalMetadata[key] = value - } - } - thread, window := stringMetadataValue(clientMetadata, "thread_id"), stringMetadataValue(clientMetadata, "x-codex-window-id") - if thread == "" { - thread = stringMetadataValue(embedded, "thread_id") - } - if window == "" { - window = stringMetadataValue(embedded, "window_id") - } - ids.captureThreadAndWindow(thread, window) + ids.captureCanonicalMetadata(clientMetadata) } -func alignCodexSandboxMetadata(metadata map[string]any, target string) bool { - // These describe actual execution permissions, not an OS-only fingerprint. - mode := stringMetadataValue(metadata, "sandbox_mode") - sandbox := stringMetadataValue(metadata, "sandbox") - if mode == "external-sandbox" { - metadata["sandbox"] = "external" - return sandbox != "external" - } - if sandbox == "none" || sandbox == "external" { - return false - } - if mode == "danger-full-access" && sandbox == "" { - return false - } - switch sandbox { - case "seatbelt", "seccomp", "windows_sandbox", "windows_elevated", "windows_mxc": - if strings.HasPrefix(sandbox, "windows_") && strings.HasPrefix(target, "windows_") { - return false - } - if target != "" && target != sandbox { - metadata["sandbox"] = target - return true - } - } - return false -} +func alignCodexSandboxMetadata(metadata map[string]any, target string) bool { return false } func alignCodexSandboxJSON(raw, target string) (string, bool) { var metadata map[string]any - if json.Unmarshal([]byte(raw), &metadata) != nil || metadata == nil || !alignCodexSandboxMetadata(metadata, target) { + if decodeCodexMetadataObject([]byte(raw), &metadata) != nil || metadata == nil || !alignCodexSandboxMetadata(metadata, target) { return raw, false } encoded, err := marshalCodexTurnMetadata(metadata) @@ -664,11 +558,11 @@ func resolveCodexFingerprintIDsWithSource(routed, source *Account, headers http. func (ids *codexFingerprintIDs) rewriteHeaderTurnMetadata(headers http.Header) { raw := headers.Get("x-codex-turn-metadata") - if raw == "" { + if raw == "" && len(ids.logicalMetadata) == 0 { return } var metadata map[string]any - if json.Unmarshal([]byte(raw), &metadata) != nil || metadata == nil { + if decodeCodexMetadataObject([]byte(raw), &metadata) != nil || metadata == nil { metadata = map[string]any{} } ids.rewriteTurnMetadata(metadata) @@ -683,7 +577,7 @@ func (ids *codexFingerprintIDs) rewriteEmbeddedTurnMetadata(clientMetadata map[s return } var metadata map[string]any - if json.Unmarshal([]byte(raw), &metadata) != nil || metadata == nil { + if decodeCodexMetadataObject([]byte(raw), &metadata) != nil || metadata == nil { metadata = map[string]any{} } ids.rewriteTurnMetadata(metadata) @@ -724,7 +618,7 @@ func shouldRewriteCodexFingerprintPromptCacheKey(ids *codexFingerprintIDs, promp if ids == nil || !ids.originalBodySessionIDCaptured || ids.originalBodySessionID == "" || ids.sessionID == "" { return false } - if ids.mode != codexFingerprintSession && ids.mode != codexFingerprintFull { + if ids.mode == codexFingerprintOff { return false } return promptCacheKey == ids.originalBodySessionID @@ -767,7 +661,7 @@ func applyCodexFingerprintClientMetadataRaw(body []byte, ids *codexFingerprintID existing := map[string]any{} if cm := gjson.GetBytes(body, "client_metadata"); cm.IsObject() { captureCodexFingerprintOriginalBodySessionIDRaw(ids, gjson.GetBytes(body, "client_metadata.session_id")) - if err := json.Unmarshal([]byte(cm.Raw), &existing); err != nil { + if err := decodeCodexMetadataObject([]byte(cm.Raw), &existing); err != nil { return body, false, fmt.Errorf("decode client_metadata for fingerprint: %w", err) } } else { @@ -788,6 +682,9 @@ func applyCodexFingerprintClientMetadataRaw(body []byte, ids *codexFingerprintID } modified = true } + if ids.err != nil { + return body, false, ids.err + } promptCacheKey := gjson.GetBytes(body, "prompt_cache_key") if promptCacheKey.Exists() && promptCacheKey.Type == gjson.String && strings.TrimSpace(promptCacheKey.String()) != "" && shouldRewriteCodexFingerprintPromptCacheKey(ids, promptCacheKey.String()) { rewritten, err := sjson.SetBytes(next, "prompt_cache_key", ids.sessionID) diff --git a/backend/internal/service/openai_codex_fingerprint_test.go b/backend/internal/service/openai_codex_fingerprint_test.go index 906a5bde0f8c..e0536be3cb44 100644 --- a/backend/internal/service/openai_codex_fingerprint_test.go +++ b/backend/internal/service/openai_codex_fingerprint_test.go @@ -64,7 +64,7 @@ func TestGetCodexFingerprintMode(t *testing.T) { }{ {"nil 账号", nil, codexFingerprintOff}, {"非 OAuth 账号", &Account{Platform: PlatformOpenAI, Type: "api_key"}, codexFingerprintOff}, - {"OpenAI setup token", &Account{Platform: PlatformOpenAI, Type: AccountTypeSetupToken, Extra: map[string]any{codexFingerprintModeExtraKey: "session"}}, codexFingerprintSession}, + {"OpenAI setup token", &Account{Platform: PlatformOpenAI, Type: AccountTypeSetupToken, Extra: map[string]any{codexFingerprintModeExtraKey: "session"}}, codexFingerprintDevice}, {"Anthropic setup token", &Account{Platform: PlatformAnthropic, Type: AccountTypeSetupToken, Extra: map[string]any{codexFingerprintModeExtraKey: "session"}}, codexFingerprintOff}, // 缺省/空/非法一律 device：一个 OAuth 账号对上游只表现为一台设备； // 显式 off 仍然生效。 @@ -73,8 +73,8 @@ func TestGetCodexFingerprintMode(t *testing.T) { {"非法值默认 device", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "invalid"}), codexFingerprintDevice}, {"显式 off", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "off"}), codexFingerprintOff}, {"device", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "device"}), codexFingerprintDevice}, - {"session", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "session"}), codexFingerprintSession}, - {"full", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "full"}), codexFingerprintFull}, + {"session", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "session"}), codexFingerprintDevice}, + {"full", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "full"}), codexFingerprintDevice}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -205,13 +205,11 @@ func TestApplyCodexFingerprintHeaders_DeviceMode(t *testing.T) { assert.Empty(t, h.Get("x-codex-installation-id"), "installation id 只在 body client_metadata，不以头形式发送") assert.Equal(t, "user-window:0", h.Get("x-codex-window-id"), "device 模式不改写 window_id") - identity, ok := account.CodexClientIdentity() - require.True(t, ok) var meta map[string]any require.NoError(t, json.Unmarshal([]byte(h.Get("x-codex-turn-metadata")), &meta)) assert.Equal(t, "converged-device", meta["installation_id"]) assert.Equal(t, "user-session", meta["session_id"], "device 模式不改写 session_id") - assert.Equal(t, identity.Sandbox, meta["sandbox"], "sandbox 跟随账号身份的操作系统") + assert.Equal(t, "seccomp", meta["sandbox"], "preserve actual execution diagnostics") } // --- applyCodexFingerprintHeaders: session 模式 --- @@ -236,8 +234,8 @@ func TestApplyCodexFingerprintHeaders_SessionMode(t *testing.T) { seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) convergedInstall := resolveConvergedInstallationID(account, seed) - convergedSession := resolveConvergedSessionID(seed) - convergedThread := resolveConvergedThreadID(seed, "client-session-aaa") + convergedSession := "client-session-aaa" + convergedThread := "client-session-aaa" assert.Empty(t, h.Get("x-codex-installation-id"), "installation id 只在 body client_metadata，不以头形式发送") assert.Equal(t, convergedSession, h.Get("session-id")) @@ -246,15 +244,13 @@ func TestApplyCodexFingerprintHeaders_SessionMode(t *testing.T) { assert.Equal(t, convergedThread, h.Get("x-client-request-id")) assert.Equal(t, convergedThread+":0", h.Get("x-codex-window-id")) - identity, ok := account.CodexClientIdentity() - require.True(t, ok) var meta map[string]any require.NoError(t, json.Unmarshal([]byte(h.Get("x-codex-turn-metadata")), &meta)) assert.Equal(t, convergedInstall, meta["installation_id"]) assert.Equal(t, convergedSession, meta["session_id"]) assert.Equal(t, convergedThread, meta["thread_id"]) assert.Equal(t, "user-turn", meta["turn_id"], "真实回合标识保持原样") - assert.Equal(t, identity.Sandbox, meta["sandbox"], "sandbox 跟随账号身份的操作系统") + assert.Equal(t, "seccomp", meta["sandbox"], "preserve actual execution diagnostics") assert.Equal(t, "user", meta["thread_source"], "thread_source 保留原样") } @@ -283,7 +279,7 @@ func TestApplyCodexFingerprintHeaders_SessionMode_DifferentClients(t *testing.T) hB.Set("x-codex-turn-metadata", makeTurnMeta()) applyCodexFingerprintHeaders(hB, idsB) - assert.Equal(t, hA.Get("session-id"), hB.Get("session-id"), "session_id 应相同") + assert.NotEqual(t, hA.Get("session-id"), hB.Get("session-id"), "independent roots remain separate") assert.NotEqual(t, hA.Get("thread-id"), hB.Get("thread-id"), "不同客户端 thread_id 应不同") assert.NotEqual(t, hA.Get("x-codex-window-id"), hB.Get("x-codex-window-id"), "不同客户端 window_id 应不同") assert.Equal(t, hA.Get("x-codex-installation-id"), hB.Get("x-codex-installation-id")) @@ -295,9 +291,7 @@ func TestApplyCodexFingerprintHeaders_FullMode(t *testing.T) { account := newTestOAuthAccount(1, map[string]any{ codexFingerprintModeExtraKey: "full", }) - seed, ok := codexFingerprintSeed(account.Extra) - require.True(t, ok) - convergedSession := resolveConvergedSessionID(seed) + convergedSession := "client-A" clientA := http.Header{} clientA.Set("session-id", "client-A") @@ -313,9 +307,9 @@ func TestApplyCodexFingerprintHeaders_FullMode(t *testing.T) { hB.Set("x-codex-turn-metadata", `{"installation_id":"x","session_id":"x","thread_id":"x","turn_id":"x","window_id":"x:0"}`) applyCodexFingerprintHeaders(hB, idsB) - assert.Equal(t, hA.Get("thread-id"), hB.Get("thread-id"), "full 模式 thread_id 应相同") + assert.NotEqual(t, hA.Get("thread-id"), hB.Get("thread-id"), "legacy full preserves independent roots") assert.Equal(t, convergedSession, hA.Get("thread-id"), "full 模式 thread_id 应等于 session_id") - assert.Equal(t, hA.Get("x-codex-window-id"), hB.Get("x-codex-window-id"), "full 模式 window_id 应相同") + assert.NotEqual(t, hA.Get("x-codex-window-id"), hB.Get("x-codex-window-id"), "independent windows remain separate") } // --- H1 修复验证：头和体的 turn_id 一致性 --- @@ -379,39 +373,15 @@ func TestFingerprintIDs_HeaderAndBody_TurnID_Consistent(t *testing.T) { func TestFingerprintIDs_MalformedEmbeddedMetadataRebuiltConsistently(t *testing.T) { account := newTestOAuthAccount(2, map[string]any{codexFingerprintModeExtraKey: "session"}) - clientHeaders := make(http.Header) - clientHeaders.Set("session-id", "client-session-malformed") - ids := resolveCodexFingerprintIDsFromRequest(account, clientHeaders) + ids := resolveCodexFingerprintIDsFromRequest(account, http.Header{"Session-Id": {"client-session"}}) require.NotNil(t, ids) - - h := make(http.Header) - h.Set("x-codex-turn-metadata", "{malformed") - applyCodexFingerprintHeaders(h, ids) - - reqBody := map[string]any{ - "client_metadata": map[string]any{ - "session_id": "client-session-malformed", - "x-codex-turn-metadata": "[malformed", - }, - } - require.True(t, applyCodexFingerprintClientMetadata(reqBody, ids)) - - var headerMeta map[string]any - require.NoError(t, json.Unmarshal([]byte(h.Get("x-codex-turn-metadata")), &headerMeta)) - clientMetadata, ok := reqBody["client_metadata"].(map[string]any) - require.True(t, ok) - bodyRaw, ok := clientMetadata["x-codex-turn-metadata"].(string) - require.True(t, ok) - var bodyMeta map[string]any - require.NoError(t, json.Unmarshal([]byte(bodyRaw), &bodyMeta)) - - for _, key := range []string{"installation_id", "session_id", "thread_id", "turn_id", "window_id", "turn_started_at_unix_ms"} { - assert.Equal(t, headerMeta[key], bodyMeta[key], "rebuilt metadata field %s must match", key) - } + body := map[string]any{"client_metadata": map[string]any{"x-codex-turn-metadata": "[malformed"}} + require.False(t, applyCodexFingerprintClientMetadata(body, ids)) + require.Error(t, ids.err, "malformed identity cannot be silently rebuilt") + _, _, err := applyCodexFingerprintClientMetadataRaw([]byte(`{"client_metadata":{"x-codex-turn-metadata":"[malformed"}}`), resolveCodexFingerprintIDsFromRequest(account, nil)) + require.Error(t, err) } -// --- applyCodexFingerprintClientMetadata --- - func TestApplyCodexFingerprintClientMetadata_OffMode(t *testing.T) { reqBody := map[string]any{ "client_metadata": map[string]any{ @@ -452,9 +422,7 @@ func TestApplyCodexFingerprintClientMetadata_DeviceMode(t *testing.T) { var meta map[string]any require.NoError(t, json.Unmarshal([]byte(turnMetaStr), &meta)) assert.Equal(t, "converged-device", meta["installation_id"]) - identity, ok := account.CodexClientIdentity() - require.True(t, ok) - assert.Equal(t, identity.Sandbox, meta["sandbox"], "sandbox 跟随账号身份的操作系统") + assert.Equal(t, "seccomp", meta["sandbox"], "preserve actual execution diagnostics") } func TestApplyCodexFingerprintClientMetadata_SessionMode(t *testing.T) { @@ -484,8 +452,8 @@ func TestApplyCodexFingerprintClientMetadata_SessionMode(t *testing.T) { seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) convergedInstall := resolveConvergedInstallationID(account, seed) - convergedSession := resolveConvergedSessionID(seed) - convergedThread := resolveConvergedThreadID(seed, "x") + convergedSession := "x" + convergedThread := "x" assert.Equal(t, convergedInstall, cm["x-codex-installation-id"]) assert.Equal(t, convergedSession, cm["session_id"]) @@ -498,9 +466,7 @@ func TestApplyCodexFingerprintClientMetadata_SessionMode(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(turnMetaStr), &meta)) assert.Equal(t, convergedInstall, meta["installation_id"]) assert.Equal(t, convergedSession, meta["session_id"]) - identity, ok := account.CodexClientIdentity() - require.True(t, ok) - assert.Equal(t, identity.Sandbox, meta["sandbox"], "sandbox 跟随账号身份的操作系统") + assert.Equal(t, "seccomp", meta["sandbox"], "preserve actual execution diagnostics") } func TestApplyCodexFingerprintClientMetadata_FullMode(t *testing.T) { @@ -526,9 +492,7 @@ func TestApplyCodexFingerprintClientMetadata_FullMode(t *testing.T) { cm, ok := reqBody["client_metadata"].(map[string]any) require.True(t, ok) - seed, ok := codexFingerprintSeed(account.Extra) - require.True(t, ok) - convergedSession := resolveConvergedSessionID(seed) + convergedSession := "x" assert.Equal(t, convergedSession, cm["session_id"]) assert.Equal(t, convergedSession, cm["thread_id"], "full 模式 thread_id 应等于 session_id") @@ -621,10 +585,10 @@ func TestApplyCodexFingerprintPromptCacheKey_MapRawEquivalence(t *testing.T) { mapBody, rawBody := applyMapAndRawFingerprintBodiesForTest(t, body, ids) require.Equal(t, mapBody["prompt_cache_key"], rawBody["prompt_cache_key"]) - require.Equal(t, ids.sessionID, mapBody["prompt_cache_key"]) + require.Equal(t, "body-session", mapBody["prompt_cache_key"]) mapCM, _ := mapBody["client_metadata"].(map[string]any) rawCM, _ := rawBody["client_metadata"].(map[string]any) - require.Equal(t, ids.sessionID, mapCM["session_id"]) + require.Equal(t, "body-session", mapCM["session_id"]) require.Equal(t, mapCM["session_id"], rawCM["session_id"]) require.Equal(t, "keep", rawCM["trace"]) }) @@ -642,8 +606,8 @@ func TestApplyCodexFingerprintPromptCacheKey_MapRawEquivalence(t *testing.T) { require.Equal(t, "explicit-cache", rawBody["prompt_cache_key"]) mapCM, _ := mapBody["client_metadata"].(map[string]any) rawCM, _ := rawBody["client_metadata"].(map[string]any) - require.Equal(t, ids.sessionID, mapCM["session_id"]) - require.Equal(t, ids.sessionID, rawCM["session_id"]) + require.Equal(t, "body-session", mapCM["session_id"]) + require.Equal(t, "body-session", rawCM["session_id"]) }) } @@ -697,13 +661,6 @@ func TestApplyCodexFingerprintPromptCacheKey_Negatives(t *testing.T) { wantExists: true, wantCacheKey: "body-session", }, - { - name: "non-string source session preserves key", - body: []byte(`{"prompt_cache_key":"123","client_metadata":{"session_id":123}}`), - ids: sessionIDs, - wantExists: true, - wantCacheKey: "123", - }, { name: "non-object source metadata preserves key", body: []byte(`{"prompt_cache_key":"body-session","client_metadata":"bad"}`), diff --git a/backend/internal/service/openai_codex_identity_snapshot.go b/backend/internal/service/openai_codex_identity_snapshot.go index 7a74361704ad..602bda643263 100644 --- a/backend/internal/service/openai_codex_identity_snapshot.go +++ b/backend/internal/service/openai_codex_identity_snapshot.go @@ -2,6 +2,7 @@ package service import ( "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "net/http" "strconv" "strings" @@ -15,29 +16,32 @@ import ( // process state published by NewOpenAIGatewayService, so the snapshot describes // the policy that was actually active in this process. type codexIdentitySnapshot struct { - SchemaVersion int `json:"v"` - AccountID int64 `json:"account_id"` - IdentityAccountID int64 `json:"identity_account_id"` - ShadowParentID *int64 `json:"shadow_parent_id,omitempty"` - AccountRevision time.Time `json:"account_revision"` - ProxyID *int64 `json:"proxy_id,omitempty"` - Transport string `json:"transport"` - IdentitySource string `json:"identity_source"` - IdentityPersisted bool `json:"identity_persisted"` - EnforcementEnabled bool `json:"enforcement_enabled"` - ForceCodexCLI bool `json:"force_codex_cli"` - UserAgent string `json:"user_agent"` - UAOverridePresent bool `json:"ua_override_present"` - Originator string `json:"originator"` - Version string `json:"version"` - FingerprintModeConfigured string `json:"fingerprint_mode_configured"` - FingerprintModeEffective string `json:"fingerprint_mode_effective"` - FingerprintReason string `json:"fingerprint_reason"` - SessionIDPresent bool `json:"session_id_present"` - ClientRequestIDPresent bool `json:"client_request_id_present"` - ZstdEnabled bool `json:"zstd_enabled"` - ZstdApplied bool `json:"zstd_applied"` - ZstdReason string `json:"zstd_reason"` + TransportProfile *codextransport.Binding `json:"transport_profile,omitempty"` + TransportProfileError string `json:"transport_profile_error,omitempty"` + WireIdentity *CodexWireIdentity `json:"wire_identity,omitempty"` + SchemaVersion int `json:"v"` + AccountID int64 `json:"account_id"` + IdentityAccountID int64 `json:"identity_account_id"` + ShadowParentID *int64 `json:"shadow_parent_id,omitempty"` + AccountRevision time.Time `json:"account_revision"` + ProxyID *int64 `json:"proxy_id,omitempty"` + Transport string `json:"transport"` + IdentitySource string `json:"identity_source"` + IdentityPersisted bool `json:"identity_persisted"` + EnforcementEnabled bool `json:"enforcement_enabled"` + ForceCodexCLI bool `json:"force_codex_cli"` + UserAgent string `json:"user_agent"` + UAOverridePresent bool `json:"ua_override_present"` + Originator string `json:"originator"` + Version string `json:"version"` + FingerprintModeConfigured string `json:"fingerprint_mode_configured"` + FingerprintModeEffective string `json:"fingerprint_mode_effective"` + FingerprintReason string `json:"fingerprint_reason"` + SessionIDPresent bool `json:"session_id_present"` + ClientRequestIDPresent bool `json:"client_request_id_present"` + ZstdEnabled bool `json:"zstd_enabled"` + ZstdApplied bool `json:"zstd_applied"` + ZstdReason string `json:"zstd_reason"` } // resolveCodexIdentitySnapshot computes the identity and fingerprint state @@ -83,6 +87,9 @@ func resolveCodexIdentitySnapshot(routed, source *Account, overrideUA string, po } if snapshot.FingerprintReason == "" { snapshot.FingerprintReason = "ok" + if snapshot.FingerprintModeConfigured == "session" || snapshot.FingerprintModeConfigured == "full" { + snapshot.FingerprintReason = "legacy_mode_normalized" + } } } if source != nil { @@ -98,6 +105,14 @@ func resolveCodexIdentitySnapshot(routed, source *Account, overrideUA string, po snapshot.UserAgent = identity.userAgent snapshot.Originator = identity.originator snapshot.Version = identity.version + if policy.enabled && source != nil && source.IsOpenAIOAuthLike() { + binding, err := codextransport.Select(identity.userAgent, codexTransportScope(source), "http") + if err != nil { + snapshot.TransportProfileError = err.Error() + } else { + snapshot.TransportProfile = &binding + } + } snapshot.IdentitySource = "canonical" if !policy.enabled { snapshot.IdentitySource = "protocol_fallback" @@ -169,3 +184,11 @@ func (s codexIdentitySnapshot) summary() string { return fmt.Sprintf("codex identity: transport=%s source=%s originator=%s version=%s fingerprint=%s zstd=%s proxy=%s", s.Transport, s.IdentitySource, s.Originator, s.Version, s.FingerprintModeEffective, s.ZstdReason, proxy) } + +func (s codexIdentitySnapshot) withRequest(transport string, request *http.Request) codexIdentitySnapshot { + if binding, ok := codextransport.FromContext(request.Context()); ok { + s.ZstdEnabled = binding.RequestZstd + s.TransportProfile = &binding + } + return s.withWire(transport, request.Header) +} diff --git a/backend/internal/service/openai_codex_metadata.go b/backend/internal/service/openai_codex_metadata.go new file mode 100644 index 000000000000..5e9343814295 --- /dev/null +++ b/backend/internal/service/openai_codex_metadata.go @@ -0,0 +1,174 @@ +package service + +import ( + "encoding/json" + "fmt" + "github.com/gin-gonic/gin" + "log/slog" + "maps" + "net/http" + "slices" + "strings" +) + +// The embedded object is authoritative. These are compatibility projections, +// not additional identity owners. Unknown embedded fields stay in the body. +var codexMetadataFlatFields = map[string]string{ + "x-codex-installation-id": "installation_id", "installation_id": "installation_id", + "session_id": "session_id", "thread_id": "thread_id", "x-codex-window-id": "window_id", + "turn_id": "turn_id", "parent_thread_id": "parent_thread_id", + "x-codex-parent-thread-id": "parent_thread_id", "parent_turn_id": "parent_turn_id", + "root_turn_id": "root_turn_id", "forked_from_thread_id": "forked_from_thread_id", +} + +func canonicalCodexMetadata(flat map[string]any) (map[string]any, []string, error) { + canonical := map[string]any{} + if value, exists := flat["x-codex-turn-metadata"]; exists { + raw, ok := value.(string) + if !ok || strings.TrimSpace(raw) == "" || decodeCodexMetadataObject([]byte(raw), &canonical) != nil || canonical == nil { + return nil, nil, fmt.Errorf("client_metadata.x-codex-turn-metadata must contain a JSON object") + } + } + conflicts := []string{} + keys := make([]string, 0, len(codexMetadataFlatFields)) + for key := range codexMetadataFlatFields { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + name := codexMetadataFlatFields[key] + value, exists := flat[key] + if !exists { + continue + } + text, ok := value.(string) + if !ok { + return nil, nil, fmt.Errorf("client_metadata.%s must be a string", key) + } + if original, exists := canonical[name]; exists { + if original != text { + conflicts = append(conflicts, key) + } + } else { + canonical[name] = text + } + } + for _, name := range []string{"installation_id", "session_id", "thread_id", "window_id", "turn_id", "parent_thread_id", "parent_turn_id", "root_turn_id", "forked_from_thread_id", "request_kind", "subagent_kind", "thread_source", "sandbox", "sandbox_mode"} { + if value, exists := canonical[name]; exists { + if _, ok := value.(string); !ok { + return nil, nil, fmt.Errorf("Codex turn metadata %s must be a string", name) + } + } + } + for _, name := range []string{"installation_id", "session_id", "thread_id", "window_id", "parent_thread_id", "forked_from_thread_id"} { + if value, ok := canonical[name].(string); ok { + canonical[name] = strings.TrimSpace(value) + } + } + return canonical, conflicts, nil +} + +func normalizeCodexMetadataProjection(flat map[string]any) ([]string, error) { + canonical, conflicts, err := canonicalCodexMetadata(flat) + if err != nil { + return nil, err + } + for key, name := range codexMetadataFlatFields { + if value, exists := canonical[name]; exists { + // Do not manufacture optional aliases that the caller did not send. + if _, exists := flat[key]; exists || key == "session_id" || key == "thread_id" || key == "x-codex-window-id" { + flat[key] = value + } + } + } + return conflicts, nil +} + +func codexMetadataClientError(err error) error { + return &UpstreamFailoverError{ + Scope: GatewayFailureScopeRequest, NextAccountAction: NextAccountStop, + SuppressAccountHealthPenalty: true, ClientStatusCode: http.StatusBadRequest, + ClientErrorType: "invalid_request_error", ClientErrorCode: "invalid_codex_client_metadata", + ClientErrorParam: "client_metadata", ClientMessage: err.Error(), + } +} + +// headers contain a bounded compatibility projection. Large body-only fields +// (tool namespaces, attribution and workspace inventories) never enter headers. +var codexHeaderMetadataFields = []string{ + "installation_id", "session_id", "thread_id", "agent_name", "turn_id", "window_id", "window_number", + "context_window_id", "request_kind", "history_initialization", "forked_from_thread_id", + "forked_from_ordinal_exclusive", "parent_thread_id", "parent_turn_id", "root_turn_id", "subagent_kind", + "thread_source", "turn_trigger", "sandbox", "sandbox_mode", "auto_review_enabled", + "node_repl_auto_review_required", "node_repl_disabled", "turn_started_at_unix_ms", "analytics_enabled", +} + +func (ids *codexFingerprintIDs) headerMetadata() map[string]any { + metadata := map[string]any{} + for _, key := range codexHeaderMetadataFields { + if value, exists := ids.logicalMetadata[key]; exists { + metadata[key] = value + } + } + metadata["installation_id"] = ids.installationID + if ids.sessionID != "" { + metadata["session_id"] = ids.sessionID + } + if ids.threadID != "" { + metadata["thread_id"] = ids.threadID + } + if ids.windowID != "" { + metadata["window_id"] = ids.windowID + } + return metadata +} + +func (ids *codexFingerprintIDs) captureCanonicalMetadata(flat map[string]any) { + if ids.bodyIdentityCaptured { + return + } + ids.bodyIdentityCaptured = true + canonical, conflicts, err := canonicalCodexMetadata(flat) + if err != nil { + ids.err = codexMetadataClientError(err) + return + } + merged := maps.Clone(ids.logicalMetadata) + if merged == nil { + merged = map[string]any{} + } + for key, value := range canonical { + merged[key] = value + } + ids.logicalMetadata = merged + ids.metadataConflicts = append(ids.metadataConflicts, conflicts...) + if value := stringMetadataValue(canonical, "session_id"); value != "" { + ids.sessionID = value + if ids.synthetic && stringMetadataValue(canonical, "thread_id") == "" { + ids.threadID = value + ids.windowID = value + ":0" + } + } else if ids.synthetic && stringMetadataValue(canonical, "thread_id") != "" { + ids.sessionID = stringMetadataValue(canonical, "thread_id") + } + + ids.captureThreadAndWindow(stringMetadataValue(canonical, "thread_id"), stringMetadataValue(canonical, "window_id")) +} + +func decodeCodexMetadataObject(raw []byte, target *map[string]any) error { + if !json.Valid(raw) { + return fmt.Errorf("invalid Codex metadata JSON") + } + return decodeOpenAIJSONUseNumber(raw, target) +} + +const codexMetadataConflictsContextKey = "codex_metadata_input_conflicts" + +func recordCodexMetadataConflicts(c *gin.Context, conflicts []string) { + if c != nil { + c.Set(codexMetadataConflictsContextKey, slices.Clone(conflicts)) + } + if len(conflicts) > 0 { + slog.Debug("codex metadata canonical projection", "source", "body_embedded", "conflicting_fields", conflicts) + } +} diff --git a/backend/internal/service/openai_codex_models_service.go b/backend/internal/service/openai_codex_models_service.go index d392e472b883..de7c827e0560 100644 --- a/backend/internal/service/openai_codex_models_service.go +++ b/backend/internal/service/openai_codex_models_service.go @@ -1960,6 +1960,11 @@ func (s *OpenAIGatewayService) fetchOpenAIModelsUpstream(ctx context.Context, re req.Header.Set("If-None-Match", ifNoneMatch) } + req, err = bindCodexTransport(req, request.credentialAccount, currentCodexFingerprintPolicyForAccount(request.credentialAccount), "http") + if err != nil { + return nil, err + } + var resp *http.Response if request.useAPIKeyUpstream { if s.httpUpstream == nil { @@ -1967,6 +1972,15 @@ func (s *OpenAIGatewayService) fetchOpenAIModelsUpstream(ctx context.Context, re } req = req.WithContext(WithHTTPUpstreamProfile(req.Context(), HTTPUpstreamProfileOpenAI)) resp, err = s.httpUpstream.Do(req, request.proxyURL, request.accountID, request.accountConcurrency) + } else if request.credentialAccount != nil && request.credentialAccount.IsOpenAIOAuthLike() && currentCodexFingerprintPolicyForAccount(request.credentialAccount).enabled { + if s.pluginManager != nil && s.pluginManager.ShouldRouteOpenAIOAuth(request.credentialAccount) { + return nil, codexStrictClientError(fmt.Errorf("third-party model transport cannot attest the verified Codex profile")) + } + if s.httpUpstream == nil { + return nil, fmt.Errorf("strict Codex models transport is not configured") + } + req = req.WithContext(WithHTTPUpstreamProfile(req.Context(), HTTPUpstreamProfileOpenAI)) + resp, err = s.httpUpstream.Do(req, request.proxyURL, request.accountID, request.accountConcurrency) } else { handled := false if s.pluginManager != nil { diff --git a/backend/internal/service/openai_codex_request_compression.go b/backend/internal/service/openai_codex_request_compression.go index 452a346e6e83..476c6366fb85 100644 --- a/backend/internal/service/openai_codex_request_compression.go +++ b/backend/internal/service/openai_codex_request_compression.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "io" "log/slog" "net/http" @@ -85,7 +86,14 @@ func isCodexStreamingResponsesRequest(req *http.Request) bool { // 重新填充。读取失败时绝不把已读出的前缀当作完整请求发送：可重放请求退回原请求明文发送， // 不可重放请求显式返回错误（该请求本就无法完整发出）。编码失败一律退回明文。 func prepareOpenAICodexWireRequest(req *http.Request, account *Account) (*http.Request, error) { - if !codexRequestZstd.Load() || account == nil || !account.IsOpenAIOAuthLike() { + if req == nil { + return nil, nil + } + enabled := codexRequestZstd.Load() + if binding, ok := codextransport.FromContext(req.Context()); ok { + enabled = binding.RequestZstd + } + if !enabled || account == nil || !account.IsOpenAIOAuthLike() { return req, nil } if !isCodexStreamingResponsesRequest(req) || req.Body == nil || req.Body == http.NoBody { @@ -144,6 +152,11 @@ func prepareOpenAICodexWireRequest(req *http.Request, account *Account) (*http.R // doOpenAICodexUpstream 是 OpenAI 网关所有 Responses 端点 POST 的统一发送入口：先按上述 // 规则生成线上请求（满足条件时压缩），再交给 httpUpstream。读取失败作为传输错误返回。 func (s *OpenAIGatewayService) doOpenAICodexUpstream(req *http.Request, account *Account, proxyURL string) (*http.Response, error) { + var bindErr error + req, bindErr = bindCodexTransport(req, account, currentCodexFingerprintPolicyForAccount(account), "http") + if bindErr != nil { + return nil, bindErr + } wire, err := prepareOpenAICodexWireRequest(req, account) if err != nil { return nil, err diff --git a/backend/internal/service/openai_codex_test_identity.go b/backend/internal/service/openai_codex_test_identity.go new file mode 100644 index 000000000000..122e57403af1 --- /dev/null +++ b/backend/internal/service/openai_codex_test_identity.go @@ -0,0 +1,106 @@ +package service + +import ( + "encoding/json" + "maps" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type codexSyntheticRoot struct { + accountID int64 + threadID string + turnID string + startedMS int64 +} + +// Retries of one administrator test reuse one real synthetic root. No identifier +// depends on a deployment-relative account number. +func projectCodexAccountTest(c *gin.Context, routed, source *Account, body map[string]any) (http.Header, error) { + const contextKey = "codex_synthetic_test_root" + root := codexSyntheticRoot{} + if value, exists := c.Get(contextKey); exists { + root, _ = value.(codexSyntheticRoot) + } + if root.accountID != routed.ID || root.threadID == "" { + thread, err := uuid.NewV7() + if err != nil { + return nil, err + } + turn, err := uuid.NewV7() + if err != nil { + return nil, err + } + root = codexSyntheticRoot{routed.ID, thread.String(), turn.String(), time.Now().UnixMilli()} + c.Set(contextKey, root) + } + policy := codexFingerprintPolicyForContext(c, routed) + metadata := map[string]any{ + "session_id": root.threadID, "thread_id": root.threadID, "window_id": root.threadID + ":0", + "turn_id": root.turnID, "root_turn_id": root.turnID, "turn_started_at_unix_ms": root.startedMS, + "request_kind": "turn", "agent_name": "/root", "window_number": 0, + } + encoded, err := marshalCodexTurnMetadata(metadata) + if err != nil { + return nil, err + } + flat := map[string]any{"session_id": root.threadID, "thread_id": root.threadID, + "x-codex-window-id": root.threadID + ":0", "turn_id": root.turnID, "x-codex-turn-metadata": string(encoded)} + body["client_metadata"] = flat + body["prompt_cache_key"] = root.threadID + account := *source + account.Extra = maps.Clone(source.Extra) + if account.Extra == nil { + account.Extra = map[string]any{} + } + account.Extra[codexFingerprintModeExtraKey] = string(routed.GetCodexFingerprintMode()) + applyCodexAccountIdentityClientMetadataMap(body, &account, 0, policy) + headers := http.Header{"Session-Id": {root.threadID}, "Thread-Id": {root.threadID}, + "X-Codex-Window-Id": {root.threadID + ":0"}, "X-Client-Request-Id": {root.threadID}, + "X-Codex-Turn-Metadata": {string(encoded)}} + applyCodexAccountIdentityHeaders(headers, &account, 0, policy) + ids := resolveCodexFingerprintIDsWithSource(routed, source, headers, policy) + if ids != nil { + applyCodexFingerprintClientMetadata(body, ids) + if ids.err != nil { + return nil, ids.err + } + applyCodexFingerprintHeaders(headers, ids) + } + return headers, nil +} + +type CodexWireIdentity struct { + InstallationID string `json:"installation_id"` + SessionID string `json:"session_id"` + ThreadID string `json:"thread_id"` + WindowID string `json:"window_id"` + Consistent bool `json:"consistent"` + Conflicts []string `json:"conflicts"` + InputConflicts []string `json:"input_conflicts,omitempty"` +} + +func codexWireIdentity(body map[string]any, headers http.Header) *CodexWireIdentity { + flat, ok := body["client_metadata"].(map[string]any) + if !ok { + return nil + } + metadata, conflicts, err := canonicalCodexMetadata(flat) + if err != nil { + return &CodexWireIdentity{Conflicts: []string{err.Error()}} + } + result := &CodexWireIdentity{InstallationID: stringMetadataValue(metadata, "installation_id"), + SessionID: stringMetadataValue(metadata, "session_id"), ThreadID: stringMetadataValue(metadata, "thread_id"), + WindowID: stringMetadataValue(metadata, "window_id"), Conflicts: conflicts} + result.Consistent = len(conflicts) == 0 && result.SessionID == headers.Get("session-id") && + result.ThreadID == headers.Get("thread-id") && result.ThreadID == headers.Get("x-client-request-id") && + result.WindowID == headers.Get("x-codex-window-id") + var headerMetadata map[string]any + if json.Unmarshal([]byte(headers.Get("x-codex-turn-metadata")), &headerMetadata) == nil { + result.Consistent = result.Consistent && result.InstallationID == stringMetadataValue(headerMetadata, "installation_id") + } + return result +} diff --git a/backend/internal/service/openai_codex_transport.go b/backend/internal/service/openai_codex_transport.go new file mode 100644 index 000000000000..2a54b9033b81 --- /dev/null +++ b/backend/internal/service/openai_codex_transport.go @@ -0,0 +1,89 @@ +package service + +import ( + "context" + "crypto/sha256" + "fmt" + "net/http" + + "encoding/json" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +func codexTransportScope(account *Account) string { + if account == nil { + return "initial-authorization" + } + seed, _ := codexFingerprintSeed(account.Extra) + installation := resolveConvergedInstallationID(account, seed) + sum := sha256.Sum256([]byte(codexAccountIdentityNamespace(account) + "\x00" + installation)) + return fmt.Sprintf("%x", sum[:16]) +} + +func codexStrictClientError(err error) error { + return &UpstreamFailoverError{Scope: GatewayFailureScopeRequest, NextAccountAction: NextAccountStop, + SuppressAccountHealthPenalty: true, ClientStatusCode: http.StatusBadGateway, + ClientErrorType: "codex_transport_error", ClientErrorCode: "codex_transport_profile_unavailable", ClientMessage: err.Error()} +} + +func bindCodexTransport(request *http.Request, source *Account, policy *codexFingerprintPolicy, plane string) (*http.Request, error) { + if request == nil || source == nil || !source.IsOpenAIOAuthLike() || !policy.enabled { + return request, nil + } + if _, ok := codextransport.FromContext(request.Context()); ok { + return request, nil + } + binding, err := codextransport.Select(request.Header.Get("User-Agent"), codexTransportScope(source), plane) + if err != nil { + return nil, codexStrictClientError(err) + } + binding.RequestZstd = codexRequestZstd.Load() + return request.WithContext(codextransport.WithBinding(request.Context(), binding)), nil +} + +func bindCodexAttemptRequest(c *gin.Context, request *http.Request, routed *Account) (*http.Request, error) { + source := codexAccountIdentitySource(c, routed) + policy := codexFingerprintPolicyForContext(c, routed) + if source != nil && source.IsOpenAIOAuthLike() && policy.enabled && request.Header.Get("originator") == "" { + identity := resolveCodexOutboundIdentityForAccount(source, codexAccountIdentityOverrideUA(source), policy) + request.Header.Set("User-Agent", identity.userAgent) + } + return bindCodexTransport(request, source, policy, "http") +} + +func codexTransportContext(ctx context.Context, headers http.Header, source *Account, policy *codexFingerprintPolicy, plane string) (context.Context, error) { + if source == nil || !source.IsOpenAIOAuthLike() || !policy.enabled { + return ctx, nil + } + binding, err := codextransport.Select(headers.Get("User-Agent"), codexTransportScope(source), plane) + if err != nil { + return nil, codexStrictClientError(err) + } + return codextransport.WithBinding(ctx, binding), nil +} + +func codexWSBodyIdentityKey(body []byte) string { + cm := gjson.GetBytes(body, "client_metadata") + if !cm.IsObject() { + return "" + } + var flat map[string]any + if json.Unmarshal([]byte(cm.Raw), &flat) != nil { + return "" + } + metadata, _, err := canonicalCodexMetadata(flat) + if err != nil { + return "" + } + values := []string{} + for _, key := range []string{"installation_id", "session_id", "thread_id", "window_id"} { + values = append(values, stringMetadataValue(metadata, key)) + } + raw, _ := json.Marshal(values) + sum := sha256.Sum256(raw) + return fmt.Sprintf("%x", sum[:16]) +} + +func CodexFingerprintSimulationEnabled() bool { return currentCodexFingerprintPolicy().enabled } diff --git a/backend/internal/service/openai_gateway_chat_completions.go b/backend/internal/service/openai_gateway_chat_completions.go index ce5f8e9076cd..345b5b663677 100644 --- a/backend/internal/service/openai_gateway_chat_completions.go +++ b/backend/internal/service/openai_gateway_chat_completions.go @@ -294,7 +294,9 @@ func (s *OpenAIGatewayService) forwardAsChatCompletions( } else if promptCacheKey != "" { reqBody["prompt_cache_key"] = promptCacheKey } - applyCodexAccountIdentityClientMetadataMap(reqBody, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c)) + if _, err := s.scopeCodexRequestMetadataMap(c, account, reqBody); err != nil { + return nil, err + } responsesBody, err = json.Marshal(reqBody) if err != nil { return nil, fmt.Errorf("remarshal after codex transform: %w", err) diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index 2d2b9c813b92..94c9b424cd5e 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/google/uuid" "io" "net/http" "strings" @@ -558,9 +559,16 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco if !isCompactRequest && codexFingerprintPolicyForContext(c, account).enabled && applyCodexClientMetadata(decoded, account) { markDecodedModified() } - if !isCompactRequest && applyCodexAccountIdentityClientMetadataMap(decoded, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) { - markDecodedModified() + if !isCompactRequest { + scoped, scopeErr := s.scopeCodexRequestMetadataMap(c, account, decoded) + if scopeErr != nil { + return nil, scopeErr + } + if scoped { + markDecodedModified() + } } + stageCodexFingerprintIDs(c, nil) // 指纹收敛：一次性解析收敛 ID，请求体和出站头共享同一份 IDs（保证 turn_id 等随机字段一致）。 // fingerprintIDs 在此处解析，后续 buildUpstreamRequest 中使用同一份。 @@ -575,6 +583,9 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco markDecodedModified() } } + if fpIDs != nil && fpIDs.err != nil { + return nil, fpIDs.err + } // 将 fpIDs 存入 gin context，供 buildUpstreamRequest 中头改写使用。 // 无条件覆写（含 nil）：failover 从收敛账号切到 off 账号时，上一 // 账号的 IDs 不得残留（stageCodexFingerprintIDs 注释）。 @@ -1551,7 +1562,7 @@ func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin. req.Header.Set("user-agent", CodexCanonicalUserAgent()) } - applyCodexAccountIdentityHeaders(req.Header, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + applyCodexAccountIdentityHeaders(req.Header, codexAccountProjectionSource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) // 指纹收敛：使用 Forward() 中预计算的收敛 ID 改写出站头，与请求体使用同一份 IDs。 applyStagedCodexFingerprintHeaders(c, account, req.Header) // 真实 Codex 必带的连字符会话头：入站缺失时以最终请求体的 prompt_cache_key @@ -1583,7 +1594,7 @@ func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin. if err := applyMappedGPT55LiteCompatibility(req, account, body); err != nil { return nil, err } - return req, nil + return bindCodexAttemptRequest(c, req, account) } // codexIdentityOverrideUA 返回账号级显式配置的出站 User-Agent，供强制统一身份时作为覆写来源。 @@ -1601,7 +1612,7 @@ func (s *OpenAIGatewayService) codexIdentityOverrideUA(account *Account) string // 按 ForceCodexCLI 策略生效——否则父/子系统不同或显式 UA 覆盖时 sandbox 会与 UA 分裂。 func (s *OpenAIGatewayService) resolveStagedCodexFingerprintIDs(c *gin.Context, account *Account, clientHeaders http.Header) *codexFingerprintIDs { policy := codexFingerprintPolicyForContext(c, account) - source := codexAccountIdentitySource(c, account) + source := codexAccountProjectionSource(c, account) scopedHeaders := clientHeaders.Clone() if scopedHeaders == nil { scopedHeaders = http.Header{} @@ -1611,6 +1622,29 @@ func (s *OpenAIGatewayService) resolveStagedCodexFingerprintIDs(c *gin.Context, if ids == nil { return nil } + if c != nil && ids.sessionID == "" && ids.threadID == "" { + const key = "codex_synthetic_request_conversation" + root, _ := c.Get(key) + raw, _ := root.(string) + if raw == "" { + generated, err := uuid.NewV7() + if err != nil { + ids.err = err + return ids + } + raw = generated.String() + c.Set(key, raw) + } + ids.sessionID = scopeCodexAccountIdentityValue(source, getAPIKeyIDFromContext(c), raw) + ids.threadID = ids.sessionID + ids.windowID = ids.threadID + ":0" + ids.synthetic = true + } + if c != nil { + if value, ok := c.Get(codexMetadataConflictsContextKey); ok { + ids.metadataConflicts, _ = value.([]string) + } + } ids.alignSandboxWithUserAgent(s.effectiveCodexOutboundUserAgent(c, account, clientHeaders)) return ids } diff --git a/backend/internal/service/openai_gateway_messages.go b/backend/internal/service/openai_gateway_messages.go index 0d68bed92e58..27e65a7bd961 100644 --- a/backend/internal/service/openai_gateway_messages.go +++ b/backend/internal/service/openai_gateway_messages.go @@ -251,7 +251,9 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic( if codexResult.PromptCacheKey != "" { promptCacheKey = codexResult.PromptCacheKey } - applyCodexAccountIdentityClientMetadataMap(reqBody, codexAccountIdentitySource(c, account), apiKeyID) + if _, err := s.scopeCodexRequestMetadataMap(c, account, reqBody); err != nil { + return nil, err + } delete(reqBody, "prompt_cache_key") if shouldAutoInjectPromptCacheKeyForCompat(upstreamModel) { compatTurnState = s.getOpenAICompatSessionTurnState(ctx, c, account, compatSessionSeed) diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 6d0bd8da6283..d318bc09b42f 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -196,7 +196,7 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough( // 账号作用域：body client_metadata / prompt_cache_key 与出站头共用同一映射， // 否则头已按账号改写而 body 仍是客户端原值，同一请求会暴露两套身份。 - accountScopedBody, accountScoped, scopeErr := applyCodexAccountIdentityClientMetadataRaw(body, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + accountScopedBody, accountScoped, scopeErr := s.scopeCodexRequestMetadataRaw(c, account, body) if scopeErr != nil { return nil, scopeErr } @@ -767,7 +767,7 @@ func (s *OpenAIGatewayService) buildUpstreamRequestOpenAIPassthrough( if s.cfg != nil && s.cfg.Gateway.ForceCodexCLI { req.Header.Set("user-agent", CodexCanonicalUserAgent()) } - applyCodexAccountIdentityHeaders(req.Header, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + applyCodexAccountIdentityHeaders(req.Header, codexAccountProjectionSource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) // 指纹收敛：使用 forwardOpenAIPassthrough 中预计算的收敛 ID 改写出站头， // 与请求体 client_metadata 共享同一份 IDs（与非透传路径相同的相对位置： // 会话隔离之后、终态身份收口之前）。 @@ -797,7 +797,7 @@ func (s *OpenAIGatewayService) buildUpstreamRequestOpenAIPassthrough( if err := applyMappedGPT55LiteCompatibility(req, account, body); err != nil { return nil, err } - return req, nil + return bindCodexAttemptRequest(c, req, account) } func stripOpenAILegacyResponsesBeta(headers http.Header) { diff --git a/backend/internal/service/openai_gateway_service.go b/backend/internal/service/openai_gateway_service.go index 5e217705ff3c..3b0db2e845f9 100644 --- a/backend/internal/service/openai_gateway_service.go +++ b/backend/internal/service/openai_gateway_service.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "log/slog" "math/rand" "net/http" @@ -61,7 +62,7 @@ const ( // 陈旧版本会被优先丢弃（HTTP 200 + 流内 server_is_overloaded）；非官方客户端配不出 // 官方身份时整体回退到本常量，因此它必须跟随官方 CLI 的当前发布版本， // 落后多个版本会让这些请求稳定落在被优先丢弃的一侧。 - codexCLIVersion = "0.161.0" + codexCLIVersion = "0.162.1" // Codex 限额快照仅用于后台展示/诊断，不需要每个成功请求都立即落库。 openAICodexSnapshotPersistMinInterval = 30 * time.Second // 配额自动暂停时，超过该时长仍未刷新的 used% 快照视为陈旧，不再据此暂停账号。 @@ -786,6 +787,13 @@ func (s *OpenAIGatewayService) getOpenAIWSProtocolResolver() OpenAIWSProtocolRes } func classifyOpenAIWSReconnectReason(err error) (string, bool) { + if codextransport.IsFailure(err) { + return "codex_strict_transport", false + } + var strict *UpstreamFailoverError + if errors.As(err, &strict) && strict.ClientErrorType == "codex_transport_error" { + return "codex_strict_transport", false + } if err == nil { return "", false } diff --git a/backend/internal/service/openai_oauth_service.go b/backend/internal/service/openai_oauth_service.go index 58ecc752142f..58871d1200c8 100644 --- a/backend/internal/service/openai_oauth_service.go +++ b/backend/internal/service/openai_oauth_service.go @@ -3,6 +3,7 @@ package service import ( "context" "crypto/subtle" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "log/slog" "net/http" "strings" @@ -222,6 +223,14 @@ func (s *OpenAIOAuthService) RefreshTokenWithClientID(ctx context.Context, refre // 一致的账号级身份（真实 Codex 客户端以自己的 UA/originator 刷新）；身份为空或客户端 // 不支持时退回全局规范身份。 func (s *OpenAIOAuthService) refreshTokenWithIdentity(ctx context.Context, refreshToken, proxyURL, clientID string, identity codexOutboundIdentity, selectedAccount *Account) (*OpenAITokenInfo, error) { + if selectedAccount != nil && selectedAccount.IsOpenAIOAuthLike() && currentCodexFingerprintPolicyForAccount(selectedAccount).enabled { + binding, err := codextransport.Select(identity.userAgent, codexTransportScope(selectedAccount), "http") + if err != nil { + return nil, err + } + ctx = codextransport.WithBinding(ctx, binding) + } + var tokenResp *openai.TokenResponse var err error refresher, ok := s.oauthClient.(OpenAIOAuthIdentityRefresher) diff --git a/backend/internal/service/openai_plugin_transport.go b/backend/internal/service/openai_plugin_transport.go index 5dfaad6e84d9..6b378e723bd1 100644 --- a/backend/internal/service/openai_plugin_transport.go +++ b/backend/internal/service/openai_plugin_transport.go @@ -1,6 +1,9 @@ package service -import "net/http" +import ( + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + "net/http" +) func (s *OpenAIGatewayService) SetPluginManager(manager *PluginManager) { s.pluginManager = manager @@ -9,7 +12,15 @@ func (s *OpenAIGatewayService) SetPluginManager(manager *PluginManager) { // doOpenAIUpstream 只在 OpenAI OAuth 能力绑定已启用时把真实请求交给插件。 // 插件返回标准 http.Response，响应解析、错误映射、SSE 和计费仍由现有核心链处理。 func (s *OpenAIGatewayService) doOpenAIUpstream(request *http.Request, proxyURL string, account *Account) (*http.Response, error) { + var bindErr error + request, bindErr = bindCodexTransport(request, account, currentCodexFingerprintPolicyForAccount(account), "http") + if bindErr != nil { + return nil, bindErr + } if s.pluginManager != nil && s.pluginManager.ShouldRouteOpenAIOAuth(account) { + if currentCodexFingerprintPolicyForAccount(account).enabled { + return nil, codexStrictClientError(&codextransport.Error{Reason: "third-party plugin cannot attest the verified transport profile"}) + } prepared, finish, prepareErr := PreparePelicanHTTPRequest(request, account.ID, account.Concurrency, "plugin") if prepareErr != nil { return nil, prepareErr @@ -34,7 +45,15 @@ func (s *AccountTestService) doOpenAIAccountTestUpstream( account *Account, useTLSFallback bool, ) (*http.Response, error) { + var bindErr error + request, bindErr = bindCodexTransport(request, account, currentCodexFingerprintPolicyForAccount(account), "http") + if bindErr != nil { + return nil, bindErr + } if s.pluginManager != nil && s.pluginManager.ShouldRouteOpenAIOAuth(account) { + if currentCodexFingerprintPolicyForAccount(account).enabled { + return nil, codexStrictClientError(&codextransport.Error{Reason: "third-party plugin cannot attest the verified transport profile"}) + } prepared, finish, prepareErr := PreparePelicanHTTPRequest(request, account.ID, account.Concurrency, "plugin") if prepareErr != nil { return nil, prepareErr diff --git a/backend/internal/service/openai_upstream_transport_error.go b/backend/internal/service/openai_upstream_transport_error.go index ee7570f8cf45..3d07578cf41a 100644 --- a/backend/internal/service/openai_upstream_transport_error.go +++ b/backend/internal/service/openai_upstream_transport_error.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "net" "net/http" "strings" @@ -133,6 +134,10 @@ func (s *OpenAIGatewayService) handleOpenAIUpstreamTransportError(ctx context.Co Message: safeErr, }) + if codextransport.IsFailure(err) { + return codexStrictClientError(err) + } + // Client disconnected: do NOT fail over to another account and do NOT evict // this one — the upstream never had a chance to exhibit a fault. if errors.Is(err, context.Canceled) || (errors.Is(err, context.DeadlineExceeded) && errors.Is(ctx.Err(), context.DeadlineExceeded)) { diff --git a/backend/internal/service/openai_ws_client.go b/backend/internal/service/openai_ws_client.go index dfa66a1f2e60..821c132b8d78 100644 --- a/backend/internal/service/openai_ws_client.go +++ b/backend/internal/service/openai_ws_client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "io" "net/http" "net/url" @@ -123,6 +124,9 @@ func (d *coderOpenAIWSClientDialer) Dial( return nil, 0, nil, errors.New("ws url is empty") } + if binding, ok := codextransport.FromContext(ctx); ok { + return d.dialStrictCodex(ctx, targetURL, headers, proxyURL, binding) + } wrapped := &coderOpenAIWSClientConn{} opts := &coderws.DialOptions{ HTTPHeader: cloneHeader(headers), diff --git a/backend/internal/service/openai_ws_codex_transport.go b/backend/internal/service/openai_ws_codex_transport.go new file mode 100644 index 000000000000..747506596ed5 --- /dev/null +++ b/backend/internal/service/openai_ws_codex_transport.go @@ -0,0 +1,123 @@ +package service + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "sync/atomic" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" + codexws "github.com/Wei-Shaw/sub2api/internal/pkg/codexws" + coderws "github.com/coder/websocket" +) + +func (d *coderOpenAIWSClientDialer) dialStrictCodex(ctx context.Context, target string, headers http.Header, proxy string, binding codextransport.Binding) (openAIWSClientConn, int, http.Header, error) { + key := "codex:" + binding.Key() + ":" + proxy + d.proxyMu.Lock() + entry := d.proxyClients[key] + if entry == nil { + var parsed *url.URL + var err error + if proxy != "" { + parsed, err = url.Parse(proxy) + if err != nil { + d.proxyMu.Unlock() + return nil, 0, nil, err + } + } + rt, err := codextransport.New(binding, codextransport.Options{ProxyURL: parsed, MaxIdleConns: openAIWSProxyTransportMaxIdleConns, + MaxIdleConnsPerHost: openAIWSProxyTransportMaxIdleConnsPerHost, IdleConnTimeout: openAIWSProxyTransportIdleConnTimeout}) + if err != nil { + d.proxyMu.Unlock() + return nil, 0, nil, err + } + entry = &openAIWSProxyClientEntry{client: &http.Client{Transport: rt}} + d.proxyClients[key] = entry + d.proxyMisses.Add(1) + } else { + d.proxyHits.Add(1) + } + entry.lastUsedUnixNano = time.Now().UnixNano() + d.cleanupProxyClientsLocked(entry.lastUsedUnixNano) + d.ensureProxyClientCapacityLocked() + d.proxyMu.Unlock() + wrapped := &strictCodexWSConn{} + conn, resp, err := codexws.Dial(ctx, target, &codexws.DialOptions{HTTPClient: entry.client, HTTPHeader: headers, + CompressionMode: codexws.CompressionContextTakeover, ClientMaxWindowBits: true, + OnPingReceived: func(context.Context, []byte) bool { wrapped.upstreamPings.Add(1); return true }}) + status := 0 + responseHeaders := http.Header(nil) + if resp != nil { + status = resp.StatusCode + responseHeaders = cloneHeader(resp.Header) + } + if err != nil { + var body []byte + if resp != nil && resp.Body != nil { + body, _ = io.ReadAll(io.LimitReader(resp.Body, 8<<10)) + _ = resp.Body.Close() + } + if status == 0 { + err = &codextransport.Failure{Err: err} + } + return nil, status, responseHeaders, &openAIWSHandshakeError{Body: body, Err: err} + } + conn.SetReadLimit(openAIWSMessageReadLimitBytes) + wrapped.conn = conn + return wrapped, 0, responseHeaders, nil +} + +type strictCodexWSConn struct { + conn *codexws.Conn + upstreamPings atomic.Int64 +} + +func (c *strictCodexWSConn) WriteJSON(ctx context.Context, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return strictCodexWSError(c.conn.Write(ctx, codexws.MessageText, data)) +} +func (c *strictCodexWSConn) ReadMessage(ctx context.Context) ([]byte, error) { + _, data, err := c.conn.Read(ctx) + return data, strictCodexWSError(err) +} +func (c *strictCodexWSConn) ReadFrame(ctx context.Context) (coderws.MessageType, []byte, error) { + kind, data, err := c.conn.Read(ctx) + return coderws.MessageType(kind), data, strictCodexWSError(err) +} +func (c *strictCodexWSConn) WriteFrame(ctx context.Context, kind coderws.MessageType, data []byte) error { + return strictCodexWSError(c.conn.Write(ctx, codexws.MessageType(kind), data)) +} +func (c *strictCodexWSConn) Ping(ctx context.Context) error { + return strictCodexWSError(c.conn.Ping(ctx)) +} +func (c *strictCodexWSConn) Close() error { + _ = c.conn.Close(codexws.StatusNormalClosure, "") + return nil +} +func (c *strictCodexWSConn) CloseNow() error { return c.conn.CloseNow() } +func (c *strictCodexWSConn) RequiresReaderLoop() bool { return true } +func (c *strictCodexWSConn) SupportsIdlePingWithoutReader() bool { return false } +func (c *strictCodexWSConn) UpstreamPingCount() int64 { return c.upstreamPings.Load() } + +type codexCompatibleCloseError struct { + original error + compat coderws.CloseError +} + +func (e *codexCompatibleCloseError) Error() string { return e.original.Error() } +func (e *codexCompatibleCloseError) Unwrap() []error { return []error{e.original, e.compat} } +func strictCodexWSError(err error) error { + if err == nil { + return nil + } + if status := codexws.CloseStatus(err); status != -1 { + err = &codexCompatibleCloseError{original: err, compat: coderws.CloseError{Code: coderws.StatusCode(status), Reason: err.Error()}} + } + return &codextransport.Failure{Err: err} +} diff --git a/backend/internal/service/openai_ws_forwarder.go b/backend/internal/service/openai_ws_forwarder.go index 1ca63816c389..9f27155060f6 100644 --- a/backend/internal/service/openai_ws_forwarder.go +++ b/backend/internal/service/openai_ws_forwarder.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "math/rand" "strings" "time" @@ -80,6 +81,9 @@ func (e *openAIWSFallbackError) Unwrap() error { } func wrapOpenAIWSFallback(reason string, err error) error { + if codextransport.IsFailure(err) { + return codexStrictClientError(err) + } return &openAIWSFallbackError{Reason: strings.TrimSpace(reason), Err: err} } diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 8b60f0a80960..ceae4fc06c81 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -337,7 +337,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( normalized = next } accountIdentitySourceRaw := append([]byte(nil), normalized...) - accountScopedPayload, accountScoped, scopeErr := applyCodexAccountIdentityClientMetadataRaw(normalized, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + accountScopedPayload, accountScoped, scopeErr := s.scopeCodexRequestMetadataRaw(c, account, normalized) if scopeErr != nil { return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket identity metadata", scopeErr) } @@ -885,6 +885,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( Account: account, WSURL: wsURL, Headers: wsHeaders, + BodyIdentityKey: codexWSBodyIdentityKey(firstPayload.payloadRaw), HeadersFactory: func(factoryCtx context.Context, headers http.Header) (http.Header, error) { return s.refreshOpenAIAgentIdentityHeaders(factoryCtx, account, headers) }, @@ -960,7 +961,11 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( req.ForcePreferredConn = forcePreferredConn // dedicated 模式及重试均新建连接，避免复用陈旧上下文。 req.ForceNewConn = dedicatedMode || forceNewConn || (turn == 1 && initialTransportRetryUsed) - acquireCtx, acquireCancel := context.WithTimeout(ctx, acquireTimeout) + boundCtx, bindErr := codexTransportContext(ctx, req.Headers, codexAccountIdentitySource(c, account), codexFingerprintPolicyForContext(c, account), "ws") + if bindErr != nil { + return nil, bindErr + } + acquireCtx, acquireCancel := context.WithTimeout(boundCtx, acquireTimeout) lease, acquireErr := pool.Acquire(acquireCtx, req) acquireCancel() var dialErr *openAIWSDialError @@ -1799,6 +1804,12 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( } baseAcquireReq.FingerprintPolicy = codexFingerprintPolicyForContext(c, account) baseAcquireReq.Headers = finalHeaders + baseAcquireReq.BodyIdentityKey = codexWSBodyIdentityKey(wirePayload) + if sessionLease != nil && !sessionLease.conn.matchesHandshakeCompatibility(normalizeOpenAIWSAcquireCompatibility(baseAcquireReq)) { + resetSessionLease(false) + return NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, openAIWSNonInitialTurnRetryCloseReason, + errors.New("codex conversation identity changed; a new handshake is required")) + } forcePreferredConn := isStrictAffinityTurn(currentPayload) && strings.TrimSpace(preferredConnID) != "" if sessionLease == nil { diff --git a/backend/internal/service/openai_ws_forwarder_payload.go b/backend/internal/service/openai_ws_forwarder_payload.go index a364db29b62c..311374bb8e00 100644 --- a/backend/internal/service/openai_ws_forwarder_payload.go +++ b/backend/internal/service/openai_ws_forwarder_payload.go @@ -136,7 +136,7 @@ func (s *OpenAIGatewayService) buildOpenAIWSHeaders( if metadata := strings.TrimSpace(turnMetadata); metadata != "" { headers.Set(openAIWSTurnMetadataHeader, metadata) } - applyCodexAccountIdentityHeaders(headers, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + applyCodexAccountIdentityHeaders(headers, codexAccountProjectionSource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) applyStagedCodexFingerprintHeaders(c, account, headers) if account != nil && account.UsesOpenAICodexProtocol() { // 入站缺失连字符会话头时补齐：来自帧体 prompt_cache_key 的回退值由调用方传入最终线上值 diff --git a/backend/internal/service/openai_ws_forwarder_v2.go b/backend/internal/service/openai_ws_forwarder_v2.go index ecb862660aba..2adccef76809 100644 --- a/backend/internal/service/openai_ws_forwarder_v2.go +++ b/backend/internal/service/openai_ws_forwarder_v2.go @@ -228,7 +228,11 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2( account.ProxyID != nil && account.Proxy != nil, ) - acquireCtx, acquireCancel := context.WithTimeout(ctx, s.openAIWSAcquireTimeout()) + boundCtx, bindErr := codexTransportContext(ctx, wsHeaders, codexAccountIdentitySource(c, account), codexFingerprintPolicyForContext(c, account), "ws") + if bindErr != nil { + return nil, bindErr + } + acquireCtx, acquireCancel := context.WithTimeout(boundCtx, s.openAIWSAcquireTimeout()) defer acquireCancel() acquireReq := openAIWSAcquireRequest{ @@ -236,6 +240,7 @@ func (s *OpenAIGatewayService) forwardOpenAIWSV2( Account: account, WSURL: wsURL, Headers: wsHeaders, + BodyIdentityKey: codexWSBodyIdentityKey(wirePayload), HeadersFactory: func(factoryCtx context.Context, headers http.Header) (http.Header, error) { return s.refreshOpenAIAgentIdentityHeaders(factoryCtx, account, headers) }, diff --git a/backend/internal/service/openai_ws_pool.go b/backend/internal/service/openai_ws_pool.go index 524efd71acd3..44cbaa0eb530 100644 --- a/backend/internal/service/openai_ws_pool.go +++ b/backend/internal/service/openai_ws_pool.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "errors" "fmt" "math" @@ -68,6 +69,7 @@ func (e *openAIWSDialError) Unwrap() error { } type openAIWSAcquireRequest struct { + BodyIdentityKey string FingerprintPolicy *codexFingerprintPolicy Account *Account WSURL string @@ -88,6 +90,7 @@ type openAIWSAcquireRequest struct { } type openAIWSHandshakeCompatibilityKey struct { + bodyIdentityKey string codexPolicyRevision uint64 codexMode string userAgent string @@ -2357,6 +2360,7 @@ func sameOpenAIWSPrewarmTarget(a, b openAIWSAcquireRequest) bool { func normalizeOpenAIWSAcquireCompatibility(req openAIWSAcquireRequest) openAIWSHandshakeCompatibilityKey { key := normalizeOpenAIWSHandshakeCompatibility(req.Account, req.Headers, req.FingerprintPolicy) key.anchorScope = stringsTrim(req.AnchorScope) + key.bodyIdentityKey = req.BodyIdentityKey return key } @@ -2400,17 +2404,17 @@ func normalizeOpenAIWSHandshakeCompatibility(account *Account, headers http.Head key.originator = headers.Get("originator") key.version = headers.Get("version") } - if !policy.enabled || account == nil { - return key - } - mode := activeCodexFingerprintMode(account, policy) - if mode == codexFingerprintOff { + if account == nil { return key } key.codexInstallationID = normalizeOpenAIWSStableIdentityHeader(headers, "x-codex-installation-id") - if mode == codexFingerprintDevice { - return key + if raw := headers.Get("x-codex-turn-metadata"); raw != "" { + var metadata map[string]any + if json.Unmarshal([]byte(raw), &metadata) == nil { + key.codexInstallationID = stringMetadataValue(metadata, "installation_id") + } } + key.sessionIDHyphen = normalizeOpenAIWSStableIdentityHeader(headers, "session-id") key.sessionIDUnderscore = normalizeOpenAIWSStableIdentityHeader(headers, "session_id") key.threadID = normalizeOpenAIWSStableIdentityHeader(headers, "thread-id") diff --git a/backend/internal/service/openai_ws_v2_passthrough_adapter.go b/backend/internal/service/openai_ws_v2_passthrough_adapter.go index a5b3688007de..2aff04184ba8 100644 --- a/backend/internal/service/openai_ws_v2_passthrough_adapter.go +++ b/backend/internal/service/openai_ws_v2_passthrough_adapter.go @@ -786,7 +786,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( } else if changed { firstClientMessage = normalized } - accountScopedFirst, accountScoped, scopeErr := applyCodexAccountIdentityClientMetadataRaw(firstClientMessage, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + accountScopedFirst, accountScoped, scopeErr := s.scopeCodexRequestMetadataRaw(c, account, firstClientMessage) if scopeErr != nil { return NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket identity metadata", scopeErr) } @@ -922,7 +922,11 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( if err != nil { return fmt.Errorf("refresh ws authentication headers: %w", err) } - dialCtx, cancelDial := context.WithTimeout(ctx, s.openAIWSDialTimeout()) + boundCtx, bindErr := codexTransportContext(ctx, headers, codexAccountIdentitySource(c, account), codexFingerprintPolicyForContext(c, account), "ws") + if bindErr != nil { + return bindErr + } + dialCtx, cancelDial := context.WithTimeout(boundCtx, s.openAIWSDialTimeout()) upstreamConn, statusCode, handshakeHeaders, err = dialer.Dial(dialCtx, wsURL, headers, proxyURL) cancelDial() if err == nil { @@ -1077,7 +1081,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2PassthroughAttempt( } } if isResponseCreate || eventType == "session.update" { - accountScopedPayload, accountScoped, scopeErr := applyCodexAccountIdentityClientMetadataRaw(payload, codexAccountIdentitySource(c, account), getAPIKeyIDFromContext(c), codexFingerprintPolicyForContext(c, account)) + accountScopedPayload, accountScoped, scopeErr := s.scopeCodexRequestMetadataRaw(c, account, payload) if scopeErr != nil { return payload, nil, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket identity metadata", scopeErr) } diff --git a/backend/internal/service/setting_codex_fingerprint.go b/backend/internal/service/setting_codex_fingerprint.go index 20198a7ee98d..c9ab23f4662e 100644 --- a/backend/internal/service/setting_codex_fingerprint.go +++ b/backend/internal/service/setting_codex_fingerprint.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "io" "strconv" "strings" @@ -29,9 +30,12 @@ type CodexFingerprintSettings struct { type CodexFingerprintSettingsView struct { CodexFingerprintSettings - SyncedVersion string `json:"synced_version"` - EffectiveVersion string `json:"effective_version"` - EffectiveUserAgent string `json:"effective_user_agent"` + SyncedVersion string `json:"synced_version"` + VerifiedVersion string `json:"verified_version"` + VersionPendingValidation bool `json:"version_pending_validation"` + TransportPolicy string `json:"transport_policy"` + EffectiveVersion string `json:"effective_version"` + EffectiveUserAgent string `json:"effective_user_agent"` } // An immutable policy is shared by the body, headers and connection selection of @@ -180,8 +184,11 @@ func (s *SettingService) codexFingerprintSettingsFromValues(values map[string]st if version == "" { version = synced } + if config.ClientVersion == "" && !codextransport.SupportsVersion(version) { + version = codextransport.LatestVersion() + } if version == "" || CompareVersions(version, codexUpstreamMinVersion) < 0 { - version = codexCLIVersion + version = codextransport.LatestVersion() } ua := buildCodexCLIUserAgent(version) if _, paired, ok := openai.PairCodexClientIdentity(config.UserAgent); ok { @@ -189,7 +196,7 @@ func (s *SettingService) codexFingerprintSettingsFromValues(values map[string]st ua = rebuilt } } - return CodexFingerprintSettingsView{CodexFingerprintSettings: config, SyncedVersion: synced, EffectiveVersion: version, EffectiveUserAgent: ua}, nil + return CodexFingerprintSettingsView{CodexFingerprintSettings: config, SyncedVersion: synced, EffectiveVersion: version, EffectiveUserAgent: ua, VerifiedVersion: codextransport.LatestVersion(), VersionPendingValidation: synced != "" && CompareVersions(synced, version) > 0 && !codextransport.SupportsVersion(synced), TransportPolicy: "strict"}, nil } func publishCodexFingerprintSettings(view CodexFingerprintSettingsView) { @@ -220,6 +227,9 @@ func (s *SettingService) LoadCodexFingerprintSettings(ctx context.Context) error } func (s *SettingService) UpdateCodexFingerprintSettings(ctx context.Context, config CodexFingerprintSettings) (CodexFingerprintSettingsView, error) { + if config.ClientVersion != "" && !codextransport.SupportsVersion(config.ClientVersion) { + return CodexFingerprintSettingsView{}, fmt.Errorf("Codex version %s has no verified transport profile", config.ClientVersion) + } s.codexFingerprintMu.Lock() defer s.codexFingerprintMu.Unlock() values, err := s.settingRepo.GetMultiple(ctx, codexFingerprintSettingKeys) diff --git a/backend/internal/service/setting_gateway_runtime.go b/backend/internal/service/setting_gateway_runtime.go index dc8fb959d28f..50c6ed5c57c4 100644 --- a/backend/internal/service/setting_gateway_runtime.go +++ b/backend/internal/service/setting_gateway_runtime.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "log/slog" "strconv" "strings" @@ -498,6 +499,9 @@ func (s *SettingService) GetOpenAICodexClientVersion(ctx context.Context) string version := NormalizeCodexClientVersion(values[SettingKeyOpenAICodexClientVersion]) if version == "" { version = NormalizeCodexClientVersion(values[SettingKeyOpenAICodexClientVersionSynced]) + if !codextransport.SupportsVersion(version) { + version = codextransport.LatestVersion() + } } if version == "" { version = fallback diff --git a/backend/internal/service/setting_update.go b/backend/internal/service/setting_update.go index 1b4a38a5ccca..fabb46d9cb55 100644 --- a/backend/internal/service/setting_update.go +++ b/backend/internal/service/setting_update.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "log/slog" "math" "strconv" @@ -514,6 +515,9 @@ func (s *SettingService) buildSystemSettingsUpdates(ctx context.Context, setting updates[SettingKeyEnableClientDatelineNormalization] = strconv.FormatBool(settings.EnableClientDatelineNormalization) updates[SettingKeyAntigravityUserAgentVersion] = antigravity.NormalizeUserAgentVersion(settings.AntigravityUserAgentVersion) updates[SettingKeyOpenAICodexUserAgent] = strings.TrimSpace(settings.OpenAICodexUserAgent) + if settings.OpenAICodexClientVersion != "" && !codextransport.SupportsVersion(NormalizeCodexClientVersion(settings.OpenAICodexClientVersion)) { + return nil, fmt.Errorf("Codex version has no verified transport profile") + } updates[SettingKeyOpenAICodexClientVersion] = NormalizeCodexClientVersion(settings.OpenAICodexClientVersion) updates[SettingKeyOpenAICodexVersionAutoSyncEnabled] = strconv.FormatBool(settings.OpenAICodexVersionAutoSyncEnabled) // SettingKeyOpenAICodexClientVersionSynced 由自动同步任务独占写入，此处不得覆盖， diff --git a/backend/internal/service/upstream_models.go b/backend/internal/service/upstream_models.go index ee563e6bc14d..0fb8b30d36a5 100644 --- a/backend/internal/service/upstream_models.go +++ b/backend/internal/service/upstream_models.go @@ -1607,6 +1607,11 @@ func (s *AccountTestService) fetchAntigravityOAuthUpstreamModels(ctx context.Con } func (s *AccountTestService) doUpstreamModelsRequest(req *http.Request, proxyURL string, account *Account) (*http.Response, error) { + var bindErr error + req, bindErr = bindCodexTransport(req, account, currentCodexFingerprintPolicyForAccount(account), "http") + if bindErr != nil { + return nil, bindErr + } if s.tlsFPProfileService == nil { return s.httpUpstream.DoWithTLS(req, proxyURL, account.ID, account.Concurrency, nil) } diff --git a/frontend/src/api/admin/codexFingerprint.ts b/frontend/src/api/admin/codexFingerprint.ts index 682679078f4b..d0cfb5716ea6 100644 --- a/frontend/src/api/admin/codexFingerprint.ts +++ b/frontend/src/api/admin/codexFingerprint.ts @@ -10,6 +10,9 @@ export interface CodexFingerprintSettings { } export interface CodexFingerprintSettingsView extends CodexFingerprintSettings { + verified_version?: string + version_pending_validation?: boolean + transport_policy?: 'strict' synced_version: string effective_version: string effective_user_agent: string @@ -28,6 +31,8 @@ export interface CodexFingerprintAccountView { fingerprint_mode_configured: string fingerprint_mode_effective: CodexFingerprintMode fingerprint_reason: string + transport_profile?: { profile_id: string; version: string; source_commit: string; system: string; arch: string; strict: boolean; validated: boolean; plane: string } + transport_profile_error?: string } device_identity: { v: number diff --git a/frontend/src/components/admin/CodexFingerprintIdentity.vue b/frontend/src/components/admin/CodexFingerprintIdentity.vue index 12d35792eb0c..e9705aa452eb 100644 --- a/frontend/src/components/admin/CodexFingerprintIdentity.vue +++ b/frontend/src/components/admin/CodexFingerprintIdentity.vue @@ -9,7 +9,7 @@
{{ field.label }}
{{ field.value }}
-

{{ text('平台沙箱标签用于指纹兼容映射；实际权限与沙箱状态由请求决定。', 'The platform sandbox label is used for fingerprint compatibility mapping; actual permissions and sandbox state come from the request.') }}

+

{{ text('设备平台标签仅供参考；真实权限与 sandbox 诊断来自请求，转发时保留。', 'Device platform labels are descriptive. Actual permissions and sandbox diagnostics come from the request and are preserved.') }}

@@ -30,6 +30,19 @@
+
+

{{ text('严格传输档案', 'Strict transport profile') }}

+ +
+
{{ text('档案', 'Profile') }}
{{ view.identity.transport_profile.profile_id }}
+
{{ text('已验证客户端', 'Verified client') }}
{{ view.identity.transport_profile.version }}
+
{{ text('平台', 'Platform') }}
{{ view.identity.transport_profile.system }} / {{ view.identity.transport_profile.arch }}
+
{{ text('失败策略', 'Failure policy') }}
{{ text('严格：报错，不回退', 'Strict: fail without fallback') }}
+
{{ text('官方源码', 'Official source') }}
{{ view.identity.transport_profile.source_commit }}
+
+

{{ text('这里展示档案验证状态；账号测试另列实际发送标识和连接协商结果。', 'This describes profile validation. Account tests separately report sent identifiers and connection negotiation.') }}

+
+

{{ text('标识处理规则', 'Identifier handling rules') }}

@@ -78,13 +91,13 @@ const effectiveFields = computed(() => { { label: text('UA 声明的系统版本', 'OS version declared by UA'), value: device?.os_version ?? unknown }, { label: text('UA 声明的架构', 'Architecture declared by UA'), value: device?.arch ?? unknown }, { label: text('UA 声明的终端', 'Terminal declared by UA'), value: device?.terminal ?? unknown }, - { label: text('沙箱兼容映射目标', 'Sandbox compatibility mapping target'), value: device?.platform_sandbox ?? unknown } + { label: text('设备平台参考标签', 'Device platform reference label'), value: device?.platform_sandbox ?? unknown } ] }) const identitySource = computed(() => ({ account: text('账号固定 TUI', 'Fixed account TUI'), override_ua: text('账号自定义 UA', 'Account UA override'), canonical: text('全局默认', 'Global default'), protocol_fallback: text('协议兜底', 'Protocol fallback') })[props.view.identity.identity_source] || props.view.identity.identity_source) -const reason = computed(() => ({ ok: text('按当前策略生效', 'Active under the current policy'), seed_missing: text('身份种子缺失，未收敛', 'Missing identity seed; no convergence'), explicit_off: text('保留设备／会话标识；身份模拟仍由总开关控制', 'Preserve device/session identifiers; identity simulation follows the master switch'), simulation_disabled: text('总开关关闭，保留客户端身份与标识', 'Master switch off; retain client identity and identifiers'), non_oauth: text('该账号类型不适用', 'Not applicable to this account type') })[props.view.identity.fingerprint_reason] || props.view.identity.fingerprint_reason) +const reason = computed(() => ({ legacy_mode_normalized: text('旧模式已兼容为固定设备、多独立对话', 'Legacy mode normalized to a fixed device with independent conversations'), ok: text('按当前策略生效', 'Active under the current policy'), seed_missing: text('身份种子缺失，未收敛', 'Missing identity seed; no convergence'), explicit_off: text('保留设备／会话标识；身份模拟仍由总开关控制', 'Preserve device/session identifiers; identity simulation follows the master switch'), simulation_disabled: text('总开关关闭，保留客户端身份与标识', 'Master switch off; retain client identity and identifiers'), non_oauth: text('该账号类型不适用', 'Not applicable to this account type') })[props.view.identity.fingerprint_reason] || props.view.identity.fingerprint_reason) function modeLabel(mode: CodexFingerprintMode): string { - return { off: text('保留设备／会话标识', 'Preserve device/session identifiers'), device: text('仅设备', 'Device only'), session: text('设备＋会话', 'Device + session'), full: text('设备＋会话＋线程', 'Device + session + thread') }[mode] || mode + return { off: text('保留设备／会话标识', 'Preserve device/session identifiers'), device: text('固定设备，多独立对话', 'Fixed device, independent conversations'), session: text('旧 session（兼容为固定设备多对话）', 'Legacy session (fixed device, independent conversations)'), full: text('旧 full（兼容为固定设备多对话）', 'Legacy full (fixed device, independent conversations)') }[mode] || mode } const identifierFields = computed(() => [ { key: 'installation_id', label: text('设备标识', 'Device identifier') }, @@ -97,6 +110,10 @@ function ruleLabel(rule: string): string { const labels: Record = { preserve_client: text('保留客户端原值', 'Preserve the client value'), account_device: text('使用账号固定设备标识；写入请求元数据', 'Use the fixed account device identifier in request metadata'), + scoped_real_session: text('按真实根会话一一映射，不合并独立对话', 'Map each real root session without merging conversations'), + scoped_real_thread: text('按真实线程一一映射，保留 UUIDv7 时间位', 'Map each real thread and preserve UUIDv7 timestamps'), + scoped_real_parent: text('与线程使用同一映射，保留真实父子关系', 'Use the thread mapping and preserve real parent relationships'), + scoped_real_window: text('映射真实线程前缀，保留窗口序号', 'Map the real thread prefix and preserve the window index'), account_session: text('使用账号固定会话标识', 'Use the fixed account session identifier'), mapped_thread: text('按真实线程确定性映射；缺失线程时采用请求会话来源', 'Map each real thread deterministically; use the request session source when no thread is present'), account_thread: text('合并为账号固定线程标识', 'Merge into the fixed account thread identifier'), diff --git a/frontend/src/views/admin/CodexFingerprintView.vue b/frontend/src/views/admin/CodexFingerprintView.vue index 2d0d1c3eabb8..e0952fc2110c 100644 --- a/frontend/src/views/admin/CodexFingerprintView.vue +++ b/frontend/src/views/admin/CodexFingerprintView.vue @@ -13,7 +13,7 @@ {{ text('启用客户端身份模拟', 'Enable client identity simulation') }} -

{{ text('开启：使用每账号固定 TUI 身份，并按账号模式收敛标识。关闭：新请求保留客户端指纹；保存的配置和账号身份继续保留。', 'On: use a fixed TUI identity per account and converge identifiers according to its mode. Off: new requests retain the client fingerprint; saved configuration and account identities are preserved.') }}

+

{{ text('开启：每个凭据来源使用固定设备身份，独立对话保留各自会话与线程关系；使用已验证的严格传输，失败不回退。关闭：保留客户端身份与标识，已保存的设备档案继续保留。', 'On: use a fixed device per credential source, preserve independent conversations, and require verified transport without fallback. Off: retain caller identity and identifiers; saved devices are preserved.') }}

{{ text('同步版本', 'Synced version') }}
{{ settings.synced_version || '—' }}
+
{{ text('已验证版本', 'Verified version') }}
{{ settings.verified_version || '—' }}
+
{{ text('已发现新版，传输档案尚未验证；继续使用已验证版本。', 'A newer release is available; its transport profile is pending validation. The verified version remains effective.') }}
{{ settings.enabled ? text('生效模拟版本', 'Effective simulated version') : text('待启用模拟版本', 'Saved simulation version') }}
{{ settings.effective_version }}
{{ text('全局默认身份', 'Global default identity') }}
{{ settings.effective_user_agent }}
@@ -126,9 +128,7 @@ type DetailTask = { id: number; epoch: number; serial: number; list: number; gen const detailQueue: DetailTask[] = [] const modes = computed(() => [ { value: 'off' as const, label: text('保留设备／会话标识', 'Preserve device/session identifiers') }, - { value: 'device' as const, label: text('固定设备（推荐）', 'Fixed device (recommended)') }, - { value: 'session' as const, label: text('固定设备，分别映射会话', 'Fixed device, separate mapped sessions') }, - { value: 'full' as const, label: text('固定设备，合并会话与线程', 'Fixed device, merged sessions and threads') } + { value: 'device' as const, label: text('固定设备，多独立对话', 'Fixed device, independent conversations') } ]) const settingsDirty = computed(() => settings.value && (draft.enabled !== settings.value.enabled || draft.user_agent !== settings.value.user_agent || draft.client_version !== settings.value.client_version || draft.version_auto_sync_enabled !== settings.value.version_auto_sync_enabled)) const allPageSelected = computed(() => rows.value.length > 0 && rows.value.every(row => selectedIDs.value.includes(row.id))) @@ -160,7 +160,7 @@ function applyAccountView(account: AccountListItem, view: CodexFingerprintAccoun storedModes[account.id] = view.mode account.extra = { ...account.extra, codex_fingerprint_mode: view.mode } accountViews[account.id] = view - if (!dirty) draftModes[account.id] = view.mode + if (!dirty) draftModes[account.id] = savedMode(account) } function pumpDetails() { while (activeDetails < 3 && detailQueue.length) { diff --git a/frontend/src/views/admin/__tests__/CodexFingerprintView.spec.ts b/frontend/src/views/admin/__tests__/CodexFingerprintView.spec.ts index 6b2ea7531923..d0552e07238a 100644 --- a/frontend/src/views/admin/__tests__/CodexFingerprintView.spec.ts +++ b/frontend/src/views/admin/__tests__/CodexFingerprintView.spec.ts @@ -19,33 +19,31 @@ vi.mock('vue-i18n', async importOriginal => ({ useI18n: () => ({ t: (key: string) => key, locale: ref(language) }) })) -const settings = { enabled: true, user_agent: '', client_version: '', version_auto_sync_enabled: true, synced_version: '0.161.0', effective_version: '0.161.0', effective_user_agent: 'codex_cli_rs/0.161.0 (Ubuntu 24.4.0; x86_64) xterm-256color' } +const settings = { enabled: true, user_agent: '', client_version: '', version_auto_sync_enabled: true, synced_version: '0.162.1', effective_version: '0.162.1', effective_user_agent: 'codex_cli_rs/0.162.1 (Ubuntu 24.4.0; x86_64) xterm-256color' } const accountView: CodexFingerprintAccountView = { mode: 'full', simulation_enabled: true, - identity: { user_agent: 'codex-tui/0.161.0 (Windows 10.0.26100; x86_64) vscode/1.104.0 (codex-tui; 0.161.0)', originator: 'codex-tui', version: '0.161.0', identity_source: 'account', identity_account_id: 1, identity_persisted: true, fingerprint_mode_configured: 'full', fingerprint_mode_effective: 'full', fingerprint_reason: 'ok' }, + identity: { user_agent: 'codex-tui/0.162.1 (Windows 10.0.26100; x86_64) vscode/1.104.0 (codex-tui; 0.162.1)', originator: 'codex-tui', version: '0.162.1', identity_source: 'account', identity_account_id: 1, identity_persisted: true, fingerprint_mode_configured: 'full', fingerprint_mode_effective: 'device', fingerprint_reason: 'legacy_mode_normalized', transport_profile: { profile_id: 'codex-rust-0.162.1-r1', version: '0.162.1', source_commit: '092d3acd', system: 'Windows', arch: 'x86_64', strict: true, validated: true, plane: 'http' } }, device_identity: { v: 1, os_type: 'Windows', os_version: '10.0.26100', arch: 'x86_64', terminal: 'vscode/1.104.0', sandbox: 'windows_sandbox', generated_at: '2026-10-08T08:00:00Z' }, effective_device: { os_type: 'Windows', os_version: '10.0.26100', arch: 'x86_64', terminal: 'vscode/1.104.0', platform_sandbox: 'windows_sandbox' }, identifier_policy: { installation_id: { behavior: 'fixed', rule: 'account_device', value: 'fixed-installation' }, - session_id: { behavior: 'fixed', rule: 'account_session', value: 'fixed-session' }, - thread_id: { behavior: 'fixed', rule: 'account_thread', value: 'fixed-session' }, - parent_thread_id: { behavior: 'request_derived', rule: 'remove_parent_self_reference', value: null }, - window_id: { behavior: 'request_derived', rule: 'merged_window', value: null } + session_id: { behavior: 'request_derived', rule: 'scoped_real_session', value: null }, + thread_id: { behavior: 'request_derived', rule: 'scoped_real_thread', value: null }, + parent_thread_id: { behavior: 'request_derived', rule: 'scoped_real_parent', value: null }, + window_id: { behavior: 'request_derived', rule: 'scoped_real_window', value: null } } } function viewFor(id: number, mode: CodexFingerprintMode = serverModes[id] || 'device'): CodexFingerprintAccountView { const view = structuredClone(accountView) view.mode = mode; view.simulation_enabled = simulationEnabled view.identity.identity_account_id = id; view.identity.fingerprint_mode_configured = mode - view.identity.fingerprint_mode_effective = simulationEnabled ? mode : 'off' + view.identity.fingerprint_mode_effective = simulationEnabled ? (mode === 'off' ? 'off' : 'device') : 'off' if (!simulationEnabled) { view.effective_device = null; view.identity.identity_source = 'protocol_fallback'; view.identity.fingerprint_reason = 'simulation_disabled' view.identity.user_agent = settings.effective_user_agent } if (mode === 'off' || !simulationEnabled) { for (const key of Object.keys(view.identifier_policy) as Array) view.identifier_policy[key] = { behavior: 'passthrough', rule: 'preserve_client', value: null } - } else if (mode !== 'full') { - view.identifier_policy.thread_id = { behavior: mode === 'session' ? 'request_derived' : 'passthrough', rule: mode === 'session' ? 'mapped_thread' : 'preserve_client', value: null } } return view } @@ -71,7 +69,7 @@ describe('CodexFingerprintView', () => { it('loads the effective version and saves only the four global settings without resetting account identity', async () => { const wrapper = await open() - expect(wrapper.get('[data-test="fingerprint-effective-version"]').text()).toBe('0.161.0') + expect(wrapper.get('[data-test="fingerprint-effective-version"]').text()).toBe('0.162.1') await wrapper.get('[data-test="fingerprint-enabled"]').setValue(false) await wrapper.find('form').trigger('submit.prevent') await flushPromises() @@ -83,10 +81,10 @@ describe('CodexFingerprintView', () => { it('updates a single mode through the bounded endpoint and displays the shared effective identity', async () => { const wrapper = await open() - await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('session') + await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('off') await wrapper.get('[data-test="fingerprint-account-save-1"]').trigger('click') await flushPromises() - expect(saveAccount).toHaveBeenCalledWith(1, 'session') + expect(saveAccount).toHaveBeenCalledWith(1, 'off') expect(wrapper.text()).toContain(settings.effective_user_agent) expect(wrapper.get('[data-test="fingerprint-account-save-1"]').element.disabled).toBe(true) await wrapper.get('[data-test="fingerprint-account-refresh-1"]').trigger('click'); await flushPromises() @@ -100,7 +98,7 @@ describe('CodexFingerprintView', () => { const identity = wrapper.findAll('[data-test="fingerprint-identity"]')[0]! for (const value of ['固定设备身份', '已持久保存', 'Windows', '10.0.26100', 'x86_64', 'vscode/1.104.0', 'windows_sandbox', '生成时间', '身份结构版本', '当前策略，适用于新请求', '保存的收敛模式', '生效收敛模式']) expect(identity.text()).toContain(value) expect(identity.get('[data-test="fingerprint-installation_id"]').text()).toBe('fixed-installation') - expect(identity.text()).toContain('保留请求中的窗口序号') + expect(identity.text()).toContain('保留窗口序号') expect(wrapper.text()).not.toContain('查看生效身份') expect(wrapper.get('[data-test="fingerprint-account-refresh-1"]').attributes('aria-label')).toBe('刷新账号身份 First') wrapper.unmount() @@ -130,18 +128,18 @@ describe('CodexFingerprintView', () => { let finish!: (view: CodexFingerprintAccountView) => void getAccount.mockImplementationOnce(() => new Promise(resolve => { finish = resolve })) const wrapper = await open() - await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('session') + await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('off') finish(viewFor(1)); await flushPromises() - expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('session') + expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('off') getAccount.mockRejectedValueOnce(new Error('identity read failed')) await wrapper.get('[data-test="fingerprint-account-refresh-1"]').trigger('click'); await flushPromises() expect(wrapper.get('[role="alert"]').text()).toContain('identity read failed') expect(wrapper.findAll('[data-test="fingerprint-identity"]')).toHaveLength(2) await wrapper.get('[data-test="fingerprint-account-retry-1"]').trigger('click'); await flushPromises() expect(wrapper.find('[role="alert"]').exists()).toBe(false) - expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('session') + expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('off') await wrapper.findAll('button').find(button => button.text() === '刷新账号')!.trigger('click'); await flushPromises() - expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('session') + expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('off') wrapper.unmount() }) @@ -180,11 +178,11 @@ describe('CodexFingerprintView', () => { let finish!: (view: CodexFingerprintAccountView) => void getAccount.mockImplementationOnce(() => new Promise(resolve => { finish = resolve })) const wrapper = await open() - await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('session') + await wrapper.get('[data-test="fingerprint-mode-1"]').setValue('off') await wrapper.get('[data-test="fingerprint-account-save-1"]').trigger('click'); await flushPromises() const obsolete = structuredClone(accountView); obsolete.identity.user_agent = 'obsolete-mode-identity' finish(obsolete); await flushPromises() - expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('session') + expect(wrapper.get('[data-test="fingerprint-mode-1"]').element.value).toBe('off') expect(wrapper.get('[data-test="fingerprint-account-save-1"]').element.disabled).toBe(true) expect(wrapper.text()).not.toContain('obsolete-mode-identity') wrapper.unmount() From 199744ec556468a11826c7862606152ac32cb210 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 13:19:42 +0800 Subject: [PATCH 08/11] fix: close strict transport retry and validation gaps --- backend/cmd/codex-transport-probe/main.go | 2 +- backend/internal/pkg/codextransport/http1.go | 16 ++-- .../internal/pkg/codextransport/proxy_test.go | 30 +++++-- .../internal/pkg/codextransport/transport.go | 31 ++++++- .../pkg/codextransport/transport_test.go | 30 +++++++ backend/internal/pkg/codexws/UPSTREAM.md | 5 +- backend/internal/pkg/codexws/accept.go | 6 +- .../pkg/codexws/codex_compress_test.go | 2 +- backend/internal/pkg/codexws/compress.go | 5 +- backend/internal/pkg/codexws/conn.go | 6 +- backend/internal/pkg/codexws/dial.go | 4 +- .../pkg/codexws/internal/bpool/bpool.go | 7 +- backend/internal/pkg/codexws/netconn.go | 9 +- backend/internal/pkg/codexws/read.go | 4 +- backend/internal/pkg/codexws/write.go | 6 +- backend/internal/pkg/codexws/wsjson/wsjson.go | 2 +- .../service/account_available_models_test.go | 2 +- .../service/account_test_models_test.go | 14 +-- .../internal/service/account_test_service.go | 1 + ...ccount_test_service_openai_compact_test.go | 7 +- .../service/openai_alpha_search_test.go | 2 +- .../openai_codex_conversations_test.go | 53 ++++++++++- .../service/openai_codex_fingerprint.go | 6 -- .../internal/service/openai_codex_metadata.go | 2 +- .../service/openai_codex_models_service.go | 3 +- .../openai_codex_models_service_test.go | 88 +++++++++++-------- .../service/openai_codex_transport.go | 19 +++- .../service/openai_gateway_passthrough.go | 6 ++ .../openai_gateway_response_handling.go | 16 ++++ backend/internal/service/openai_ws_pool.go | 11 --- .../service/setting_codex_fingerprint.go | 2 +- backend/internal/service/setting_update.go | 2 +- 32 files changed, 283 insertions(+), 116 deletions(-) diff --git a/backend/cmd/codex-transport-probe/main.go b/backend/cmd/codex-transport-probe/main.go index 20ebe8c98b7a..5fa501ec3964 100644 --- a/backend/cmd/codex-transport-probe/main.go +++ b/backend/cmd/codex-transport-probe/main.go @@ -67,7 +67,7 @@ func run() error { if err != nil { return err } - defer conn.CloseNow() + defer func() { _ = conn.CloseNow() }() if err = conn.Write(ctx, websocket.MessageText, []byte(`{"type":"response.create","model":"local-fixture","input":[]}`)); err != nil { return err } diff --git a/backend/internal/pkg/codextransport/http1.go b/backend/internal/pkg/codextransport/http1.go index 8dbab70ad7cc..2c1f962152c2 100644 --- a/backend/internal/pkg/codextransport/http1.go +++ b/backend/internal/pkg/codextransport/http1.go @@ -34,7 +34,7 @@ func (c *http1WireConn) Write(input []byte) (int, error) { n = c.bodyRemaining } if _, err := c.Conn.Write(p[:n]); err != nil { - _ = c.Conn.Close() + _ = c.Close() return 0, err } c.bodyRemaining -= n @@ -44,11 +44,11 @@ func (c *http1WireConn) Write(input []byte) (int, error) { if c.chunked { n, err := c.consumeChunk(p) if err != nil { - _ = c.Conn.Close() + _ = c.Close() return 0, err } if _, err = c.Conn.Write(p[:n]); err != nil { - _ = c.Conn.Close() + _ = c.Close() return 0, err } p = p[n:] @@ -56,11 +56,11 @@ func (c *http1WireConn) Write(input []byte) (int, error) { } // Headers may be split across writes. Bound only the header allocation, // retaining net/http's existing default header-size boundary. - c.header.WriteByte(p[0]) + _ = c.header.WriteByte(p[0]) // bytes.Buffer.WriteByte cannot fail. p = p[1:] if c.header.Len() > 1<<20 { - _ = c.Conn.Close() - return 0, fmt.Errorf("Codex HTTP1 headers exceed 1 MiB") + _ = c.Close() + return 0, fmt.Errorf("codex HTTP1 headers exceed 1 MiB") } if !bytes.HasSuffix(c.header.Bytes(), []byte("\r\n\r\n")) { continue @@ -81,7 +81,7 @@ func (c *http1WireConn) Write(input []byte) (int, error) { case "content-length": length, err := strconv.ParseInt(value, 10, 64) if err != nil || length < 0 { - _ = c.Conn.Close() + _ = c.Close() return 0, fmt.Errorf("invalid HTTP1 Content-Length") } c.bodyRemaining = length @@ -93,7 +93,7 @@ func (c *http1WireConn) Write(input []byte) (int, error) { } } if _, err := c.Conn.Write(raw); err != nil { - _ = c.Conn.Close() + _ = c.Close() return 0, err } c.header.Reset() diff --git a/backend/internal/pkg/codextransport/proxy_test.go b/backend/internal/pkg/codextransport/proxy_test.go index 3d1d26d8f87b..cea8363c1a6d 100644 --- a/backend/internal/pkg/codextransport/proxy_test.go +++ b/backend/internal/pkg/codextransport/proxy_test.go @@ -52,13 +52,13 @@ func TestStrictCodexProxyVariantsPreserveAuthenticationAndStreaming(t *testing.T if err != nil { t.Fatal(err) } - defer listener.Close() + defer func() { _ = listener.Close() }() go func() { client, err := listener.Accept() if err != nil { return } - defer client.Close() + defer func() { _ = client.Close() }() _ = client.SetDeadline(time.Now().Add(5 * time.Second)) head := make([]byte, 2) if _, err = io.ReadFull(client, head); err != nil { @@ -88,10 +88,11 @@ func TestStrictCodexProxyVariantsPreserveAuthenticationAndStreaming(t *testing.T return } size := 4 - if request[3] == 3 { + switch request[3] { + case 3: _, _ = io.ReadFull(client, one) size = int(one[0]) - } else if request[3] == 4 { + case 4: size = 16 } address := make([]byte, size+2) @@ -124,10 +125,16 @@ func TestStrictCodexProxyVariantsPreserveAuthenticationAndStreaming(t *testing.T } upstream, err := net.DialTimeout("tcp", targetURL.Host, time.Second) if err != nil { - http.Error(w, "tunnel failed", 502) + http.Error(w, "tunnel failed", http.StatusBadGateway) return } - client, buffer, err := w.(http.Hijacker).Hijack() + hijacker, ok := w.(http.Hijacker) + if !ok { + _ = upstream.Close() + t.Error("fixture does not support hijacking") + return + } + client, buffer, err := hijacker.Hijack() if err != nil { _ = upstream.Close() return @@ -175,7 +182,12 @@ func TestStrictCodexDoesNotReplaySentPOSTOrCanceledCONNECT(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { posts.Add(1) _, _ = io.Copy(io.Discard, r.Body) - conn, _, err := w.(http.Hijacker).Hijack() + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("fixture does not support hijacking") + return + } + conn, _, err := hijacker.Hijack() if err == nil { _ = conn.Close() } @@ -203,13 +215,13 @@ func TestStrictCodexDoesNotReplaySentPOSTOrCanceledCONNECT(t *testing.T) { if err != nil { t.Fatal(err) } - defer listener.Close() + defer func() { _ = listener.Close() }() go func() { conn, err := listener.Accept() if err != nil { return } - defer conn.Close() + defer func() { _ = conn.Close() }() _, _ = http.ReadRequest(bufio.NewReader(conn)) _, _ = io.Copy(io.Discard, conn) }() diff --git a/backend/internal/pkg/codextransport/transport.go b/backend/internal/pkg/codextransport/transport.go index ee57bfad3e74..967d45abfd1c 100644 --- a/backend/internal/pkg/codextransport/transport.go +++ b/backend/internal/pkg/codextransport/transport.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "encoding/hex" "fmt" + "io" "net" "net/http" "net/url" @@ -104,7 +105,7 @@ func New(binding Binding, opts Options) (*req.Transport, error) { defer cancel() if err = conn.HandshakeContext(handshake); err != nil { _ = plain.Close() - return nil, nil, fmt.Errorf("Codex TLS handshake: %w", err) + return nil, nil, fmt.Errorf("codex TLS handshake: %w", err) } if err = conn.SetDeadline(time.Time{}); err != nil { _ = conn.Close() @@ -112,8 +113,9 @@ func New(binding Binding, opts Options) (*req.Transport, error) { } cs := conn.ConnectionState() state := &tls.ConnectionState{Version: cs.Version, HandshakeComplete: cs.HandshakeComplete, DidResume: cs.DidResume, - CipherSuite: cs.CipherSuite, NegotiatedProtocol: cs.NegotiatedProtocol, NegotiatedProtocolIsMutual: cs.NegotiatedProtocolIsMutual, - ServerName: cs.ServerName, PeerCertificates: cs.PeerCertificates, VerifiedChains: cs.VerifiedChains, + CipherSuite: cs.CipherSuite, NegotiatedProtocol: cs.NegotiatedProtocol, + NegotiatedProtocolIsMutual: cs.NegotiatedProtocolIsMutual, //nolint:staticcheck // req/v3 still requires this legacy flag to activate HTTP/2. + ServerName: cs.ServerName, PeerCertificates: cs.PeerCertificates, VerifiedChains: cs.VerifiedChains, SignedCertificateTimestamps: cs.SignedCertificateTimestamps, OCSPResponse: cs.OCSPResponse} verifiedConn := &verifiedTLSConn{UConn: conn, state: *state} if cs.NegotiatedProtocol != "h2" { @@ -135,12 +137,35 @@ func New(binding Binding, opts Options) (*req.Transport, error) { if err != nil { return response, &Failure{Err: err} } + response.Request = wire + if response.Body != nil { + body := &strictResponseBody{ReadCloser: response.Body} + if writer, ok := response.Body.(io.Writer); ok { + // Upgraded WebSockets must retain their duplex body interface. + response.Body = &struct { + *strictResponseBody + io.Writer + }{body, writer} + } else { + response.Body = body + } + } return response, nil } }) return transport, nil } +type strictResponseBody struct{ io.ReadCloser } + +func (b *strictResponseBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if err != nil && err != io.EOF && !IsFailure(err) { + err = &Failure{Err: err} + } + return n, err +} + func clientHello(h hello) *utls.ClientHelloSpec { extensions := make([]utls.TLSExtension, 0, len(h.Extensions)) for _, kind := range h.Extensions { diff --git a/backend/internal/pkg/codextransport/transport_test.go b/backend/internal/pkg/codextransport/transport_test.go index 2474bc0c0d8a..0c71332a1119 100644 --- a/backend/internal/pkg/codextransport/transport_test.go +++ b/backend/internal/pkg/codextransport/transport_test.go @@ -108,3 +108,33 @@ func TestStrictCodexUnsupportedAndCanceledRequests(t *testing.T) { t.Fatal("sent a canceled POST") } } + +func TestStrictCodexInterruptedResponseRetainsFailureWithoutReplay(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + posts.Add(1) + w.Header().Set("Content-Length", "64") + _, _ = w.Write([]byte("incomplete response")) + })) + defer server.Close() + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + rt, err := New(testBinding(t), Options{RootCAs: roots}) + if err != nil { + t.Fatal(err) + } + defer rt.CloseIdleConnections() + request, _ := http.NewRequest(http.MethodPost, server.URL, strings.NewReader("already sent")) + response, err := (&http.Client{Transport: rt, Timeout: time.Second}).Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { _ = response.Body.Close() }() + _, err = io.ReadAll(response.Body) + if !IsFailure(err) { + t.Fatalf("interrupted body lost strict failure classification: %v", err) + } + if posts.Load() != 1 { + t.Fatalf("interrupted POST was replayed %d times", posts.Load()) + } +} diff --git a/backend/internal/pkg/codexws/UPSTREAM.md b/backend/internal/pkg/codexws/UPSTREAM.md index e178f88724f7..0bbd95337c9c 100644 --- a/backend/internal/pkg/codexws/UPSTREAM.md +++ b/backend/internal/pkg/codexws/UPSTREAM.md @@ -4,5 +4,6 @@ Original license: LICENSE.txt. Other providers retain the original dependency. Local changes: opt-in client_max_window_bits offer and validation; bounded-window compression with the already locked klauspost/compress implementation; narrow -window writers never enter the default 32 KiB writer pool. No protocol parser -or connection-lifecycle behavior is otherwise changed. +window writers never enter the default 32 KiB writer pool. Cleanup error handling +is explicit for the host lint rules; pooled readers and buffers validate their +type before reuse. No protocol parser behavior is otherwise changed. diff --git a/backend/internal/pkg/codexws/accept.go b/backend/internal/pkg/codexws/accept.go index bea93fc2b5a5..1cb9f1512d7f 100644 --- a/backend/internal/pkg/codexws/accept.go +++ b/backend/internal/pkg/codexws/accept.go @@ -164,7 +164,7 @@ func accept(w http.ResponseWriter, r *http.Request, opts *AcceptOptions) (_ *Con } // https://github.com/golang/go/issues/32314 - b, _ := brw.Reader.Peek(brw.Reader.Buffered()) + b, _ := brw.Peek(brw.Reader.Buffered()) brw.Reader.Reset(io.MultiReader(bytes.NewReader(b), netConn)) return newConn(connConfig{ @@ -371,8 +371,8 @@ var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") func secWebSocketAccept(secWebSocketKey string) string { h := sha1.New() - h.Write([]byte(secWebSocketKey)) - h.Write(keyGUID) + _, _ = h.Write([]byte(secWebSocketKey)) // hash writes cannot fail. + _, _ = h.Write(keyGUID) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } diff --git a/backend/internal/pkg/codexws/codex_compress_test.go b/backend/internal/pkg/codexws/codex_compress_test.go index 1a7cb30a948d..a6b7010f00be 100644 --- a/backend/internal/pkg/codexws/codex_compress_test.go +++ b/backend/internal/pkg/codexws/codex_compress_test.go @@ -31,7 +31,7 @@ func TestCodexBoundedWindowNegotiationAndCompression(t *testing.T) { t.Fatal(err) } // RFC 7692 stream tail, followed by the final empty DEFLATE block. - wire.Write([]byte{1, 0, 0, 255, 255}) + _, _ = wire.Write([]byte{1, 0, 0, 255, 255}) reader := flate.NewReader(bytes.NewReader(wire.Bytes())) decoded, err := io.ReadAll(reader) _ = reader.Close() diff --git a/backend/internal/pkg/codexws/compress.go b/backend/internal/pkg/codexws/compress.go index 29db77f467e1..00a8509666b9 100644 --- a/backend/internal/pkg/codexws/compress.go +++ b/backend/internal/pkg/codexws/compress.go @@ -144,7 +144,10 @@ func getFlateReader(r io.Reader, dict []byte) io.Reader { if !ok { return flate.NewReaderDict(r, dict) } - fr.(flate.Resetter).Reset(r, dict) + resetter, ok := fr.(flate.Resetter) + if !ok || resetter.Reset(r, dict) != nil { + return flate.NewReaderDict(r, dict) + } return fr } diff --git a/backend/internal/pkg/codexws/conn.go b/backend/internal/pkg/codexws/conn.go index 092348712d8a..9e8b901ae880 100644 --- a/backend/internal/pkg/codexws/conn.go +++ b/backend/internal/pkg/codexws/conn.go @@ -136,7 +136,7 @@ func newConn(cfg connConfig) *Conn { } runtime.SetFinalizer(c, func(c *Conn) { - c.close() + _ = c.close() }) return c @@ -171,7 +171,7 @@ func (c *Conn) close() error { func (c *Conn) setupWriteTimeout(ctx context.Context) { stop := context.AfterFunc(ctx, func() { c.clearWriteTimeout() - c.close() + _ = c.close() }) swapTimeoutStop(&c.writeTimeoutStop, &stop) } @@ -183,7 +183,7 @@ func (c *Conn) clearWriteTimeout() { func (c *Conn) setupReadTimeout(ctx context.Context) { stop := context.AfterFunc(ctx, func() { c.clearReadTimeout() - c.close() + _ = c.close() }) swapTimeoutStop(&c.readTimeoutStop, &stop) } diff --git a/backend/internal/pkg/codexws/dial.go b/backend/internal/pkg/codexws/dial.go index f76b62f76e1d..7347119c5835 100644 --- a/backend/internal/pkg/codexws/dial.go +++ b/backend/internal/pkg/codexws/dial.go @@ -157,12 +157,12 @@ func dial(ctx context.Context, urls string, opts *DialOptions, rand io.Reader) ( r := io.LimitReader(respBody, 1024) timer := time.AfterFunc(time.Second*3, func() { - respBody.Close() + _ = respBody.Close() }) defer timer.Stop() b, _ := io.ReadAll(r) - respBody.Close() + _ = respBody.Close() resp.Body = io.NopCloser(bytes.NewReader(b)) } }() diff --git a/backend/internal/pkg/codexws/internal/bpool/bpool.go b/backend/internal/pkg/codexws/internal/bpool/bpool.go index 12cf577a9f7a..70592529fee1 100644 --- a/backend/internal/pkg/codexws/internal/bpool/bpool.go +++ b/backend/internal/pkg/codexws/internal/bpool/bpool.go @@ -14,8 +14,11 @@ var bpool = sync.Pool{ // Get returns a buffer from the pool or creates a new one if // the pool is empty. func Get() *bytes.Buffer { - b := bpool.Get() - return b.(*bytes.Buffer) + b, ok := bpool.Get().(*bytes.Buffer) + if !ok { + return &bytes.Buffer{} + } + return b } // Put returns a buffer into the pool. diff --git a/backend/internal/pkg/codexws/netconn.go b/backend/internal/pkg/codexws/netconn.go index 1f73b04be6c7..d36a7afd1578 100644 --- a/backend/internal/pkg/codexws/netconn.go +++ b/backend/internal/pkg/codexws/netconn.go @@ -174,7 +174,7 @@ func (nc *netConn) read(p []byte) (int, error) { } if typ != nc.msgType { err := fmt.Errorf("unexpected frame type read (expected %v): %v", nc.msgType, typ) - nc.c.Close(StatusUnsupportedData, err.Error()) + _ = nc.c.Close(StatusUnsupportedData, err.Error()) return 0, err } nc.reader = r @@ -199,9 +199,10 @@ func (a websocketAddr) String() string { } func (nc *netConn) SetDeadline(t time.Time) error { - nc.SetWriteDeadline(t) - nc.SetReadDeadline(t) - return nil + if err := nc.SetWriteDeadline(t); err != nil { + return err + } + return nc.SetReadDeadline(t) } func (nc *netConn) SetWriteDeadline(t time.Time) error { diff --git a/backend/internal/pkg/codexws/read.go b/backend/internal/pkg/codexws/read.go index 782881113b52..63291c88b817 100644 --- a/backend/internal/pkg/codexws/read.go +++ b/backend/internal/pkg/codexws/read.go @@ -76,10 +76,10 @@ func (c *Conn) CloseRead(ctx context.Context) context.Context { go func() { defer close(c.closeReadDone) defer cancel() - defer c.close() + defer func() { _ = c.close() }() _, _, err := c.Reader(ctx) if err == nil { - c.Close(StatusPolicyViolation, "unexpected data message") + _ = c.Close(StatusPolicyViolation, "unexpected data message") } }() return ctx diff --git a/backend/internal/pkg/codexws/write.go b/backend/internal/pkg/codexws/write.go index d6c05d66d98a..790bb55a5b88 100644 --- a/backend/internal/pkg/codexws/write.go +++ b/backend/internal/pkg/codexws/write.go @@ -376,8 +376,8 @@ func extractBufioWriterBuf(bw *bufio.Writer, w io.Writer) []byte { return len(p2), nil })) - bw.WriteByte(0) - bw.Flush() + _ = bw.WriteByte(0) // The temporary in-memory writer above cannot fail. + _ = bw.Flush() bw.Reset(w) @@ -385,5 +385,5 @@ func extractBufioWriterBuf(bw *bufio.Writer, w io.Writer) []byte { } func (c *Conn) writeError(code StatusCode, err error) { - c.writeClose(code, err.Error()) + _ = c.writeClose(code, err.Error()) } diff --git a/backend/internal/pkg/codexws/wsjson/wsjson.go b/backend/internal/pkg/codexws/wsjson/wsjson.go index 3ad3bf2f4308..336ad94211dd 100644 --- a/backend/internal/pkg/codexws/wsjson/wsjson.go +++ b/backend/internal/pkg/codexws/wsjson/wsjson.go @@ -36,7 +36,7 @@ func read(ctx context.Context, c *websocket.Conn, v any) (err error) { err = json.Unmarshal(b.Bytes(), v) if err != nil { - c.Close(websocket.StatusInvalidFramePayloadData, "failed to unmarshal JSON") + _ = c.Close(websocket.StatusInvalidFramePayloadData, "failed to unmarshal JSON") return fmt.Errorf("failed to unmarshal JSON: %w", err) } diff --git a/backend/internal/service/account_available_models_test.go b/backend/internal/service/account_available_models_test.go index d1e730db0b55..2888885cafbb 100644 --- a/backend/internal/service/account_available_models_test.go +++ b/backend/internal/service/account_available_models_test.go @@ -63,7 +63,7 @@ func TestAccountAvailableModelsProjectsDiscoveryWithoutChangingCache(t *testing. func TestAccountAvailableModelsProjectsOAuthDiscovery(t *testing.T) { _, calls := newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"Case/OAuth-A","display_name":"upstream descriptor"},{"slug":"oauth-second"}]}`) - gateway := &OpenAIGatewayService{} + gateway := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} svc := &AccountTestService{} svc.SetOpenAIGatewayService(gateway) account := newCodexModelsTestAccount() diff --git a/backend/internal/service/account_test_models_test.go b/backend/internal/service/account_test_models_test.go index 8d739b811d1d..9dfdbc73bbcd 100644 --- a/backend/internal/service/account_test_models_test.go +++ b/backend/internal/service/account_test_models_test.go @@ -18,7 +18,7 @@ func TestFetchOpenAIAccountModelsOAuthPopulatesPickerFields(t *testing.T) { {"slug":"blank-display-name","display_name":" "}, {"slug":"gpt-6-astra"} ]}`) - gateway := &OpenAIGatewayService{} + gateway := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} svc := &AccountTestService{} svc.SetOpenAIGatewayService(gateway) account := newCodexModelsTestAccount() @@ -121,7 +121,7 @@ func TestFetchOpenAIAccountModelsPreservesEmptyCatalog(t *testing.T) { func TestFetchOpenAIAccountModelsOAuthLabelsLocalImageModelsLikeUpstream(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-5.6-sol"}]}`) - svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{}} + svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}} account := newCodexModelsTestAccount() // This case is about label sources, so it must configure the upstream slug it // asserts on: the picker is scoped to the account mapping, and an unconfigured @@ -143,7 +143,7 @@ func TestFetchOpenAIAccountModelsOAuthLabelsLocalImageModelsLikeUpstream(t *test func TestFetchOpenAIAccountModelsOAuthRespectsImageAllowlist(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-6-astra"}]}`) svc := &AccountTestService{} - svc.SetOpenAIGatewayService(&OpenAIGatewayService{}) + svc.SetOpenAIGatewayService(&OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}) account := newCodexModelsTestAccount() account.Credentials["model_mapping"] = map[string]any{"gpt-image-2.5-flare": "gpt-image-2.5-flare"} models, err := svc.FetchOpenAIAccountModels(context.Background(), account) @@ -168,7 +168,7 @@ func pickerModelIDs(models []openai.Model) []string { // not configure must not become a testable choice. func TestFetchOpenAIAccountModelsOAuthExcludesUnconfiguredTextModels(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-5.6-sol"},{"slug":"gpt-6-astra"}]}`) - svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{}} + svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}} account := newCodexModelsTestAccount() account.Credentials["model_mapping"] = map[string]any{"gpt-5.6-sol": "gpt-5.6-sol"} @@ -181,7 +181,7 @@ func TestFetchOpenAIAccountModelsOAuthExcludesUnconfiguredTextModels(t *testing. // the picker has to resolve the target before deciding to offer the public name. func TestFetchOpenAIAccountModelsOAuthLocalImageAlias(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-6-astra"}]}`) - svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{}} + svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}} account := newCodexModelsTestAccount() account.Credentials["model_mapping"] = map[string]any{"paint": "gpt-image-2.5-flare"} @@ -198,7 +198,7 @@ func TestFetchOpenAIAccountModelsOAuthImageNamesMappedToText(t *testing.T) { for _, target := range []string{"text-target-missing", "gpt-6-astra"} { t.Run(publicID+"/"+target, func(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-6-astra","display_name":"Upstream Text Model"}]}`) - svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{}} + svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}} account := newCodexModelsTestAccount() account.Credentials["model_mapping"] = map[string]any{publicID: target} @@ -219,7 +219,7 @@ func TestFetchOpenAIAccountModelsOAuthPassthroughIgnoresMappingTargets(t *testin for _, flag := range []string{"openai_passthrough", "openai_oauth_passthrough"} { t.Run(flag, func(t *testing.T) { newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-6-astra"}]}`) - svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{}} + svc := &AccountTestService{openaiGatewayService: &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()}} account := newCodexModelsTestAccount() account.Extra = map[string]any{flag: true} account.Credentials["model_mapping"] = map[string]any{ diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index 83f882733e8e..b4977fd172a7 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -2438,6 +2438,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if isOAuth { // 与真实转发一致：使用该账号的 Codex TUI 身份，账号级自定义 UA 经 ForceCodexCLI // 策略过滤后作为管理员显式配置传入（同普通 OAuth 连接测试）。 + ensureCodexIdentityHeaders(req.Header, codexFingerprintPolicyForContext(c, account)) enforceCodexIdentityHeadersForAccount(req.Header, credentialAccount, codexAccountIdentityOverrideUA(credentialAccount), codexFingerprintPolicyForContext(c, account)) } if isOAuth { diff --git a/backend/internal/service/account_test_service_openai_compact_test.go b/backend/internal/service/account_test_service_openai_compact_test.go index a07967f923df..866260902f04 100644 --- a/backend/internal/service/account_test_service_openai_compact_test.go +++ b/backend/internal/service/account_test_service_openai_compact_test.go @@ -66,13 +66,18 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactOAuthSuccessPersi require.Equal(t, "chatgpt.com", upstream.lastReq.Host) require.Equal(t, "text/event-stream", upstream.lastReq.Header.Get("Accept")) require.Contains(t, upstream.lastReq.Header.Get("x-codex-beta-features"), "remote_compaction_v2") - require.NotEmpty(t, upstream.lastReq.Header.Get("Session_Id")) + sessionID := upstream.lastReq.Header.Get("Session-Id") + require.NotEmpty(t, sessionID) + require.Equal(t, uuid.Version(7), uuid.MustParse(sessionID).Version()) + require.Equal(t, sessionID, upstream.lastReq.Header.Get("Thread-Id")) require.Equal(t, HTTPUpstreamProfileOpenAI, HTTPUpstreamProfileFromContext(upstream.lastReq.Context())) require.Equal(t, codexCLIUserAgent, upstream.lastReq.Header.Get("User-Agent")) require.Equal(t, "chatgpt-acc", upstream.lastReq.Header.Get("chatgpt-account-id")) require.Equal(t, "true", upstream.lastReq.Header.Get("x-openai-fedramp")) require.Equal(t, "zstd", upstream.lastReq.Header.Get("Content-Encoding")) probeBody := zstdDecodeForTest(t, upstream.lastBody) + require.Equal(t, sessionID, gjson.GetBytes(probeBody, "client_metadata.session_id").String()) + require.Equal(t, sessionID, gjson.GetBytes(probeBody, "client_metadata.thread_id").String()) require.Equal(t, "gpt-5.4", gjson.GetBytes(probeBody, "model").String()) require.True(t, gjson.GetBytes(probeBody, "stream").Bool()) require.False(t, gjson.GetBytes(probeBody, "store").Bool()) diff --git a/backend/internal/service/openai_alpha_search_test.go b/backend/internal/service/openai_alpha_search_test.go index e3cb2dd6a8ee..b5d2d842a979 100644 --- a/backend/internal/service/openai_alpha_search_test.go +++ b/backend/internal/service/openai_alpha_search_test.go @@ -95,7 +95,7 @@ func TestForwardAlphaSearchOAuthPreservesWire(t *testing.T) { require.Equal(t, codexCLIVersion, upstream.lastReq.Header.Get("Version")) require.Empty(t, upstream.lastReq.Header.Get("OpenAI-Beta")) require.Equal(t, - "search-session", + scopeCodexAccountIdentityValue(account, 0, "search-session"), gjson.Get(upstream.lastReq.Header.Get("X-Codex-Turn-Metadata"), "session_id").String(), ) require.Equal(t, diff --git a/backend/internal/service/openai_codex_conversations_test.go b/backend/internal/service/openai_codex_conversations_test.go index 751b9f02684a..870b9e69477f 100644 --- a/backend/internal/service/openai_codex_conversations_test.go +++ b/backend/internal/service/openai_codex_conversations_test.go @@ -1,13 +1,17 @@ package service import ( + "context" "encoding/json" "errors" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" "io" "net/http" "net/http/httptest" + "strings" "testing" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -45,7 +49,8 @@ func TestCodexConversationsPreserveRootAndChildGraph(t *testing.T) { require.True(t, applyCodexFingerprintClientMetadata(body, ids)) require.NoError(t, ids.err) applyCodexFingerprintHeaders(headers, ids) - flat := body["client_metadata"].(map[string]any) + flat, ok := body["client_metadata"].(map[string]any) + require.True(t, ok) canonical, _, err := canonicalCodexMetadata(flat) require.NoError(t, err) mappedThread := scopeCodexAccountIdentityValue(account, 9, thread) @@ -186,7 +191,9 @@ func TestCodexLegacyWindowRetainsTheThreadMappingAndIndex(t *testing.T) { for _, raw := range []string{"legacy-thread", "project:legacy", uuid.NewString(), uuid.Must(uuid.NewV7()).String()} { metadata := map[string]any{"thread_id": raw, "window_id": raw + ":19"} require.True(t, applyCodexAccountIdentityFields(metadata, account, 9, policy)) - require.Equal(t, metadata["thread_id"].(string)+":19", metadata["window_id"]) + thread, ok := metadata["thread_id"].(string) + require.True(t, ok) + require.Equal(t, thread+":19", metadata["window_id"]) } } @@ -201,3 +208,45 @@ func TestCodexWSLeaseCannotCrossBodyDeviceIdentity(t *testing.T) { require.NotEqual(t, first.BodyIdentityKey, next.BodyIdentityKey) require.False(t, conn.matchesHandshakeCompatibility(normalizeOpenAIWSAcquireCompatibility(next))) } + +func TestCodexStrictStreamInterruptionCannotBecomeReplay(t *testing.T) { + binding, err := codextransport.Select(codexCLIUserAgent, "stream-test", "http") + require.NoError(t, err) + for _, passthrough := range []bool{false, true} { + for _, truncated := range []bool{false, true} { + name := "native" + if passthrough { + name = "passthrough" + } + if truncated { + name += "/truncated" + } else { + name += "/missing-terminal" + } + t.Run(name, func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + request := c.Request.WithContext(codextransport.WithBinding(context.Background(), binding)) + var body io.Reader = strings.NewReader("") + if truncated { + body = &prefixThenErrorReader{err: io.ErrUnexpectedEOF} + } + response := &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"text/event-stream"}}, + Body: io.NopCloser(body), Request: request} + s := &OpenAIGatewayService{cfg: &config.Config{}} + account := fingerprintTestAccount("device") + if passthrough { + _, err = s.handleStreamingResponsePassthrough(context.Background(), response, c, account, time.Now(), "gpt-5.6-sol", "gpt-5.6-sol") + } else { + _, err = s.handleStreamingResponse(context.Background(), response, c, account, time.Now(), "gpt-5.6-sol", "gpt-5.6-sol") + } + var failure *UpstreamFailoverError + require.ErrorAs(t, err, &failure) + require.Equal(t, GatewayFailureScopeRequest, failure.Scope) + require.Equal(t, NextAccountStop, failure.NextAccountAction) + require.True(t, failure.SuppressAccountHealthPenalty) + require.Equal(t, "codex_transport_failed", failure.ClientErrorCode) + }) + } + } +} diff --git a/backend/internal/service/openai_codex_fingerprint.go b/backend/internal/service/openai_codex_fingerprint.go index 8653db638916..002f83951db7 100644 --- a/backend/internal/service/openai_codex_fingerprint.go +++ b/backend/internal/service/openai_codex_fingerprint.go @@ -470,8 +470,6 @@ func stringMetadataValue(metadata map[string]any, key string) string { return value } -func (ids *codexFingerprintIDs) mapThread(original string) string { return original } - func (ids *codexFingerprintIDs) captureThreadAndWindow(thread, window string) { if strings.TrimSpace(thread) != "" { ids.threadID = thread @@ -486,10 +484,6 @@ func (ids *codexFingerprintIDs) captureThreadAndWindow(thread, window string) { } } -func (ids *codexFingerprintIDs) rewriteThreadReferences(metadata map[string]any) { - // The account-scoping pass preserves the complete root/child/fork graph. -} - func (ids *codexFingerprintIDs) rewriteTurnMetadata(metadata map[string]any) { for key, value := range ids.headerMetadata() { metadata[key] = value diff --git a/backend/internal/service/openai_codex_metadata.go b/backend/internal/service/openai_codex_metadata.go index 5e9343814295..9a591005cfb4 100644 --- a/backend/internal/service/openai_codex_metadata.go +++ b/backend/internal/service/openai_codex_metadata.go @@ -56,7 +56,7 @@ func canonicalCodexMetadata(flat map[string]any) (map[string]any, []string, erro for _, name := range []string{"installation_id", "session_id", "thread_id", "window_id", "turn_id", "parent_thread_id", "parent_turn_id", "root_turn_id", "forked_from_thread_id", "request_kind", "subagent_kind", "thread_source", "sandbox", "sandbox_mode"} { if value, exists := canonical[name]; exists { if _, ok := value.(string); !ok { - return nil, nil, fmt.Errorf("Codex turn metadata %s must be a string", name) + return nil, nil, fmt.Errorf("codex turn metadata %s must be a string", name) } } } diff --git a/backend/internal/service/openai_codex_models_service.go b/backend/internal/service/openai_codex_models_service.go index de7c827e0560..43a744efdfad 100644 --- a/backend/internal/service/openai_codex_models_service.go +++ b/backend/internal/service/openai_codex_models_service.go @@ -18,6 +18,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/pkg/claude" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/httpclient" "github.com/Wei-Shaw/sub2api/internal/pkg/openai" @@ -1551,7 +1552,7 @@ func isRetryableCodexModelsManifestStatus(statusCode int, useAPIKeyUpstream bool } func isRetryableCodexModelsManifestTransportError(err error) bool { - if err == nil || errors.Is(err, context.Canceled) { + if err == nil || errors.Is(err, context.Canceled) || codextransport.IsFailure(err) { return false } if errors.Is(err, context.DeadlineExceeded) || diff --git a/backend/internal/service/openai_codex_models_service_test.go b/backend/internal/service/openai_codex_models_service_test.go index 773eca79128a..d6421bcff355 100644 --- a/backend/internal/service/openai_codex_models_service_test.go +++ b/backend/internal/service/openai_codex_models_service_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/codextransport" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" @@ -29,6 +30,18 @@ type codexModelsHTTPUpstreamStub struct { do func(req *http.Request, proxyURL string, accountID int64, accountConcurrency int) (*http.Response, error) } +// Model projection/cache tests inject a loopback sender at the same boundary as +// production. Wire-profile conformance is verified in codextransport, separately. +func newCodexModelsLoopbackUpstream() HTTPUpstream { + return &codexModelsHTTPUpstreamStub{do: func(req *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + ip := net.ParseIP(req.URL.Hostname()) + if ip == nil || !ip.IsLoopback() { + return nil, fmt.Errorf("model fixture must use a loopback endpoint") + } + return http.DefaultClient.Do(req) + }} +} + type codexModelsVisibilityAccountRepo struct { AccountRepository byGroup map[int64][]Account @@ -1191,7 +1204,7 @@ func TestMergeGroupConfiguredCodexModelsInjectsCurrentGroupAliases(t *testing.T) t.Parallel() const groupID int64 = 71 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: { { @@ -1267,7 +1280,7 @@ func TestBuildGroupConfiguredCodexModelsManifestUsesAdministratorConfiguration(t CapacitySource: ModelContextSourceUpstream, }, }}) - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: { { @@ -1317,7 +1330,7 @@ func TestBuildGroupConfiguredCodexModelsManifestUsesResolvedTargetsForOrdinaryGP "gpt-5.6-terra": "provider-terra", "gpt-5.6-luna": "provider-luna", } - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{groupID: {*account}}, }} @@ -1377,7 +1390,7 @@ func TestConfiguredCodexManifestCapacityComesFromResolver(t *testing.T) { account.Extra = map[string]any{ModelContextOverridesExtraKey: map[string]int64{test.target: test.override}} } repo := codexModelsVisibilityAccountRepo{byGroup: map[int64][]Account{groupID: {*account}}} - svc := &OpenAIGatewayService{accountRepo: repo} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: repo} group := &Group{ID: groupID, Platform: PlatformOpenAI} var body []byte switch path { @@ -1428,7 +1441,7 @@ func TestMergeGroupConfiguredCodexModelsAppliesReferenceCapacityAfterFinalMerge( const groupID int64 = 79 account := newCodexModelsAPIKeyTestAccount("https://openai-compatible.example.test/v1") - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{groupID: {*account}}, }} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[ @@ -1480,7 +1493,7 @@ func TestBuildGroupConfiguredCodexModelsManifestExpandsSelectedModelCoveredByWil t.Parallel() const groupID int64 = 80 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: {{ Platform: PlatformOpenAI, @@ -1533,7 +1546,7 @@ func TestBuildGroupConfiguredCodexModelsManifestIntersectsTransientlyUnschedulab true, map[string]any{"exclusive-model": "exclusive-upstream"}, ) - svc := &OpenAIGatewayService{accountRepo: splitCodexModelsAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: splitCodexModelsAccountRepo{ schedulable: map[int64][]Account{groupID: {schedulable}}, catalog: map[int64][]Account{groupID: {schedulable, transientlyUnschedulable}}, }} @@ -1580,7 +1593,7 @@ func TestBuildGroupConfiguredCodexModelsManifestIgnoresPersistentlyDisabledMappe false, nil, ) - svc := &OpenAIGatewayService{accountRepo: splitCodexModelsAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: splitCodexModelsAccountRepo{ schedulable: map[int64][]Account{groupID: {enabled}}, catalog: map[int64][]Account{groupID: {enabled}}, all: map[int64][]Account{groupID: {enabled, disabled}}, @@ -1607,7 +1620,7 @@ func TestBuildGroupConfiguredCodexModelsManifestFallsThroughWithoutConfiguration t.Parallel() const groupID int64 = 78 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: {{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}}, }, @@ -1627,7 +1640,7 @@ func TestMergeGroupConfiguredCodexModelsFiltersAutoReviewByDefault(t *testing.T) t.Parallel() const groupID int64 = 74 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{}} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{}} manifest := &OpenAIModelsResponse{ Body: []byte(`{"models":[{"slug":"codex-auto-review","visibility":"list"},{"slug":"codex-auto-future","visibility":"list"},{"slug":"gpt-image-2","visibility":"list"},{"slug":"gpt-5.6","visibility":"list"}]}`), } @@ -1649,7 +1662,7 @@ func TestMergeGroupConfiguredCodexModelsFiltersAccountMappedAutoReviewByDefault( t.Parallel() const groupID int64 = 75 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: { { @@ -1681,7 +1694,7 @@ func TestMergeGroupConfiguredCodexModelsKeepsExplicitAutoReviewSelection(t *test t.Parallel() const groupID int64 = 76 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{}} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{}} manifest := &OpenAIModelsResponse{ Body: []byte(`{"models":[{"slug":"codex-auto-review","visibility":"list"},{"slug":"gpt-5.6","visibility":"list"}]}`), } @@ -1702,7 +1715,7 @@ func TestMergeGroupConfiguredCodexModelsHonorsCustomListAndFinalETag(t *testing. t.Parallel() const groupID int64 = 73 - svc := &OpenAIGatewayService{accountRepo: codexModelsVisibilityAccountRepo{ + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ groupID: { { @@ -1780,6 +1793,8 @@ func TestIsRetryableCodexModelsManifestTransportError(t *testing.T) { {name: "upstream configuration error", err: errors.New("upstream error: invalid proxy")}, {name: "proxy connection configuration error", err: errors.New("proxy connection error: invalid configuration")}, {name: "canceled request", err: context.Canceled}, + {name: "strict transport deadline", err: &codextransport.Failure{Err: context.DeadlineExceeded}}, + {name: "strict transport truncated response", err: &codextransport.Failure{Err: io.ErrUnexpectedEOF}}, { name: "redirect policy error", err: &url.Error{ @@ -1891,7 +1906,7 @@ func TestFetchCodexModelsManifestPassthrough(t *testing.T) { chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest, err := s.FetchCodexModelsManifest(context.Background(), newCodexModelsTestAccount(), "0.137.0", "") if err != nil { t.Fatalf("FetchCodexModelsManifest returned error: %v", err) @@ -1944,7 +1959,7 @@ func TestFetchCodexModelsManifestAgentIdentityUsesAssertionWithoutOAuthToken(t * chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") if err != nil { t.Fatalf("FetchCodexModelsManifest returned error: %v", err) @@ -2003,7 +2018,7 @@ func TestFetchCodexModelsManifestAgentIdentityRecoversInvalidTaskOnce(t *testing openAIAgentIdentityAuthAPIBaseURL = server.URL t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = originalAuthBase }) - s := &OpenAIGatewayService{accountRepo: repo} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: repo} manifest, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") require.NoError(t, err) require.Equal(t, `{"models":[]}`, string(manifest.Body)) @@ -2037,7 +2052,7 @@ func TestFetchCodexModelsManifestAgentIdentityRedactsUpstreamErrors(t *testing.T chatgptCodexModelsURL = server.URL t.Cleanup(func() { chatgptCodexModelsURL = original }) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") require.Error(t, err) require.NotContains(t, err.Error(), key.runtimeID) @@ -2059,7 +2074,7 @@ func TestFetchCodexModelsManifestDefaultClientVersion(t *testing.T) { chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} if _, err := s.FetchCodexModelsManifest(context.Background(), newCodexModelsTestAccount(), "", ""); err != nil { t.Fatalf("FetchCodexModelsManifest returned error: %v", err) } @@ -2082,7 +2097,7 @@ func TestFetchCodexModelsManifestNotModified(t *testing.T) { chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() first, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") if err != nil { @@ -2122,7 +2137,7 @@ func TestFetchCodexModelsManifestUpstreamError(t *testing.T) { chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} if _, err := s.FetchCodexModelsManifest(context.Background(), newCodexModelsTestAccount(), "0.137.0", ""); err == nil { t.Fatal("expected error for upstream 500, got nil") } @@ -2132,7 +2147,7 @@ func TestFetchCodexModelsManifestMissingToken(t *testing.T) { account := newCodexModelsTestAccount() delete(account.Credentials, "access_token") - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} if _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", ""); err == nil { t.Fatal("expected error for missing access token, got nil") } @@ -2362,7 +2377,7 @@ func TestFetchCodexModelsManifestOAuthPreservesResponsesLite(t *testing.T) { chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest, err := s.FetchCodexModelsManifest(context.Background(), newCodexModelsTestAccount(), "0.145.0", "") require.NoError(t, err) require.Equal(t, manifestBody, string(manifest.Body)) @@ -2494,7 +2509,7 @@ func TestFetchCodexModelsManifestAcceptsConfiguredLimitAboveLegacyBoundary(t *te chatgptCodexModelsURL = server.URL defer func() { chatgptCodexModelsURL = original }() - s := &OpenAIGatewayService{cfg: &config.Config{}} + s := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: newCodexModelsLoopbackUpstream()} s.cfg.Gateway.ModelsListReadMaxBytes = 16 << 20 manifest, err := s.FetchCodexModelsManifest(context.Background(), newCodexModelsTestAccount(), "0.144.0", "") require.NoError(t, err) @@ -3159,7 +3174,7 @@ func (r *codexModelsAccountStateRepo) SetTempUnschedulable(_ context.Context, _ func newCodexModels401TestService(repo AccountRepository) *OpenAIGatewayService { rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) - s := &OpenAIGatewayService{rateLimitService: rateLimitService} + s := &OpenAIGatewayService{rateLimitService: rateLimitService, httpUpstream: newCodexModelsLoopbackUpstream()} rateLimitService.SetAccountRuntimeBlocker(s) return s } @@ -3325,7 +3340,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientFillsMissingProviderFieldsWit "context_window":64000, "max_context_window":64000 }]}`)} - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} require.NoError(t, svc.CompleteAPIKeyCodexModelsManifestForClient(manifest, account)) models := decodeCodexManifestModels(t, manifest.Body) @@ -3344,7 +3359,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientFillsMissingProviderFieldsWit func TestCompleteAPIKeyCodexModelsManifestForClientFiltersOfficialNonAgentModels(t *testing.T) { t.Parallel() - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[{"slug":"gpt-5.6-sol"},{"slug":"gpt-4o-realtime-preview"},{"slug":"gpt-4o-mini-tts"},{"slug":"text-embedding-3-large"},{"slug":"omni-moderation-latest"},{"slug":"o4-mini"},{"slug":"codex-mini-latest"}]}`)} account := newCodexModelsAPIKeyTestAccount("") @@ -3356,7 +3371,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientFiltersOfficialNonAgentModels func TestCompleteAPIKeyCodexModelsManifestForClientMarksOnlyOfficialVisionGPTImageInput(t *testing.T) { t.Parallel() - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[{"slug":"gpt-5.6-sol"},{"slug":"gpt-4o"},{"slug":"gpt-3.5-turbo"},{"slug":"gpt-4"}]}`)} account := newCodexModelsAPIKeyTestAccount("") @@ -3384,7 +3399,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientMarksOnlyOfficialVisionGPTIma func TestCompleteAPIKeyCodexModelsManifestForClientPreservesProviderMetadata(t *testing.T) { t.Parallel() - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{ Body: []byte(`{"models":[{"slug":"grok-4.6","description":"Provider supplied","service_tiers":[{"id":"provider-priority","name":"Provider Fast","description":"Provider supplied tier."}],"model_messages":{"auto_review":{"enabled":true}},"truncation_policy":{"mode":"tokens"},"unknown":{"kept":true}}],"metadata":{"source":"upstream"}}`), } @@ -3483,7 +3498,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientUsesCurrentSnapshotForCachedN func TestCompleteAPIKeyCodexModelsManifestForClientUsesKnownGPTImageFallback(t *testing.T) { t.Parallel() - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[ {"slug":"gpt-5.6-sol"}, {"slug":"company-coding-model"}, @@ -3509,7 +3524,7 @@ func TestCompleteAPIKeyCodexModelsManifestForClientUsesKnownGPTImageFallback(t * func TestCompleteAPIKeyCodexModelsManifestForClientDoesNotApplyLegacyFixedSolContext(t *testing.T) { t.Parallel() - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[ {"slug":"gpt-5.6-sol"}, {"slug":"gpt-5.6-terra"}, @@ -3910,7 +3925,7 @@ func expireCodexModelsManifestCache(s *OpenAIGatewayService, age time.Duration) func TestFetchCodexModelsManifestOAuthFreshWindowZeroUpstreamRequests(t *testing.T) { _, calls := newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-5.5"}]}`) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() for i := 0; i < 10; i++ { @@ -3940,7 +3955,7 @@ func TestFetchCodexModelsManifestOAuthStaleServesOldValueAndRefreshesInBackgroun chatgptCodexModelsURL = server.URL t.Cleanup(func() { chatgptCodexModelsURL = original }) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() first, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") require.NoError(t, err) @@ -4006,7 +4021,7 @@ func TestFetchCodexModelsManifestOAuthOverdueSynchronousRefreshAndFailure(t *tes chatgptCodexModelsURL = server.URL t.Cleanup(func() { chatgptCodexModelsURL = original }) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") require.NoError(t, err) @@ -4042,7 +4057,7 @@ func TestFetchCodexModelsManifestOAuthOverdueSynchronousRefreshAndFailure(t *tes chatgptCodexModelsURL = server.URL t.Cleanup(func() { chatgptCodexModelsURL = original }) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") require.NoError(t, err) @@ -4058,7 +4073,7 @@ func TestFetchCodexModelsManifestOAuthOverdueSynchronousRefreshAndFailure(t *tes func TestFetchCodexModelsManifestOAuthTokenChangeCacheMiss(t *testing.T) { _, calls := newCodexModelsOAuthCacheServer(t, `{"models":[]}`) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() _, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") @@ -4091,6 +4106,7 @@ func TestFetchCodexModelsManifestOAuthSharedAcrossGroupsWithIndependentFiltering t.Cleanup(func() { chatgptCodexModelsURL = original }) s := &OpenAIGatewayService{ + httpUpstream: newCodexModelsLoopbackUpstream(), accountRepo: codexModelsVisibilityAccountRepo{ byGroup: map[int64][]Account{ 91: {}, @@ -4205,7 +4221,7 @@ func TestCodexGPTIdentityPatternsCoverBundledPrompts(t *testing.T) { func TestGPT6SolLunaCatalogKeepsAuthoritativeCapabilities(t *testing.T) { for _, id := range []string{"gpt-6.1-sol", "gpt-6-sol", "gpt-6-luna"} { - svc := &OpenAIGatewayService{} + svc := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} manifest := &OpenAIModelsResponse{Body: []byte(`{"models":[{"slug":"` + id + `","supported_reasoning_levels":[{"effort":"ultra"}],"default_reasoning_level":"ultra","multi_agent_reasoning_effort":"xhigh","service_tiers":[{"id":"ultrafast"}],"context_window":300000,"max_context_window":900000,"supports_search_tool":false,"apply_patch_tool_type":null}]}`)} account := newCodexModelsAPIKeyTestAccount("https://api.openai.com/v1") require.NoError(t, svc.CompleteAPIKeyCodexModelsManifestForClient(manifest, account)) diff --git a/backend/internal/service/openai_codex_transport.go b/backend/internal/service/openai_codex_transport.go index 2a54b9033b81..49f72dc4eee8 100644 --- a/backend/internal/service/openai_codex_transport.go +++ b/backend/internal/service/openai_codex_transport.go @@ -22,10 +22,25 @@ func codexTransportScope(account *Account) string { return fmt.Sprintf("%x", sum[:16]) } -func codexStrictClientError(err error) error { +func codexStrictClientError(err error) *UpstreamFailoverError { + code := "codex_transport_profile_unavailable" + if codextransport.IsFailure(err) { + code = "codex_transport_failed" + } return &UpstreamFailoverError{Scope: GatewayFailureScopeRequest, NextAccountAction: NextAccountStop, SuppressAccountHealthPenalty: true, ClientStatusCode: http.StatusBadGateway, - ClientErrorType: "codex_transport_error", ClientErrorCode: "codex_transport_profile_unavailable", ClientMessage: err.Error()} + ClientErrorType: "codex_transport_error", ClientErrorCode: code, ClientMessage: err.Error()} +} + +// A successful handshake cannot make an interrupted, already-sent turn replayable. +func codexStrictResponseFailure(response *http.Response, err error) *UpstreamFailoverError { + if response == nil || response.Request == nil { + return nil + } + if _, ok := codextransport.FromContext(response.Request.Context()); !ok { + return nil + } + return codexStrictClientError(&codextransport.Failure{Err: err}) } func bindCodexTransport(request *http.Request, source *Account, policy *codexFingerprintPolicy, plane string) (*http.Request, error) { diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index d318bc09b42f..56b21c90c25a 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -2535,6 +2535,9 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( if sawFailedEvent { return resultWithUsage(), fmt.Errorf("upstream response failed: %s", failedMessage) } + if strictErr := codexStrictResponseFailure(resp, err); strictErr != nil { + return resultWithUsage(), strictErr + } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return resultWithUsage(), fmt.Errorf("stream usage incomplete: %w", err) } @@ -2572,6 +2575,9 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( if ctx.Err() != nil { return resultWithUsage(), fmt.Errorf("stream usage incomplete: %w", ctx.Err()) } + if strictErr := codexStrictResponseFailure(resp, errors.New("stream ended before a terminal event")); strictErr != nil { + return resultWithUsage(), strictErr + } logger.FromContext(ctx).With( zap.String("component", "service.openai_gateway"), zap.Int64("account_id", account.ID), diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index d60d8b1a8f59..2aa4d12d5f90 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -396,6 +396,11 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. message = "OpenAI first-output staging limit exceeded" } logger.LegacyPrintf("service.openai_gateway", "%s: account=%d model=%s error=%v", message, account.ID, originalModel, err) + if strictErr := codexStrictResponseFailure(resp, err); strictErr != nil { + streamEarlyErr = strictErr + _ = resp.Body.Close() + return + } failoverErr := s.newOpenAIStreamFailoverError(c, account, false, upstreamRequestID, nil, message, resp.StatusCode) failoverErr.SafeToFailoverAfterWrite = true streamEarlyErr = failoverErr @@ -519,6 +524,11 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. if sawTerminalEvent && !sawFailedEvent && terminalResponseErr == nil { s.clearOpenAIProxyStreamDisconnectInContext(ctx, account) } + if !sawTerminalEvent { + if strictErr := codexStrictResponseFailure(resp, errors.New("stream ended before a terminal event")); strictErr != nil { + return resultWithUsage(), strictErr + } + } if !sawTerminalEvent && !openAIStreamClientOutputStarted(c, clientOutputStarted) && !eventShouldFlush { return resultWithUsage(), s.newOpenAIStreamFailoverError( c, @@ -554,6 +564,9 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. if scanErr == nil { return nil, nil, false } + if strictErr := codexStrictResponseFailure(resp, scanErr); strictErr != nil { + return resultWithUsage(), strictErr, true + } if errors.Is(scanErr, errOpenAIFirstOutputScannerLimit) && !firstOutputProgressObserved { logger.LegacyPrintf("service.openai_gateway", "SSE token exceeded guarded first-output limit: account=%d limit=%d error=%v", account.ID, openAIFirstOutputStageMaxBytes+openAIFirstOutputScannerFramingAllowance, scanErr) failoverErr := s.newOpenAIStreamFailoverError( @@ -1247,6 +1260,9 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. for ev := range events { markEventProcessed(ev) } + if strictErr := codexStrictResponseFailure(resp, errors.New("first output timeout")); strictErr != nil { + return resultWithUsage(), strictErr + } return resultWithUsage(), s.newOpenAIFirstOutputTimeoutError( ctx, c, account, opsUpstreamProxyID(account), opsUpstreamProxyName(account), startTime, originalModel, reasoningEffort, diff --git a/backend/internal/service/openai_ws_pool.go b/backend/internal/service/openai_ws_pool.go index 44cbaa0eb530..c49449071f54 100644 --- a/backend/internal/service/openai_ws_pool.go +++ b/backend/internal/service/openai_ws_pool.go @@ -2423,17 +2423,6 @@ func normalizeOpenAIWSHandshakeCompatibility(account *Account, headers http.Head return key } -func activeCodexFingerprintMode(account *Account, policies ...*codexFingerprintPolicy) codexFingerprintMode { - policy := selectCodexFingerprintPolicy(policies) - if account == nil || !policy.enabled || account.GetCodexFingerprintMode() == codexFingerprintOff { - return codexFingerprintOff - } - if _, ok := codexFingerprintSeed(account.Extra); !ok { - return codexFingerprintOff - } - return account.GetCodexFingerprintMode() -} - func normalizeOpenAIWSStableIdentityHeader(headers http.Header, name string) string { if headers == nil { return "" diff --git a/backend/internal/service/setting_codex_fingerprint.go b/backend/internal/service/setting_codex_fingerprint.go index c9ab23f4662e..730d8a1c732c 100644 --- a/backend/internal/service/setting_codex_fingerprint.go +++ b/backend/internal/service/setting_codex_fingerprint.go @@ -228,7 +228,7 @@ func (s *SettingService) LoadCodexFingerprintSettings(ctx context.Context) error func (s *SettingService) UpdateCodexFingerprintSettings(ctx context.Context, config CodexFingerprintSettings) (CodexFingerprintSettingsView, error) { if config.ClientVersion != "" && !codextransport.SupportsVersion(config.ClientVersion) { - return CodexFingerprintSettingsView{}, fmt.Errorf("Codex version %s has no verified transport profile", config.ClientVersion) + return CodexFingerprintSettingsView{}, fmt.Errorf("codex version %s has no verified transport profile", config.ClientVersion) } s.codexFingerprintMu.Lock() defer s.codexFingerprintMu.Unlock() diff --git a/backend/internal/service/setting_update.go b/backend/internal/service/setting_update.go index fabb46d9cb55..628f6b62af36 100644 --- a/backend/internal/service/setting_update.go +++ b/backend/internal/service/setting_update.go @@ -516,7 +516,7 @@ func (s *SettingService) buildSystemSettingsUpdates(ctx context.Context, setting updates[SettingKeyAntigravityUserAgentVersion] = antigravity.NormalizeUserAgentVersion(settings.AntigravityUserAgentVersion) updates[SettingKeyOpenAICodexUserAgent] = strings.TrimSpace(settings.OpenAICodexUserAgent) if settings.OpenAICodexClientVersion != "" && !codextransport.SupportsVersion(NormalizeCodexClientVersion(settings.OpenAICodexClientVersion)) { - return nil, fmt.Errorf("Codex version has no verified transport profile") + return nil, fmt.Errorf("codex version has no verified transport profile") } updates[SettingKeyOpenAICodexClientVersion] = NormalizeCodexClientVersion(settings.OpenAICodexClientVersion) updates[SettingKeyOpenAICodexVersionAutoSyncEnabled] = strconv.FormatBool(settings.OpenAICodexVersionAutoSyncEnabled) From 62b99ecbbdcd3fa5a30ad835eba6cdafca14ffcf Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 13:30:06 +0800 Subject: [PATCH 09/11] test: align remaining Codex fixtures with independent conversations --- ...ccount_test_service_openai_compact_test.go | 24 +++++++------------ .../internal/service/openai_compact_probe.go | 12 ---------- .../service/openai_oauth_passthrough_test.go | 22 +++++++++-------- .../openai_ws_forwarder_success_test.go | 9 +++---- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/backend/internal/service/account_test_service_openai_compact_test.go b/backend/internal/service/account_test_service_openai_compact_test.go index 866260902f04..5b5ce00d3f3a 100644 --- a/backend/internal/service/account_test_service_openai_compact_test.go +++ b/backend/internal/service/account_test_service_openai_compact_test.go @@ -275,7 +275,7 @@ func TestAccountTestService_TestAccountConnection_OpenAICompact2xxWithoutItemMar } // 探测与真实转发走同一 /responses 端点，出站身份必须与真实 Codex 同构： -// session/thread 为 UUID、携带 x-codex-installation-id（收敛账号用收敛值）。 +// session/thread 使用本次测试的 UUIDv7 根，installation 使用固定设备值。 func TestAccountTestService_TestAccountConnection_OpenAICompactProbeIdentityMatchesRealTraffic(t *testing.T) { gin.SetMode(gin.TestMode) @@ -315,12 +315,16 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactProbeIdentityMatc require.NoError(t, svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact)) - // 显式 session 收敛模式：出站身份 = 账号级收敛值 + // 旧 session 配置归一为固定设备、多独立根对话，不能再合并为账号会话。 seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) - converged := resolveConvergedSessionID(seed) - require.Equal(t, converged, upstream.lastReq.Header.Get("session-id")) - require.Equal(t, converged, upstream.lastReq.Header.Get("session_id")) + sessionID := upstream.lastReq.Header.Get("session-id") + require.Equal(t, uuid.Version(7), uuid.MustParse(sessionID).Version()) + require.Equal(t, sessionID, upstream.lastReq.Header.Get("thread-id")) + require.Equal(t, sessionID, gjson.GetBytes(upstream.lastBody, "client_metadata.session_id").String()) + require.Equal(t, sessionID, gjson.GetBytes(upstream.lastBody, "client_metadata.thread_id").String()) + metadata := gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-turn-metadata").String() + require.Equal(t, resolveConvergedInstallationID(&account, seed), gjson.Get(metadata, "installation_id").String()) require.Empty(t, upstream.lastReq.Header.Get("x-codex-installation-id"), "真实 Codex 从不以请求头形式发送 installation id") require.NotContains(t, upstream.lastReq.Header.Get("session-id"), "probe_compact", @@ -329,13 +333,3 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactProbeIdentityMatc require.Equal(t, resolveCodexOutboundIdentityForAccount(&account, "").userAgent, upstream.lastReq.Header.Get("User-Agent")) <-updateCalls } - -func TestCompactProbeSessionID_IsUUIDShaped(t *testing.T) { - for _, id := range []int64{0, 1, 987654} { - got := compactProbeSessionID(id) - _, err := uuid.Parse(got) - require.NoError(t, err, "探测会话标识必须是 UUID 形态: %s", got) - } - require.Equal(t, compactProbeSessionID(7), compactProbeSessionID(7), "同账号应稳定复用同一会话") - require.NotEqual(t, compactProbeSessionID(7), compactProbeSessionID(8)) -} diff --git a/backend/internal/service/openai_compact_probe.go b/backend/internal/service/openai_compact_probe.go index 9d5e8ccad444..c8c538a85c70 100644 --- a/backend/internal/service/openai_compact_probe.go +++ b/backend/internal/service/openai_compact_probe.go @@ -152,15 +152,3 @@ func mergeExtraUpdates(base map[string]any, more map[string]any) map[string]any } return out } - -// compactProbeSessionID 返回探测请求使用的会话标识。真实 Codex 的 -// session-id / thread-id 恒为 UUID（codex-protocol ThreadId 是 UUIDv7）， -// 探测既然与真实流量走同一个 /responses 端点，标识形态就必须同构—— -// 否则上游能凭 "probe_compact_5" 这类字面量一眼区分出探测流量。 -// 账号级稳定派生：重复探测复用同一会话，而不是每次新开一个。 -func compactProbeSessionID(accountID int64) string { - if accountID <= 0 { - return deriveStableUUIDv4("sub2api:codex-compact-probe:v1:anonymous") - } - return deriveStableUUIDv4("sub2api:codex-compact-probe:v1:" + strconv.FormatInt(accountID, 10)) -} diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index e9c80b660932..0724cee242d4 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -2562,8 +2562,9 @@ func TestOpenAIGatewayService_CodexFingerprintHTTPTransformedHeaderBodyParityAnd seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) wantInstall := resolveConvergedInstallationID(account, seed) - wantSession := resolveConvergedSessionID(seed) - wantThread := resolveConvergedThreadID(seed, scopeCodexAccountIdentityValue(account, 0, "body-thread")) + wantSession := scopeCodexAccountIdentityValue(account, 0, "body-session") + wantThread := scopeCodexAccountIdentityValue(account, 0, "body-thread") + wantWindow := scopeCodexWindowIdentity(account, 0, "body-window") require.Empty(t, upstream.lastReq.Header.Get("x-codex-installation-id"), "installation id 只在 body client_metadata") require.Equal(t, wantInstall, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-installation-id").String()) @@ -2571,13 +2572,13 @@ func TestOpenAIGatewayService_CodexFingerprintHTTPTransformedHeaderBodyParityAnd require.Empty(t, upstream.lastReq.Header.Get("session_id"), "真实 Codex 不发下划线 session_id") require.Equal(t, wantThread, upstream.lastReq.Header.Get("thread-id")) require.Equal(t, wantThread, upstream.lastReq.Header.Get("x-client-request-id")) - require.Equal(t, wantThread+":0", upstream.lastReq.Header.Get("x-codex-window-id")) + require.Equal(t, wantWindow, upstream.lastReq.Header.Get("x-codex-window-id")) require.Equal(t, wantSession, gjson.GetBytes(upstream.lastBody, "prompt_cache_key").String()) require.Equal(t, wantInstall, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-installation-id").String()) require.Equal(t, wantSession, gjson.GetBytes(upstream.lastBody, "client_metadata.session_id").String()) require.Equal(t, wantThread, gjson.GetBytes(upstream.lastBody, "client_metadata.thread_id").String()) - require.Equal(t, wantThread+":0", gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-window-id").String()) + require.Equal(t, wantWindow, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-window-id").String()) bodyTurnMetadata := gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-turn-metadata").String() headerTurnMetadata := upstream.lastReq.Header.Get("x-codex-turn-metadata") @@ -2625,8 +2626,9 @@ func TestOpenAIGatewayService_CodexFingerprintHTTPRawPassthroughHeaderBodyParity seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) wantInstall := resolveConvergedInstallationID(account, seed) - wantSession := resolveConvergedSessionID(seed) - wantThread := resolveConvergedThreadID(seed, scopeCodexAccountIdentityValue(account, 0, "body-thread")) + wantSession := scopeCodexAccountIdentityValue(account, 0, "body-session") + wantThread := scopeCodexAccountIdentityValue(account, 0, "body-thread") + wantWindow := scopeCodexWindowIdentity(account, 0, "body-window") require.Empty(t, upstream.lastReq.Header.Get("x-codex-installation-id"), "installation id 只在 body client_metadata") require.Equal(t, wantInstall, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-installation-id").String()) @@ -2634,13 +2636,13 @@ func TestOpenAIGatewayService_CodexFingerprintHTTPRawPassthroughHeaderBodyParity require.Empty(t, upstream.lastReq.Header.Get("session_id"), "真实 Codex 不发下划线 session_id") require.Equal(t, wantThread, upstream.lastReq.Header.Get("thread-id")) require.Equal(t, wantThread, upstream.lastReq.Header.Get("x-client-request-id")) - require.Equal(t, wantThread+":0", upstream.lastReq.Header.Get("x-codex-window-id")) + require.Equal(t, wantWindow, upstream.lastReq.Header.Get("x-codex-window-id")) require.Equal(t, wantSession, gjson.GetBytes(upstream.lastBody, "prompt_cache_key").String()) require.Equal(t, wantInstall, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-installation-id").String()) require.Equal(t, wantSession, gjson.GetBytes(upstream.lastBody, "client_metadata.session_id").String()) require.Equal(t, wantThread, gjson.GetBytes(upstream.lastBody, "client_metadata.thread_id").String()) - require.Equal(t, wantThread+":0", gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-window-id").String()) + require.Equal(t, wantWindow, gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-window-id").String()) bodyTurnMetadata := gjson.GetBytes(upstream.lastBody, "client_metadata.x-codex-turn-metadata").String() headerTurnMetadata := upstream.lastReq.Header.Get("x-codex-turn-metadata") @@ -2725,9 +2727,9 @@ func TestOpenAIGatewayService_CodexFingerprintMessagesBridgeDoesNotInjectBodyPro require.NoError(t, err) require.NotNil(t, upstream.lastReq) - seed, ok := codexFingerprintSeed(account.Extra) + _, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) - wantSession := resolveConvergedSessionID(seed) + wantSession := scopeCodexAccountIdentityValue(account, 0, "anthropic-metadata-session-1") require.False(t, gjson.GetBytes(upstream.lastBody, "prompt_cache_key").Exists()) require.Equal(t, wantSession, gjson.GetBytes(upstream.lastBody, "client_metadata.session_id").String()) require.Equal(t, wantSession, upstream.lastReq.Header.Get("session-id")) diff --git a/backend/internal/service/openai_ws_forwarder_success_test.go b/backend/internal/service/openai_ws_forwarder_success_test.go index e56d6f4e5cbb..b71061b8e5bf 100644 --- a/backend/internal/service/openai_ws_forwarder_success_test.go +++ b/backend/internal/service/openai_ws_forwarder_success_test.go @@ -1157,8 +1157,9 @@ func TestOpenAIGatewayService_Forward_WSv2_CodexFingerprintHandshakeBodyParityAn seed, ok := codexFingerprintSeed(account.Extra) require.True(t, ok) wantInstall := resolveConvergedInstallationID(account, seed) - wantSession := resolveConvergedSessionID(seed) - wantThread := resolveConvergedThreadID(seed, scopeCodexAccountIdentityValue(account, 0, "body-thread")) + wantSession := scopeCodexAccountIdentityValue(account, 0, "body-session") + wantThread := scopeCodexAccountIdentityValue(account, 0, "body-thread") + wantWindow := scopeCodexWindowIdentity(account, 0, "body-window") payloadJSON := requestToJSONString(captureConn.lastWrite) require.Empty(t, captureDialer.lastHeaders.Get("x-codex-installation-id"), "installation id 只在 body client_metadata") @@ -1167,13 +1168,13 @@ func TestOpenAIGatewayService_Forward_WSv2_CodexFingerprintHandshakeBodyParityAn require.Empty(t, captureDialer.lastHeaders.Get("session_id")) require.Equal(t, wantThread, captureDialer.lastHeaders.Get("thread-id")) require.Equal(t, wantThread, captureDialer.lastHeaders.Get("x-client-request-id")) - require.Equal(t, wantThread+":0", captureDialer.lastHeaders.Get("x-codex-window-id")) + require.Equal(t, wantWindow, captureDialer.lastHeaders.Get("x-codex-window-id")) require.Equal(t, wantSession, gjson.Get(payloadJSON, "prompt_cache_key").String()) require.Equal(t, wantInstall, gjson.Get(payloadJSON, "client_metadata.x-codex-installation-id").String()) require.Equal(t, wantSession, gjson.Get(payloadJSON, "client_metadata.session_id").String()) require.Equal(t, wantThread, gjson.Get(payloadJSON, "client_metadata.thread_id").String()) - require.Equal(t, wantThread+":0", gjson.Get(payloadJSON, "client_metadata.x-codex-window-id").String()) + require.Equal(t, wantWindow, gjson.Get(payloadJSON, "client_metadata.x-codex-window-id").String()) bodyTurnMetadata := gjson.Get(payloadJSON, "client_metadata.x-codex-turn-metadata").String() headerTurnMetadata := captureDialer.lastHeaders.Get("x-codex-turn-metadata") From 4310cba8e3d6e09873bc139958dae78e1a539c54 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 13:44:00 +0800 Subject: [PATCH 10/11] fix: preserve strict identity across count and websocket entrypoints --- .../service/openai_codex_fingerprint.go | 2 +- .../service/openai_codex_fingerprint_test.go | 8 ++--- .../openai_codex_version_sync_service_test.go | 16 +++++----- .../service/openai_compat_model_test.go | 4 +-- .../service/openai_gateway_count_tokens.go | 5 +++- .../service/openai_gateway_forward.go | 13 ++++----- .../service/openai_images_direct_test.go | 2 +- .../service/openai_models_list_test.go | 9 ++++-- .../service/openai_ws_forwarder_ingress.go | 3 +- ...penai_ws_forwarder_ingress_session_test.go | 29 ++++++++++++------- .../openai_ws_forwarder_success_test.go | 16 +++++++--- ...ai_ws_forwarder_v2_execution_scope_test.go | 7 +++-- .../openai_ws_http_bridge_resume_test.go | 8 ++--- backend/internal/service/openai_ws_pool.go | 29 +++++++++++++++++++ .../internal/service/openai_ws_pool_test.go | 16 +++++++--- .../service/plugin_manager_routing_test.go | 6 +++- 16 files changed, 120 insertions(+), 53 deletions(-) diff --git a/backend/internal/service/openai_codex_fingerprint.go b/backend/internal/service/openai_codex_fingerprint.go index 002f83951db7..9f378eaf160f 100644 --- a/backend/internal/service/openai_codex_fingerprint.go +++ b/backend/internal/service/openai_codex_fingerprint.go @@ -552,7 +552,7 @@ func resolveCodexFingerprintIDsWithSource(routed, source *Account, headers http. func (ids *codexFingerprintIDs) rewriteHeaderTurnMetadata(headers http.Header) { raw := headers.Get("x-codex-turn-metadata") - if raw == "" && len(ids.logicalMetadata) == 0 { + if raw == "" && len(ids.logicalMetadata) == 0 && ids.installationID == "" && ids.sessionID == "" && ids.threadID == "" { return } var metadata map[string]any diff --git a/backend/internal/service/openai_codex_fingerprint_test.go b/backend/internal/service/openai_codex_fingerprint_test.go index e0536be3cb44..e2315eb74a69 100644 --- a/backend/internal/service/openai_codex_fingerprint_test.go +++ b/backend/internal/service/openai_codex_fingerprint_test.go @@ -131,8 +131,7 @@ func TestResolveCodexFingerprintIDsFromRequest_ExplicitOff(t *testing.T) { assert.Nil(t, ids, "显式 off 模式应返回 nil") } -// 未显式配置的存量账号不得被收敛（#5610）：默认返回 nil，出站身份保持 -// v0.1.175 之前的客户端原值。 +// Seeded accounts use fixed-device mode by default. func TestResolveCodexFingerprintIDsFromRequest_DefaultIsDevice(t *testing.T) { account := newTestOAuthAccount(1, nil) ids := resolveCodexFingerprintIDsFromRequest(account, nil) @@ -141,14 +140,15 @@ func TestResolveCodexFingerprintIDsFromRequest_DefaultIsDevice(t *testing.T) { assert.NotEmpty(t, ids.installationID) } -// 管理员显式 opt-in 的账号行为不变。 +// Historical enabled modes retain their stored input and normalize to device. func TestResolveCodexFingerprintIDsFromRequest_ExplicitOptInHonored(t *testing.T) { for _, mode := range []string{"device", "session", "full"} { t.Run(mode, func(t *testing.T) { account := newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: mode}) ids := resolveCodexFingerprintIDsFromRequest(account, nil) require.NotNil(t, ids, "显式配置必须生效") - assert.Equal(t, codexFingerprintMode(mode), ids.mode) + assert.Equal(t, codexFingerprintDevice, ids.mode) + assert.Equal(t, mode, account.Extra[codexFingerprintModeExtraKey], "保留历史配置输入") assert.NotEmpty(t, ids.installationID) }) } diff --git a/backend/internal/service/openai_codex_version_sync_service_test.go b/backend/internal/service/openai_codex_version_sync_service_test.go index b78cd8cc9435..444607a032c5 100644 --- a/backend/internal/service/openai_codex_version_sync_service_test.go +++ b/backend/internal/service/openai_codex_version_sync_service_test.go @@ -303,9 +303,9 @@ func TestGetOpenAICodexClientVersionPriority(t *testing.T) { want string }{ {name: "面板覆写优先", override: "0.150.0", synced: "0.146.0", want: "0.150.0"}, - {name: "覆写为空时用同步值", synced: "0.146.0", want: "0.146.0"}, + {name: "未验证同步值保持待支持", synced: "0.146.0", want: codexCLIVersion}, {name: "两者皆空时用内置常量", want: codexCLIVersion}, - {name: "非法覆写回退同步值", override: "latest", synced: "0.146.0", want: "0.146.0"}, + {name: "非法覆写且同步未验证时用已验证版本", override: "latest", synced: "0.146.0", want: codexCLIVersion}, {name: "非法同步值回退内置常量", synced: "not-a-version", want: codexCLIVersion}, } @@ -334,7 +334,7 @@ func TestGetOpenAICodexCanonicalUserAgentBuildsFromVersion(t *testing.T) { }}, nil) require.Equal(t, - "codex-tui/0.200.1"+codexCLIUserAgentSuffix, + "codex-tui/"+codexCLIVersion+codexCLIUserAgentSuffix, svc.GetOpenAICodexCanonicalUserAgent(context.Background()), ) } @@ -351,7 +351,7 @@ func TestGetOpenAICodexCanonicalUserAgentRebuildsPanelUAVersion(t *testing.T) { }}, nil) require.Equal(t, - "codex_cli_rs/0.200.1 (Ubuntu 22.4.0; x86_64) xterm-256color", + "codex_cli_rs/"+codexCLIVersion+" (Ubuntu 22.4.0; x86_64) xterm-256color", svc.GetOpenAICodexCanonicalUserAgent(context.Background()), ) }) @@ -363,7 +363,7 @@ func TestGetOpenAICodexCanonicalUserAgentRebuildsPanelUAVersion(t *testing.T) { }}, nil) require.Equal(t, - "codex_cli_rs/0.200.1 (Mac OS X 15.1.0; arm64) iTerm.app", + "codex_cli_rs/"+codexCLIVersion+" (Mac OS X 15.1.0; arm64) iTerm.app", svc.GetOpenAICodexCanonicalUserAgent(context.Background()), ) }) @@ -375,7 +375,7 @@ func TestGetOpenAICodexCanonicalUserAgentRebuildsPanelUAVersion(t *testing.T) { }}, nil) require.Equal(t, - "codex-tui/0.200.1 (Ubuntu 22.4.0; x86_64) WindowsTerminal (codex-tui; 0.200.1)", + "codex-tui/"+codexCLIVersion+" (Ubuntu 22.4.0; x86_64) WindowsTerminal (codex-tui; "+codexCLIVersion+")", svc.GetOpenAICodexCanonicalUserAgent(context.Background()), ) }) @@ -489,8 +489,8 @@ func TestGetOpenAICodexCanonicalUserAgentOutboundIdentity(t *testing.T) { name, manual, synced, want string }{ {"manual", "0.154.0", "0.155.1", "0.154.0"}, - {"synced", "", "0.155.1", "0.155.1"}, - {"invalid_manual", "latest", "0.155.1", "0.155.1"}, + {"synced", "", "0.155.1", codexCLIVersion}, + {"invalid_manual", "latest", "0.155.1", codexCLIVersion}, {"default", "", "", codexCLIVersion}, {"invalid_versions", "latest", "invalid", codexCLIVersion}, } { diff --git a/backend/internal/service/openai_compat_model_test.go b/backend/internal/service/openai_compat_model_test.go index 862f781f5fe1..898b5aef185b 100644 --- a/backend/internal/service/openai_compat_model_test.go +++ b/backend/internal/service/openai_compat_model_test.go @@ -1127,8 +1127,8 @@ func TestForwardAsAnthropic_KeepsInboundHyphenSessionHeaderOverCacheKeyFallback( require.NoError(t, err) require.NotNil(t, result) require.Len(t, upstream.requests, 1) - scoped := inboundSession - require.Equal(t, scoped, upstream.requests[0].Header.Get("session-id"), "设备模式保留入站 R，不被 K 回退覆盖") + scoped := scopeCodexAccountIdentityValue(account, 0, inboundSession) + require.Equal(t, scoped, upstream.requests[0].Header.Get("session-id"), "设备模式一致映射入站 R，不被 K 回退覆盖") require.Equal(t, scoped, upstream.requests[0].Header.Get("thread-id")) require.NotEqual(t, generateSessionUUID(isolateOpenAIUpstreamSessionID(0, account, "stable-cache-key")), upstream.requests[0].Header.Get("session-id")) require.Empty(t, upstream.requests[0].Header.Get("session_id")) diff --git a/backend/internal/service/openai_gateway_count_tokens.go b/backend/internal/service/openai_gateway_count_tokens.go index f10d3aa66db3..f2e593ce154e 100644 --- a/backend/internal/service/openai_gateway_count_tokens.go +++ b/backend/internal/service/openai_gateway_count_tokens.go @@ -494,7 +494,10 @@ func (s *OpenAIGatewayService) buildInputTokensUpstreamRequest( // 账号级请求头覆写（仅 openai api_key 账号启用时生效；OAuth 路径 no-op） account.ApplyHeaderOverrides(req.Header) - return req, nil + if _, err := s.prepareCodexAccountIdentitySource(ctx, c, account); err != nil { + return nil, err + } + return bindCodexAttemptRequest(c, req, account) } func writeAnthropicCountTokensError(c *gin.Context, status int, errType, message string) { diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index 94c9b424cd5e..81c2571b1cee 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -1607,9 +1607,9 @@ func (s *OpenAIGatewayService) codexIdentityOverrideUA(account *Account) string return codexAccountIdentityOverrideUA(account) } -// resolveStagedCodexFingerprintIDs 解析本次请求的收敛 ID，并让 turn metadata 的 sandbox 跟随 -// 最终出站 UA：UA 来源与构造器/收口点一致——影子账号取 staged 的父账号身份，账号显式 UA -// 按 ForceCodexCLI 策略生效——否则父/子系统不同或显式 UA 覆盖时 sandbox 会与 UA 分裂。 +// resolveStagedCodexFingerprintIDs uses the staged credential source and policy. +// Missing conversation IDs receive one real temporary root per logical request; +// the device label never changes actual sandbox or permission diagnostics. func (s *OpenAIGatewayService) resolveStagedCodexFingerprintIDs(c *gin.Context, account *Account, clientHeaders http.Header) *codexFingerprintIDs { policy := codexFingerprintPolicyForContext(c, account) source := codexAccountProjectionSource(c, account) @@ -1696,10 +1696,9 @@ func (s *OpenAIGatewayService) effectiveCodexOutboundUserAgent(c *gin.Context, a return resolveCodexOutboundIdentity("").userAgent } -// stageCodexFingerprintForWSFrame 为原生 WS 入口的每一帧解析、暂存并应用收敛 ID：原生 -// Proxy 不经过 HTTP Forward，握手头（buildOpenAIWSHeaders → applyStagedCodexFingerprintHeaders） -// 与帧体 client_metadata 由此共用同一份 IDs；每帧重算使 session/full 模式的 turn_id 逐轮更新， -// device 模式只收敛 installation 与 sandbox。返回改写后的帧（未变化时原样返回）。 +// stageCodexFingerprintForWSFrame shares each frame's final device and graph +// snapshot with its handshake. Legacy enabled modes use the same independent +// conversation mapping; genuine turn and execution diagnostics stay intact. func (s *OpenAIGatewayService) stageCodexFingerprintForWSFrame(c *gin.Context, account *Account, frame []byte) ([]byte, error) { var clientHeaders http.Header if c != nil && c.Request != nil { diff --git a/backend/internal/service/openai_images_direct_test.go b/backend/internal/service/openai_images_direct_test.go index ea723f50144c..3f0592bc2808 100644 --- a/backend/internal/service/openai_images_direct_test.go +++ b/backend/internal/service/openai_images_direct_test.go @@ -220,7 +220,7 @@ func TestCodexDirectImagesAccountTestAndWhitelist(t *testing.T) { require.Equal(t, "/backend-api/codex/images/generations", upstream.lastReq.URL.Path) require.Contains(t, rec.Body.String(), `"success":true`) newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"gpt-5.6-luna"}]}`) - svc.openaiGatewayService = &OpenAIGatewayService{} + svc.openaiGatewayService = &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() account.Credentials["model_mapping"] = map[string]any{"gpt-image-2.5-flare": "gpt-image-2.5-flare"} models, err := svc.FetchOpenAIAccountModels(context.Background(), account) diff --git a/backend/internal/service/openai_models_list_test.go b/backend/internal/service/openai_models_list_test.go index edb50a569e50..6fb37c40c0ec 100644 --- a/backend/internal/service/openai_models_list_test.go +++ b/backend/internal/service/openai_models_list_test.go @@ -60,7 +60,7 @@ func TestFetchOpenAIModelsListUsesStandardRequestAndIsolatesCodexCache(t *testin func TestFetchOpenAIModelsListOAuthSharesManifestCache(t *testing.T) { _, calls := newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"special-oauth-model","display_name":"Special OAuth Model","description":"manifest only"},{"slug":"gpt-image-1"}]}`) - s := &OpenAIGatewayService{} + s := &OpenAIGatewayService{httpUpstream: newCodexModelsLoopbackUpstream()} account := newCodexModelsTestAccount() response, err := s.FetchOpenAIModelsList(context.Background(), account) require.NoError(t, err) @@ -235,7 +235,10 @@ func TestFetchOpenAIModelsListEmptyAndMalformedResponses(t *testing.T) { func TestPinnedOpenAIModelsListMixedAccountsShareColdCacheAcrossGroups(t *testing.T) { _, oauthCalls := newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"shared-model"},{"slug":"oauth-special"}]}`) var apiCalls atomic.Int32 - s := newCodexModelsAPIKeyTestService(&codexModelsHTTPUpstreamStub{do: func(_ *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + s := newCodexModelsAPIKeyTestService(&codexModelsHTTPUpstreamStub{do: func(req *http.Request, proxy string, accountID int64, concurrency int) (*http.Response, error) { + if accountID == 1 { + return newCodexModelsLoopbackUpstream().Do(req, proxy, accountID, concurrency) + } apiCalls.Add(1) return ordinaryModelsUpstreamResponse(`{"data":[{"id":"shared-model","owned_by":"api-provider"},{"id":"api-special"}]}`), nil }}) @@ -297,7 +300,7 @@ func TestFetchOpenAIModelsListResolvesShadowOAuthCredentials(t *testing.T) { _, calls := newCodexModelsOAuthCacheServer(t, `{"models":[{"slug":"parent-model"}]}`) parent := newCodexModelsTestAccount() shadow := &Account{ID: 9, Platform: PlatformOpenAI, Type: AccountTypeOAuth, ParentAccountID: &parent.ID} - s := &OpenAIGatewayService{accountRepo: newStubCredRepo(parent)} + s := &OpenAIGatewayService{accountRepo: newStubCredRepo(parent), httpUpstream: newCodexModelsLoopbackUpstream()} response, err := s.FetchOpenAIModelsList(context.Background(), shadow) require.NoError(t, err) require.Contains(t, string(response.Body), `"id":"parent-model"`) diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index ceae4fc06c81..4863280c9b07 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -1806,9 +1806,10 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( baseAcquireReq.Headers = finalHeaders baseAcquireReq.BodyIdentityKey = codexWSBodyIdentityKey(wirePayload) if sessionLease != nil && !sessionLease.conn.matchesHandshakeCompatibility(normalizeOpenAIWSAcquireCompatibility(baseAcquireReq)) { + changedFields := sessionLease.conn.handshakeCompatibility.changedFields(normalizeOpenAIWSAcquireCompatibility(baseAcquireReq)) resetSessionLease(false) return NewOpenAIWSClientCloseError(coderws.StatusTryAgainLater, openAIWSNonInitialTurnRetryCloseReason, - errors.New("codex conversation identity changed; a new handshake is required")) + fmt.Errorf("codex conversation identity changed (%s); a new handshake is required", changedFields)) } forcePreferredConn := isStrictAffinityTurn(currentPayload) && strings.TrimSpace(preferredConnID) != "" diff --git a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go index 20d6d854f265..53fbce89d589 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go +++ b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go @@ -963,6 +963,13 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_CodexImageBridge readCtx, cancelRead = context.WithTimeout(context.Background(), 3*time.Second) msgType, message, err = clientConn.Read(readCtx) cancelRead() + if err != nil { + select { + case serverErr := <-serverErrCh: + t.Logf("websocket server returned: %v", serverErr) + default: + } + } require.NoError(t, err) require.Equal(t, coderws.MessageText, msgType) require.Equal(t, "resp_codex_image_lite", gjson.GetBytes(message, "response.id").String()) @@ -996,6 +1003,7 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_CodexImageBridge } require.Len(t, captureConn.writes, 3) + require.Equal(t, 1, captureDialer.DialCount(), "同一会话的可选元数据变化不得触发设备重握手") nonLitePayload := requestToJSONString(captureConn.writes[0]) require.True(t, gjson.Get(nonLitePayload, `tools.#(type=="image_generation")`).Exists()) require.Equal(t, "png", gjson.Get(nonLitePayload, `tools.#(type=="image_generation").output_format`).String()) @@ -1023,15 +1031,15 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_CodexImageBridge require.False(t, gjson.Get(functionPayload, "tool_choice").Exists()) require.NotContains(t, gjson.Get(functionPayload, "instructions").String(), codexImageGenerationBridgeMarker) - // 原生 ctx_pool 入口：默认 device 收敛在首帧与后续帧都落地，sandbox 跟随账号身份（Mac→seatbelt）， - // workspaces 不改写。 + // Native ctx_pool keeps the same device across turns while retaining actual + // sandbox diagnostics and workspaces. expectedInstallation := resolveConvergedInstallationID(account, fingerprintSeed) require.Equal(t, expectedInstallation, gjson.Get(nonLitePayload, "client_metadata.x-codex-installation-id").String(), "首帧 installation 收敛为账号级设备") require.Equal(t, expectedInstallation, gjson.Get(functionPayload, "client_metadata.x-codex-installation-id").String(), "后续帧同一设备") require.Equal(t, expectedInstallation, gjson.Get(gjson.Get(nonLitePayload, "client_metadata.x-codex-turn-metadata").String(), "installation_id").String()) require.Equal(t, expectedInstallation, gjson.Get(gjson.Get(functionPayload, "client_metadata.x-codex-turn-metadata").String(), "installation_id").String()) - require.Equal(t, codexClientSandboxMac, gjson.Get(gjson.Get(nonLitePayload, "client_metadata.x-codex-turn-metadata").String(), "sandbox").String(), "首帧 sandbox 跟随最终 UA 的系统") - require.Equal(t, codexClientSandboxMac, gjson.Get(gjson.Get(functionPayload, "client_metadata.x-codex-turn-metadata").String(), "sandbox").String(), "后续帧 sandbox 同样改写") + require.Equal(t, "seccomp", gjson.Get(gjson.Get(nonLitePayload, "client_metadata.x-codex-turn-metadata").String(), "sandbox").String(), "保留首帧实际 sandbox") + require.Equal(t, "seccomp", gjson.Get(gjson.Get(functionPayload, "client_metadata.x-codex-turn-metadata").String(), "sandbox").String(), "保留后续帧实际 sandbox") require.False(t, gjson.Get(nonLitePayload, "client_metadata.workspaces").Exists(), "workspaces 保持原样（本例未提供）") } @@ -1516,7 +1524,7 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_PassthroughHeade req.Header = req.Header.Clone() req.Header.Set("User-Agent", "codex_cli_rs/0.98.0") req.Header.Set(openAIWSTurnStateHeader, "turn-state-1") - req.Header.Set(openAIWSTurnMetadataHeader, "turn-meta-1") + req.Header.Set(openAIWSTurnMetadataHeader, `{"turn_id":"turn-meta-1"}`) ginCtx.Request = req readCtx, cancel := context.WithTimeout(r.Context(), 3*time.Second) @@ -1592,17 +1600,17 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_PassthroughHeade } require.Empty(t, captureDialer.lastHeaders.Get("session_id"), "真实 Codex 不发下划线 session_id") - require.Equal(t, "pcache_passthrough", captureDialer.lastHeaders.Get("session-id"), "设备模式保留客户端会话标识") + require.NotEmpty(t, captureDialer.lastHeaders.Get("session-id")) require.Empty(t, captureDialer.lastHeaders.Get(openAIWSTurnStateHeader), "client turn state without account provenance must fail closed") - // 握手 turn metadata 经指纹层重建：非 JSON 的客户端值被替换为最小合法 metadata， - // installation 收敛为账号级设备，sandbox 跟随最终 UA 的系统。 + // 合法输入保留真实 turn；installation 取账号设备，缺失 sandbox 不伪造。 handshakeTurnMetadata := captureDialer.lastHeaders.Get(openAIWSTurnMetadataHeader) require.True(t, gjson.Valid(handshakeTurnMetadata), handshakeTurnMetadata) require.Equal(t, resolveConvergedInstallationID(account, "5d2e8c1a-7b4f-4e3d-9a6c-0f1e2d3c4b5a"), gjson.Get(handshakeTurnMetadata, "installation_id").String()) require.False(t, gjson.Get(handshakeTurnMetadata, "sandbox").Exists(), "未提供实际沙箱时不生成 OS 推断值") require.Len(t, upstreamConn.writes, 2) forwarded := requestToJSONString(upstreamConn.writes[0]) - require.Equal(t, gjson.Get(forwarded, "prompt_cache_key").String(), captureDialer.lastHeaders.Get("session-id"), "握手回退与首帧 prompt_cache_key 同源") + require.Equal(t, "pcache_passthrough", gjson.Get(forwarded, "prompt_cache_key").String(), "保留显式自定义缓存键") + require.Equal(t, gjson.Get(forwarded, "client_metadata.session_id").String(), captureDialer.lastHeaders.Get("session-id"), "握手与 body 根会话同源") require.Equal(t, resolveConvergedInstallationID(account, "5d2e8c1a-7b4f-4e3d-9a6c-0f1e2d3c4b5a"), gjson.Get(forwarded, "client_metadata.x-codex-installation-id").String(), "passthrough 首帧默认 device 收敛") require.Equal(t, gjson.Get(forwarded, "client_metadata.x-codex-installation-id").String(), gjson.Get(requestToJSONString(upstreamConn.writes[1]), "client_metadata.x-codex-installation-id").String(), "passthrough 后续帧与首帧同一设备") require.False(t, gjson.Get(forwarded, `tools.#(type=="namespace")`).Exists()) @@ -4864,6 +4872,7 @@ type openAIWSQueueDialer struct { mu sync.Mutex conns []openAIWSClientConn dialCount int + handshake http.Header } func (d *openAIWSQueueDialer) Dial( @@ -4886,7 +4895,7 @@ func (d *openAIWSQueueDialer) Dial( if len(d.conns) > 1 { d.conns = d.conns[1:] } - return conn, 0, nil, nil + return conn, 0, cloneHeader(d.handshake), nil } func (d *openAIWSQueueDialer) DialCount() int { diff --git a/backend/internal/service/openai_ws_forwarder_success_test.go b/backend/internal/service/openai_ws_forwarder_success_test.go index b71061b8e5bf..866475f4301b 100644 --- a/backend/internal/service/openai_ws_forwarder_success_test.go +++ b/backend/internal/service/openai_ws_forwarder_success_test.go @@ -467,7 +467,7 @@ func TestOpenAIGatewayService_BuildOpenAIWSHeadersDeviceModePreservesNamespacedC require.Equal(t, "client-window", headers.Get("x-codex-window-id")) require.Equal(t, "client-session", headers.Get("session-id")) require.Equal(t, "client-thread", headers.Get("thread-id")) - require.Equal(t, "client-request", headers.Get("x-client-request-id")) + require.Equal(t, "client-thread", headers.Get("x-client-request-id")) } func TestLogOpenAIWSBindResponseAccountWarn(t *testing.T) { @@ -1745,7 +1745,7 @@ func TestOpenAIGatewayService_Forward_WSv2StoreFalseSessionConnIsolation(t *test require.Equal(t, int64(2), upgradeCount.Load(), "不同 session(store=false) 应隔离连接，避免续链状态互相覆盖") } -func TestOpenAIGatewayService_Forward_WSv2StoreFalseDisableForceNewConnAllowsReuse(t *testing.T) { +func TestOpenAIGatewayService_Forward_WSv2StoreFalseReusesOnlySameSession(t *testing.T) { gin.SetMode(gin.TestMode) var upgradeCount atomic.Int64 @@ -1836,11 +1836,19 @@ func TestOpenAIGatewayService_Forward_WSv2StoreFalseDisableForceNewConnAllowsReu rec2 := httptest.NewRecorder() c2, _ := gin.CreateTestContext(rec2) c2.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) - c2.Request.Header.Set("session_id", "session_store_false_reuse_b") + c2.Request.Header.Set("session_id", "session_store_false_reuse_a") result2, err := svc.Forward(context.Background(), c2, account, body) require.NoError(t, err) require.NotNil(t, result2) - require.Equal(t, int64(1), upgradeCount.Load(), "关闭强制新连后，不同 session(store=false) 可复用连接") + require.Equal(t, int64(1), upgradeCount.Load(), "同一 session(store=false) 继续复用连接") + + c3, _ := gin.CreateTestContext(httptest.NewRecorder()) + c3.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/responses", nil) + c3.Request.Header.Set("session_id", "session_store_false_reuse_b") + result3, err := svc.Forward(context.Background(), c3, account, body) + require.NoError(t, err) + require.NotNil(t, result3) + require.Equal(t, int64(2), upgradeCount.Load(), "独立 session 必须重新握手") } func TestOpenAIGatewayService_Forward_WSv2ReadTimeoutAppliesPerRead(t *testing.T) { diff --git a/backend/internal/service/openai_ws_forwarder_v2_execution_scope_test.go b/backend/internal/service/openai_ws_forwarder_v2_execution_scope_test.go index da13cd1fcbdf..bab18bb5e456 100644 --- a/backend/internal/service/openai_ws_forwarder_v2_execution_scope_test.go +++ b/backend/internal/service/openai_ws_forwarder_v2_execution_scope_test.go @@ -125,11 +125,14 @@ func TestOpenAIGatewayService_Forward_WSv2_ExecutionScopeUsesOriginalIdentity(t completed := func(id string) []byte { return []byte(`{"type":"response.completed","response":{"id":"` + id + `","model":"gpt-5.1","usage":{"input_tokens":1,"output_tokens":1}}}`) } - captureConn := &openAIWSCaptureConn{events: [][]byte{completed("resp_a"), completed("resp_b"), completed("resp_c")}} handshake := http.Header{} handshake.Set(openAIWSTurnStateHeader, "turn-state-from-upstream") pool := newOpenAIWSConnPool(cfg) - pool.setClientDialerForTest(&openAIWSCaptureDialer{conn: captureConn, handshake: handshake}) + pool.setClientDialerForTest(&openAIWSQueueDialer{handshake: handshake, conns: []openAIWSClientConn{ + &openAIWSCaptureConn{events: [][]byte{completed("resp_a")}}, + &openAIWSCaptureConn{events: [][]byte{completed("resp_b")}}, + &openAIWSCaptureConn{events: [][]byte{completed("resp_c")}}, + }}) defer pool.Close() stateStore := NewOpenAIWSStateStore(nil) diff --git a/backend/internal/service/openai_ws_http_bridge_resume_test.go b/backend/internal/service/openai_ws_http_bridge_resume_test.go index 21afea49ebcf..3dc427919286 100644 --- a/backend/internal/service/openai_ws_http_bridge_resume_test.go +++ b/backend/internal/service/openai_ws_http_bridge_resume_test.go @@ -253,11 +253,11 @@ func TestOpenAIWSHTTPBridgeLaterTurn429RetriesCurrentTurnOnReplacementAccount(t } require.Len(t, upstream.bodies, 3) require.Contains(t, string(upstream.bodies[0]), "first") - require.Equal(t, "client-session", gjson.GetBytes(upstream.bodies[1], "client_metadata.session_id").String()) - require.Equal(t, "client-thread", gjson.GetBytes(upstream.bodies[1], "client_metadata.thread_id").String()) + require.Equal(t, scopeCodexAccountIdentityValue(account, 0, "client-session"), gjson.GetBytes(upstream.bodies[1], "client_metadata.session_id").String()) + require.Equal(t, scopeCodexAccountIdentityValue(account, 0, "client-thread"), gjson.GetBytes(upstream.bodies[1], "client_metadata.thread_id").String()) require.NotContains(t, string(upstream.bodies[2]), "previous_response_id") require.Contains(t, string(upstream.bodies[2]), "second") - require.Equal(t, "client-session", gjson.GetBytes(upstream.bodies[2], "client_metadata.session_id").String()) - require.Equal(t, "client-thread", gjson.GetBytes(upstream.bodies[2], "client_metadata.thread_id").String()) + require.Equal(t, scopeCodexAccountIdentityValue(&nextAccount, 0, "client-session"), gjson.GetBytes(upstream.bodies[2], "client_metadata.session_id").String()) + require.Equal(t, scopeCodexAccountIdentityValue(&nextAccount, 0, "client-thread"), gjson.GetBytes(upstream.bodies[2], "client_metadata.thread_id").String()) require.Empty(t, upstream.requests[2].Header.Get(openAIWSTurnStateHeader)) } diff --git a/backend/internal/service/openai_ws_pool.go b/backend/internal/service/openai_ws_pool.go index c49449071f54..66b24fa7f8a8 100644 --- a/backend/internal/service/openai_ws_pool.go +++ b/backend/internal/service/openai_ws_pool.go @@ -2364,6 +2364,35 @@ func normalizeOpenAIWSAcquireCompatibility(req openAIWSAcquireRequest) openAIWSH return key } +func (key openAIWSHandshakeCompatibilityKey) changedFields(next openAIWSHandshakeCompatibilityKey) string { + fields := []struct { + name string + changed bool + }{ + {"body", key.bodyIdentityKey != next.bodyIdentityKey}, + {"policy", key.codexPolicyRevision != next.codexPolicyRevision}, + {"mode", key.codexMode != next.codexMode}, + {"user_agent", key.userAgent != next.userAgent}, + {"originator", key.originator != next.originator}, + {"version", key.version != next.version}, + {"anchor", key.anchorScope != next.anchorScope}, + {"beta_features", key.betaFeatures != next.betaFeatures}, + {"installation", key.codexInstallationID != next.codexInstallationID}, + {"session", key.sessionIDHyphen != next.sessionIDHyphen}, + {"session_alias", key.sessionIDUnderscore != next.sessionIDUnderscore}, + {"thread", key.threadID != next.threadID}, + {"client_request", key.clientRequestID != next.clientRequestID}, + {"window", key.codexWindowID != next.codexWindowID}, + } + changed := []string{} + for _, field := range fields { + if field.changed { + changed = append(changed, field.name) + } + } + return strings.Join(changed, ",") +} + func normalizeOpenAIWSBetaFeatures(headers http.Header) string { features := make(map[string]struct{}) for name, values := range headers { diff --git a/backend/internal/service/openai_ws_pool_test.go b/backend/internal/service/openai_ws_pool_test.go index a57c922a9194..a395e0c01ffc 100644 --- a/backend/internal/service/openai_ws_pool_test.go +++ b/backend/internal/service/openai_ws_pool_test.go @@ -945,7 +945,7 @@ func TestOpenAIWSConnPool_AcquireRoutingHintRemainsSoftAffinity(t *testing.T) { require.Equal(t, 1, dialer.DialCount()) } -func TestOpenAIWSConnPool_DeviceModeKeysOnlyInstallationIdentity(t *testing.T) { +func TestOpenAIWSConnPool_DeviceModeKeysDeviceAndConversationIdentity(t *testing.T) { cfg := &config.Config{} cfg.Gateway.OpenAIWS.MaxConnsPerAccount = 2 cfg.Gateway.OpenAIWS.MinIdlePerAccount = 0 @@ -979,9 +979,17 @@ func TestOpenAIWSConnPool_DeviceModeKeysOnlyInstallationIdentity(t *testing.T) { Headers: sessionChanged, }) require.NoError(t, err) - require.True(t, second.Reused()) - require.Equal(t, firstConnID, second.ConnID()) + require.False(t, second.Reused()) + require.NotEqual(t, firstConnID, second.ConnID()) + secondConnID := second.ConnID() second.Release() + same, err := pool.Acquire(context.Background(), openAIWSAcquireRequest{ + Account: account, WSURL: "wss://example.com/v1/responses", Headers: sessionChanged, + }) + require.NoError(t, err) + require.True(t, same.Reused()) + require.Equal(t, secondConnID, same.ConnID()) + same.Release() installationChanged := sessionChanged.Clone() installationChanged.Set("x-codex-installation-id", "install-b") @@ -994,7 +1002,7 @@ func TestOpenAIWSConnPool_DeviceModeKeysOnlyInstallationIdentity(t *testing.T) { require.False(t, third.Reused()) require.NotEqual(t, firstConnID, third.ConnID()) third.Release() - require.Equal(t, 2, dialer.DialCount()) + require.Equal(t, 3, dialer.DialCount()) } func TestOpenAIWSConnPool_AcquireReplacesIdleConnWithDifferentBetaFeatures(t *testing.T) { diff --git a/backend/internal/service/plugin_manager_routing_test.go b/backend/internal/service/plugin_manager_routing_test.go index 8a6d3a9e2549..479798b9fab5 100644 --- a/backend/internal/service/plugin_manager_routing_test.go +++ b/backend/internal/service/plugin_manager_routing_test.go @@ -96,12 +96,16 @@ func TestOpenAIGatewayPluginRoutingPreservesAPIKeyAndFailsClosedForOAuth(t *test oauthRequest, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "https://example.com/v1/responses", nil) require.NoError(t, err) + oauthRequest.Header.Set("User-Agent", codexCLIUserAgent) oauthResponse, err := service.doOpenAIUpstream(oauthRequest, "", &Account{ ID: 2, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Concurrency: 1, }) require.Error(t, err) assert.Nil(t, oauthResponse) - assert.Contains(t, err.Error(), "插件不可用") + var strictErr *UpstreamFailoverError + require.ErrorAs(t, err, &strictErr) + assert.Contains(t, strictErr.ClientMessage, "third-party plugin cannot attest") + assert.Equal(t, NextAccountStop, strictErr.NextAccountAction) assert.Equal(t, 1, upstream.doCalls) } From a5ed37479bf6880a47f44f71188827bbb6e33a1a Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 15 Oct 2026 13:47:18 +0800 Subject: [PATCH 11/11] fix: retain canonical body precedence on Codex websocket ingress --- .../service/openai_codex_conversations_test.go | 8 ++++++++ backend/internal/service/openai_codex_metadata.go | 10 ++++++++-- backend/internal/service/openai_gateway_forward.go | 1 + .../internal/service/openai_ws_forwarder_ingress.go | 3 ++- .../openai_ws_forwarder_ingress_session_test.go | 1 + 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/backend/internal/service/openai_codex_conversations_test.go b/backend/internal/service/openai_codex_conversations_test.go index 870b9e69477f..7ac9c2336859 100644 --- a/backend/internal/service/openai_codex_conversations_test.go +++ b/backend/internal/service/openai_codex_conversations_test.go @@ -87,6 +87,14 @@ func TestCodexConversationsCanonicalPrecedenceAndInvalidMetadata(t *testing.T) { require.NoError(t, err) require.Contains(t, conflicts, "session_id") require.Equal(t, root, flat["session_id"]) + headers := http.Header{"X-Codex-Turn-Metadata": {`{"session_id":"stale-header","thread_id":"stale-header"}`}} + ids := resolveCodexFingerprintIDsFromRequest(account, headers, policy) + require.NotNil(t, ids) + require.True(t, applyCodexFingerprintClientMetadata(map[string]any{"client_metadata": flat}, ids)) + require.NoError(t, ids.err) + require.Contains(t, ids.metadataConflicts, "header.session_id") + applyCodexFingerprintHeaders(headers, ids) + require.Equal(t, root, headers.Get("session-id")) for _, raw := range []string{`{"client_metadata":{"x-codex-turn-metadata":"{invalid"}}`, `{"client_metadata":[]}`} { _, _, err := applyCodexAccountIdentityClientMetadataRaw([]byte(raw), account, 1, policy) require.Error(t, err) diff --git a/backend/internal/service/openai_codex_metadata.go b/backend/internal/service/openai_codex_metadata.go index 9a591005cfb4..5713983c3e5d 100644 --- a/backend/internal/service/openai_codex_metadata.go +++ b/backend/internal/service/openai_codex_metadata.go @@ -7,6 +7,7 @@ import ( "log/slog" "maps" "net/http" + "reflect" "slices" "strings" ) @@ -138,10 +139,15 @@ func (ids *codexFingerprintIDs) captureCanonicalMetadata(flat map[string]any) { merged = map[string]any{} } for key, value := range canonical { + if previous, exists := merged[key]; exists && !reflect.DeepEqual(previous, value) { + conflicts = append(conflicts, "header."+key) + } merged[key] = value } ids.logicalMetadata = merged - ids.metadataConflicts = append(ids.metadataConflicts, conflicts...) + ids.metadataConflicts = append(append([]string(nil), ids.metadataConflicts...), conflicts...) + slices.Sort(ids.metadataConflicts) + ids.metadataConflicts = slices.Compact(ids.metadataConflicts) if value := stringMetadataValue(canonical, "session_id"); value != "" { ids.sessionID = value if ids.synthetic && stringMetadataValue(canonical, "thread_id") == "" { @@ -169,6 +175,6 @@ func recordCodexMetadataConflicts(c *gin.Context, conflicts []string) { c.Set(codexMetadataConflictsContextKey, slices.Clone(conflicts)) } if len(conflicts) > 0 { - slog.Debug("codex metadata canonical projection", "source", "body_embedded", "conflicting_fields", conflicts) + slog.Debug("codex metadata canonical projection", "source", "canonical_body", "conflicting_fields", conflicts) } } diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index 81c2571b1cee..31d21ed317ec 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -1716,6 +1716,7 @@ func (s *OpenAIGatewayService) stageCodexFingerprintForWSFrame(c *gin.Context, a if err != nil { return frame, err } + recordCodexMetadataConflicts(c, ids.metadataConflicts) if changed { return next, nil } diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 4863280c9b07..40a84c1f0c49 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -329,7 +329,8 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( } promptCacheKey := strings.TrimSpace(values[2].String()) - if turnMetadata := strings.TrimSpace(c.GetHeader(openAIWSTurnMetadataHeader)); turnMetadata != "" { + if turnMetadata := strings.TrimSpace(c.GetHeader(openAIWSTurnMetadataHeader)); turnMetadata != "" && + !gjson.GetBytes(normalized, "client_metadata."+openAIWSTurnMetadataHeader).Exists() { next, setErr := applyPayloadMutation(normalized, "client_metadata."+openAIWSTurnMetadataHeader, turnMetadata) if setErr != nil { return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", setErr) diff --git a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go index 53fbce89d589..ec7fa5ee4c97 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress_session_test.go +++ b/backend/internal/service/openai_ws_forwarder_ingress_session_test.go @@ -903,6 +903,7 @@ func TestOpenAIGatewayService_ProxyResponsesWebSocketFromClient_CodexImageBridge req := r.Clone(r.Context()) req.Header = req.Header.Clone() req.Header.Set("User-Agent", "codex_cli_rs/0.98.0") + req.Header.Set(openAIWSTurnMetadataHeader, `{"sandbox":"seatbelt"}`) ginCtx.Request = req ginCtx.Set("api_key", apiKey)