From 36789677cc290632e0cefd3504e7d32b6964b3a3 Mon Sep 17 00:00:00 2001 From: orut34iop Date: Mon, 7 Sep 2026 14:50:41 +0800 Subject: [PATCH] fix(video): route recovery-point H264 VOD through seek-compatible decoding --- CHANGELOG.md | 9 +++- Scripts/test-h264-recovery-point.sh | 10 ++++ .../tests/H264RecoveryPointStandalone.swift | 28 ++++++++++ Sources/AetherEngine/AetherEngine.swift | 29 +++++++++++ .../Decoder/H264RecoveryPoint.swift | 50 ++++++++++++++++++ .../Decoder/H264RecoveryPointProbe.swift | 51 +++++++++++++++++++ .../Diagnostics/EngineDiagnostics.swift | 4 ++ .../DocumentedConstantsTests.swift | 12 +++++ .../H264RecoveryPointTests.swift | 28 ++++++++++ docs/api.md | 1 + docs/formats.md | 23 +++++++++ 11 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 Scripts/test-h264-recovery-point.sh create mode 100644 Scripts/tests/H264RecoveryPointStandalone.swift create mode 100644 Sources/AetherEngine/Decoder/H264RecoveryPoint.swift create mode 100644 Sources/AetherEngine/Decoder/H264RecoveryPointProbe.swift create mode 100644 Tests/AetherEngineTests/H264RecoveryPointTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 767414dc9..87223a791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,14 @@ the public-API contract. ## [Unreleased] -_Nothing yet._ +### Fixed + +- H.264 VOD with repeated non-IDR immediate/exact recovery points uses the existing + software decoder for seek compatibility. On three reporting sources the native + HLS path remained around 3 fps after a seek despite a full buffer. A bounded + positive-evidence probe distinguishes this shape from ordinary IDR H.264; live + sessions and already-software sources are unchanged. Exposes the identity-free + `diagnostics.h264RecoveryPointKeyCount` sample result for hosts. ## [6.71.0] - 2026-09-06 diff --git a/Scripts/test-h264-recovery-point.sh b/Scripts/test-h264-recovery-point.sh new file mode 100644 index 000000000..6cd98a30f --- /dev/null +++ b/Scripts/test-h264-recovery-point.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail +TASK_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +TASK_TMP=$(mktemp -d "${TMPDIR:-/tmp}/aether-recovery-point.XXXXXX") +trap 'rm -f "$TASK_TMP/check"; rmdir "$TASK_TMP"' EXIT +# Pure Foundation host check, not a macOS/iOS application target. +xcrun swiftc \ + "$TASK_ROOT/Sources/AetherEngine/Decoder/H264RecoveryPoint.swift" \ + "$TASK_ROOT/Scripts/tests/H264RecoveryPointStandalone.swift" -o "$TASK_TMP/check" +"$TASK_TMP/check" diff --git a/Scripts/tests/H264RecoveryPointStandalone.swift b/Scripts/tests/H264RecoveryPointStandalone.swift new file mode 100644 index 000000000..480064cc6 --- /dev/null +++ b/Scripts/tests/H264RecoveryPointStandalone.swift @@ -0,0 +1,28 @@ +import Foundation + +@main +struct RecoveryPointTests { + static func main() { + let classify = H264RecoveryPoint.isImmediateExactRecovery + precondition(classify([6, 6, 1, 0xc4, 0x80])) + precondition(classify([6, 5, 2, 1, 2, 6, 1, 0xc4, 0x80])) + for bytes: [UInt8] in [[], [6], [6, 255], [6, 6, 2, 0xc4], + [5, 6, 1, 0xc4, 0x80], [6, 6, 1, 0x84, 0x80], + [6, 6, 1, 0x54, 0x80], [0x86, 6, 1, 0xc4, 0x80]] { + precondition(!classify(bytes)) + } + precondition(!classify(Array(repeating: 6, count: 4097))) + var evidence = H264RecoveryPoint.Evidence() + for _ in 0..<100 { + evidence.observe(containerKey: true, hasIDR: true, immediateExactRecovery: true) + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: false) + evidence.observe(containerKey: false, hasIDR: false, immediateExactRecovery: true) + } + precondition(!evidence.requiresCompatibilityPath) + for count in 1...3 { + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: true) + precondition(evidence.requiresCompatibilityPath == (count == 3)) + } + print("PASS: exact/delayed recovery, IDR exclusion, container flags, malformed/oversized input, repeated evidence") + } +} diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 33eef582e..4a462198f 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -3970,6 +3970,34 @@ public final class AetherEngine: ObservableObject { ) } } + // Recovery-point H.264 can play linearly through AVPlayer yet retain only + // ~3 fps after a cached seek. Container key flags include non-IDR recovery + // points; the loopback HLS path cuts on those flags and promises independent + // segments. Use libavcodec for repeated, positively identified recovery + // points, not for all H.264 or merely because an IDR was absent in a sample. + if !useSoftwarePath, probeOpened, !options.isLive, probe.isSourceSeekable, + detectedCodecID == AV_CODEC_ID_H264, options.preferredDecodePath != .software { + let videoIdx = probe.videoStreamIndex + let (result, rewound) = await Task.detached(priority: .userInitiated) { [probe] in + let result = H264RecoveryPointProbe.run(demuxer: probe, streamIndex: videoIdx) + return (result, probe.seek(to: 0)) + }.value + if loadGeneration != gen { + probe.markClosed() + Task.detached { [probe] in probe.close() } + try checkLoadCurrent(gen) + } + guard rewound else { + probe.markClosed() + Task.detached { [probe] in probe.close() } + throw DemuxerError.readFailed(code: -5) + } + diagnostics.h264RecoveryPointKeyCount = result.evidence.recoveryKeys + useSoftwarePath = result.evidence.requiresCompatibilityPath + EngineLog.emit("[AetherEngine] H264 recovery-point sample: video=\(result.videoPackets) " + + "recoveryKeys=\(result.evidence.recoveryKeys) " + + "compatibility=\(useSoftwarePath)", category: .engine) + } // #2: an H.264 / HEVC format AVPlayer accepts at the HLS CODECS level but VideoToolbox can't // hardware-decode (H.264 High 4:2:2/4:4:4/High-10, HEVC Rext on Intel Macs / older Apple TV) reaches // readyToPlay then renders nothing on the native path. QuickTime plays it via its own software decoder; @@ -6016,6 +6044,7 @@ public final class AetherEngine: ObservableObject { liveTelemetrySampler?.stop() liveTelemetrySampler = nil diagnostics.liveTelemetry = nil + diagnostics.h264RecoveryPointKeyCount = nil nativeCancellables.removeAll() // AE#158: keepCurrentItem defers the item detach to the next host.load(inPlaceSwap:) so a // system PiP window never sees a nil-item gap across a native->native load. Only meaningful diff --git a/Sources/AetherEngine/Decoder/H264RecoveryPoint.swift b/Sources/AetherEngine/Decoder/H264RecoveryPoint.swift new file mode 100644 index 000000000..90b43663e --- /dev/null +++ b/Sources/AetherEngine/Decoder/H264RecoveryPoint.swift @@ -0,0 +1,50 @@ +import Foundation + +/// Recovery-point SEI is not an IDR NAL. A muxer's container key flag must not +/// erase that distinction when selecting Apple's independently decodable HLS path. +enum H264RecoveryPoint { + /// Narrow, bounded recognizer for recovery_frame_cnt=0 and exact_match_flag=1. + /// Unknown/malformed SEI is not evidence for changing the playback route. + static func isImmediateExactRecovery(seiNAL: [UInt8]) -> Bool { + guard seiNAL.count > 3, seiNAL.count <= 4096, + seiNAL[0] & 0x80 == 0, seiNAL[0] & 31 == 6 else { return false } + var body: [UInt8] = [] + var zeros = 0 + for byte in seiNAL.dropFirst() { + if zeros >= 2 && byte == 3 { zeros = 0; continue } + body.append(byte) + zeros = byte == 0 ? zeros + 1 : 0 + } + var index = 0 + func extendedValue() -> Int? { + var value = 0 + while index < body.count { + let byte = body[index] + index += 1 + value += Int(byte) + if byte != 255 { return value } + } + return nil + } + while index < body.count { + if index == body.count - 1 && body[index] == 0x80 { break } + guard let type = extendedValue(), let size = extendedValue(), + size <= body.count - index else { return false } + // ue(0) is the single bit 1; followed by exact_match_flag=1, + // broken_link_flag and two changing_slice_group_idc bits (5 bits). + if type == 6, size >= 1, body[index] & 0xc0 == 0xc0 { return true } + index += size + } + return false + } + + struct Evidence { + private(set) var recoveryKeys = 0 + mutating func observe(containerKey: Bool, hasIDR: Bool, immediateExactRecovery: Bool) { + if containerKey && !hasIDR && immediateExactRecovery { recoveryKeys += 1 } + } + /// Repeated actual recovery points, never a codec name, filename, frame + /// rate, missing IDR alone, or a malformed sample, select compatibility. + var requiresCompatibilityPath: Bool { recoveryKeys >= 3 } + } +} diff --git a/Sources/AetherEngine/Decoder/H264RecoveryPointProbe.swift b/Sources/AetherEngine/Decoder/H264RecoveryPointProbe.swift new file mode 100644 index 000000000..a9614b78d --- /dev/null +++ b/Sources/AetherEngine/Decoder/H264RecoveryPointProbe.swift @@ -0,0 +1,51 @@ +import Foundation +import AetherLibavcodec +import AetherLibavutil + +enum H264RecoveryPointProbe { + struct Result { + var evidence = H264RecoveryPoint.Evidence() + var videoPackets = 0 + var packetsRead = 0 + } + + /// Consumes a bounded packet sample; the caller must rewind the reused demuxer. + /// Like InterlaceProbe, this does not race AVIO prefetch by changing its deadline. + static func run(demuxer: Demuxer, streamIndex: Int32) -> Result { + var result = Result() + guard streamIndex >= 0, let stream = demuxer.stream(at: streamIndex), + let parameters = stream.pointee.codecpar, + parameters.pointee.codec_id == AV_CODEC_ID_H264 else { return result } + let framing = A53SEIParser.nalFraming(codec: .h264, + extradata: parameters.pointee.extradata, size: Int(parameters.pointee.extradata_size)) + let deadline = Date(timeIntervalSinceNow: 3) + while result.videoPackets < 180, result.packetsRead < 600, Date() < deadline { + guard let packet = try? demuxer.readPacket() else { break } + result.packetsRead += 1 + defer { + av_packet_unref(packet) + av_packet_free_safe(packet) + } + guard packet.pointee.stream_index == streamIndex else { continue } + result.videoPackets += 1 + guard packet.pointee.flags & AV_PKT_FLAG_KEY != 0, + let data = packet.pointee.data, packet.pointee.size > 0 else { continue } + var hasIDR = false + var hasNonIDRSlice = false + var recovery = false + A53SEIParser.forEachNAL(data, Int(packet.pointee.size), framing) { nal, size in + guard size > 0 else { return } + if nal[0] & 31 == 5 { hasIDR = true } + if nal[0] & 31 == 1, nal[0] & 0x80 == 0 { hasNonIDRSlice = true } + if nal[0] & 31 == 6, size <= 4096 { + recovery = recovery || H264RecoveryPoint.isImmediateExactRecovery( + seiNAL: Array(UnsafeBufferPointer(start: nal, count: size))) + } + } + result.evidence.observe(containerKey: true, hasIDR: hasIDR, + immediateExactRecovery: recovery && hasNonIDRSlice) + if result.evidence.requiresCompatibilityPath { break } + } + return result + } +} diff --git a/Sources/AetherEngine/Diagnostics/EngineDiagnostics.swift b/Sources/AetherEngine/Diagnostics/EngineDiagnostics.swift index 80cd231ae..79438bb7f 100644 --- a/Sources/AetherEngine/Diagnostics/EngineDiagnostics.swift +++ b/Sources/AetherEngine/Diagnostics/EngineDiagnostics.swift @@ -7,6 +7,10 @@ import Combine @MainActor public final class EngineDiagnostics: ObservableObject { + /// Positive non-IDR immediate/exact recovery keys in the bounded VOD routing sample. + /// nil means no sample was taken; cleared when the session stops. + @Published public internal(set) var h264RecoveryPointKeyCount: Int? + /// 1 Hz snapshot while playing/paused; nil while idle. Cleared in stopInternal so sessions don't inherit stale numbers. @Published public internal(set) var liveTelemetry: LiveTelemetry? } diff --git a/Tests/AetherEngineTests/DocumentedConstantsTests.swift b/Tests/AetherEngineTests/DocumentedConstantsTests.swift index 409214b70..e08fe7abc 100644 --- a/Tests/AetherEngineTests/DocumentedConstantsTests.swift +++ b/Tests/AetherEngineTests/DocumentedConstantsTests.swift @@ -106,6 +106,18 @@ final class DocumentedConstantsTests: XCTestCase { // MARK: - Probe budgets + func testRecoveryPointRoutingMatchesDocumentedThreshold() throws { + let docs = try documentation() + var evidence = H264RecoveryPoint.Evidence() + for _ in 0..<2 { + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: true) + } + XCTAssertFalse(evidence.requiresCompatibilityPath) + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: true) + XCTAssertTrue(evidence.requiresCompatibilityPath) + assertDocumented("Three positive samples select software compatibility", docs) + } + /// docs/api.md states the defaults a host overrides with `probesize` / `maxAnalyzeDuration`. func testProbeBudgetDefaultsAreWhatTheDocsSay() throws { let docs = try documentation() diff --git a/Tests/AetherEngineTests/H264RecoveryPointTests.swift b/Tests/AetherEngineTests/H264RecoveryPointTests.swift new file mode 100644 index 000000000..6d122d495 --- /dev/null +++ b/Tests/AetherEngineTests/H264RecoveryPointTests.swift @@ -0,0 +1,28 @@ +import Testing +@testable import AetherEngine + +@Suite("H.264 recovery points are not HLS IDRs") +struct H264RecoveryPointTests { + @Test func exactRecovery() { + #expect(H264RecoveryPoint.isImmediateExactRecovery(seiNAL: [6, 6, 1, 0xc4, 0x80])) + #expect(!H264RecoveryPoint.isImmediateExactRecovery(seiNAL: [6, 6, 1, 0x84, 0x80])) + #expect(!H264RecoveryPoint.isImmediateExactRecovery(seiNAL: [6, 6, 1, 0x54, 0x80])) + #expect(!H264RecoveryPoint.isImmediateExactRecovery(seiNAL: [6, 6, 2, 0xc4])) + #expect(!H264RecoveryPoint.isImmediateExactRecovery(seiNAL: [5, 6, 1, 0xc4, 0x80])) + } + @Test func requiresRepeatedPositiveEvidence() { + var evidence = H264RecoveryPoint.Evidence() + for _ in 0..<100 { + evidence.observe(containerKey: true, hasIDR: true, immediateExactRecovery: true) + evidence.observe(containerKey: false, hasIDR: false, immediateExactRecovery: true) + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: false) + } + #expect(!evidence.requiresCompatibilityPath) + for _ in 0..<2 { + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: true) + } + #expect(!evidence.requiresCompatibilityPath) + evidence.observe(containerKey: true, hasIDR: false, immediateExactRecovery: true) + #expect(evidence.requiresCompatibilityPath) + } +} diff --git a/docs/api.md b/docs/api.md index f4df16569..88e4914c8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -659,6 +659,7 @@ as well. | Symbol | Notes | | --- | --- | | `diagnostics.liveTelemetry` | 1 Hz `LiveTelemetry?` snapshot while playing or paused, nil while idle. On a separate `ObservableObject` so its ticks cannot re-render a host observing the engine. | +| `diagnostics.h264RecoveryPointKeyCount` | Positive non-IDR immediate/exact recovery keys in the bounded H.264 VOD routing sample. `nil` when no sample was taken; cleared on stop. Three positive samples select software compatibility; this count is not a decode-performance measurement. | | `EngineLog.handler` | Mirror every info-level line into a host capture path. Fires from whatever thread emitted it, so it must be thread-safe and non-blocking. | | `EngineLog.subsystem`, `EngineLog.Category` | `de.superuser404.AetherEngine`, one category per subsystem: `engine`, `ffmpeg`, `session`, `muxer`, `demux`, `hls.server`, `audio.bridge`, `sw.playback`, `scrub`. | | `EngineLog.Level` | `.info` reaches os_log and the host handler; `.verbose` is per-segment trace and reaches os_log's debug level **only**, never the handler, which is what keeps a mirrored stream readable. Read the verbose ones with `log stream --level debug`. | diff --git a/docs/formats.md b/docs/formats.md index 9847e0bfe..e67203d76 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -23,6 +23,29 @@ That list is the supported set, not the compiled set. The FFmpeg build also carr Interlaced sources (DVD-rip MPEG-2, SD / HD broadcast H.264) are deinterlaced through a persistent bwdif graph (yadif fallback) that engages on the first interlaced frame and costs nothing on progressive content. The dispatch decision lives in `AetherEngine.load` (`VideoRoutingPolicy`), gated per source on `VTCapabilityProbe`, codec id, declared field order, and on VOD the decode sample that verifies it. +### H.264 recovery-point VOD seek compatibility + +Some H.264 VOD streams mark immediate/exact non-IDR recovery points as container +keys. The local HLS cutter accepts those flags as independent segment boundaries, +yet the reporting Apple TV retained only about 3 fps after a seek in three such +streams, despite a full buffer and normal source timestamps. VidHub played those +same private files normally; its internal decode route is not known. + +The engine samples positive evidence before selecting compatibility: repeated +container-key packets with non-IDR slices and recovery SEI `recovery_frame_cnt=0` +and `exact_match_flag=1`. Merely lacking an IDR, a malformed SEI or a non-key packet +cannot trigger the route. The sample is bounded in packets and bytes of SEI parsed, +with a between-read wall-clock limit; individual reads keep their transport timeout. +The reused demuxer is rewound and load-generation ownership is rechecked. + +Confirmed sources use the existing libavcodec software path. This is a compatibility +route, not a claim that all non-IDR input is undecodable by Apple hardware or a fix +to native HLS random access. CPU/power costs differ; compressed payloads and source +timestamps are not rewritten. Live/non-H.264 and already-software sessions avoid +the probe. `diagnostics.h264RecoveryPointKeyCount` records the numeric decision. +Physical acceptance covers three private MP4/H.264 sources on Apple TV, from-head, +seek and pause/resume. Separate software read-ahead/cache work is not in this change. + ### MP4 without composition offsets Some writers emit a sample table with no `ctts` while the H.264 bitstream still reorders pictures.