diff --git a/.github/workflows/android-release.yml b/.github/workflows/android-release.yml index 9b05d43..12a2a8c 100644 --- a/.github/workflows/android-release.yml +++ b/.github/workflows/android-release.yml @@ -48,6 +48,11 @@ jobs: - name: Set up Android SDK uses: android-actions/setup-android@v4 + with: + # The action's default is "tools platform-tools", but the obsolete "tools" + # package has been removed from Google's SDK repository and now fails with + # "Failed to find package 'tools'". Gradle/AGP fetch what else they need. + packages: platform-tools - name: Set up Gradle uses: gradle/actions/setup-gradle@v6 diff --git a/.github/workflows/ci-pull-request-merge-main.yml b/.github/workflows/ci-pull-request-merge-main.yml index 626e5e7..fb2088d 100644 --- a/.github/workflows/ci-pull-request-merge-main.yml +++ b/.github/workflows/ci-pull-request-merge-main.yml @@ -30,6 +30,11 @@ jobs: - name: Set up Android SDK uses: android-actions/setup-android@v4 + with: + # The action's default is "tools platform-tools", but the obsolete "tools" + # package has been removed from Google's SDK repository and now fails with + # "Failed to find package 'tools'". Gradle/AGP fetch what else they need. + packages: platform-tools - name: Set up Gradle uses: gradle/actions/setup-gradle@v6 diff --git a/Consolation/.idea/misc.xml b/Consolation/.idea/misc.xml index b2c751a..74dd639 100644 --- a/Consolation/.idea/misc.xml +++ b/Consolation/.idea/misc.xml @@ -1,3 +1,4 @@ + diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/MainActivity.kt b/Consolation/app/src/main/java/org/centennialoss/consolation/MainActivity.kt index fbd5649..ee82daf 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/MainActivity.kt +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/MainActivity.kt @@ -17,6 +17,7 @@ import android.text.SpannableString import android.text.style.ForegroundColorSpan import android.view.ContextThemeWrapper import android.view.Menu +import android.view.SurfaceView import android.view.TextureView import android.view.View import android.view.WindowManager @@ -122,9 +123,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlin.math.abs import java.util.Locale -import kotlin.math.cos import kotlin.math.roundToInt -import kotlin.math.sin private val ConsolationColorScheme = darkColorScheme( primary = Color(0xFFCC11BB), @@ -132,7 +131,9 @@ private val ConsolationColorScheme = darkColorScheme( ) class MainActivity : ComponentActivity() { - private lateinit var previewTexture: TextureView + /** SurfaceView for every format except H264 (MediaCodec renders straight into the + * surface, so only a TextureView can still be flipped/zoomed by the View system). */ + private lateinit var previewTexture: View private lateinit var rootView: View private lateinit var deviceRepository: UsbCaptureDeviceRepository private lateinit var previewBackend: UsbVideoPreviewBackend @@ -154,6 +155,7 @@ class MainActivity : ComponentActivity() { private var permissionTimeoutJob: Job? = null private var telemetryJob: Job? = null private var controlsAutoHideJob: Job? = null + private var oneToOneNoticeJob: Job? = null private var connectingWatchdogJob: Job? = null private var hasRetriedConnectingSession = false private var lastUsbPermissionGrantedAtMs = 0L @@ -175,11 +177,21 @@ class MainActivity : ComponentActivity() { private var currentRotation by mutableIntStateOf(0) private var isFlippedHorizontal by mutableStateOf(false) private var isFlippedVertical by mutableStateOf(false) - private var currentZoom by mutableIntStateOf(0) + /** Zoom slider position 0..100; [ZOOM_FIT_POSITION] is fit-to-screen, below shrinks, above zooms in. */ + private var currentZoom by mutableIntStateOf(ZOOM_FIT_POSITION) + /** When on, the preview is scaled so one stream pixel maps to one screen pixel (overrides the slider). */ + private var isOneToOneZoom by mutableStateOf(false) + private var isOneToOneNoticeVisible by mutableStateOf(false) + private var previewStreamWidth by mutableIntStateOf(0) + private var previewStreamHeight by mutableIntStateOf(0) private var zoomPanOffsetX by mutableFloatStateOf(0f) private var zoomPanOffsetY by mutableFloatStateOf(0f) - private var previewLayoutWidthPx by mutableFloatStateOf(0f) - private var previewLayoutHeightPx by mutableFloatStateOf(0f) + /** Size of the full-screen preview area (the SurfaceView fills all of it). */ + private var previewContainerWidthPx by mutableFloatStateOf(0f) + private var previewContainerHeightPx by mutableFloatStateOf(0f) + /** Fit-to-screen content box inside the preview area, at the stream's (rotated) aspect. */ + private val previewLayoutWidthPx: Float get() = fittedPreviewBoxSize().first + private val previewLayoutHeightPx: Float get() = fittedPreviewBoxSize().second private var audioVolumePercent by mutableIntStateOf(100) private var audioMuted by mutableStateOf(false) @@ -565,7 +577,7 @@ class MainActivity : ComponentActivity() { private fun replacePreviewTextureAfterUsbRemoval() { if (::previewTexture.isInitialized) { - previewTexture.surfaceTextureListener = null + (previewTexture as? TextureView)?.surfaceTextureListener = null previewTexture.isVisible = false } previewTextureGeneration++ @@ -1021,6 +1033,8 @@ class MainActivity : ComponentActivity() { private fun updateAspectRatio(width: Int, height: Int) { if (width > 0 && height > 0) { previewAspectRatio = width.toFloat() / height.toFloat() + previewStreamWidth = width + previewStreamHeight = height } } @@ -1181,55 +1195,129 @@ class MainActivity : ComponentActivity() { } } + /** + * Turns 1:1 pixel scaling on or off. Turning it on (re)shows the explanatory notice + * for [ONE_TO_ONE_NOTICE_MS], restarting the timer; turning it off hides the notice. + */ + private fun applyOneToOneZoom(enabled: Boolean) { + isOneToOneZoom = enabled + oneToOneNoticeJob?.cancel() + oneToOneNoticeJob = null + isOneToOneNoticeVisible = enabled + if (enabled) { + oneToOneNoticeJob = lifecycleScope.launch { + delay(ONE_TO_ONE_NOTICE_MS) + isOneToOneNoticeVisible = false + } + } + } + private fun updatePreviewScale() { // Compose applies scale/rotation from state. } + /** Scale the preview is drawn at: the 1:1 pixel scale when enabled, else the slider scale. */ private fun previewZoomScale(): Float { - return 1.0f + (currentZoom / 100.0f) * (PREVIEW_MAX_ZOOM_SCALE - 1.0f) + if (isOneToOneZoom) { + oneToOneZoomScale()?.let { return it } + } + return zoomPositionToScale(currentZoom) + } + + private fun zoomPositionToScale(position: Int): Float { + val p = position.coerceIn(0, 100) + return if (p >= ZOOM_FIT_POSITION) { + val t = (p - ZOOM_FIT_POSITION) / (100f - ZOOM_FIT_POSITION) + 1.0f + t * (PREVIEW_MAX_ZOOM_SCALE - 1.0f) + } else { + val t = p / ZOOM_FIT_POSITION.toFloat() + PREVIEW_MIN_ZOOM_SCALE + t * (1.0f - PREVIEW_MIN_ZOOM_SCALE) + } + } + + private fun zoomScaleToPosition(scale: Float): Float { + return if (scale >= 1.0f) { + ZOOM_FIT_POSITION + (100f - ZOOM_FIT_POSITION) * (scale - 1.0f) / (PREVIEW_MAX_ZOOM_SCALE - 1.0f) + } else { + ZOOM_FIT_POSITION * (scale - PREVIEW_MIN_ZOOM_SCALE) / (1.0f - PREVIEW_MIN_ZOOM_SCALE) + }.coerceIn(0f, 100f) + } + + /** + * Scale at which one stream pixel covers one screen pixel, relative to the fit-to-screen + * preview box, or null until both the stream size and the box size are known. + */ + private fun oneToOneZoomScale(): Float? { + val w = previewLayoutWidthPx + val h = previewLayoutHeightPx + if (previewStreamWidth <= 0 || previewStreamHeight <= 0 || w <= 0f || h <= 0f) return null + /* The SurfaceView box takes the rotated aspect (the renderer rotates the content), so + * the stream's width then runs along the box height. The TextureView box is unrotated. */ + val boxSpanForStreamWidth = if (usesSurfaceViewPreview() && currentRotation % 180 == 90) h else w + return previewStreamWidth / boxSpanForStreamWidth } - private fun effectivePreviewDimensionsForPan(widthPx: Float, heightPx: Float): Pair { - return if (currentRotation % 180 == 90) { - heightPx to widthPx + /** Every format but H264 renders through the native GPU renderer into a SurfaceView. */ + private fun usesSurfaceViewPreview(): Boolean = + selectedPixelFormatPreference != PixelFormatPreference.H264 + + /** + * Aspect of the fit-to-screen content box. With a SurfaceView the renderer rotates the + * content, so the box takes the rotated aspect; the TextureView path rotates the whole view. + */ + private fun previewBoxAspect(): Float { + return if (usesSurfaceViewPreview() && currentRotation % 180 == 90) { + 1f / previewAspectRatio } else { - widthPx to heightPx + previewAspectRatio } } - private fun maxZoomPanOffsetX(widthPx: Float, heightPx: Float, scale: Float): Float { - val (effectiveWidth, _) = effectivePreviewDimensionsForPan(widthPx, heightPx) - return (effectiveWidth * (scale - 1f) / 2f).coerceAtLeast(0f) + private fun fittedPreviewBoxSize(): Pair { + val cw = previewContainerWidthPx + val ch = previewContainerHeightPx + val aspect = previewBoxAspect() + if (cw <= 0f || ch <= 0f || aspect <= 0f) return 0f to 0f + return if (cw / ch > aspect) (ch * aspect) to ch else cw to (cw / aspect) } - private fun maxZoomPanOffsetY(widthPx: Float, heightPx: Float, scale: Float): Float { - val (_, effectiveHeight) = effectivePreviewDimensionsForPan(widthPx, heightPx) - return (effectiveHeight * (scale - 1f) / 2f).coerceAtLeast(0f) + private fun isPreviewZoomedIn(scale: Float): Boolean = scale > 1.0f + ZOOM_PAN_EPSILON + + /** + * On-screen size of the zoomed content. The TextureView is laid out unrotated and then + * rotated as a whole, so at 90/270 its box spans swap on screen. + */ + private fun displayedPreviewSize(scale: Float): Pair { + val w = previewLayoutWidthPx * scale + val h = previewLayoutHeightPx * scale + return if (!usesSurfaceViewPreview() && currentRotation % 180 == 90) h to w else w to h } - private fun clampZoomPanOffsets(widthPx: Float, heightPx: Float, scale: Float) { - if (currentZoom <= 0 || widthPx <= 0f || heightPx <= 0f) { + /* + * Pan offsets are screen pixels (the pan is applied after rotation and mirroring), limited + * so the zoomed content keeps covering the screen along any axis where it is larger than + * the screen; a letterboxed axis stays centered until the zoom fills it. + */ + private fun maxZoomPanOffsetX(scale: Float): Float { + return ((displayedPreviewSize(scale).first - previewContainerWidthPx) / 2f).coerceAtLeast(0f) + } + + private fun maxZoomPanOffsetY(scale: Float): Float { + return ((displayedPreviewSize(scale).second - previewContainerHeightPx) / 2f).coerceAtLeast(0f) + } + + private fun clampZoomPanOffsets(scale: Float) { + if (!isPreviewZoomedIn(scale) || previewLayoutWidthPx <= 0f || previewLayoutHeightPx <= 0f) { zoomPanOffsetX = 0f zoomPanOffsetY = 0f return } - val maxX = maxZoomPanOffsetX(widthPx, heightPx, scale) - val maxY = maxZoomPanOffsetY(widthPx, heightPx, scale) + val maxX = maxZoomPanOffsetX(scale) + val maxY = maxZoomPanOffsetY(scale) zoomPanOffsetX = zoomPanOffsetX.coerceIn(-maxX, maxX) zoomPanOffsetY = zoomPanOffsetY.coerceIn(-maxY, maxY) } - private fun transformDragToPanDelta(dragX: Float, dragY: Float): Pair { - var x = dragX - var y = dragY - if (isFlippedHorizontal) x = -x - if (isFlippedVertical) y = -y - val radians = Math.toRadians(-currentRotation.toDouble()) - val c = cos(radians).toFloat() - val s = sin(radians).toFloat() - return (x * c - y * s) to (x * s + y * c) - } - private fun showSettingsDialog() { activeSheet = ActiveSheet.SETTINGS } @@ -1534,7 +1622,13 @@ class MainActivity : ComponentActivity() { currentRotation = prefs.getInt(KEY_ROTATION, 0) isFlippedHorizontal = prefs.getBoolean(KEY_FLIP_H, false) isFlippedVertical = prefs.getBoolean(KEY_FLIP_V, false) - currentZoom = prefs.getInt(KEY_ZOOM, 0).coerceIn(0, 100) + currentZoom = if (prefs.contains(KEY_ZOOM_POSITION)) { + prefs.getInt(KEY_ZOOM_POSITION, ZOOM_FIT_POSITION) + } else { + // Legacy slider: 0 = fit, 100 = max zoom; now the upper half of the slider. + ZOOM_FIT_POSITION + prefs.getInt(KEY_ZOOM, 0).coerceIn(0, 100) * (100 - ZOOM_FIT_POSITION) / 100 + }.coerceIn(0, 100) + isOneToOneZoom = prefs.getBoolean(KEY_ZOOM_ONE_TO_ONE, false) audioVolumePercent = prefs.getInt(KEY_VOLUME, 100).coerceIn(0, 100) audioMuted = prefs.getBoolean(KEY_MUTED, false) } @@ -1549,7 +1643,8 @@ class MainActivity : ComponentActivity() { .putInt(KEY_ROTATION, currentRotation) .putBoolean(KEY_FLIP_H, isFlippedHorizontal) .putBoolean(KEY_FLIP_V, isFlippedVertical) - .putInt(KEY_ZOOM, currentZoom) + .putInt(KEY_ZOOM_POSITION, currentZoom) + .putBoolean(KEY_ZOOM_ONE_TO_ONE, isOneToOneZoom) .putInt(KEY_VOLUME, audioVolumePercent) .putBoolean(KEY_MUTED, audioMuted) .apply() @@ -1559,14 +1654,15 @@ class MainActivity : ComponentActivity() { private fun MainScreen() { val baseScale = previewZoomScale() LaunchedEffect( - currentZoom, + baseScale, currentRotation, isFlippedHorizontal, isFlippedVertical, - previewLayoutWidthPx, - previewLayoutHeightPx, + previewContainerWidthPx, + previewContainerHeightPx, + previewAspectRatio, ) { - clampZoomPanOffsets(previewLayoutWidthPx, previewLayoutHeightPx, baseScale) + clampZoomPanOffsets(baseScale) } Box( modifier = Modifier @@ -1581,59 +1677,81 @@ class MainActivity : ComponentActivity() { } }, ) { - Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { - key(previewTextureGeneration) { + val useSurfaceView = usesSurfaceViewPreview() + val boxAspect = previewBoxAspect() + if (useSurfaceView) { + /* The SurfaceView fills the screen so zooming can grow into the letterbox; the + * renderer fits the content box into it, then pans in surface NDC. */ + LaunchedEffect( + currentRotation, isFlippedHorizontal, isFlippedVertical, baseScale, + zoomPanOffsetX, zoomPanOffsetY, + previewContainerWidthPx, previewContainerHeightPx, boxAspect, + ) { + val cw = previewContainerWidthPx + val ch = previewContainerHeightPx + val zoomedIn = isPreviewZoomedIn(baseScale) + val panX = if (zoomedIn && cw > 0f) 2f * zoomPanOffsetX / cw else 0f + val panY = if (zoomedIn && ch > 0f) -2f * zoomPanOffsetY / ch else 0f + val fitX = if (cw > 0f) (previewLayoutWidthPx / cw).coerceIn(0f, 1f) else 1f + val fitY = if (ch > 0f) (previewLayoutHeightPx / ch).coerceIn(0f, 1f) else 1f + previewBackend.setPreviewTransform( + currentRotation, isFlippedHorizontal, isFlippedVertical, baseScale, panX, panY, + fitX, fitY, + ) + } + } + Box( + modifier = Modifier + .fillMaxSize() + .onSizeChanged { + previewContainerWidthPx = it.width.toFloat() + previewContainerHeightPx = it.height.toFloat() + }, + contentAlignment = Alignment.Center, + ) { + key(previewTextureGeneration, useSurfaceView) { AndroidView( factory = { context -> - TextureView(context).also { + val view: View = if (useSurfaceView) SurfaceView(context) else TextureView(context) + view.also { previewTexture = it it.isVisible = isPlaybackRunningUi } }, modifier = Modifier - .fillMaxHeight() - .aspectRatio(previewAspectRatio) - .onSizeChanged { - previewLayoutWidthPx = it.width.toFloat() - previewLayoutHeightPx = it.height.toFloat() - clampZoomPanOffsets( - previewLayoutWidthPx, - previewLayoutHeightPx, - baseScale, - ) - } + .then( + if (useSurfaceView) { + Modifier.fillMaxSize() + } else { + Modifier.fillMaxHeight().aspectRatio(boxAspect) + }, + ) .pointerInput( - currentZoom, currentRotation, isFlippedHorizontal, isFlippedVertical, baseScale, ) { - if (currentZoom <= 0) return@pointerInput + if (!isPreviewZoomedIn(baseScale)) return@pointerInput detectDragGestures { change, dragAmount -> change.consume() - val (dx, dy) = transformDragToPanDelta(dragAmount.x, dragAmount.y) - val maxX = maxZoomPanOffsetX( - previewLayoutWidthPx, - previewLayoutHeightPx, - baseScale, - ) - val maxY = maxZoomPanOffsetY( - previewLayoutWidthPx, - previewLayoutHeightPx, - baseScale, - ) - zoomPanOffsetX = (zoomPanOffsetX + dx).coerceIn(-maxX, maxX) - zoomPanOffsetY = (zoomPanOffsetY + dy).coerceIn(-maxY, maxY) + // Offsets are screen pixels, so the content simply follows the finger. + val maxX = maxZoomPanOffsetX(baseScale) + val maxY = maxZoomPanOffsetY(baseScale) + zoomPanOffsetX = (zoomPanOffsetX + dragAmount.x).coerceIn(-maxX, maxX) + zoomPanOffsetY = (zoomPanOffsetY + dragAmount.y).coerceIn(-maxY, maxY) resetControlsTimer() } } .graphicsLayer { - scaleX = baseScale * if (isFlippedHorizontal) -1f else 1f - scaleY = baseScale * if (isFlippedVertical) -1f else 1f - rotationZ = currentRotation.toFloat() - translationX = if (currentZoom > 0) zoomPanOffsetX else 0f - translationY = if (currentZoom > 0) zoomPanOffsetY else 0f + if (!useSurfaceView) { + scaleX = baseScale * if (isFlippedHorizontal) -1f else 1f + scaleY = baseScale * if (isFlippedVertical) -1f else 1f + rotationZ = currentRotation.toFloat() + val zoomedIn = isPreviewZoomedIn(baseScale) + translationX = if (zoomedIn) zoomPanOffsetX else 0f + translationY = if (zoomedIn) zoomPanOffsetY else 0f + } }, update = { it.isVisible = isPlaybackRunningUi @@ -1676,6 +1794,19 @@ class MainActivity : ComponentActivity() { ) } + if (isOneToOneNoticeVisible && isPlaybackRunningUi) { + Text( + text = getString(R.string.notice_zoom_one_to_one), + color = Color.White, + fontSize = 16.sp, + modifier = Modifier + .align(Alignment.TopStart) + .padding(8.dp) + .background(Color.DarkGray, RoundedCornerShape(8.dp)) + .padding(horizontal = 12.dp, vertical = 8.dp), + ) + } + if (isLowFpsWarningVisible) { val alignRight = isStatsVisible && statsPosition == StatsPosition.BOTTOM_LEFT Text( @@ -2043,19 +2174,18 @@ class MainActivity : ComponentActivity() { BarDivider() Icon(painterResource(R.drawable.ic_zoom_out), null, tint = Color.White, modifier = Modifier.size(24.dp)) WhiteSlider( - value = currentZoom.toFloat(), + value = if (isOneToOneZoom) zoomScaleToPosition(previewZoomScale()) else currentZoom.toFloat(), onValueChange = { - currentZoom = it.roundToInt().coerceIn(0, 100) - if (currentZoom == 0) { - zoomPanOffsetX = 0f - zoomPanOffsetY = 0f + // Dragging the slider takes over from 1:1 mode. + if (isOneToOneZoom) applyOneToOneZoom(false) + val position = it.roundToInt().coerceIn(0, 100) + // Snap to fit-to-screen so the preview can't sit at a barely-off scale. + currentZoom = if (abs(position - ZOOM_FIT_POSITION) <= ZOOM_FIT_SNAP_RANGE) { + ZOOM_FIT_POSITION } else { - clampZoomPanOffsets( - previewLayoutWidthPx, - previewLayoutHeightPx, - previewZoomScale(), - ) + position } + clampZoomPanOffsets(previewZoomScale()) resetControlsTimer() }, onValueChangeFinished = { @@ -2064,6 +2194,8 @@ class MainActivity : ComponentActivity() { }, ) Icon(painterResource(R.drawable.ic_zoom_in), null, tint = Color.White, modifier = Modifier.size(24.dp)) + Spacer(modifier = Modifier.width(12.dp)) + OneToOneZoomToggle() BarDivider() IconButton( onClick = { @@ -2077,6 +2209,37 @@ class MainActivity : ComponentActivity() { } } + @Composable + private fun OneToOneZoomToggle() { + val accent = ConsolationColorScheme.primary + Box( + contentAlignment = Alignment.Center, + modifier = Modifier + .size(34.dp) + .clip(CircleShape) + .border( + 1.dp, + if (isOneToOneZoom) accent else Color(0x99FFFFFF), + CircleShape, + ) + .clickable( + onClickLabel = getString(R.string.action_toggle_zoom_one_to_one), + ) { + applyOneToOneZoom(!isOneToOneZoom) + clampZoomPanOffsets(previewZoomScale()) + persistSettings() + resetControlsTimer() + }, + ) { + Text( + text = "1:1", + color = if (isOneToOneZoom) accent else Color.White, + fontSize = 14.sp, + fontWeight = FontWeight.Bold, + ) + } + } + @Composable private fun WhiteSlider( value: Float, @@ -2731,8 +2894,16 @@ class MainActivity : ComponentActivity() { private const val KEY_ROTATION = "rotation" private const val KEY_FLIP_H = "flip_h" private const val KEY_FLIP_V = "flip_v" + /** Legacy zoom pref (0 = fit); migrated to [KEY_ZOOM_POSITION] on load. */ private const val KEY_ZOOM = "zoom" + private const val KEY_ZOOM_POSITION = "zoom_position" + private const val KEY_ZOOM_ONE_TO_ONE = "zoom_one_to_one" private const val PREVIEW_MAX_ZOOM_SCALE = 1.7625f // 1.175 * 1.5 + private const val PREVIEW_MIN_ZOOM_SCALE = 0.5f + private const val ZOOM_FIT_POSITION = 50 + private const val ZOOM_FIT_SNAP_RANGE = 4 + private const val ONE_TO_ONE_NOTICE_MS = 5_000L + private const val ZOOM_PAN_EPSILON = 0.001f private const val KEY_VOLUME = "volume" private const val KEY_MUTED = "muted" private const val KEY_DEVICE_FORMAT_PREFIX = "device_format:" diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/audio/UsbCaptureAudioLoop.kt b/Consolation/app/src/main/java/org/centennialoss/consolation/audio/UsbCaptureAudioLoop.kt index 9800773..2a67f37 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/audio/UsbCaptureAudioLoop.kt +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/audio/UsbCaptureAudioLoop.kt @@ -4,6 +4,7 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import android.hardware.usb.UsbDevice +import android.os.Build import android.os.SystemClock import android.media.AudioAttributes import android.media.AudioDeviceInfo @@ -109,7 +110,13 @@ class UsbCaptureAudioLoop( val inputCapture: AudioRecord? = try { AudioRecord.Builder() - .setContext(appContext) + .apply { + // setContext() is API 31+. Below that AudioRecord uses the process context, + // which is what appContext is anyway, so there is nothing to substitute. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + setContext(appContext) + } + } .setAudioSource(MediaRecorder.AudioSource.UNPROCESSED) .setAudioFormat( AudioFormat.Builder() diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UsbVideoPreviewBackend.kt b/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UsbVideoPreviewBackend.kt index 4589a4e..7bb4408 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UsbVideoPreviewBackend.kt +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UsbVideoPreviewBackend.kt @@ -59,6 +59,23 @@ interface UsbVideoPreviewBackend : PreviewRenderer { fun setRotation(degrees: Int) fun setFlip(horizontal: Boolean, vertical: Boolean) + /** + * Full preview transform for backends that render into a SurfaceView, where + * the View system cannot rotate or mirror the content: rotation in degrees, + * mirror flags, zoom scale, pan in NDC units (-1..1 across the surface), and the + * fit-to-screen content box as a fraction of the surface ([fitScaleX], [fitScaleY] <= 1). + */ + fun setPreviewTransform( + rotationDegrees: Int, + flipHorizontal: Boolean, + flipVertical: Boolean, + scale: Float, + panXNdc: Float, + panYNdc: Float, + fitScaleX: Float, + fitScaleY: Float, + ) = Unit + fun dispose() /** diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UvccameraLibPreviewBackend.kt b/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UvccameraLibPreviewBackend.kt index 3297594..15f2961 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UvccameraLibPreviewBackend.kt +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/preview/backend/UvccameraLibPreviewBackend.kt @@ -11,6 +11,8 @@ import android.os.Handler import android.os.Looper import android.os.SystemClock import android.view.Surface +import android.view.SurfaceHolder +import android.view.SurfaceView import android.view.TextureView import android.widget.Toast import org.centennialoss.consolation.uvc.Size @@ -47,6 +49,18 @@ class UvccameraLibPreviewBackend( private val appContext = context.applicationContext private var preferredDevice: CaptureDevice? = null private var textureView: TextureView? = null + /** SurfaceView target (all formats except H264, which stays on TextureView). */ + private var surfaceView: SurfaceView? = null + + /** Latest transform from the UI; re-applied whenever a camera starts. */ + private var xformRotation = 0 + private var xformFlipH = false + private var xformFlipV = false + private var xformScale = 1f + private var xformPanX = 0f + private var xformPanY = 0f + private var xformFitX = 1f + private var xformFitY = 1f private var usbMonitorRef: USBMonitor? = null @@ -116,6 +130,8 @@ class UvccameraLibPreviewBackend( private var previewRunning: Boolean = false private val h264DecoderLock = Any() private var h264Decoder: MediaCodec? = null + /** True when [h264OutputSurface] was created by us (TextureView path) and must be released. */ + private var h264OutputSurfaceOwned = false private var h264OutputSurface: Surface? = null private var h264FramePtsUs = 0L @@ -174,9 +190,21 @@ class UvccameraLibPreviewBackend( private var lastNativeIsIsochronous: Boolean = false private var lastNativePublishedCountRaw: Long = 0L + private val holderCallback = object : SurfaceHolder.Callback { + override fun surfaceCreated(holder: SurfaceHolder) { + requestStartPreview(PreviewTarget.Holder(holder)) + } + + override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) = Unit + + override fun surfaceDestroyed(holder: SurfaceHolder) { + stopUvcStreamingAsync() + } + } + private val surfaceListener = object : TextureView.SurfaceTextureListener { override fun onSurfaceTextureAvailable(surface: SurfaceTexture, width: Int, height: Int) { - requestStartPreview(surface) + requestStartPreview(PreviewTarget.Texture(surface)) } override fun onSurfaceTextureSizeChanged(surface: SurfaceTexture, width: Int, height: Int) = @@ -283,35 +311,102 @@ class UvccameraLibPreviewBackend( return null } + /** + * Where frames go. A SurfaceView surface is composited directly by SurfaceFlinger, + * skipping the UI-thread vsync hop a TextureView needs; the native GPU renderer then + * applies rotation/flip/zoom itself (see [setPreviewTransform]). + */ + private sealed class PreviewTarget { + data class Texture(val surfaceTexture: SurfaceTexture) : PreviewTarget() + data class Holder(val holder: SurfaceHolder) : PreviewTarget() + } + override fun bindPreviewSurface(surfaceHandle: Any?) { - textureView = surfaceHandle as? TextureView ?: return - val view = textureView ?: return - view.applyForUsbPreviewLatency() - view.surfaceTextureListener = surfaceListener - if (view.isAvailable) { - val tex = view.surfaceTexture ?: return - requestStartPreview(tex) + when (surfaceHandle) { + is SurfaceView -> { + textureView?.surfaceTextureListener = null + textureView = null + surfaceView = surfaceHandle + val holder = surfaceHandle.holder + holder.addCallback(holderCallback) + if (holder.surface?.isValid == true) { + requestStartPreview(PreviewTarget.Holder(holder)) + } + } + is TextureView -> { + surfaceView?.holder?.removeCallback(holderCallback) + surfaceView = null + textureView = surfaceHandle + surfaceHandle.applyForUsbPreviewLatency() + surfaceHandle.surfaceTextureListener = surfaceListener + if (surfaceHandle.isAvailable) { + val tex = surfaceHandle.surfaceTexture ?: return + requestStartPreview(PreviewTarget.Texture(tex)) + } + } + else -> return } } - override fun unbindPreviewSurface() { + private fun detachPreviewViews() { textureView?.surfaceTextureListener = null textureView = null + surfaceView?.holder?.removeCallback(holderCallback) + surfaceView = null + } + + override fun unbindPreviewSurface() { + detachPreviewViews() stopUvcStreamingAsync() } override fun unbindPreviewSurfaceBlocking() { - textureView?.surfaceTextureListener = null - textureView = null + detachPreviewViews() stopUvcStreamingBlocking() } + override fun setPreviewTransform( + rotationDegrees: Int, + flipHorizontal: Boolean, + flipVertical: Boolean, + scale: Float, + panXNdc: Float, + panYNdc: Float, + fitScaleX: Float, + fitScaleY: Float, + ) { + xformRotation = rotationDegrees + xformFlipH = flipHorizontal + xformFlipV = flipVertical + xformScale = scale + xformPanX = panXNdc + xformPanY = panYNdc + xformFitX = fitScaleX + xformFitY = fitScaleY + uvcCamera?.setPreviewTransform( + rotationDegrees, flipHorizontal, flipVertical, scale, panXNdc, panYNdc, fitScaleX, fitScaleY, + ) + } + + /** The target currently bound to a view, or null. */ + private fun currentPreviewTarget(): PreviewTarget? { + surfaceView?.holder?.let { h -> if (h.surface?.isValid == true) return PreviewTarget.Holder(h) } + textureView?.surfaceTexture?.let { return PreviewTarget.Texture(it) } + return null + } + + /** True while [target] is still the surface the bound view is showing. */ + private fun isCurrentTarget(target: PreviewTarget): Boolean = when (target) { + is PreviewTarget.Texture -> textureView?.surfaceTexture === target.surfaceTexture + is PreviewTarget.Holder -> surfaceView?.holder === target.holder && target.holder.surface?.isValid == true + } + /** - * [surface] must be the active TextureView surface (typically obtained on the main thread). + * [target] must be the active view surface (typically obtained on the main thread). */ - private fun requestStartPreview(surface: SurfaceTexture) { + private fun requestStartPreview(target: PreviewTarget) { uvcPreviewExecutor.execute { - startPreviewIfReadyWithSurface(surface) + startPreviewIfReadyWithSurface(target) } } @@ -534,8 +629,8 @@ class UvccameraLibPreviewBackend( if (uvcCamera != null) { stopUvcStreamingBlocking() mainHandler.post { - val tex = textureView?.surfaceTexture ?: return@post - requestStartPreview(tex) + val target = currentPreviewTarget() ?: return@post + requestStartPreview(target) } } } @@ -674,8 +769,7 @@ class UvccameraLibPreviewBackend( } override fun dispose() { - textureView?.surfaceTextureListener = null - textureView = null + detachPreviewViews() val stopped = uvcPreviewExecutor.submit { stopUvcStreamingBody() } @@ -791,7 +885,7 @@ class UvccameraLibPreviewBackend( * native start/stop: the main thread posts delayed USB-audio runnables that must never block * behind a held monitor (frozen UI when native startPreview blocks). */ - private fun startPreviewIfReadyWithSurface(surfaceTexture: SurfaceTexture) { + private fun startPreviewIfReadyWithSurface(target: PreviewTarget) { val tSession = SystemClock.elapsedRealtime() Log.i( logTag, @@ -814,12 +908,12 @@ class UvccameraLibPreviewBackend( Log.d(logTag, "playback: abort no USB permission") return } - if (textureView == null) { - Log.d(logTag, "playback: abort no textureView") + if (textureView == null && surfaceView == null) { + Log.d(logTag, "playback: abort no preview view") return } - if (textureView?.surfaceTexture !== surfaceTexture) { - Log.i(logTag, "playback: abort stale SurfaceTexture before UVC open") + if (!isCurrentTarget(target)) { + Log.i(logTag, "playback: abort stale preview surface before UVC open") return } @@ -887,9 +981,23 @@ class UvccameraLibPreviewBackend( Log.i(logTag, "playback: usb bandwidth hint unavailable; auto prefers uncompressed") } val autoOrder = buildAutoFormatOrder(nativeFrameFormat, lowBandwidthHint == true) - val selectedOrder = preferredPixelFormatOverride?.let { pref -> + val requestedOrder = preferredPixelFormatOverride?.let { pref -> listOf(pref) + autoOrder.filter { it != pref } } ?: autoOrder + /* The UI binds a SurfaceView for every non-H.264 request and relies on the native + * renderer for rotation/mirror/zoom/pan. MediaCodec output bypasses that renderer (and + * Compose cannot transform a SurfaceView), so never fall back to H.264 on a holder. */ + val selectedOrder = if (target is PreviewTarget.Holder && + preferredPixelFormatOverride != UVCCamera.FRAME_FORMAT_H264 + ) { + requestedOrder.filter { it != UVCCamera.FRAME_FORMAT_H264 }.also { + if (it.size != requestedOrder.size) { + Log.i(logTag, "playback: H.264 fallback excluded for SurfaceView target") + } + } + } else { + requestedOrder + } val selectedFrameFormat = selectedOrder.firstNotNullOfOrNull { format -> trySetPreviewSize(camera, width, height, minFps, maxFps, fps, format, bwFactor).also { if (it != null) { @@ -905,21 +1013,42 @@ class UvccameraLibPreviewBackend( currentFpsConfigured = fps currentPixelFormat = frameFormatName(selectedFrameFormat) - if (textureView?.surfaceTexture !== surfaceTexture) { - Log.i(logTag, "playback: abort stale SurfaceTexture after UVC negotiation") + if (!isCurrentTarget(target)) { + Log.i(logTag, "playback: abort stale preview surface after UVC negotiation") stopUvcStreamingBody() return } - surfaceTexture.setDefaultBufferSize(width, height) - if (selectedFrameFormat == UVCCamera.FRAME_FORMAT_H264) { - t1 = SystemClock.elapsedRealtime() - startH264Decoder(surfaceTexture, width, height) - Log.i(logTag, "playback: startH264Decoder ${SystemClock.elapsedRealtime() - t1}ms") - } else { - t1 = SystemClock.elapsedRealtime() - camera.setPreviewTexture(surfaceTexture) - Log.i(logTag, "playback: setPreviewTexture ${SystemClock.elapsedRealtime() - t1}ms") + when (target) { + is PreviewTarget.Texture -> { + target.surfaceTexture.setDefaultBufferSize(width, height) + if (selectedFrameFormat == UVCCamera.FRAME_FORMAT_H264) { + t1 = SystemClock.elapsedRealtime() + startH264Decoder(Surface(target.surfaceTexture), width, height, ownsSurface = true) + Log.i(logTag, "playback: startH264Decoder ${SystemClock.elapsedRealtime() - t1}ms") + } else { + t1 = SystemClock.elapsedRealtime() + camera.setPreviewTexture(target.surfaceTexture) + Log.i(logTag, "playback: setPreviewTexture ${SystemClock.elapsedRealtime() - t1}ms") + } + } + is PreviewTarget.Holder -> { + /* Buffer geometry is set natively (ANativeWindow_setBuffersGeometry); + * SurfaceFlinger scales the frame-sized buffers to the view. */ + if (selectedFrameFormat == UVCCamera.FRAME_FORMAT_H264) { + t1 = SystemClock.elapsedRealtime() + startH264Decoder(target.holder.surface, width, height, ownsSurface = false) + Log.i(logTag, "playback: startH264Decoder(surface) ${SystemClock.elapsedRealtime() - t1}ms") + } else { + t1 = SystemClock.elapsedRealtime() + camera.setPreviewDisplay(target.holder.surface) + Log.i(logTag, "playback: setPreviewDisplay ${SystemClock.elapsedRealtime() - t1}ms") + } + camera.setPreviewTransform( + xformRotation, xformFlipH, xformFlipV, xformScale, xformPanX, xformPanY, + xformFitX, xformFitY, + ) + } } loggedFirstVideoFrame.set(false) @@ -1102,9 +1231,9 @@ class UvccameraLibPreviewBackend( ) } - private fun startH264Decoder(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + private fun startH264Decoder(surface: Surface, width: Int, height: Int, ownsSurface: Boolean) { stopH264Decoder() - val surface = Surface(surfaceTexture) + h264OutputSurfaceOwned = ownsSurface val format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height) val decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC) decoder.configure(format, surface, null, 0) @@ -1129,7 +1258,10 @@ class UvccameraLibPreviewBackend( } runCatching { decoder?.stop() } runCatching { decoder?.release() } - runCatching { surface?.release() } + if (h264OutputSurfaceOwned) { + runCatching { surface?.release() } + } + h264OutputSurfaceOwned = false } private fun queueH264Frame(encodedFrame: ByteBuffer) { diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/usb/UsbCaptureDeviceRepository.kt b/Consolation/app/src/main/java/org/centennialoss/consolation/usb/UsbCaptureDeviceRepository.kt index 397e13e..356d13c 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/usb/UsbCaptureDeviceRepository.kt +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/usb/UsbCaptureDeviceRepository.kt @@ -11,6 +11,7 @@ import android.hardware.usb.UsbManager import android.os.Build import android.os.Parcelable import androidx.core.content.ContextCompat +import androidx.core.content.IntentCompat import org.centennialoss.consolation.core.capture.CaptureDevice import org.centennialoss.consolation.logging.AppLog as Log import kotlinx.coroutines.flow.MutableSharedFlow @@ -310,4 +311,4 @@ class UsbCaptureDeviceRepository( } private inline fun Intent.parcelableExtraCompat(name: String): T? = - getParcelableExtra(name, T::class.java) + IntentCompat.getParcelableExtra(this, name, T::class.java) diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/USBMonitor.java b/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/USBMonitor.java index b5c050c..4045670 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/USBMonitor.java +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/USBMonitor.java @@ -48,6 +48,8 @@ import android.util.Log; import android.util.SparseArray; +import androidx.core.content.IntentCompat; + import org.centennialoss.consolation.uvc.util.HandlerThreadHandler; public final class USBMonitor { @@ -504,6 +506,14 @@ public void releaseCachedDevice(final UsbDevice device) { } } + /** + * Reads the {@link UsbDevice} extra from a USB broadcast. IntentCompat keeps this working + * below API 33, where the typed Intent#getParcelableExtra(String, Class) overload is absent. + */ + private static UsbDevice getUsbDeviceExtra(final Intent intent) { + return IntentCompat.getParcelableExtra(intent, UsbManager.EXTRA_DEVICE, UsbDevice.class); + } + /** * BroadcastReceiver for USB permission */ @@ -516,7 +526,7 @@ public void onReceive(final Context context, final Intent intent) { if (ACTION_USB_PERMISSION.equals(action)) { // when received the result of requesting USB permission synchronized (USBMonitor.this) { - final UsbDevice device = intent.getParcelableExtra(UsbManager.EXTRA_DEVICE, UsbDevice.class); + final UsbDevice device = getUsbDeviceExtra(intent); if (intent.getBooleanExtra(UsbManager.EXTRA_PERMISSION_GRANTED, false)) { if (device != null) { // get permission, call onConnect @@ -528,12 +538,12 @@ public void onReceive(final Context context, final Intent intent) { } } } else if (UsbManager.ACTION_USB_DEVICE_ATTACHED.equals(action)) { - final UsbDevice device = intent.getParcelableExtra(UsbManager.EXTRA_DEVICE, UsbDevice.class); + final UsbDevice device = getUsbDeviceExtra(intent); updatePermission(device, hasPermission(device)); processAttach(device); } else if (UsbManager.ACTION_USB_DEVICE_DETACHED.equals(action)) { // when device removed - final UsbDevice device = intent.getParcelableExtra(UsbManager.EXTRA_DEVICE, UsbDevice.class); + final UsbDevice device = getUsbDeviceExtra(intent); if (device != null) { UsbControlBlock ctrlBlock = mCtrlBlocks.remove(device); if (ctrlBlock != null) { diff --git a/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/UVCCamera.java b/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/UVCCamera.java index d3db7b3..dcad37d 100644 --- a/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/UVCCamera.java +++ b/Consolation/app/src/main/java/org/centennialoss/consolation/uvc/UVCCamera.java @@ -481,12 +481,28 @@ public synchronized void setPreviewDisplay(final Surface surface) { nativeSetPreviewDisplay(mNativePtr, surface); } + /** + * Rotation (0/90/180/270), mirror flags, zoom scale, pan (in NDC units, + * -1..1 across the surface) and the fit-to-screen content box as a fraction + * of the surface (fitX, fitY <= 1) applied by the native GPU renderer. + * Synchronized with {@link #destroy()}: the UI thread calls this while the + * preview executor may be freeing the native object. + */ + public synchronized void setPreviewTransform(final int rotationDegrees, final boolean flipH, + final boolean flipV, final float scale, final float panXNdc, final float panYNdc, + final float fitX, final float fitY) { + if (mNativePtr != 0) { + nativeSetPreviewTransform(mNativePtr, rotationDegrees, flipH, flipV, scale, panXNdc, panYNdc, + fitX, fitY); + } + } + /** * set frame callback * @param callback * @param pixelFormat */ - public void setFrameCallback(final IFrameCallback callback, final int pixelFormat) { + public synchronized void setFrameCallback(final IFrameCallback callback, final int pixelFormat) { if (mNativePtr != 0) { nativeSetFrameCallback(mNativePtr, callback, pixelFormat); } @@ -499,13 +515,13 @@ public void setFrameCallback(final IFrameCallback callback, final int pixelForma * deliver lightweight rendered-frame notifications so apps can count frames without entering * the capture path. */ - public void setPreviewFrameCallback(final IFrameCallback callback, final int pixelFormat) { + public synchronized void setPreviewFrameCallback(final IFrameCallback callback, final int pixelFormat) { if (mNativePtr != 0) { nativeSetPreviewFrameCallback(mNativePtr, callback, pixelFormat); } } - public long[] getAndResetProcessingStats() { + public synchronized long[] getAndResetProcessingStats() { if (mNativePtr != 0) { final long[] stats = nativeGetAndResetProcessingStats(mNativePtr); if (stats != null) { @@ -1151,6 +1167,9 @@ private final String getUSBFSName(final UsbControlBlock ctrlBlock) { private static final native int nativeStartPreview(final long id_camera); private static final native int nativeStopPreview(final long id_camera); private static final native int nativeSetPreviewDisplay(final long id_camera, final Surface surface); + private static final native int nativeSetPreviewTransform(final long id_camera, final int rotation, + final boolean flipH, final boolean flipV, final float scale, final float panX, final float panY, + final float fitX, final float fitY); private static final native int nativeSetPreviewFrameCallback(final long mNativePtr, final IFrameCallback callback, final int pixelFormat); private static final native int nativeSetFrameCallback(final long mNativePtr, final IFrameCallback callback, final int pixelFormat); private static final native long[] nativeGetAndResetProcessingStats(final long mNativePtr); diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCCamera.cpp b/Consolation/app/src/main/jni/UVCCamera/UVCCamera.cpp index 39266ce..f7fec0e 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCCamera.cpp +++ b/Consolation/app/src/main/jni/UVCCamera/UVCCamera.cpp @@ -290,6 +290,15 @@ int UVCCamera::setPreviewDisplay(ANativeWindow *preview_window) { RETURN(result, int); } +int UVCCamera::setPreviewTransform(int rotation_degrees, bool flip_h, bool flip_v, + float scale, float pan_x_ndc, float pan_y_ndc, float fit_x, float fit_y) { + int result = EXIT_FAILURE; + if (mPreview) + result = mPreview->setPreviewTransform(rotation_degrees, flip_h, flip_v, + scale, pan_x_ndc, pan_y_ndc, fit_x, fit_y); + return result; +} + int UVCCamera::setFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format) { ENTER(); int result = EXIT_FAILURE; diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCCamera.h b/Consolation/app/src/main/jni/UVCCamera/UVCCamera.h index ad0f70e..4383d25 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCCamera.h +++ b/Consolation/app/src/main/jni/UVCCamera/UVCCamera.h @@ -192,6 +192,7 @@ class UVCCamera { char *getDescriptions(); int setPreviewSize(int width, int height, int min_fps, int max_fps, int mode, float bandwidth = DEFAULT_BANDWIDTH); int setPreviewDisplay(ANativeWindow *preview_window); + int setPreviewTransform(int rotation_degrees, bool flip_h, bool flip_v, float scale, float pan_x_ndc, float pan_y_ndc, float fit_x, float fit_y); int setPreviewFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format); int setFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format); int startPreview(); diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.cpp b/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.cpp index 6305a3a..2593f73 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.cpp +++ b/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace { @@ -24,10 +25,11 @@ static const char *vertex_shader_src = "#version 300 es\n" "layout(location=0) in vec2 aPos;\n" "layout(location=1) in vec2 aTex;\n" + "uniform mat3 uXform;\n" "out vec2 vTex;\n" "void main() {\n" " vTex = aTex;\n" - " gl_Position = vec4(aPos, 0.0, 1.0);\n" + " gl_Position = vec4((uXform * vec3(aPos, 1.0)).xy, 0.0, 1.0);\n" "}\n"; static const char *yuyv_fragment_shader_src = @@ -150,6 +152,47 @@ static const char *mjpeg_planar_fragment_shader_src = " fragColor = vec4(clamp(yuvToRgb(yy, uu, vv), 0.0, 1.0), 1.0);\n" "}\n"; +/* Same as the planar shader, but each plane lives in an RGBA8 texture a + * quarter as wide: byte x of a row is component (x & 3) of texel x >> 2. */ +static const char *mjpeg_planar_packed_fragment_shader_src = + "#version 300 es\n" + "precision highp float;\n" + "precision highp int;\n" + "uniform sampler2D uY;\n" + "uniform sampler2D uU;\n" + "uniform sampler2D uV;\n" + "uniform int uWidth;\n" + "uniform int uHeight;\n" + "uniform int uChromaWidth;\n" + "uniform int uChromaHeight;\n" + "uniform int uGray;\n" + "in vec2 vTex;\n" + "out vec4 fragColor;\n" + "float planeByte(sampler2D s, int x, int y) {\n" + " vec4 v = texelFetch(s, ivec2(x >> 2, y), 0);\n" + " int c = x & 3;\n" + " return c == 0 ? v.r : (c == 1 ? v.g : (c == 2 ? v.b : v.a));\n" + "}\n" + "vec3 yuvToRgb(float y, float u, float v) {\n" + " u -= 0.5;\n" + " v -= 0.5;\n" + " return vec3(y + 1.402 * v, y - 0.344136 * u - 0.714136 * v, y + 1.772 * u);\n" + "}\n" + "void main() {\n" + " int x = clamp(int(vTex.x * float(uWidth)), 0, uWidth - 1);\n" + " int yrow = clamp(int(vTex.y * float(uHeight)), 0, uHeight - 1);\n" + " float yy = planeByte(uY, x, yrow);\n" + " if (uGray != 0) {\n" + " fragColor = vec4(yy, yy, yy, 1.0);\n" + " return;\n" + " }\n" + " int cx = clamp((x * uChromaWidth) / uWidth, 0, uChromaWidth - 1);\n" + " int cy = clamp((yrow * uChromaHeight) / uHeight, 0, uChromaHeight - 1);\n" + " float uu = planeByte(uU, cx, cy);\n" + " float vv = planeByte(uV, cx, cy);\n" + " fragColor = vec4(clamp(yuvToRgb(yy, uu, vv), 0.0, 1.0), 1.0);\n" + "}\n"; + static const char *bgr_fragment_shader_src = "#version 300 es\n" "precision highp float;\n" @@ -262,9 +305,14 @@ enum ProgramKind { PROGRAM_BGR, PROGRAM_P010, PROGRAM_HARDWARE_LINEAR, + PROGRAM_MJPEG_PLANAR_PACKED, PROGRAM_COUNT }; +/* Re-query the EGL surface size this often (frames); it only changes on a + * geometry change, which also resets the surface in practice. */ +static const uint32_t SURFACE_SIZE_REFRESH_FRAMES = 64; + static GLuint compile_shader(GLenum type, const char *src) { GLuint shader = glCreateShader(type); @@ -317,21 +365,60 @@ static size_t frame_actual_bytes(const uvc_frame_t *frame) } // namespace struct UVCGpuPreviewRenderer::Impl { + /* Uniform locations resolved once at link time (glGetUniformLocation is a + * string lookup in the driver; it was being done ~8x per frame). -1 = absent. */ + struct Uniforms { + GLint tex0 = -1, tex1 = -1, tex2 = -1; + GLint width = -1, height = -1; + GLint chromaWidth = -1, chromaHeight = -1, gray = -1, uyvy = -1; + GLint storageWidth = -1, format = -1; + GLint xform = -1; + }; + /* Identity until UVCPreview pushes rotation/flip/zoom/pan (column-major). */ + float xform[9] = { 1, 0, 0, 0, 1, 0, 0, 0, 1 }; + EGLDisplay display = EGL_NO_DISPLAY; EGLContext context = EGL_NO_CONTEXT; EGLSurface surface = EGL_NO_SURFACE; EGLConfig config = nullptr; ANativeWindow *window = nullptr; GLuint programs[PROGRAM_COUNT] = {}; + Uniforms uniforms[PROGRAM_COUNT]; GLuint textures[3] = {}; - int mjpegTextureWidths[3] = {}; - int mjpegTextureHeights[3] = {}; + /* Storage spec currently allocated for each of the 3 texture units, so the + * steady state is one glTexSubImage2D per plane (no re-specification). */ + int texWidth[3] = {}; + int texHeight[3] = {}; + GLenum texInternal[3] = {}; + GLint texFilter[3] = {}; GLuint hardwareTexture = 0; EGLImageKHR hardwareImage = EGL_NO_IMAGE_KHR; void *hardwareBuffer = nullptr; GLuint vbo = 0; + GLuint vao = 0; int surface_width = 0; int surface_height = 0; + uint32_t frames_since_size_query = 0; + /* EGL/GLES extension entry points, resolved once per EGL init. */ + PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC pGetNativeClientBuffer = nullptr; + PFNEGLCREATEIMAGEKHRPROC pCreateImage = nullptr; + PFNEGLDESTROYIMAGEKHRPROC pDestroyImage = nullptr; + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC pImageTargetTexture = nullptr; + PFNEGLCREATESYNCKHRPROC pCreateSync = nullptr; + PFNEGLDESTROYSYNCKHRPROC pDestroySync = nullptr; + PFNEGLDUPNATIVEFENCEFDANDROIDPROC pDupNativeFenceFD = nullptr; + + /* EGLImage + texture per decoder plane buffer, keyed by the frame's + * allocation id (pointers get recycled; ids never do). */ + struct PlaneImage { + uint64_t id = 0; + EGLImageKHR image = EGL_NO_IMAGE_KHR; + GLuint tex = 0; + }; + static const int PLANE_IMAGE_CACHE = 16; + PlaneImage planeImages[PLANE_IMAGE_CACHE]; + int planeImageCount = 0; + int lastRenderFenceFd = -1; bool ensureEgl(ANativeWindow *target); bool ensureSurface(ANativeWindow *target); @@ -339,18 +426,17 @@ struct UVCGpuPreviewRenderer::Impl { void destroyGl(); GLuint program(ProgramKind kind); void setupGeometry(); + void refreshSurfaceSize(bool force); + void drawQuad(ProgramKind kind); bool drawHardwareBuffer(uvc_frame_t *frame); + bool drawPlanarHardware(uvc_frame_t *frame); + GLuint planeTextureFor(void *ahb, uint64_t id); + void destroyPlaneImages(); + void captureRenderFence(); bool uploadAndDraw(uvc_frame_t *frame); - void resetMjpegTextureStorage(); - bool uploadR8(GLuint tex, int width, int height, const void *data); - bool uploadR8Stride(GLuint tex, int width, int height, int stride, - const void *data); - bool uploadMjpegPlane(int plane, int width, int height, int stride, - const void *data); - bool uploadRG8(GLuint tex, int width, int height, const void *data); - bool uploadRGB8(GLuint tex, int width, int height, const void *data); - bool uploadR16UI(GLuint tex, int width, int height, const void *data); - bool uploadRG16UI(GLuint tex, int width, int height, const void *data); + void resetTextureStorage(); + bool uploadTexture(int unit, GLenum internal, GLenum format, GLenum type, + GLint filter, int width, int height, int stride_px, const void *data); }; UVCGpuPreviewRenderer::UVCGpuPreviewRenderer() @@ -368,10 +454,28 @@ UVCGpuPreviewRenderer::~UVCGpuPreviewRenderer() bool UVCGpuPreviewRenderer::render(uvc_frame_t *frame, ANativeWindow *window, uint64_t *frame_ready_ns) { - if (!impl || !frame || !window || !frame->data) + if (!impl || !frame || !window) + return false; + if (!frame->data && !frame->yuv_hardware_buffers[0]) return false; if (!impl->ensureEgl(window) || !impl->ensureSurface(window)) return false; + if (frame->frame_format == UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR + && frame->yuv_hardware_buffers[0]) { + /* Zero-copy path: planes are already in GPU-sampleable memory. */ + if (!impl->drawPlanarHardware(frame)) + return false; + const uint64_t ready_ns = now_ns(); + if (eglSwapBuffers(impl->display, impl->surface) != EGL_TRUE) { + LOGW("gpu-preview: eglSwapBuffers failed err=0x%x", eglGetError()); + impl->destroySurface(); + return false; + } + impl->captureRenderFence(); + if (frame_ready_ns) + *frame_ready_ns = ready_ns; + return true; + } if (frame->library_hardware_buffer) { if (impl->drawHardwareBuffer(frame)) { const uint64_t ready_ns = now_ns(); @@ -403,6 +507,27 @@ bool UVCGpuPreviewRenderer::render(uvc_frame_t *frame, ANativeWindow *window, return true; } +int UVCGpuPreviewRenderer::takeRenderFenceFd() +{ + if (!impl) + return -1; + const int fd = impl->lastRenderFenceFd; + impl->lastRenderFenceFd = -1; + return fd; +} + +void UVCGpuPreviewRenderer::setTransform(const float m[9]) +{ + if (impl && m) + memcpy(impl->xform, m, sizeof(impl->xform)); +} + +void UVCGpuPreviewRenderer::invalidateSurfaceSize() +{ + if (impl) + impl->frames_since_size_query = SURFACE_SIZE_REFRESH_FRAMES; +} + void UVCGpuPreviewRenderer::resetSurface() { if (impl) @@ -464,6 +589,17 @@ bool UVCGpuPreviewRenderer::Impl::ensureEgl(ANativeWindow *target) return false; } + pGetNativeClientBuffer = (PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC) + eglGetProcAddress("eglGetNativeClientBufferANDROID"); + pCreateImage = (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); + pDestroyImage = (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); + pImageTargetTexture = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC) + eglGetProcAddress("glEGLImageTargetTexture2DOES"); + pCreateSync = (PFNEGLCREATESYNCKHRPROC)eglGetProcAddress("eglCreateSyncKHR"); + pDestroySync = (PFNEGLDESTROYSYNCKHRPROC)eglGetProcAddress("eglDestroySyncKHR"); + pDupNativeFenceFD = (PFNEGLDUPNATIVEFENCEFDANDROIDPROC) + eglGetProcAddress("eglDupNativeFenceFDANDROID"); + (void)target; return true; } @@ -487,7 +623,9 @@ bool UVCGpuPreviewRenderer::Impl::ensureSurface(ANativeWindow *target) } eglSwapInterval(display, 0); glGenTextures(3, textures); + resetTextureStorage(); setupGeometry(); + refreshSurfaceSize(true); return true; } @@ -499,13 +637,11 @@ void UVCGpuPreviewRenderer::Impl::destroySurface() if (textures[0] || textures[1] || textures[2]) { glDeleteTextures(3, textures); memset(textures, 0, sizeof(textures)); - resetMjpegTextureStorage(); + resetTextureStorage(); } if (hardwareImage != EGL_NO_IMAGE_KHR) { - PFNEGLDESTROYIMAGEKHRPROC destroyImage = - (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); - if (destroyImage) - destroyImage(display, hardwareImage); + if (pDestroyImage) + pDestroyImage(display, hardwareImage); hardwareImage = EGL_NO_IMAGE_KHR; hardwareBuffer = nullptr; } @@ -513,6 +649,15 @@ void UVCGpuPreviewRenderer::Impl::destroySurface() glDeleteTextures(1, &hardwareTexture); hardwareTexture = 0; } + destroyPlaneImages(); + if (lastRenderFenceFd >= 0) { + close(lastRenderFenceFd); + lastRenderFenceFd = -1; + } + if (vao) { + glDeleteVertexArrays(1, &vao); + vao = 0; + } if (vbo) { glDeleteBuffers(1, &vbo); vbo = 0; @@ -536,12 +681,17 @@ void UVCGpuPreviewRenderer::Impl::destroyGl() glDeleteProgram(programs[i]); programs[i] = 0; } + uniforms[i] = Uniforms(); } if (display != EGL_NO_DISPLAY && context != EGL_NO_CONTEXT) { eglDestroyContext(display, context); context = EGL_NO_CONTEXT; } config = nullptr; + pGetNativeClientBuffer = nullptr; + pCreateImage = nullptr; + pDestroyImage = nullptr; + pImageTargetTexture = nullptr; } GLuint UVCGpuPreviewRenderer::Impl::program(ProgramKind kind) @@ -572,38 +722,100 @@ GLuint UVCGpuPreviewRenderer::Impl::program(ProgramKind kind) case PROGRAM_HARDWARE_LINEAR: src = hardware_linear_fragment_shader_src; break; + case PROGRAM_MJPEG_PLANAR_PACKED: + src = mjpeg_planar_packed_fragment_shader_src; + break; default: return 0; } - programs[kind] = link_program(src); - return programs[kind]; + const GLuint prog = link_program(src); + programs[kind] = prog; + if (!prog) + return 0; + + /* Resolve every uniform this program might declare; absent ones are -1 and + * glUniform1i(-1, ...) is a defined no-op. Sampler bindings never change, + * so set them once here. */ + Uniforms &u = uniforms[kind]; + static const char *tex0_names[PROGRAM_COUNT] = { + "uPacked", "uY", "uY", "uY", "uBgr", "uY16", "uStorage", "uY" }; + static const char *tex1_names[PROGRAM_COUNT] = { + nullptr, "uUV", "uU", "uU", nullptr, "uUV16", nullptr, "uU" }; + static const char *tex2_names[PROGRAM_COUNT] = { + nullptr, nullptr, "uV", "uV", nullptr, nullptr, nullptr, "uV" }; + u.tex0 = tex0_names[kind] ? glGetUniformLocation(prog, tex0_names[kind]) : -1; + u.tex1 = tex1_names[kind] ? glGetUniformLocation(prog, tex1_names[kind]) : -1; + u.tex2 = tex2_names[kind] ? glGetUniformLocation(prog, tex2_names[kind]) : -1; + u.width = glGetUniformLocation(prog, "uWidth"); + u.height = glGetUniformLocation(prog, "uHeight"); + u.chromaWidth = glGetUniformLocation(prog, "uChromaWidth"); + u.chromaHeight = glGetUniformLocation(prog, "uChromaHeight"); + u.gray = glGetUniformLocation(prog, "uGray"); + u.uyvy = glGetUniformLocation(prog, "uUyvy"); + u.storageWidth = glGetUniformLocation(prog, "uStorageWidth"); + u.format = glGetUniformLocation(prog, "uFormat"); + u.xform = glGetUniformLocation(prog, "uXform"); + glUseProgram(prog); + glUniform1i(u.tex0, 0); + glUniform1i(u.tex1, 1); + glUniform1i(u.tex2, 2); + return prog; } void UVCGpuPreviewRenderer::Impl::setupGeometry() { if (!vbo) glGenBuffers(1, &vbo); + if (!vao) + glGenVertexArrays(1, &vao); static const GLfloat vertices[] = { -1.0f, 1.0f, 0.0f, 0.0f, -1.0f, -1.0f, 0.0f, 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f, -1.0f, 1.0f, 1.0f, }; + /* Vertex layout is captured in the VAO once; per frame it is one bind. */ + glBindVertexArray(vao); glBindBuffer(GL_ARRAY_BUFFER, vbo); glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + glEnableVertexAttribArray(0); + glEnableVertexAttribArray(1); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (const void *)0); + glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), + (const void *)(2 * sizeof(GLfloat))); + glBindVertexArray(0); +} + +void UVCGpuPreviewRenderer::Impl::refreshSurfaceSize(bool force) +{ + if (!force && ++frames_since_size_query < SURFACE_SIZE_REFRESH_FRAMES) + return; + frames_since_size_query = 0; + EGLint sw = 0; + EGLint sh = 0; + eglQuerySurface(display, surface, EGL_WIDTH, &sw); + eglQuerySurface(display, surface, EGL_HEIGHT, &sh); + surface_width = sw; + surface_height = sh; +} + +void UVCGpuPreviewRenderer::Impl::drawQuad(ProgramKind kind) +{ + refreshSurfaceSize(false); + glViewport(0, 0, surface_width, surface_height); + /* A zoom scale below 1 (zoomed out / 1:1 on a small stream) leaves part of the + * surface uncovered by the quad; clear it so the borders are black. */ + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + glUniformMatrix3fv(uniforms[kind].xform, 1, GL_FALSE, xform); + glBindVertexArray(vao); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + glBindVertexArray(0); } bool UVCGpuPreviewRenderer::Impl::drawHardwareBuffer(uvc_frame_t *frame) { - PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC getNativeClientBuffer = - (PFNEGLGETNATIVECLIENTBUFFERANDROIDPROC)eglGetProcAddress( - "eglGetNativeClientBufferANDROID"); - PFNEGLCREATEIMAGEKHRPROC createImage = - (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); - PFNGLEGLIMAGETARGETTEXTURE2DOESPROC imageTargetTexture = - (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress( - "glEGLImageTargetTexture2DOES"); - if (!getNativeClientBuffer || !createImage || !imageTargetTexture) + if (!pGetNativeClientBuffer || !pCreateImage || !pImageTargetTexture) return false; AHardwareBuffer_Desc desc; @@ -620,17 +832,15 @@ bool UVCGpuPreviewRenderer::Impl::drawHardwareBuffer(uvc_frame_t *frame) do { if (hardwareBuffer != frame->library_hardware_buffer) { if (hardwareImage != EGL_NO_IMAGE_KHR) { - PFNEGLDESTROYIMAGEKHRPROC destroyImage = - (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); - if (destroyImage) - destroyImage(display, hardwareImage); + if (pDestroyImage) + pDestroyImage(display, hardwareImage); hardwareImage = EGL_NO_IMAGE_KHR; } - EGLClientBuffer clientBuffer = getNativeClientBuffer( + EGLClientBuffer clientBuffer = pGetNativeClientBuffer( (AHardwareBuffer *)frame->library_hardware_buffer); if (!clientBuffer) break; - hardwareImage = createImage(display, EGL_NO_CONTEXT, + hardwareImage = pCreateImage(display, EGL_NO_CONTEXT, EGL_NATIVE_BUFFER_ANDROID, clientBuffer, NULL); if (hardwareImage == EGL_NO_IMAGE_KHR) { LOGW("gpu-preview: eglCreateImageKHR(AHB) failed err=0x%x", @@ -640,41 +850,31 @@ bool UVCGpuPreviewRenderer::Impl::drawHardwareBuffer(uvc_frame_t *frame) hardwareBuffer = frame->library_hardware_buffer; } - if (!hardwareTexture) + if (!hardwareTexture) { glGenTextures(1, &hardwareTexture); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, hardwareTexture); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - imageTargetTexture(GL_TEXTURE_2D, hardwareImage); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, hardwareTexture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } else { + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, hardwareTexture); + } + pImageTargetTexture(GL_TEXTURE_2D, hardwareImage); GLuint prog = program(PROGRAM_HARDWARE_LINEAR); if (!prog) break; + const Uniforms &u = uniforms[PROGRAM_HARDWARE_LINEAR]; glUseProgram(prog); - glUniform1i(glGetUniformLocation(prog, "uStorage"), 0); - glUniform1i(glGetUniformLocation(prog, "uWidth"), (int)frame->width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), (int)frame->height); - glUniform1i(glGetUniformLocation(prog, "uStorageWidth"), (int)desc.width); - glUniform1i(glGetUniformLocation(prog, "uFormat"), (int)frame->frame_format); - - EGLint sw = 0; - EGLint sh = 0; - eglQuerySurface(display, surface, EGL_WIDTH, &sw); - eglQuerySurface(display, surface, EGL_HEIGHT, &sh); - glViewport(0, 0, sw, sh); - glBindBuffer(GL_ARRAY_BUFFER, vbo); - glEnableVertexAttribArray(0); - glEnableVertexAttribArray(1); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), - (const void *)0); - glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), - (const void *)(2 * sizeof(GLfloat))); - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - glDisableVertexAttribArray(0); - glDisableVertexAttribArray(1); + glUniform1i(u.width, (int)frame->width); + glUniform1i(u.height, (int)frame->height); + glUniform1i(u.storageWidth, (int)desc.width); + glUniform1i(u.format, (int)frame->frame_format); + + drawQuad(PROGRAM_HARDWARE_LINEAR); const GLenum err = glGetError(); if (err != GL_NO_ERROR) { @@ -693,114 +893,172 @@ bool UVCGpuPreviewRenderer::Impl::drawHardwareBuffer(uvc_frame_t *frame) return ok; } -void UVCGpuPreviewRenderer::Impl::resetMjpegTextureStorage() +void UVCGpuPreviewRenderer::Impl::destroyPlaneImages() { - memset(mjpegTextureWidths, 0, sizeof(mjpegTextureWidths)); - memset(mjpegTextureHeights, 0, sizeof(mjpegTextureHeights)); + for (int i = 0; i < planeImageCount; i++) { + if (planeImages[i].tex) + glDeleteTextures(1, &planeImages[i].tex); + if (planeImages[i].image != EGL_NO_IMAGE_KHR && pDestroyImage) + pDestroyImage(display, planeImages[i].image); + planeImages[i] = PlaneImage(); + } + planeImageCount = 0; } -bool UVCGpuPreviewRenderer::Impl::uploadR8(GLuint tex, int width, int height, - const void *data) +GLuint UVCGpuPreviewRenderer::Impl::planeTextureFor(void *ahb, uint64_t id) { + for (int i = 0; i < planeImageCount; i++) + if (planeImages[i].id == id) + return planeImages[i].tex; + if (!pGetNativeClientBuffer || !pCreateImage || !pImageTargetTexture) + return 0; + if (planeImageCount == PLANE_IMAGE_CACHE) { + /* Evict the oldest entry; the decoder pool is far smaller than this. */ + if (planeImages[0].tex) + glDeleteTextures(1, &planeImages[0].tex); + if (planeImages[0].image != EGL_NO_IMAGE_KHR && pDestroyImage) + pDestroyImage(display, planeImages[0].image); + memmove(&planeImages[0], &planeImages[1], + sizeof(PlaneImage) * (PLANE_IMAGE_CACHE - 1)); + planeImages[PLANE_IMAGE_CACHE - 1] = PlaneImage(); + planeImageCount--; + } + EGLClientBuffer clientBuffer = pGetNativeClientBuffer((AHardwareBuffer *)ahb); + if (!clientBuffer) + return 0; + EGLImageKHR image = pCreateImage(display, EGL_NO_CONTEXT, + EGL_NATIVE_BUFFER_ANDROID, clientBuffer, NULL); + if (image == EGL_NO_IMAGE_KHR) { + LOGW("gpu-preview: eglCreateImageKHR(plane AHB) failed err=0x%x", eglGetError()); + return 0; + } + GLuint tex = 0; + glGenTextures(1, &tex); glBindTexture(GL_TEXTURE_2D, tex); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, - GL_UNSIGNED_BYTE, data); - return glGetError() == GL_NO_ERROR; + pImageTargetTexture(GL_TEXTURE_2D, image); + if (glGetError() != GL_NO_ERROR) { + glDeleteTextures(1, &tex); + if (pDestroyImage) + pDestroyImage(display, image); + LOGW("gpu-preview: glEGLImageTargetTexture2DOES(plane) failed"); + return 0; + } + PlaneImage &e = planeImages[planeImageCount++]; + e.id = id; + e.image = image; + e.tex = tex; + return tex; } -bool UVCGpuPreviewRenderer::Impl::uploadR8Stride(GLuint tex, int width, int height, - int stride, const void *data) +/* Record a native fence for the commands just issued so the decoder can wait + * for the GPU to finish reading these planes before overwriting them. */ +void UVCGpuPreviewRenderer::Impl::captureRenderFence() { - glPixelStorei(GL_UNPACK_ROW_LENGTH, stride > width ? stride : 0); - const bool ok = uploadR8(tex, width, height, data); - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - return ok; + if (lastRenderFenceFd >= 0) { + close(lastRenderFenceFd); + lastRenderFenceFd = -1; + } + if (!pCreateSync || !pDupNativeFenceFD || !pDestroySync) { + glFinish(); /* no native fences: fall back to a full GPU wait */ + return; + } + EGLSyncKHR sync = pCreateSync(display, EGL_SYNC_NATIVE_FENCE_ANDROID, NULL); + if (sync == EGL_NO_SYNC_KHR) { + glFinish(); + return; + } + glFlush(); /* the fence fd only becomes valid once the sync is flushed */ + lastRenderFenceFd = pDupNativeFenceFD(display, sync); + pDestroySync(display, sync); + if (lastRenderFenceFd < 0) { + lastRenderFenceFd = -1; + glFinish(); + } } -bool UVCGpuPreviewRenderer::Impl::uploadMjpegPlane(int plane, int width, - int height, int stride, const void *data) +bool UVCGpuPreviewRenderer::Impl::drawPlanarHardware(uvc_frame_t *frame) { - if (plane < 0 || plane >= 3 || width <= 0 || height <= 0 || !data) + const bool gray = frame->yuv_hardware_buffers[1] == nullptr; + const int width = (int)frame->width; + const int height = (int)frame->height; + if (width <= 0 || height <= 0) return false; - - GLuint tex = textures[plane]; - glBindTexture(GL_TEXTURE_2D, tex); - if (mjpegTextureWidths[plane] != width || mjpegTextureHeights[plane] != height) { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, - GL_UNSIGNED_BYTE, NULL); - if (glGetError() != GL_NO_ERROR) { - mjpegTextureWidths[plane] = 0; - mjpegTextureHeights[plane] = 0; + const ProgramKind kind = frame->yuv_hardware_buffer_bytes_per_texel == 4 + ? PROGRAM_MJPEG_PLANAR_PACKED : PROGRAM_MJPEG_PLANAR; + GLuint prog = program(kind); + if (!prog) + return false; + for (int i = 0; i < (gray ? 1 : 3); i++) { + GLuint tex = planeTextureFor(frame->yuv_hardware_buffers[i], + frame->yuv_hardware_buffer_ids[i]); + if (!tex) return false; - } - mjpegTextureWidths[plane] = width; - mjpegTextureHeights[plane] = height; + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_2D, tex); } - - glPixelStorei(GL_UNPACK_ROW_LENGTH, stride > width ? stride : 0); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RED, - GL_UNSIGNED_BYTE, data); - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - return glGetError() == GL_NO_ERROR; -} - -bool UVCGpuPreviewRenderer::Impl::uploadRG8(GLuint tex, int width, int height, - const void *data) -{ - glBindTexture(GL_TEXTURE_2D, tex); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, width, height, 0, GL_RG, - GL_UNSIGNED_BYTE, data); - return glGetError() == GL_NO_ERROR; -} - -bool UVCGpuPreviewRenderer::Impl::uploadRGB8(GLuint tex, int width, int height, - const void *data) -{ - glBindTexture(GL_TEXTURE_2D, tex); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, width, height, 0, GL_RGB, - GL_UNSIGNED_BYTE, data); - return glGetError() == GL_NO_ERROR; + const Uniforms &u = uniforms[kind]; + glUseProgram(prog); + glUniform1i(u.width, width); + glUniform1i(u.height, height); + glUniform1i(u.chromaWidth, gray ? 1 : (int)frame->yuv_plane_widths[1]); + glUniform1i(u.chromaHeight, gray ? 1 : (int)frame->yuv_plane_heights[1]); + glUniform1i(u.gray, gray ? 1 : 0); + drawQuad(kind); + const GLenum err = glGetError(); + if (err != GL_NO_ERROR) { + LOGW("gpu-preview: planar AHB draw failed glerr=0x%x", err); + return false; + } + return true; } -bool UVCGpuPreviewRenderer::Impl::uploadR16UI(GLuint tex, int width, int height, - const void *data) +void UVCGpuPreviewRenderer::Impl::resetTextureStorage() { - glBindTexture(GL_TEXTURE_2D, tex); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_R16UI, width, height, 0, GL_RED_INTEGER, - GL_UNSIGNED_SHORT, data); - return glGetError() == GL_NO_ERROR; + memset(texWidth, 0, sizeof(texWidth)); + memset(texHeight, 0, sizeof(texHeight)); + memset(texInternal, 0, sizeof(texInternal)); + memset(texFilter, 0, sizeof(texFilter)); } -bool UVCGpuPreviewRenderer::Impl::uploadRG16UI(GLuint tex, int width, int height, - const void *data) +/* Upload one plane into texture unit `unit`. Storage is (re)specified with + * glTexImage2D only when the size, internal format, or filter changes; the + * steady state is a single glTexSubImage2D, which lets the driver reuse the + * existing allocation instead of orphaning it every frame. */ +bool UVCGpuPreviewRenderer::Impl::uploadTexture(int unit, GLenum internal, + GLenum format, GLenum type, GLint filter, int width, int height, + int stride_px, const void *data) { - glBindTexture(GL_TEXTURE_2D, tex); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RG16UI, width, height, 0, GL_RG_INTEGER, - GL_UNSIGNED_SHORT, data); + if (unit < 0 || unit >= 3 || width <= 0 || height <= 0 || !data) + return false; + glActiveTexture(GL_TEXTURE0 + unit); + glBindTexture(GL_TEXTURE_2D, textures[unit]); + if (texWidth[unit] != width || texHeight[unit] != height + || texInternal[unit] != internal || texFilter[unit] != filter) { + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexImage2D(GL_TEXTURE_2D, 0, (GLint)internal, width, height, 0, format, + type, NULL); + if (glGetError() != GL_NO_ERROR) { + texWidth[unit] = 0; + texHeight[unit] = 0; + texInternal[unit] = 0; + texFilter[unit] = 0; + return false; + } + texWidth[unit] = width; + texHeight[unit] = height; + texInternal[unit] = internal; + texFilter[unit] = filter; + } + glPixelStorei(GL_UNPACK_ROW_LENGTH, stride_px > width ? stride_px : 0); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, format, type, data); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); return glGetError() == GL_NO_ERROR; } @@ -810,10 +1068,9 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) const int height = (int)frame->height; if (width <= 0 || height <= 0) return false; - if (frame->frame_format != UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR) - resetMjpegTextureStorage(); GLuint prog = 0; + ProgramKind kind = PROGRAM_COUNT; const uint8_t *data = (const uint8_t *)frame->data; const size_t actual = frame_actual_bytes(frame); @@ -824,16 +1081,15 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) const size_t need = (size_t)width * (size_t)height * 2u; if (actual < need) return false; - prog = program(PROGRAM_YUYV); - if (!prog || !uploadR8(textures[0], width * 2, height, data)) + kind = PROGRAM_YUYV; + prog = program(kind); + if (!prog || !uploadTexture(0, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, + width * 2, height, 0, data)) return false; glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uPacked"), 0); - glUniform1i(glGetUniformLocation(prog, "uWidth"), width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), height); - glUniform1i(glGetUniformLocation(prog, "uUyvy"), + glUniform1i(uniforms[kind].width, width); + glUniform1i(uniforms[kind].height, height); + glUniform1i(uniforms[kind].uyvy, frame->frame_format == UVC_FRAME_FORMAT_UYVY ? 1 : 0); break; } @@ -842,20 +1098,17 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) const size_t need = y_bytes + y_bytes / 2u; if (actual < need || (width & 1) || (height & 1)) return false; - prog = program(PROGRAM_NV12); + kind = PROGRAM_NV12; + prog = program(kind); if (!prog - || !uploadR8(textures[0], width, height, data) - || !uploadRG8(textures[1], width / 2, height / 2, data + y_bytes)) + || !uploadTexture(0, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, + width, height, 0, data) + || !uploadTexture(1, GL_RG8, GL_RG, GL_UNSIGNED_BYTE, GL_NEAREST, + width / 2, height / 2, 0, data + y_bytes)) return false; glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uY"), 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures[1]); - glUniform1i(glGetUniformLocation(prog, "uUV"), 1); - glUniform1i(glGetUniformLocation(prog, "uWidth"), width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), height); + glUniform1i(uniforms[kind].width, width); + glUniform1i(uniforms[kind].height, height); break; } case UVC_FRAME_FORMAT_YU12: { @@ -864,25 +1117,19 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) const size_t need = y_bytes + chroma_bytes * 2u; if (actual < need || (width & 1) || (height & 1)) return false; - prog = program(PROGRAM_YU12); + kind = PROGRAM_YU12; + prog = program(kind); if (!prog - || !uploadR8(textures[0], width, height, data) - || !uploadR8(textures[1], width / 2, height / 2, data + y_bytes) - || !uploadR8(textures[2], width / 2, height / 2, - data + y_bytes + chroma_bytes)) + || !uploadTexture(0, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, + width, height, 0, data) + || !uploadTexture(1, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, + width / 2, height / 2, 0, data + y_bytes) + || !uploadTexture(2, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, + width / 2, height / 2, 0, data + y_bytes + chroma_bytes)) return false; glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uY"), 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures[1]); - glUniform1i(glGetUniformLocation(prog, "uU"), 1); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, textures[2]); - glUniform1i(glGetUniformLocation(prog, "uV"), 2); - glUniform1i(glGetUniformLocation(prog, "uWidth"), width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), height); + glUniform1i(uniforms[kind].width, width); + glUniform1i(uniforms[kind].height, height); break; } case UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR: { @@ -891,21 +1138,22 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) if (actual < frame->actual_bytes || !frame->yuv_plane_widths[0] || !frame->yuv_plane_heights[0]) return false; - prog = program(PROGRAM_MJPEG_PLANAR); + kind = PROGRAM_MJPEG_PLANAR; + prog = program(kind); if (!prog - || !uploadMjpegPlane(0, + || !uploadTexture(0, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, (int)frame->yuv_plane_widths[0], (int)frame->yuv_plane_heights[0], (int)frame->yuv_plane_strides[0], data + frame->yuv_plane_offsets[0])) return false; if (!gray) { - if (!uploadMjpegPlane(1, + if (!uploadTexture(1, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, (int)frame->yuv_plane_widths[1], (int)frame->yuv_plane_heights[1], (int)frame->yuv_plane_strides[1], data + frame->yuv_plane_offsets[1]) - || !uploadMjpegPlane(2, + || !uploadTexture(2, GL_R8, GL_RED, GL_UNSIGNED_BYTE, GL_NEAREST, (int)frame->yuv_plane_widths[2], (int)frame->yuv_plane_heights[2], (int)frame->yuv_plane_strides[2], @@ -913,35 +1161,25 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) return false; } glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uY"), 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures[1]); - glUniform1i(glGetUniformLocation(prog, "uU"), 1); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, textures[2]); - glUniform1i(glGetUniformLocation(prog, "uV"), 2); - glUniform1i(glGetUniformLocation(prog, "uWidth"), width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), height); - glUniform1i(glGetUniformLocation(prog, "uChromaWidth"), + glUniform1i(uniforms[kind].width, width); + glUniform1i(uniforms[kind].height, height); + glUniform1i(uniforms[kind].chromaWidth, gray ? 1 : (int)frame->yuv_plane_widths[1]); - glUniform1i(glGetUniformLocation(prog, "uChromaHeight"), + glUniform1i(uniforms[kind].chromaHeight, gray ? 1 : (int)frame->yuv_plane_heights[1]); - glUniform1i(glGetUniformLocation(prog, "uGray"), gray ? 1 : 0); + glUniform1i(uniforms[kind].gray, gray ? 1 : 0); break; } case UVC_FRAME_FORMAT_BGR: { const size_t need = (size_t)width * (size_t)height * 3u; if (actual < need) return false; - prog = program(PROGRAM_BGR); - if (!prog || !uploadRGB8(textures[0], width, height, data)) + kind = PROGRAM_BGR; + prog = program(kind); + if (!prog || !uploadTexture(0, GL_RGB8, GL_RGB, GL_UNSIGNED_BYTE, GL_LINEAR, + width, height, 0, data)) return false; glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uBgr"), 0); break; } case UVC_FRAME_FORMAT_P010: { @@ -949,40 +1187,24 @@ bool UVCGpuPreviewRenderer::Impl::uploadAndDraw(uvc_frame_t *frame) const size_t need = y_bytes + y_bytes / 2u; if (actual < need || (width & 1) || (height & 1)) return false; - prog = program(PROGRAM_P010); + kind = PROGRAM_P010; + prog = program(kind); if (!prog - || !uploadR16UI(textures[0], width, height, data) - || !uploadRG16UI(textures[1], width / 2, height / 2, data + y_bytes)) + || !uploadTexture(0, GL_R16UI, GL_RED_INTEGER, GL_UNSIGNED_SHORT, GL_NEAREST, + width, height, 0, data) + || !uploadTexture(1, GL_RG16UI, GL_RG_INTEGER, GL_UNSIGNED_SHORT, GL_NEAREST, + width / 2, height / 2, 0, data + y_bytes)) return false; glUseProgram(prog); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures[0]); - glUniform1i(glGetUniformLocation(prog, "uY16"), 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures[1]); - glUniform1i(glGetUniformLocation(prog, "uUV16"), 1); - glUniform1i(glGetUniformLocation(prog, "uWidth"), width); - glUniform1i(glGetUniformLocation(prog, "uHeight"), height); + glUniform1i(uniforms[kind].width, width); + glUniform1i(uniforms[kind].height, height); break; } default: return false; } - EGLint sw = 0; - EGLint sh = 0; - eglQuerySurface(display, surface, EGL_WIDTH, &sw); - eglQuerySurface(display, surface, EGL_HEIGHT, &sh); - glViewport(0, 0, sw, sh); - glBindBuffer(GL_ARRAY_BUFFER, vbo); - glEnableVertexAttribArray(0); - glEnableVertexAttribArray(1); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (const void *)0); - glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), - (const void *)(2 * sizeof(GLfloat))); - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - glDisableVertexAttribArray(0); - glDisableVertexAttribArray(1); + drawQuad(kind); const GLenum err = glGetError(); if (err != GL_NO_ERROR) { diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.h b/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.h index db0e7a1..00b8a0b 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.h +++ b/Consolation/app/src/main/jni/UVCCamera/UVCGpuPreviewRenderer.h @@ -11,6 +11,15 @@ class UVCGpuPreviewRenderer { ~UVCGpuPreviewRenderer(); bool render(uvc_frame_t *frame, ANativeWindow *window, uint64_t *frame_ready_ns); + /** After rendering a frame whose planes are AHardwareBuffers: a native + * fence fd that signals when the GPU has finished reading them, or -1. + * Ownership passes to the caller (hand it to AHardwareBuffer_lock). */ + int takeRenderFenceFd(); + /** 3x3 column-major NDC transform applied to the quad (rotation, flip, + * zoom, pan). Called under the same lock as render(). */ + void setTransform(const float m[9]); + /** Re-read the surface size on the next draw (the window may have resized). */ + void invalidateSurfaceSize(); void resetSurface(); void shutdown(); diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCPreview.cpp b/Consolation/app/src/main/jni/UVCCamera/UVCPreview.cpp index c4b215c..981b2fb 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCPreview.cpp +++ b/Consolation/app/src/main/jni/UVCCamera/UVCPreview.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -45,6 +46,10 @@ #include "libuvc_internal.h" #include +#include +#include +#include +#include #ifdef __ANDROID__ #include #endif @@ -98,6 +103,99 @@ static inline void atomic_store_max(std::atomic &target, uint64_t valu } // namespace +/* Frame-buffer integrity probe: each hop re-samples the bytes it is about to + * read and compares with the fingerprint taken when they were produced. A + * mismatch means something wrote into the buffer while it was in flight, and + * the stage name says between which two threads. Reads ~1 KiB per check. */ +#ifndef UVC_FRAME_INTEGRITY_CHECK +#define UVC_FRAME_INTEGRITY_CHECK 1 +#endif + +namespace { +static bool frame_integrity_ok(const uvc_frame_t *frame, const char *stage, + std::atomic &mismatches) +{ +#if UVC_FRAME_INTEGRITY_CHECK + if (!frame || !frame->data || !frame->integrity_sample_hash) + return true; + const size_t len = frame->actual_bytes ? frame->actual_bytes : frame->data_bytes; + const uint32_t now = uvc_frame_sample_hash(frame->data, len); + if (LIKELY(now == frame->integrity_sample_hash)) + return true; + const uint32_t n = mismatches.fetch_add(1, std::memory_order_relaxed) + 1; + if (n <= 10 || !(n % 100)) + LOGW("frame-integrity: buffer changed in flight stage=%s count=%u seq=%u bytes=%zu " + "expected=%08x now=%08x fmt=%d slot=%u", + stage, n, frame->sequence, len, frame->integrity_sample_hash, now, + frame->frame_format, frame->library_frame_slot); + return false; +#else + (void)frame; (void)stage; (void)mismatches; + return true; +#endif +} +/* Debug-only raw MJPEG dump, driven by a system property so it costs nothing + * unless armed: adb shell setprop debug.consolation.mjpeg_dump + * dumps the next N consecutive frames handed to the async decoder into + * /data/data//cache/mjpeg_dump/.jpg (pull with `adb shell run-as`). + * Setting a different value re-arms it. The property is polled once per + * 60 frames so the hot path pays one property read per second at most. */ +static void debug_mjpeg_dump_maybe(const uvc_frame_t *frame) +{ + static char s_armed_value[PROP_VALUE_MAX] = {}; + static int s_remaining = 0; + static uint32_t s_poll = 0; + static const char *s_dir = "/data/data/org.centennialoss.consolation/cache/mjpeg_dump"; + + if (!frame || !frame->data || !frame->actual_bytes) + return; + if (s_remaining <= 0) { + if ((s_poll++ % 60) != 0) + return; + char value[PROP_VALUE_MAX] = {}; + if (__system_property_get("debug.consolation.mjpeg_dump", value) <= 0 + || !value[0] || !strcmp(value, s_armed_value)) + return; + strncpy(s_armed_value, value, sizeof(s_armed_value) - 1); + s_remaining = atoi(value); + if (s_remaining <= 0) + return; + mkdir(s_dir, 0755); + LOGW("mjpeg-dump: armed for %d frames -> %s", s_remaining, s_dir); + } + char path[256]; + snprintf(path, sizeof(path), "%s/%06u.jpg", s_dir, frame->sequence); + FILE *f = fopen(path, "wb"); + if (!f) { + LOGW("mjpeg-dump: cannot open %s", path); + s_remaining = 0; + return; + } + fwrite(frame->data, 1, frame->actual_bytes, f); + fclose(f); + /* Sidecar: one line per USB packet "len flags cumulative_image_bytes". */ + snprintf(path, sizeof(path), "%s/%06u.pkts", s_dir, frame->sequence); + f = fopen(path, "w"); + if (f) { + size_t cum = 0; + for (unsigned i = 0; i < frame->iso_trace_count; i++) { + const unsigned len = frame->iso_trace_len[i]; + const unsigned fl = frame->iso_trace_flags[i]; + if (!(fl & 8u) && len > 12) + cum += len - 12; + fprintf(f, "%u %u %zu\n", len, fl, cum); + } + fclose(f); + } + if (--s_remaining == 0) + LOGW("mjpeg-dump: done"); +} + +static std::atomic g_integrity_mismatch_cb{0}; +static std::atomic g_integrity_mismatch_decode{0}; +static std::atomic g_integrity_mismatch_render{0}; +} // namespace + /** processingUvcSeqState bit63 marks the last-sequence value as valid. */ #define UVC_SEQ_STATE_VALID_BIT (1ULL << 63) @@ -224,6 +322,7 @@ UVCPreview::UVCPreview(uvc_device_handle_t *devh) ENTER(); pthread_cond_init(&preview_sync, NULL); pthread_mutex_init(&preview_mutex, NULL); + pthread_mutex_init(&preview_queue_mutex, NULL); pthread_cond_init(&mjpeg_decode_sync, NULL); pthread_mutex_init(&mjpeg_decode_mutex, NULL); // @@ -233,9 +332,320 @@ UVCPreview::UVCPreview(uvc_device_handle_t *devh) pthread_mutex_init(&pool_mutex, NULL); iframecallback_fields.onFrame = nullptr; preview_iframecallback_fields.onFrame = nullptr; + memset(mjpeg_header_slots, 0, sizeof(mjpeg_header_slots)); + memset(mjpeg_header_used, 0, sizeof(mjpeg_header_used)); + { + static const float identity[9] = { 1, 0, 0, 0, 1, 0, 0, 0, 1 }; + memcpy(mPreviewXform, identity, sizeof(mPreviewXform)); + } + mPreviewFitX = 1.0f; + mPreviewFitY = 1.0f; + mPreviewGeomWidth = 0; + mPreviewGeomHeight = 0; + mPreviewSurfaceSizeDirty = false; + memset(mGpuPlanar, 0, sizeof(mGpuPlanar)); + for (int i = 0; i < GPU_PLANAR_POOL_SZ; i++) + mGpuPlanar[i].fence_fd = -1; + mGpuPlanarNext = 0; + mGpuPlanarRenderFailures = 0; + mMergedRender = true; + { + /* Pick a GPU-sampleable buffer format for the zero-copy planar path. + * R8 is the natural fit but many gralloc implementations reject it; + * RGBA8 a quarter as wide (4 plane bytes per texel) is universal. */ + /* CPU_READ_OFTEN is requested first on purpose: gralloc then gives a + * cached CPU mapping (write-combined memory is very slow for libjpeg's + * 8x8-block write pattern) and cleans the cache on unlock for the GPU. */ + struct { uint32_t format; uint64_t usage; uint32_t bpt; const char *name; } cands[] = { + { AHARDWAREBUFFER_FORMAT_R8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN + | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, 1, "R8 (cached)" }, + { AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN + | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, 4, "packed RGBA8 (cached)" }, + { AHARDWAREBUFFER_FORMAT_R8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_CPU_READ_RARELY + | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, 1, "R8" }, + { AHARDWAREBUFFER_FORMAT_R8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, + 1, "R8 (write-only)" }, + { AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_CPU_READ_RARELY + | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, 4, "packed RGBA8" }, + { AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM, + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, + 4, "packed RGBA8 (write-only)" }, + }; + mGpuPlanarEnabled = false; + mGpuPlanarFormat = 0; + mGpuPlanarUsage = 0; + mGpuPlanarBytesPerTexel = 1; + for (size_t c = 0; c < sizeof(cands) / sizeof(cands[0]); c++) { + AHardwareBuffer_Desc probe; + memset(&probe, 0, sizeof(probe)); + probe.width = 64; + probe.height = 64; + probe.layers = 1; + probe.format = cands[c].format; + probe.usage = cands[c].usage; + /* AHardwareBuffer_isSupported() is API 29+; actually allocating a + * small buffer is the portable equivalent and answers the same + * question more honestly. This runs once at preview setup. */ + AHardwareBuffer *probe_buf = NULL; + if (AHardwareBuffer_allocate(&probe, &probe_buf) == 0 && probe_buf) { + AHardwareBuffer_release(probe_buf); + mGpuPlanarEnabled = true; + mGpuPlanarFormat = cands[c].format; + mGpuPlanarUsage = cands[c].usage; + mGpuPlanarBytesPerTexel = cands[c].bpt; + LOGI("gpu-planar: zero-copy decode targets enabled using %s AHardwareBuffers", + cands[c].name); + break; + } + } + if (!mGpuPlanarEnabled) + LOGI("gpu-planar: no GPU-sampleable CPU-writable buffer format; using texture upload"); + } EXIT(); } +/* ---- zero-copy planar MJPEG targets ------------------------------------ */ + +void UVCPreview::gpu_planar_free_slot(GpuPlanarFrame *g) +{ + for (int i = 0; i < 3; i++) { + if (g->ahb[i]) { + AHardwareBuffer_release((AHardwareBuffer *)g->ahb[i]); + g->ahb[i] = NULL; + } + g->ids[i] = 0; + g->w[i] = g->h[i] = g->stride[i] = 0; + } + if (g->fence_fd >= 0) { + close(g->fence_fd); + g->fence_fd = -1; + } +} + +bool UVCPreview::gpu_planar_alloc_plane(GpuPlanarFrame *g, int i, + uint32_t width, uint32_t height) +{ + static std::atomic s_next_id{1}; + AHardwareBuffer_Desc desc; + AHardwareBuffer *buf = NULL; + const uint32_t bpt = mGpuPlanarBytesPerTexel; + memset(&desc, 0, sizeof(desc)); + desc.width = (width + bpt - 1) / bpt; /* texels; 4 plane bytes per RGBA texel */ + desc.height = height; + desc.layers = 1; + desc.format = mGpuPlanarFormat; + desc.usage = mGpuPlanarUsage; + if (AHardwareBuffer_allocate(&desc, &buf) != 0 || !buf) + return false; + AHardwareBuffer_describe(buf, &desc); + g->ahb[i] = buf; + g->ids[i] = s_next_id.fetch_add(1, std::memory_order_relaxed); + g->w[i] = width; + g->h[i] = height; + g->stride[i] = desc.stride * bpt; /* row pitch in bytes */ + return true; +} + +uvc_frame_t *UVCPreview::gpu_planar_get(const uint32_t widths[3], const uint32_t heights[3]) +{ + GpuPlanarFrame *g = NULL; + pthread_mutex_lock(&pool_mutex); + /* Round-robin so the slot handed out is the one the GPU finished with + * longest ago; its read fence is then almost always already signalled. */ + for (int n = 0; n < GPU_PLANAR_POOL_SZ; n++) { + GpuPlanarFrame *c = &mGpuPlanar[(mGpuPlanarNext + n) % GPU_PLANAR_POOL_SZ]; + if (!c->in_use) { + c->in_use = true; + mGpuPlanarNext = (unsigned)((c - mGpuPlanar) + 1) % GPU_PLANAR_POOL_SZ; + g = c; + break; + } + } + pthread_mutex_unlock(&pool_mutex); + if (!g) + return NULL; + + for (int i = 0; i < 3; i++) { + if (!widths[i]) { + if (g->ahb[i]) { + AHardwareBuffer_release((AHardwareBuffer *)g->ahb[i]); + g->ahb[i] = NULL; + g->ids[i] = 0; + } + g->w[i] = g->h[i] = g->stride[i] = 0; + continue; + } + if (g->ahb[i] && g->w[i] == widths[i] && g->h[i] == heights[i]) + continue; + if (g->ahb[i]) { + AHardwareBuffer_release((AHardwareBuffer *)g->ahb[i]); + g->ahb[i] = NULL; + } + if (!gpu_planar_alloc_plane(g, i, widths[i], heights[i])) { + LOGW("gpu-planar: AHardwareBuffer_allocate %ux%u R8 failed", widths[i], heights[i]); + gpu_planar_put(&g->frame, -1); + return NULL; + } + } + return &g->frame; +} + +void UVCPreview::gpu_planar_put(uvc_frame_t *frame, int fence_fd) +{ + if (!frame) + return; + GpuPlanarFrame *g = reinterpret_cast(frame); /* frame is first member */ + pthread_mutex_lock(&pool_mutex); + if (g->fence_fd >= 0) + close(g->fence_fd); + g->fence_fd = fence_fd; + g->in_use = false; + pthread_mutex_unlock(&pool_mutex); +} + +void UVCPreview::gpu_planar_release_all() +{ + pthread_mutex_lock(&pool_mutex); + for (int i = 0; i < GPU_PLANAR_POOL_SZ; i++) { + gpu_planar_free_slot(&mGpuPlanar[i]); + mGpuPlanar[i].in_use = false; + } + pthread_mutex_unlock(&pool_mutex); +} + +/* Decode `frame` straight into a GPU planar target and queue it for the + * preview thread. Returns true when the frame was handled (queued or + * dropped as corrupt); false means the zero-copy path is unavailable and the + * caller should fall back to the contiguous decode + texture upload. */ +bool UVCPreview::decode_mjpeg_to_gpu_planar(uvc_frame_t *frame) +{ + uint32_t widths[3], heights[3]; + int subsamp = 0; + if (uvc_mjpeg_planar_layout(frame, widths, heights, &subsamp) != UVC_SUCCESS) + return false; /* header problem: let the normal path report it */ + + uvc_frame_t *out = gpu_planar_get(widths, heights); + if (UNLIKELY(!out)) { + if (!mGpuPlanarEnabled) + return false; + /* Pool exhausted (should not happen with latest-wins): drop this one. */ + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + return true; + } + GpuPlanarFrame *g = reinterpret_cast(out); + const int nplanes = widths[1] ? 3 : 1; + unsigned char *planes[3] = { NULL, NULL, NULL }; + int strides[3] = { 0, 0, 0 }; + int locked = 0; + bool ok = true; + + /* The first lock waits on the fence from this slot's last GPU read; the + * fence fd is consumed by the lock, so hand it over exactly once. */ + int fence = g->fence_fd; + g->fence_fd = -1; + for (int i = 0; i < nplanes; i++) { + void *ptr = NULL; + if (AHardwareBuffer_lock((AHardwareBuffer *)g->ahb[i], + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, i == 0 ? fence : -1, + NULL, &ptr) != 0 || !ptr) { + if (i == 0 && fence >= 0) + close(fence); + ok = false; + break; + } + planes[i] = (unsigned char *)ptr; + strides[i] = (int)g->stride[i]; + locked = i + 1; + } + + uvc_error_t result = UVC_ERROR_OTHER; + const uint64_t t_convert = processing_now_ns(); + if (ok) + result = uvc_mjpeg2yuv_planes(frame, planes, strides); + for (int i = 0; i < locked; i++) + AHardwareBuffer_unlock((AHardwareBuffer *)g->ahb[i], NULL); + if (ok) + recordPreviewConversionTiming(processing_now_ns() - t_convert); + + if (!ok) { + LOGW("gpu-planar: AHardwareBuffer_lock failed; falling back to texture upload"); + mGpuPlanarEnabled = false; + gpu_planar_put(out, -1); + return false; + } + if (result != UVC_SUCCESS) { + /* Corrupt or undecodable: same policy as the upload path, frame dropped. */ + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + gpu_planar_put(out, -1); + return true; + } + + out->width = frame->width; + out->height = frame->height; + out->frame_format = UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR; + out->data = NULL; + out->data_bytes = 0; + out->actual_bytes = 0; + out->library_owns_data = 0; + out->step = g->stride[0]; + out->sequence = frame->sequence; + out->capture_time = frame->capture_time; + out->arrival_monotonic_ns = frame->arrival_monotonic_ns; + out->source = frame->source; + out->yuv_subsampling = subsamp; + out->integrity_sample_hash = 0; + out->yuv_hardware_buffer_bytes_per_texel = mGpuPlanarBytesPerTexel; + for (int i = 0; i < 3; i++) { + out->yuv_plane_widths[i] = widths[i]; + out->yuv_plane_heights[i] = heights[i]; + out->yuv_plane_strides[i] = g->stride[i]; + out->yuv_plane_offsets[i] = 0; + out->yuv_hardware_buffers[i] = widths[i] ? g->ahb[i] : NULL; + out->yuv_hardware_buffer_ids[i] = widths[i] ? g->ids[i] : 0; + } + if (mMergedRender) + presentPlanarFrame(out); + else + addPreviewFrame(out); + return true; +} + +uvc_frame_t *UVCPreview::mjpeg_header_get() { + uvc_frame_t *header = NULL; + pthread_mutex_lock(&mjpeg_decode_mutex); + for (int i = 0; i < MJPEG_HEADER_POOL_SZ; i++) { + if (!mjpeg_header_used[i]) { + mjpeg_header_used[i] = true; + header = &mjpeg_header_slots[i]; + break; + } + } + pthread_mutex_unlock(&mjpeg_decode_mutex); + return header; +} + +void UVCPreview::mjpeg_header_put_locked(uvc_frame_t *header) { + if (!header) + return; + const ptrdiff_t i = header - mjpeg_header_slots; + if (LIKELY(i >= 0 && i < MJPEG_HEADER_POOL_SZ)) { + mjpeg_header_used[i] = false; + } else { + LOGW("mjpeg_header_put: foreign frame header %p", header); + } +} + +void UVCPreview::mjpeg_header_put(uvc_frame_t *header) { + pthread_mutex_lock(&mjpeg_decode_mutex); + mjpeg_header_put_locked(header); + pthread_mutex_unlock(&mjpeg_decode_mutex); +} + UVCPreview::~UVCPreview() { ENTER(); @@ -282,6 +692,7 @@ UVCPreview::~UVCPreview() { delete mGpuPreviewRenderer; mGpuPreviewRenderer = NULL; } + gpu_planar_release_all(); if (mMjpegPreviewYuvFrame) { uvc_free_frame(mMjpegPreviewYuvFrame); mMjpegPreviewYuvFrame = NULL; @@ -291,6 +702,7 @@ UVCPreview::~UVCPreview() { clearCaptureFrame(); clear_pool(); pthread_mutex_destroy(&preview_mutex); + pthread_mutex_destroy(&preview_queue_mutex); pthread_cond_destroy(&preview_sync); pthread_mutex_destroy(&mjpeg_decode_mutex); pthread_cond_destroy(&mjpeg_decode_sync); @@ -338,6 +750,10 @@ uvc_frame_t *UVCPreview::get_notification_frame() { } void UVCPreview::recycle_frame(uvc_frame_t *frame) { + if (UNLIKELY(gpu_planar_is(frame))) { + gpu_planar_put(frame, -1); + return; + } if (UNLIKELY(frame && frame->data_bytes <= 1 && frame->library_owns_data)) { pthread_mutex_lock(&pool_mutex); if (LIKELY(mNotificationFramePool.size() < FRAME_POOL_SZ)) { @@ -667,6 +1083,8 @@ int UVCPreview::setPreviewDisplay(ANativeWindow *preview_window) { if (LIKELY(mPreviewWindow)) { ANativeWindow_setBuffersGeometry(mPreviewWindow, frameWidth, frameHeight, previewFormat); + mPreviewGeomWidth = frameWidth; + mPreviewGeomHeight = frameHeight; } } else if (preview_window) { /* JNI calls ANativeWindow_fromSurface each time; if the pointer matches the @@ -678,6 +1096,37 @@ int UVCPreview::setPreviewDisplay(ANativeWindow *preview_window) { RETURN(0, int); } +int UVCPreview::setPreviewTransform(int rotation_degrees, bool flip_h, bool flip_v, + float scale, float pan_x_ndc, float pan_y_ndc, float fit_x, float fit_y) { + /* scale (incl. mirror) -> rotate -> fit -> translate. Rotation happens in the + * NDC of the fit-to-screen content box (which has the rotated aspect), the fit + * step maps that box into the full surface, and the pan is in surface NDC. + * NDC has y up while the screen has y down, so a clockwise on-screen rotation + * by r is (x, y) -> (x cos r + y sin r, -x sin r + y cos r). */ + if (!(fit_x > 0.0f) || fit_x > 1.0f) + fit_x = 1.0f; + if (!(fit_y > 0.0f) || fit_y > 1.0f) + fit_y = 1.0f; + const double r = rotation_degrees * M_PI / 180.0; + const float c = (float)cos(r); + const float sn = (float)sin(r); + const float sx = scale * (flip_h ? -1.0f : 1.0f); + const float sy = scale * (flip_v ? -1.0f : 1.0f); + /* M = T * F * R * S, column-major for glUniformMatrix3fv (F scales rows). */ + float m[9]; + m[0] = fit_x * c * sx; m[1] = -fit_y * sn * sx; m[2] = 0.0f; /* column 0 */ + m[3] = fit_x * sn * sy; m[4] = fit_y * c * sy; m[5] = 0.0f; /* column 1 */ + m[6] = pan_x_ndc; m[7] = pan_y_ndc; m[8] = 1.0f; /* column 2 */ + pthread_mutex_lock(&preview_mutex); + memcpy(mPreviewXform, m, sizeof(mPreviewXform)); + mPreviewFitX = fit_x; + mPreviewFitY = fit_y; + /* Kotlin re-sends the transform whenever the view's size changes. */ + mPreviewSurfaceSizeDirty = true; + pthread_mutex_unlock(&preview_mutex); + return 0; +} + int UVCPreview::setFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format) { ENTER(); @@ -862,11 +1311,11 @@ int UVCPreview::startPreview() { if (UNLIKELY(result != EXIT_SUCCESS)) { LOGW("UVCCamera::window does not exist/already running/could not create thread etc."); mIsRunning = false; - pthread_mutex_lock(&preview_mutex); + pthread_mutex_lock(&preview_queue_mutex); { pthread_cond_signal(&preview_sync); } - pthread_mutex_unlock(&preview_mutex); + pthread_mutex_unlock(&preview_queue_mutex); } } RETURN(result, int); @@ -877,7 +1326,11 @@ int UVCPreview::stopPreview() { bool b = isRunning(); if (LIKELY(b)) { mIsRunning = false; + /* Signal under the queue lock so a waiter that has checked isRunning() + * but not yet entered cond_wait cannot miss this wakeup. */ + pthread_mutex_lock(&preview_queue_mutex); pthread_cond_signal(&preview_sync); + pthread_mutex_unlock(&preview_queue_mutex); pthread_mutex_lock(&mjpeg_decode_mutex); pthread_cond_signal(&mjpeg_decode_sync); pthread_mutex_unlock(&mjpeg_decode_mutex); @@ -1125,19 +1578,34 @@ void UVCPreview::addMjpegDecodeFrame(uvc_frame_t *frame) { if (UNLIKELY(!frame)) return; - uvc_frame_t *queued = uvc_allocate_frame(0); - if (UNLIKELY(!queued)) + /* Stage 1: USB-thread publish -> libuvc callback thread. */ + if (UNLIKELY(!frame_integrity_ok(frame, "publish->callback", g_integrity_mismatch_cb))) { + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + return; + } + uvc_frame_t *queued = mjpeg_header_get(); + if (UNLIKELY(!queued)) { + /* Pool exhausted: ring full plus both in-flight slots busy. The ring's + * drop-oldest would have evicted anyway; count it as a queue drop. */ + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); return; + } *queued = *frame; queued->library_owns_data = 0; - uvc_frame_retain(queued); + if (UNLIKELY(!uvc_frame_retain(queued))) { + /* No slot reference: the bytes could be overwritten by the USB thread + * while the decoder reads them. Never hand such a frame to the async + * decoder (sequential overwrite shows as a shredded lower half). */ + mjpeg_header_put(queued); + return; + } pthread_mutex_lock(&mjpeg_decode_mutex); if (isRunning() && mjpeg_decode_thread_joinable) { uvc_frame_t *drop = mjpeg_decode_frame_ring.enqueue_drop_oldest_if_full(queued); if (drop) { processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); uvc_frame_release(drop); - uvc_free_frame(drop); + mjpeg_header_put_locked(drop); } queued = NULL; pthread_cond_signal(&mjpeg_decode_sync); @@ -1145,7 +1613,7 @@ void UVCPreview::addMjpegDecodeFrame(uvc_frame_t *frame) { pthread_mutex_unlock(&mjpeg_decode_mutex); if (queued) { uvc_frame_release(queued); - uvc_free_frame(queued); + mjpeg_header_put(queued); } } @@ -1167,7 +1635,7 @@ void UVCPreview::clearMjpegDecodeFrame() { while (!mjpeg_decode_frame_ring.empty()) { uvc_frame_t *frame = mjpeg_decode_frame_ring.dequeue(); uvc_frame_release(frame); - uvc_free_frame(frame); + mjpeg_header_put_locked(frame); } mjpeg_decode_frame_ring.reset_storage(); pthread_mutex_unlock(&mjpeg_decode_mutex); @@ -1184,20 +1652,85 @@ void *UVCPreview::mjpeg_decode_thread_func(void *vptr_args) { } void UVCPreview::do_mjpeg_decode() { + { + /* Merged decode+render is chosen once per stream: the EGL context can be + * current on only one thread, so it cannot move between the decode and + * preview threads while running. A/B: setprop debug.consolation.merged_render 0 + * then restart the preview. */ + char mv[PROP_VALUE_MAX] = {}; + mMergedRender = true; + if (__system_property_get("debug.consolation.merged_render", mv) > 0 && mv[0] == '0') + mMergedRender = false; + LOGI("merged-render: %s (decode thread %s the planar frame)", + mMergedRender ? "on" : "off", mMergedRender ? "renders" : "hands off"); + } for (; LIKELY(isRunning() && mjpeg_decode_thread_joinable); ) { uvc_frame_t *frame = waitMjpegDecodeFrame(); if (!frame) break; + /* Stage 2: callback thread enqueue -> decode thread dequeue (retained slot). */ + if (UNLIKELY(!frame_integrity_ok(frame, "callback->decode", g_integrity_mismatch_decode))) { + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + uvc_frame_release(frame); + mjpeg_header_put(frame); + continue; + } + debug_mjpeg_dump_maybe(frame); + { + /* A/B switch: adb shell setprop debug.consolation.gpu_planar 0|1 + * (polled every 60 frames; only honoured if the format probe passed). */ + static uint32_t s_poll; + static int s_forced = -1; /* -1 unset, 0 off, 1 on */ + if ((s_poll++ % 60) == 0) { + char value[PROP_VALUE_MAX] = {}; + if (__system_property_get("debug.consolation.gpu_planar", value) > 0 && value[0]) { + const int want = value[0] != '0'; + if (want != s_forced) { + s_forced = want; + if (mGpuPlanarFormat) { + mGpuPlanarEnabled = want != 0; + mGpuPlanarRenderFailures = 0; + LOGW("gpu-planar: %s by property", want ? "enabled" : "disabled"); + } + } + } + } + } + if (mGpuPlanarEnabled && decode_mjpeg_to_gpu_planar(frame)) { + uvc_frame_release(frame); + mjpeg_header_put(frame); + continue; + } uvc_frame_t *decoded = get_frame(0); if (LIKELY(decoded)) { const uint64_t t_convert = processing_now_ns(); const uvc_error_t result = uvc_mjpeg2yuv_planar(frame, decoded); recordPreviewConversionTiming(processing_now_ns() - t_convert); if (LIKELY(result == UVC_SUCCESS)) { - addPreviewFrame(decoded); + /* Stage 3 check happens on the preview thread: was the JPEG + * still intact once decoding finished (writer overtook reader)? */ + if (UNLIKELY(!frame_integrity_ok(frame, "during-decode", g_integrity_mismatch_decode))) { + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + recycle_frame(decoded); + decoded = NULL; + uvc_frame_release(frame); + mjpeg_header_put(frame); + continue; + } +#if UVC_FRAME_INTEGRITY_CHECK + decoded->integrity_sample_hash = + uvc_frame_sample_hash(decoded->data, decoded->actual_bytes); +#endif + if (mMergedRender) + presentPlanarFrame(decoded); + else + addPreviewFrame(decoded); decoded = NULL; } else { + /* Corrupt or undecodable frame: not shown, last good frame stays. + * Surface it in the dropped-frames telemetry counter. */ + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); UVC_DIAG_LOGI("mjpeg-diag:async-planar-decode-fail seq=%u bytes=%zu result=%d frame=%ux%u", frame->sequence, frame->actual_bytes, @@ -1209,7 +1742,30 @@ void UVCPreview::do_mjpeg_decode() { if (decoded) recycle_frame(decoded); uvc_frame_release(frame); - uvc_free_frame(frame); + mjpeg_header_put(frame); + } +} + +/* Blacks out the buffer area around a frame decoded at (off_x, off_y); RGBX 0 is black. + * Only the letterbox strips are touched, never the frame itself. */ +static void clearLetterbox(uint8_t *bits, size_t row_bytes, int32_t buf_w, int32_t buf_h, + uint32_t frame_w, uint32_t frame_h, size_t off_x, size_t off_y) { + const size_t px = PREVIEW_PIXEL_BYTES; + const size_t right_x = off_x + frame_w; + const size_t right_bytes = ((size_t) buf_w - right_x) * px; + if (off_y) + memset(bits, 0, off_y * row_bytes); + const size_t bottom_y = off_y + frame_h; + if ((size_t) buf_h > bottom_y) + memset(bits + bottom_y * row_bytes, 0, ((size_t) buf_h - bottom_y) * row_bytes); + if (off_x || right_bytes) { + for (size_t y = off_y; y < bottom_y; ++y) { + uint8_t *row = bits + y * row_bytes; + if (off_x) + memset(row, 0, off_x * px); + if (right_bytes) + memset(row + right_x * px, 0, right_bytes); + } } } @@ -1222,12 +1778,30 @@ bool UVCPreview::renderFrameDirectToSurface(uvc_frame_t *frame, ANativeWindow *target = *window; if (LIKELY(target)) { if (mGpuPreviewRenderer) { + /* The GPU path renders at the view's own size (1:1 with the screen); + * only the CPU fallback below needs stream-sized buffers. */ + if (mPreviewGeomWidth || mPreviewGeomHeight) { + ANativeWindow_setBuffersGeometry(target, 0, 0, previewFormat); + mPreviewGeomWidth = 0; + mPreviewGeomHeight = 0; + mPreviewSurfaceSizeDirty = true; + } + if (mPreviewSurfaceSizeDirty) { + mGpuPreviewRenderer->invalidateSurfaceSize(); + mPreviewSurfaceSizeDirty = false; + } + mGpuPreviewRenderer->setTransform(mPreviewXform); if (frame->frame_format != UVC_FRAME_FORMAT_MJPEG && mGpuPreviewRenderer->render(frame, target, frame_ready_ns)) { rendered = true; pthread_mutex_unlock(window_mutex); return rendered; } + if (gpu_planar_is(frame)) { + /* Planes live only in GPU memory: nothing for the CPU path. */ + pthread_mutex_unlock(window_mutex); + return false; + } if (frame->frame_format == UVC_FRAME_FORMAT_MJPEG) { if (UNLIKELY(!mMjpegPreviewYuvFrame)) mMjpegPreviewYuvFrame = uvc_allocate_frame(0); @@ -1254,6 +1828,21 @@ bool UVCPreview::renderFrameDirectToSurface(uvc_frame_t *frame, } } } + /* The view fills the screen, so pad the buffer to the screen's shape (the + * compositor then scales it uniformly) and decode the frame into its center. */ + { + int32_t geom_w = (int32_t) frame->width; + int32_t geom_h = (int32_t) frame->height; + if (mPreviewFitX < 1.0f) + geom_w = std::max(geom_w, (int32_t) lroundf(frame->width / mPreviewFitX) & ~1); + if (mPreviewFitY < 1.0f) + geom_h = std::max(geom_h, (int32_t) lroundf(frame->height / mPreviewFitY) & ~1); + if (geom_w != mPreviewGeomWidth || geom_h != mPreviewGeomHeight) { + ANativeWindow_setBuffersGeometry(target, geom_w, geom_h, previewFormat); + mPreviewGeomWidth = geom_w; + mPreviewGeomHeight = geom_h; + } + } ANativeWindow_Buffer buffer; const uint64_t lock_start_ns = processing_now_ns(); if (LIKELY(ANativeWindow_lock(target, &buffer, NULL) == 0)) { @@ -1261,10 +1850,17 @@ bool UVCPreview::renderFrameDirectToSurface(uvc_frame_t *frame, *surface_wait_ns += processing_now_ns() - lock_start_ns; if (LIKELY(buffer.bits && buffer.width >= (int32_t) frame->width && buffer.height >= (int32_t) frame->height)) { + const size_t row_bytes = (size_t) buffer.stride * PREVIEW_PIXEL_BYTES; + const size_t off_x = (size_t) (buffer.width - (int32_t) frame->width) / 2; + const size_t off_y = (size_t) (buffer.height - (int32_t) frame->height) / 2; + uint8_t *bits = (uint8_t *) buffer.bits; + if (off_x || off_y) + clearLetterbox(bits, row_bytes, buffer.width, buffer.height, + frame->width, frame->height, off_x, off_y); uvc_frame_t surface = {}; - surface.data = buffer.bits; - surface.data_bytes = (size_t) buffer.stride * (size_t) buffer.height - * PREVIEW_PIXEL_BYTES; + surface.data = bits + off_y * row_bytes + off_x * PREVIEW_PIXEL_BYTES; + surface.data_bytes = row_bytes * ((size_t) buffer.height - off_y) + - off_x * PREVIEW_PIXEL_BYTES; surface.width = frame->width; surface.height = frame->height; surface.frame_format = UVC_FRAME_FORMAT_RGBX; @@ -1281,7 +1877,7 @@ bool UVCPreview::renderFrameDirectToSurface(uvc_frame_t *frame, rendered = result == UVC_SUCCESS; if (LIKELY(rendered && frame->frame_format == UVC_FRAME_FORMAT_MJPEG)) { recordMjpegDecodedVisualSample(frame->sequence, frame->actual_bytes, - (const uint8_t *)buffer.bits, surface.step, frame->width, frame->height); + (const uint8_t *)surface.data, surface.step, frame->width, frame->height); } if (UNLIKELY(result && frame->frame_format == UVC_FRAME_FORMAT_MJPEG)) { UVC_DIAG_LOGI("mjpeg-diag:decode-fail direct seq=%u bytes=%zu result=%d surface=%dx%d stride=%d frame=%ux%u", @@ -1357,8 +1953,17 @@ uvc_frame_t *UVCPreview::convertPreviewFrameToRgbx(uvc_frame_t *frame) { void UVCPreview::addPreviewFrame(uvc_frame_t *frame) { - pthread_mutex_lock(&preview_mutex); + pthread_mutex_lock(&preview_queue_mutex); if (isRunning()) { + if (frameMode != REQUEST_MODE_H264) { + /* Latest-wins for live video: anything still queued is older than + * this frame and would only be shown late. Recycle it instead. */ + while (!preview_frame_ring.empty()) { + uvc_frame_t *stale = preview_frame_ring.dequeue(); + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + recycle_frame(stale); + } + } uvc_frame_t *drop = preview_frame_ring.enqueue_drop_oldest_if_full(frame); const uint64_t queued_backlog = preview_frame_ring.size() > 0 ? preview_frame_ring.size() - 1 : 0; @@ -1374,34 +1979,34 @@ void UVCPreview::addPreviewFrame(uvc_frame_t *frame) { frame = nullptr; pthread_cond_signal(&preview_sync); } - pthread_mutex_unlock(&preview_mutex); + pthread_mutex_unlock(&preview_queue_mutex); if (frame) recycle_frame(frame); } uvc_frame_t *UVCPreview::waitPreviewFrame() { uvc_frame_t *frame = nullptr; - pthread_mutex_lock(&preview_mutex); + pthread_mutex_lock(&preview_queue_mutex); { while (isRunning() && preview_frame_ring.empty()) - pthread_cond_wait(&preview_sync, &preview_mutex); + pthread_cond_wait(&preview_sync, &preview_queue_mutex); if (LIKELY(isRunning() && !preview_frame_ring.empty())) { frame = preview_frame_ring.dequeue(); recordPreviewQueueDepthSample(static_cast(preview_frame_ring.size())); } } - pthread_mutex_unlock(&preview_mutex); + pthread_mutex_unlock(&preview_queue_mutex); return frame; } void UVCPreview::clearPreviewFrame() { - pthread_mutex_lock(&preview_mutex); + pthread_mutex_lock(&preview_queue_mutex); { while (!preview_frame_ring.empty()) recycle_frame(preview_frame_ring.dequeue()); preview_frame_ring.reset_storage(); } - pthread_mutex_unlock(&preview_mutex); + pthread_mutex_unlock(&preview_queue_mutex); } void *UVCPreview::preview_thread_func(void *vptr_args) { @@ -1446,6 +2051,8 @@ int UVCPreview::prepare_preview(uvc_stream_ctrl_t *ctrl) { if (LIKELY(mPreviewWindow)) { ANativeWindow_setBuffersGeometry(mPreviewWindow, frameWidth, frameHeight, previewFormat); + mPreviewGeomWidth = frameWidth; + mPreviewGeomHeight = frameHeight; } pthread_mutex_unlock(&preview_mutex); } else { @@ -1474,6 +2081,58 @@ int UVCPreview::prepare_preview(uvc_stream_ctrl_t *ctrl) { RETURN(result, int); } +void UVCPreview::presentPlanarFrame(uvc_frame_t *frame) { + uint64_t frame_ready_ns = 0; + uint64_t surface_wait_ns = 0; + /* Stage 4: decode thread -> preview thread (pool frame). */ + if (UNLIKELY(!frame_integrity_ok(frame, "decode->render", g_integrity_mismatch_render))) { + processingPreviewQueueDropCount.fetch_add(1, std::memory_order_relaxed); + recycle_frame(frame); + return; + } + const uint64_t t_render = processing_now_ns(); + const bool rendered = renderFrameDirectToSurface(frame, + &mPreviewWindow, &preview_mutex, &frame_ready_ns, + &surface_wait_ns); + { + /* Per-path render cost (upload+draw+swap), logged every + * 600 frames so the zero-copy path can be A/B'd on device. */ + static uint64_t s_sum[2], s_max[2]; + static uint32_t s_n[2]; + const int path = gpu_planar_is(frame) ? 1 : 0; + const uint64_t dt = processing_now_ns() - t_render; + s_sum[path] += dt; + if (dt > s_max[path]) s_max[path] = dt; + if (++s_n[path] == 600) { + LOGI("planar-render: path=%s avg_us=%llu max_us=%llu over 600 frames", + path ? "zero-copy" : "upload", + (unsigned long long)(s_sum[path] / 600 / 1000), + (unsigned long long)(s_max[path] / 1000)); + s_sum[path] = 0; s_max[path] = 0; s_n[path] = 0; + } + } + if (rendered && frame_ready_ns && frame->arrival_monotonic_ns) { + recordEndToEndLatencyTiming(frame->arrival_monotonic_ns, + frame_ready_ns); + } + if (gpu_planar_is(frame)) { + if (rendered) { + mGpuPlanarRenderFailures = 0; + gpu_planar_put(frame, mGpuPreviewRenderer + ? mGpuPreviewRenderer->takeRenderFenceFd() : -1); + } else { + gpu_planar_put(frame, -1); + if (++mGpuPlanarRenderFailures >= 3 && mGpuPlanarEnabled) { + LOGW("gpu-planar: render failed %u times; falling back to texture upload", + mGpuPlanarRenderFailures); + mGpuPlanarEnabled = false; + } + } + } else { + recycle_frame(frame); + } +} + void UVCPreview::do_preview(uvc_stream_ctrl_t *ctrl) { ENTER(); @@ -1550,15 +2209,7 @@ void UVCPreview::do_preview(uvc_stream_ctrl_t *ctrl) { continue; } if (frame->frame_format == UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR) { - uint64_t frame_ready_ns = 0; - uint64_t surface_wait_ns = 0; - if (renderFrameDirectToSurface(frame, &mPreviewWindow, - &preview_mutex, &frame_ready_ns, &surface_wait_ns) - && frame_ready_ns && frame->arrival_monotonic_ns) { - recordEndToEndLatencyTiming(frame->arrival_monotonic_ns, - frame_ready_ns); - } - recycle_frame(frame); + presentPlanarFrame(frame); frame = NULL; continue; } diff --git a/Consolation/app/src/main/jni/UVCCamera/UVCPreview.h b/Consolation/app/src/main/jni/UVCCamera/UVCPreview.h index 2e738e4..68f63eb 100644 --- a/Consolation/app/src/main/jni/UVCCamera/UVCPreview.h +++ b/Consolation/app/src/main/jni/UVCCamera/UVCPreview.h @@ -36,10 +36,14 @@ #pragma interface -/** Preview FIFO depth; drop-oldest on overflow (see BoundedPointerRing). */ +/** Preview FIFO capacity; drop-oldest on overflow (see BoundedPointerRing). + * Only H264 uses the depth: encoded access units must not be skipped. Every + * other mode enqueues latest-wins (addPreviewFrame drains older entries), so + * a renderer that falls behind never shows a frame older than the newest. */ #define PREVIEW_QUEUE_MAX 4 -/** MJPEG decode input depth; keep tight to avoid adding frame latency. */ -#define MJPEG_DECODE_QUEUE_MAX 2 +/** MJPEG decode input depth: 1 = the decoder always picks up the newest frame, + * the ring never adds more than one frame of latency under load. */ +#define MJPEG_DECODE_QUEUE_MAX 1 #define DEFAULT_PREVIEW_WIDTH 640 #define DEFAULT_PREVIEW_HEIGHT 480 @@ -75,7 +79,12 @@ class UVCPreview { int frameMode; size_t frameBytes; pthread_t preview_thread; + /** Guards mPreviewWindow and the GPU/CPU render into it. */ pthread_mutex_t preview_mutex; + /** Guards preview_frame_ring + preview_sync only. Kept separate from + * preview_mutex so a producer's enqueue never blocks behind the preview + * thread's render + eglSwapBuffers (which hold preview_mutex). */ + pthread_mutex_t preview_queue_mutex; pthread_cond_t preview_sync; /** Incoming frames; fixed ring, O(1) enqueue with drop-oldest on overflow */ BoundedPointerRing preview_frame_ring; @@ -84,6 +93,48 @@ class UVCPreview { pthread_mutex_t mjpeg_decode_mutex; pthread_cond_t mjpeg_decode_sync; BoundedPointerRing mjpeg_decode_frame_ring; + /** Fixed pool of frame headers handed to the async MJPEG decoder (ring depth + * + one in flight on each side). Replaces a malloc/free pair per frame. */ +#define MJPEG_HEADER_POOL_SZ (MJPEG_DECODE_QUEUE_MAX + 2) + uvc_frame_t mjpeg_header_slots[MJPEG_HEADER_POOL_SZ]; + bool mjpeg_header_used[MJPEG_HEADER_POOL_SZ]; + uvc_frame_t *mjpeg_header_get(); + void mjpeg_header_put_locked(uvc_frame_t *header); + void mjpeg_header_put(uvc_frame_t *header); + /** Planar MJPEG decode targets in GPU-sampleable memory: one R8 + * AHardwareBuffer per plane, bound by the renderer as EGLImages, so a + * decoded frame reaches the GPU with no upload copy. Depth: one queued + * (latest-wins) + one rendering + one decoding + one of slack. */ +#define GPU_PLANAR_POOL_SZ 4 + struct GpuPlanarFrame { + uvc_frame_t frame; + void *ahb[3]; + uint64_t ids[3]; + uint32_t w[3], h[3], stride[3]; + int fence_fd; /**< GPU read fence from the last render, -1 = none */ + bool in_use; + }; + GpuPlanarFrame mGpuPlanar[GPU_PLANAR_POOL_SZ]; + unsigned mGpuPlanarNext; + unsigned mGpuPlanarRenderFailures; + volatile bool mGpuPlanarEnabled; + uint32_t mGpuPlanarFormat; /**< AHARDWAREBUFFER_FORMAT_* chosen by the probe */ + uint64_t mGpuPlanarUsage; + uint32_t mGpuPlanarBytesPerTexel; /**< 1 for R8, 4 for packed RGBA8 */ + static void gpu_planar_free_slot(GpuPlanarFrame *g); + bool gpu_planar_alloc_plane(GpuPlanarFrame *g, int i, uint32_t width, uint32_t height); + uvc_frame_t *gpu_planar_get(const uint32_t widths[3], const uint32_t heights[3]); + void gpu_planar_put(uvc_frame_t *frame, int fence_fd); + void gpu_planar_release_all(); + static bool gpu_planar_is(const uvc_frame_t *frame) { + return frame && frame->yuv_hardware_buffers[0] != NULL; + } + bool decode_mjpeg_to_gpu_planar(uvc_frame_t *frame); + /** Render a decoded planar frame and return it to its pool. Used by the + * preview thread, or directly by the decode thread when merged rendering + * is on (saves the ring hand-off; costs the decode/render overlap). */ + void presentPlanarFrame(uvc_frame_t *frame); + volatile bool mMergedRender; int previewFormat; size_t previewBytes; // @@ -96,6 +147,17 @@ class UVCPreview { pthread_cond_t capture_sync; uvc_frame_t *captureQueu; // keep latest frame UVCGpuPreviewRenderer *mGpuPreviewRenderer; + float mPreviewXform[9]; /**< guarded by preview_mutex */ + /** Fit-to-screen content box as a fraction of the preview surface (<= 1 per axis); + * the CPU fallback pads its buffers to this shape. Guarded by preview_mutex. */ + float mPreviewFitX; + float mPreviewFitY; + /** Buffer geometry last applied to mPreviewWindow; 0x0 = the window's own size + * (GPU path). Guarded by preview_mutex. */ + int32_t mPreviewGeomWidth; + int32_t mPreviewGeomHeight; + /** Set when the view may have resized, so the GPU renderer re-reads its surface size. */ + bool mPreviewSurfaceSizeDirty; uvc_frame_t *mMjpegPreviewYuvFrame; jobject mFrameCallbackObj; convFunc_t mFrameCallbackFunc; @@ -212,6 +274,13 @@ class UVCPreview { inline const bool isRunning() const; int setPreviewSize(int width, int height, int min_fps, int max_fps, int mode, float bandwidth = 1.0f); int setPreviewDisplay(ANativeWindow *preview_window); + /** Rotation (0/90/180/270, clockwise on screen), mirror flags, zoom scale, + * pan in surface NDC units, and the fit-to-screen content box as a fraction + * of the surface (fit_x, fit_y <= 1), applied by the GPU renderer. Lets the + * preview live in a full-screen SurfaceView, which cannot be rotated or + * mirrored by the View system, while zooming into the letterbox area. */ + int setPreviewTransform(int rotation_degrees, bool flip_h, bool flip_v, + float scale, float pan_x_ndc, float pan_y_ndc, float fit_x, float fit_y); int setPreviewFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format); int setFrameCallback(JNIEnv *env, jobject frame_callback_obj, int pixel_format); int startPreview(); diff --git a/Consolation/app/src/main/jni/UVCCamera/uvc_jni_registration.cpp b/Consolation/app/src/main/jni/UVCCamera/uvc_jni_registration.cpp index 5b9d7f2..1c86df7 100644 --- a/Consolation/app/src/main/jni/UVCCamera/uvc_jni_registration.cpp +++ b/Consolation/app/src/main/jni/UVCCamera/uvc_jni_registration.cpp @@ -299,6 +299,16 @@ static jint nativeSetPreviewDisplay(JNIEnv *env, jobject thiz, RETURN(result, jint); } +static jint nativeSetPreviewTransform(JNIEnv *env, jobject thiz, + ID_TYPE id_camera, jint rotation, jboolean flip_h, jboolean flip_v, + jfloat scale, jfloat pan_x, jfloat pan_y, jfloat fit_x, jfloat fit_y) { + UVCCamera *camera = reinterpret_cast(id_camera); + if (LIKELY(camera)) + return camera->setPreviewTransform(rotation, flip_h == JNI_TRUE, flip_v == JNI_TRUE, + scale, pan_x, pan_y, fit_x, fit_y); + return JNI_ERR; +} + static jint nativeSetFrameCallback(JNIEnv *env, jobject thiz, ID_TYPE id_camera, jobject jIFrameCallback, jint pixel_format) { @@ -2098,6 +2108,7 @@ static JNINativeMethod methods[] = { { "nativeStartPreview", "(J)I", (void *) nativeStartPreview }, { "nativeStopPreview", "(J)I", (void *) nativeStopPreview }, { "nativeSetPreviewDisplay", "(JLandroid/view/Surface;)I", (void *) nativeSetPreviewDisplay }, + { "nativeSetPreviewTransform", "(JIZZFFFFF)I", (void *) nativeSetPreviewTransform }, { "nativeSetPreviewFrameCallback", "(JLorg/centennialoss/consolation/uvc/IFrameCallback;I)I", (void *) nativeSetPreviewFrameCallback }, { "nativeSetFrameCallback", "(JLorg/centennialoss/consolation/uvc/IFrameCallback;I)I", (void *) nativeSetFrameCallback }, diff --git a/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc.h b/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc.h index 68cabae..f4d1de4 100644 --- a/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc.h +++ b/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc.h @@ -513,6 +513,28 @@ typedef struct uvc_frame { size_t library_hardware_buffer_stride; /** @internal Planar YUV metadata for UVC_FRAME_FORMAT_MJPEG_YUV_PLANAR. */ int yuv_subsampling; + /** Sparse content hash taken when the frame was published/produced. + * Consumers re-sample before use to detect the buffer changing underneath + * them (see uvc_frame_sample_hash). 0 = not recorded. */ + uint32_t integrity_sample_hash; + /** Diagnostic: USB packet trace for the payloads that built this frame + * (ISO only). len = actual_length incl. UVC header; flags: bit0 first + * packet of a libusb transfer, bit1 EOF bit set, bit2 header_len != 12, + * bit3 header-only (no image data). count saturates at the array size. */ + /** Planar-MJPEG frames decoded straight into GPU-sampleable memory: one + * R8 AHardwareBuffer per plane (NULL past the plane count, e.g. gray). + * data is NULL for such frames; the renderer binds them via EGLImage. + * ids are unique per allocation so a renderer cache never confuses a + * recycled pointer with the buffer it previously mapped. */ + void *yuv_hardware_buffers[3]; + uint64_t yuv_hardware_buffer_ids[3]; + /** 1 = R8 buffers (one byte per texel); 4 = RGBA8 buffers a quarter as + * wide with four consecutive plane bytes packed into one texel. */ + uint32_t yuv_hardware_buffer_bytes_per_texel; +#define UVC_ISO_TRACE_MAX 512 + uint16_t iso_trace_len[UVC_ISO_TRACE_MAX]; + uint8_t iso_trace_flags[UVC_ISO_TRACE_MAX]; + uint16_t iso_trace_count; size_t yuv_plane_offsets[3]; size_t yuv_plane_strides[3]; uint32_t yuv_plane_widths[3]; @@ -844,7 +866,15 @@ uvc_error_t uvc_mjpeg2bgr(uvc_frame_t *in, uvc_frame_t *out); // XXX uvc_error_t uvc_mjpeg2rgb565(uvc_frame_t *in, uvc_frame_t *out); // XXX uvc_error_t uvc_mjpeg2rgbx(uvc_frame_t *in, uvc_frame_t *out); // XXX uvc_error_t uvc_mjpeg2yuyv(uvc_frame_t *in, uvc_frame_t *out); // XXX -uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out); // XXX +uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out); +/** Parse the MJPEG header and report the planar layout tj would produce: + * widths/heights per plane (0 for absent chroma planes on gray). */ +uvc_error_t uvc_mjpeg_planar_layout(uvc_frame_t *in, uint32_t widths[3], + uint32_t heights[3], int *subsamp); +/** Decode into caller-supplied planes (row strides in bytes). planes[1..2] + * may be NULL for gray. Applies the corrupt-frame warning policy. */ +uvc_error_t uvc_mjpeg2yuv_planes(uvc_frame_t *in, unsigned char *planes[3], + const int strides[3]); // XXX #endif uvc_error_t uvc_yuyv2rgb565(uvc_frame_t *in, uvc_frame_t *out); // XXX @@ -873,8 +903,13 @@ uvc_error_t uvc_any2yuyv(uvc_frame_t *in, uvc_frame_t *out); // XXX uvc_error_t uvc_ensure_frame_size(uvc_frame_t *frame, size_t need_bytes); // XXX /** Retain/release borrowed stream frame buffers returned by callback/get_frame. - * For copied/allocated frames these are no-ops. */ -void uvc_frame_retain(uvc_frame_t *frame); + * For copied/allocated frames these are no-ops. + * uvc_frame_retain returns 1 when a library slot reference was taken, 0 when + * the frame borrows no library slot (so the buffer is NOT protected from reuse). */ +int uvc_frame_retain(uvc_frame_t *frame); +/** Sparse FNV-1a over 16 x 64-byte windows spread across [data, data+len). + * ~1 KiB of reads regardless of frame size; cheap enough for every frame. */ +uint32_t uvc_frame_sample_hash(const void *data, size_t len); void uvc_frame_release(uvc_frame_t *frame); /** Synchronize an optional AHardwareBuffer-backed borrowed frame for GPU use. * These are no-ops for malloc-backed frames and non-Android builds. diff --git a/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc_internal.h b/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc_internal.h index d4acbe3..89df5db 100644 --- a/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc_internal.h +++ b/Consolation/app/src/main/jni/libuvc/include/libuvc/libuvc_internal.h @@ -295,8 +295,10 @@ typedef struct uvc_device_info { #ifndef LIBUVC_NUM_TRANSFER_BUFS #define LIBUVC_NUM_TRANSFER_BUFS 24 #endif +/* Sized for the ISO ring (stream_iso.c: 128 transfers x 8 packets). The + * per-slot arrays are pointers and bytes, so 128 costs ~2.5 KB per stream. */ #ifndef LIBUVC_MAX_TRANSFER_BUFS -#define LIBUVC_MAX_TRANSFER_BUFS 32 +#define LIBUVC_MAX_TRANSFER_BUFS 128 #endif #if LIBUVC_NUM_TRANSFER_BUFS > LIBUVC_MAX_TRANSFER_BUFS #error "LIBUVC_NUM_TRANSFER_BUFS cannot exceed LIBUVC_MAX_TRANSFER_BUFS array capacity" @@ -328,7 +330,26 @@ struct uvc_stream_handle { uint32_t pts, hold_pts; uint32_t last_scr, hold_last_scr; uint64_t frame_start_monotonic_ns, frame_complete_monotonic_ns, hold_start_monotonic_ns; + uint32_t hold_sample_hash; + uint16_t iso_trace_len[UVC_ISO_TRACE_MAX], hold_iso_trace_len[UVC_ISO_TRACE_MAX]; + uint8_t iso_trace_flags[UVC_ISO_TRACE_MAX], hold_iso_trace_flags[UVC_ISO_TRACE_MAX]; + uint16_t iso_trace_count, hold_iso_trace_count; size_t got_bytes, hold_bytes; + /* Incremental MJPEG marker scan of outbuf (see _uvc_mjpeg_note_payload_append): + * mjpeg_scan_pos = next pair index to examine; flags accumulate per frame. + * Lets _uvc_swap_buffers validate without a second full pass over the JPEG. */ + size_t mjpeg_scan_pos; + uint8_t mjpeg_scan_found_sos; + uint8_t mjpeg_scan_embedded_soi; + /* After an MJPEG frame is published on its EOI marker, remaining payloads + * with the same FID (padding, header-only EOF packets) are ignored until the + * FID flips, so they cannot start a bogus SOI-less frame. */ + uint8_t mjpeg_eoi_skip_valid; + uint8_t mjpeg_eoi_skip_fid; + /* EOI seen but the frame is held until its trailing UVC status is known: + * a later header-only payload may still carry EOF/ERR for this frame. */ + uint8_t mjpeg_eoi_pending; + const char *mjpeg_eoi_pending_reason; size_t size_buf; // XXX add for boundary check uint8_t *outbuf, *holdbuf; uint8_t *frame_pool[LIBUVC_FRAME_POOL_SLOTS]; @@ -372,6 +393,8 @@ struct uvc_stream_handle { uint16_t diag_bulk_timeout_count_before_payload; uint32_t diag_mjpeg_publish_count; uint32_t diag_mjpeg_drop_count; + /* Payload headers seen with the BFH ERR bit set (rate-limits the log line). */ + uint32_t diag_bfh_err_packets; uint32_t diag_selected_frame_interval_100ns; int32_t diag_selected_altsetting; uint8_t diag_selected_isochronous; diff --git a/Consolation/app/src/main/jni/libuvc/include/libuvc/stream_internal.h b/Consolation/app/src/main/jni/libuvc/include/libuvc/stream_internal.h index 8fccf2a..57916ed 100644 --- a/Consolation/app/src/main/jni/libuvc/include/libuvc/stream_internal.h +++ b/Consolation/app/src/main/jni/libuvc/include/libuvc/stream_internal.h @@ -11,8 +11,21 @@ struct libusb_interface; void _uvc_stream_try_acquire_outbuf(uvc_stream_handle_t *strmh); void _uvc_swap_buffers(uvc_stream_handle_t *strmh, const char *reason); -int _uvc_mjpeg_payload_has_markers(const uvc_stream_handle_t *strmh); -void _uvc_mjpeg_note_payload_append(uvc_stream_handle_t *strmh); +int _uvc_mjpeg_payload_has_markers(uvc_stream_handle_t *strmh); +/* Returns 1 when the MJPEG frame under assembly now ends with EOI (FFD9). */ +int _uvc_mjpeg_note_payload_append(uvc_stream_handle_t *strmh); +/* EOI seen: publish the assembled MJPEG frame now when its UVC status is + * final (EOF in this payload's header); otherwise hold it until EOF, an FID + * flip or the next SOI. Call right after a successful append that returned 1. */ +void _uvc_mjpeg_publish_on_eoi(uvc_stream_handle_t *strmh, uint8_t header_info, + const char *reason); +/* 1 while a complete frame is held awaiting its trailing status. */ +int _uvc_mjpeg_eoi_pending(const uvc_stream_handle_t *strmh); +/* 1 if this payload belongs to a frame already completed on EOI: drop it. + * Also resolves a frame held on EOI (applying a trailing ERR bit). */ +int _uvc_mjpeg_payload_after_eoi(uvc_stream_handle_t *strmh, uint8_t header_info, + const uint8_t *data, size_t data_len); +void _uvc_mjpeg_scan_reset(uvc_stream_handle_t *strmh); void _uvc_diag_mjpeg_drop(uvc_stream_handle_t *strmh, const char *reason); void _uvc_diag_mjpeg_publish(uvc_stream_handle_t *strmh, const char *reason); void _uvc_diag_mjpeg_log_stream_start(const uvc_stream_ctrl_t *ctrl, diff --git a/Consolation/app/src/main/jni/libuvc/src/frame-mjpeg.c b/Consolation/app/src/main/jni/libuvc/src/frame-mjpeg.c index 62fff7d..6fb98ad 100644 --- a/Consolation/app/src/main/jni/libuvc/src/frame-mjpeg.c +++ b/Consolation/app/src/main/jni/libuvc/src/frame-mjpeg.c @@ -45,6 +45,8 @@ #include #include #include +#include +#include #ifdef __ANDROID__ #include #endif @@ -109,6 +111,15 @@ struct mjpeg_decoder_ctx { struct jpeg_decompress_struct dinfo; tjhandle tj; int initialized; + /* Planar-decode warning policy (see _mjpeg_planar_warning_tolerated). + * Lives with the decoder instance so one stream's verdict is never + * inherited by another, and is reset when a new stream is detected. */ + uint32_t warn_drops; + uint32_t warn_pad_streak; /* consecutive frames with verified EOI padding */ + uint8_t warn_pad_fill; /* fill byte of that padding */ + uint8_t warn_pad_logged; + uint32_t warn_last_seq; + uint32_t warn_width, warn_height; }; static pthread_key_t mjpeg_decoder_key; @@ -722,16 +733,12 @@ uvc_error_t uvc_mjpeg2rgbx(uvc_frame_t *in, uvc_frame_t *out) { return UVC_ERROR_OTHER+1; } -uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out) { +uvc_error_t uvc_mjpeg_planar_layout(uvc_frame_t *in, uint32_t widths[3], + uint32_t heights[3], int *subsamp_out) { struct mjpeg_decoder_ctx *decoder; tjhandle tj; int subsamp; - int strides[3] = { 0, 0, 0 }; - unsigned char *planes[3] = { NULL, NULL, NULL }; - size_t offsets[3] = { 0, 0, 0 }; - size_t total_bytes = 0; - out->actual_bytes = 0; if (UNLIKELY(in->frame_format != UVC_FRAME_FORMAT_MJPEG)) return UVC_ERROR_INVALID_PARAM; if (UNLIKELY(!in->data || !in->actual_bytes || !in->width || !in->height)) @@ -756,27 +763,182 @@ uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out) { return UVC_ERROR_INVALID_PARAM; for (int i = 0; i < 3; i++) { - const int plane_width = tj3YUVPlaneWidth(i, (int)in->width, subsamp); - const int plane_height = tj3YUVPlaneHeight(i, (int)in->height, subsamp); - size_t plane_size; if (i > 0 && subsamp == TJSAMP_GRAY) { + widths[i] = 0; + heights[i] = 0; + continue; + } + const int pw = tj3YUVPlaneWidth(i, (int)in->width, subsamp); + const int ph = tj3YUVPlaneHeight(i, (int)in->height, subsamp); + if (UNLIKELY(pw <= 0 || ph <= 0)) + return UVC_ERROR_INVALID_PARAM; + widths[i] = (uint32_t)pw; + heights[i] = (uint32_t)ph; + } + if (subsamp_out) + *subsamp_out = subsamp; + return UVC_SUCCESS; +} + +/* + * libjpeg warnings after a "successful" decode mean the bitstream was damaged: + * a few bytes inserted or dropped desyncs the Huffman decoder with no restart + * markers, so the top decodes and everything below is shredded, ending in + * "premature end" (bytes missing) or "N extraneous bytes before marker" (bytes + * inserted). Both were observed on a MUSB USB 2.0 host; such frames are + * dropped so the last good one stays on glass. + * + * The one benign case is a camera that pads every frame between the end of the + * entropy-coded data and EOI. That is only accepted when it is positively + * identified, never merely because the warning is persistent (persistent + * transport corruption warns on every frame too): + * - the first (reported) warning is exactly "N extraneous bytes before + * marker 0xd9", so nothing was wrong before the scan data ended, + * - the N bytes immediately before the trailing FFD9 are one repeated fill + * byte (leftover entropy-coded data from a desync is not uniform), and + * - UVC_MJPEG_PAD_TRUST consecutive frames had that shape with the same + * fill byte. + * Any other warning rejects the frame and restarts that count. + */ +#define UVC_MJPEG_PAD_TRUST 60 + +static void _mjpeg_planar_warning_track_stream(struct mjpeg_decoder_ctx *ctx, + const uvc_frame_t *in) { + /* New stream (sequence restarted) or new mode: forget the old verdict. */ + if (in->sequence < ctx->warn_last_seq + || in->width != ctx->warn_width || in->height != ctx->warn_height) { + ctx->warn_drops = 0; + ctx->warn_pad_streak = 0; + ctx->warn_pad_logged = 0; + ctx->warn_width = in->width; + ctx->warn_height = in->height; + } + ctx->warn_last_seq = in->sequence; +} + +static void _mjpeg_planar_warning_clean(struct mjpeg_decoder_ctx *ctx, + const uvc_frame_t *in) { + _mjpeg_planar_warning_track_stream(ctx, in); + ctx->warn_pad_streak = 0; + ctx->warn_pad_logged = 0; +} + +/* 1 and *fill set when msg/in describe uniform fill padding before EOI. */ +static int _mjpeg_warning_is_eoi_padding(const uvc_frame_t *in, const char *msg, + uint8_t *fill) { + const uint8_t *data = (const uint8_t *)in->data; + const size_t len = in->actual_bytes; + const char *p = msg ? strstr(msg, "Corrupt JPEG data: ") : NULL; + unsigned int discarded = 0, marker = 0; + size_t i; + + if (!p || sscanf(p, "Corrupt JPEG data: %u extraneous bytes before marker 0x%x", + &discarded, &marker) != 2) + return 0; + if (marker != 0xd9 || !discarded || (size_t)discarded + 4 > len) + return 0; + if (data[len - 2] != 0xff || data[len - 1] != 0xd9) + return 0; + *fill = data[len - 3]; + for (i = 0; i < discarded; i++) { + if (data[len - 3 - i] != *fill) + return 0; + } + return 1; +} + +static int _mjpeg_planar_warning_tolerated(struct mjpeg_decoder_ctx *ctx, + const uvc_frame_t *in, const char *msg) { + uint8_t fill = 0; + + _mjpeg_planar_warning_track_stream(ctx, in); + if (_mjpeg_warning_is_eoi_padding(in, msg, &fill)) { + if (ctx->warn_pad_streak && fill != ctx->warn_pad_fill) + ctx->warn_pad_streak = 0; + ctx->warn_pad_fill = fill; + if (ctx->warn_pad_streak >= UVC_MJPEG_PAD_TRUST) { + if (!ctx->warn_pad_logged) { + ctx->warn_pad_logged = 1; + LOGW("mjpeg planar decode: every frame is 0x%02x-padded before EOI (%s); accepting", + (unsigned)fill, msg); + } + return 1; + } + ctx->warn_pad_streak++; + } else { + ctx->warn_pad_streak = 0; + ctx->warn_pad_logged = 0; + } + + ctx->warn_drops++; + if (ctx->warn_drops <= 5 || !(ctx->warn_drops % 300)) + LOGW("mjpeg planar decode dropped corrupt frame count=%u seq=%u bytes=%zu warn=%s", + ctx->warn_drops, in->sequence, in->actual_bytes, msg ? msg : "?"); + return 0; +} + +uvc_error_t uvc_mjpeg2yuv_planes(uvc_frame_t *in, unsigned char *planes[3], + const int strides[3]) { + struct mjpeg_decoder_ctx *decoder; + tjhandle tj; + + decoder = _mjpeg_decoder_get(); + if (UNLIKELY(!decoder)) + return UVC_ERROR_NO_MEM; + tj = _mjpeg_tj_decoder_get(decoder); + if (UNLIKELY(!tj)) + return UVC_ERROR_NO_MEM; + + if (UNLIKELY(tj3DecompressToYUVPlanes8(tj, + (const unsigned char *)in->data, in->actual_bytes, + planes, (int *)strides) != 0)) { + UVC_DIAG_LOGI("mjpeg-diag:planar-decode-fail seq=%u bytes=%zu err=%s", + in->sequence, + in->actual_bytes, + tj3GetErrorStr(tj)); + return UVC_ERROR_OTHER; + } + if (UNLIKELY(tj3GetErrorCode(tj) == TJERR_WARNING)) { + if (!_mjpeg_planar_warning_tolerated(decoder, in, tj3GetErrorStr(tj))) + return UVC_ERROR_OTHER; + } else { + _mjpeg_planar_warning_clean(decoder, in); + } + return UVC_SUCCESS; +} + +uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out) { + uint32_t widths[3], heights[3]; + int subsamp = 0; + int strides[3] = { 0, 0, 0 }; + unsigned char *planes[3] = { NULL, NULL, NULL }; + size_t offsets[3] = { 0, 0, 0 }; + size_t total_bytes = 0; + uvc_error_t err; + + out->actual_bytes = 0; + err = uvc_mjpeg_planar_layout(in, widths, heights, &subsamp); + if (UNLIKELY(err != UVC_SUCCESS)) + return err; + + for (int i = 0; i < 3; i++) { + size_t plane_size; + if (!widths[i]) { out->yuv_plane_widths[i] = 0; out->yuv_plane_heights[i] = 0; out->yuv_plane_offsets[i] = 0; out->yuv_plane_strides[i] = 0; continue; } - if (UNLIKELY(plane_width <= 0 || plane_height <= 0)) - return UVC_ERROR_INVALID_PARAM; - strides[i] = plane_width; + strides[i] = (int)widths[i]; plane_size = tj3YUVPlaneSize(i, (int)in->width, strides[i], (int)in->height, subsamp); if (UNLIKELY(!plane_size || total_bytes > (size_t)-1 - plane_size)) return UVC_ERROR_INVALID_PARAM; offsets[i] = total_bytes; total_bytes += plane_size; - out->yuv_plane_widths[i] = (uint32_t)plane_width; - out->yuv_plane_heights[i] = (uint32_t)plane_height; + out->yuv_plane_widths[i] = widths[i]; + out->yuv_plane_heights[i] = heights[i]; out->yuv_plane_offsets[i] = offsets[i]; out->yuv_plane_strides[i] = (size_t)strides[i]; } @@ -784,23 +946,12 @@ uvc_error_t uvc_mjpeg2yuv_planar(uvc_frame_t *in, uvc_frame_t *out) { if (UNLIKELY(uvc_ensure_frame_size(out, total_bytes) < 0)) return UVC_ERROR_NO_MEM; - for (int i = 0; i < 3; i++) { - if (i > 0 && subsamp == TJSAMP_GRAY) { - planes[i] = NULL; - continue; - } - planes[i] = (unsigned char *)out->data + offsets[i]; - } + for (int i = 0; i < 3; i++) + planes[i] = widths[i] ? (unsigned char *)out->data + offsets[i] : NULL; - if (UNLIKELY(tj3DecompressToYUVPlanes8(tj, - (const unsigned char *)in->data, in->actual_bytes, - planes, strides) != 0)) { - UVC_DIAG_LOGI("mjpeg-diag:planar-decode-fail seq=%u bytes=%zu err=%s", - in->sequence, - in->actual_bytes, - tj3GetErrorStr(tj)); - return UVC_ERROR_OTHER; - } + err = uvc_mjpeg2yuv_planes(in, planes, strides); + if (UNLIKELY(err != UVC_SUCCESS)) + return err; out->width = in->width; out->height = in->height; diff --git a/Consolation/app/src/main/jni/libuvc/src/frame.c b/Consolation/app/src/main/jni/libuvc/src/frame.c index 8cf841e..34a225a 100644 --- a/Consolation/app/src/main/jni/libuvc/src/frame.c +++ b/Consolation/app/src/main/jni/libuvc/src/frame.c @@ -103,6 +103,30 @@ void uvc_set_rgbx_converter_backend(uvc_rgbx_converter_backend_t backend) { } /** @internal */ +uint32_t uvc_frame_sample_hash(const void *data, size_t len) { + const uint8_t *bytes = (const uint8_t *)data; + uint32_t hash = 2166136261u; + size_t chunk; + + if (!bytes || !len) + return 0; + for (chunk = 0; chunk < 16; chunk++) { + size_t offset = (len * chunk) / 16; + size_t end = offset + 64; + size_t i; + if (end > len) + end = len; + for (i = offset; i < end; i++) { + hash ^= bytes[i]; + hash *= 16777619u; + } + } + /* Fold in the length so a same-prefix, different-size frame differs. */ + hash ^= (uint32_t)len; + hash *= 16777619u; + return hash ? hash : 1u; +} + uvc_error_t uvc_ensure_frame_size(uvc_frame_t *frame, size_t need_bytes) { if (UNLIKELY(!need_bytes)) return UVC_ERROR_NO_MEM; diff --git a/Consolation/app/src/main/jni/libuvc/src/stream.c b/Consolation/app/src/main/jni/libuvc/src/stream.c index f116a5e..05dd53e 100644 --- a/Consolation/app/src/main/jni/libuvc/src/stream.c +++ b/Consolation/app/src/main/jni/libuvc/src/stream.c @@ -47,6 +47,8 @@ #ifdef __ANDROID__ #include #include +#include +#include #endif #include "libuvc/libuvc.h" @@ -102,14 +104,15 @@ void _uvc_stream_try_acquire_outbuf(uvc_stream_handle_t *strmh) { pthread_mutex_unlock(&strmh->cb_mutex); } -void uvc_frame_retain(uvc_frame_t *frame) { +int uvc_frame_retain(uvc_frame_t *frame) { uvc_stream_handle_t *strmh; if (!frame || !frame->library_frame_owner || !_uvc_frame_slot_valid(frame->library_frame_slot)) - return; + return 0; strmh = (uvc_stream_handle_t *)frame->library_frame_owner; pthread_mutex_lock(&strmh->cb_mutex); _uvc_frame_retain_locked(strmh, frame->library_frame_slot); pthread_mutex_unlock(&strmh->cb_mutex); + return 1; } void uvc_frame_release(uvc_frame_t *frame) { @@ -192,6 +195,8 @@ static void _uvc_discard_assembled_frame(uvc_stream_handle_t *strmh, const char strmh->pts = 0; strmh->bfh_err = 0; _uvc_diag_iso_frame_reset(strmh); + _uvc_mjpeg_scan_reset(strmh); + strmh->iso_trace_count = 0; } struct format_table_entry { @@ -738,6 +743,13 @@ void _uvc_swap_buffers(uvc_stream_handle_t *strmh, const char *reason) { ? strmh->frame_complete_monotonic_ns : uvc_diag_now_ns(); strmh->hold_slot = strmh->out_slot; strmh->holdbuf = strmh->frame_pool[strmh->hold_slot]; + /* Publish-time fingerprint; consumers compare before reading. */ + strmh->hold_sample_hash = uvc_frame_sample_hash(strmh->holdbuf, strmh->got_bytes); + strmh->hold_iso_trace_count = strmh->iso_trace_count; + memcpy(strmh->hold_iso_trace_len, strmh->iso_trace_len, + (size_t)strmh->iso_trace_count * sizeof(strmh->iso_trace_len[0])); + memcpy(strmh->hold_iso_trace_flags, strmh->iso_trace_flags, + (size_t)strmh->iso_trace_count * sizeof(strmh->iso_trace_flags[0])); strmh->hold_last_scr = strmh->last_scr; strmh->hold_pts = strmh->pts; strmh->hold_seq = strmh->seq; @@ -750,10 +762,12 @@ void _uvc_swap_buffers(uvc_stream_handle_t *strmh, const char *reason) { strmh->outbuf = NULL; strmh->bfh_err |= UVC_STREAM_ERR; } - - pthread_cond_broadcast(&strmh->cb_cond); } pthread_mutex_unlock(&strmh->cb_mutex); + /* Signal after unlock: bionic has no wait morphing, so a waiter woken + * while the mutex is still held just blocks on it again (two context + * switches instead of one). */ + pthread_cond_broadcast(&strmh->cb_cond); strmh->seq++; strmh->got_bytes = 0; @@ -763,6 +777,8 @@ void _uvc_swap_buffers(uvc_stream_handle_t *strmh, const char *reason) { strmh->pts = 0; strmh->bfh_err = 0; // XXX _uvc_diag_iso_frame_reset(strmh); + _uvc_mjpeg_scan_reset(strmh); + strmh->iso_trace_count = 0; } /* Unified transfer-slot cleanup: @@ -883,21 +899,32 @@ static void _uvc_discard_iso_transfer_gap(uvc_stream_handle_t *strmh, static void _uvc_drain_ordered_iso_transfers(uvc_stream_handle_t *strmh) { uint32_t guard = 0; - while (strmh->num_transfer_bufs - && guard++ < strmh->num_transfer_bufs - && strmh->iso_transfer_pending[strmh->next_iso_transfer_id]) { + while (strmh->num_transfer_bufs && guard++ < strmh->num_transfer_bufs) { const uint32_t transfer_id = strmh->next_iso_transfer_id; - const uint8_t pending = strmh->iso_transfer_pending[transfer_id]; struct libusb_transfer *ready = strmh->transfers[transfer_id]; - int resubmit = pending != UVC_ISO_PENDING_STOP; + uint8_t pending; + int resubmit; + + if (UNLIKELY(!ready)) { + /* Slot was deleted (submit failure / stop). A deleted slot never + * becomes pending again, so waiting on it would stall the ordered + * drain forever and freeze the stream once every other transfer + * had completed once. Skip it and keep draining behind it. */ + strmh->iso_transfer_pending[transfer_id] = 0; + strmh->next_iso_transfer_id = + (transfer_id + 1) % strmh->num_transfer_bufs; + continue; + } + + pending = strmh->iso_transfer_pending[transfer_id]; + if (!pending) + break; + resubmit = pending != UVC_ISO_PENDING_STOP; strmh->iso_transfer_pending[transfer_id] = 0; strmh->next_iso_transfer_id = (transfer_id + 1) % strmh->num_transfer_bufs; - if (UNLIKELY(!ready)) - continue; - if (pending == UVC_ISO_PENDING_COMPLETE) { _uvc_diag_first_xfer_completed(strmh, ready); _uvc_stream_try_acquire_outbuf(strmh); @@ -1469,7 +1496,11 @@ uvc_error_t uvc_stream_start_bandwidth(uvc_stream_handle_t *strmh, strmh->next_iso_transfer_id = 0; memset(strmh->stalled_transfer_slots, 0, sizeof(strmh->stalled_transfer_slots)); memset(strmh->iso_transfer_pending, 0, sizeof(strmh->iso_transfer_pending)); + strmh->diag_bfh_err_packets = 0; + strmh->mjpeg_eoi_skip_valid = 0; _uvc_diag_iso_frame_reset(strmh); + _uvc_mjpeg_scan_reset(strmh); + strmh->iso_trace_count = 0; frame_desc = uvc_find_frame_desc_stream(strmh, ctrl->bFormatIndex, ctrl->bFrameIndex); if (UNLIKELY(!frame_desc)) { @@ -1604,6 +1635,16 @@ static void *_uvc_user_caller(void *arg) { uvc_stream_handle_t *strmh = (uvc_stream_handle_t *) arg; uint32_t last_seq = 0; + int deliver; + +#if defined(__ANDROID__) + /* This thread only hands published frames to the consumer, but it sits + * between the USB thread (nice -18) and the decoder (nice -4); at default + * priority it was the one hop that UI work could preempt. Keep it above + * the decoder so a published frame is queued for decode without delay. */ + pthread_setname_np(pthread_self(), "UVC-cb"); + (void)setpriority(PRIO_PROCESS, (id_t)gettid(), -8); +#endif for (; 1 ;) { pthread_mutex_lock(&strmh->cb_mutex); @@ -1618,14 +1659,21 @@ static void *_uvc_user_caller(void *arg) { } last_seq = strmh->hold_seq; - if (LIKELY(!strmh->hold_bfh_err)) // XXX + /* Snapshot the error flag once, under the lock, and use that same + * decision for populate, callback, and release. hold_bfh_err is + * rewritten by the USB thread on every publish; re-reading it after + * unlock let a set->clear flip deliver the *previous* (already + * released) frame to the user while the USB thread refilled its + * slot, and a clear->set flip leak a slot reference. ISO on USB 2.0 + * sets the error bit often enough to hit both regularly. */ + deliver = !strmh->hold_bfh_err; + if (LIKELY(deliver)) _uvc_populate_frame(strmh); } pthread_mutex_unlock(&strmh->cb_mutex); - if (LIKELY(!strmh->hold_bfh_err)) // XXX + if (LIKELY(deliver)) { strmh->user_cb(&strmh->frame, strmh->user_ptr); // call user callback function - if (LIKELY(!strmh->hold_bfh_err)) { uvc_frame_release(&strmh->frame); strmh->frame.library_frame_owner = NULL; strmh->frame.library_hardware_buffer = NULL; @@ -1701,6 +1749,12 @@ void _uvc_populate_frame(uvc_stream_handle_t *strmh) { frame->capture_time.tv_sec = 0; frame->capture_time.tv_usec = 0; frame->arrival_monotonic_ns = strmh->hold_start_monotonic_ns; + frame->integrity_sample_hash = strmh->hold_sample_hash; + frame->iso_trace_count = strmh->hold_iso_trace_count; + memcpy(frame->iso_trace_len, strmh->hold_iso_trace_len, + (size_t)strmh->hold_iso_trace_count * sizeof(frame->iso_trace_len[0])); + memcpy(frame->iso_trace_flags, strmh->hold_iso_trace_flags, + (size_t)strmh->hold_iso_trace_count * sizeof(frame->iso_trace_flags[0])); /** @todo set the frame time */ } diff --git a/Consolation/app/src/main/jni/libuvc/src/stream_bulk.c b/Consolation/app/src/main/jni/libuvc/src/stream_bulk.c index 2a66bfb..48dc372 100644 --- a/Consolation/app/src/main/jni/libuvc/src/stream_bulk.c +++ b/Consolation/app/src/main/jni/libuvc/src/stream_bulk.c @@ -8,6 +8,7 @@ #include #endif +#include #include "libuvc/stream_log.h" #include "libuvc/libuvc.h" #include "libuvc/libuvc_internal.h" @@ -44,6 +45,8 @@ void _uvc_process_payload_bulk(uvc_stream_handle_t *strmh, const uint8_t *payloa if (UNLIKELY(header_len < 2)) { header_info = 0; + if (_uvc_mjpeg_eoi_pending(strmh)) + return; /* no status bits; never append past a held EOI */ } else { // @todo we should be checking the end-of-header bit size_t variable_offset = 2; @@ -63,15 +66,10 @@ void _uvc_process_payload_bulk(uvc_stream_handle_t *strmh, const uint8_t *payloa data_len); } - if (UNLIKELY(header_info & UVC_STREAM_ERR)) { - LOGI("startup-diag:libuvc bulk ERR bit set in header " - "(before_first_payload=%d)", - !strmh->first_video_payload_received); - /* A BFH ERR bit in a completed bulk payload is a device stream - * condition, not a USB endpoint halt. Do not issue synchronous - * control transfers from this hot path; true endpoint halts are - * reported separately as LIBUSB_TRANSFER_STALL. */ - } + /* Tail of a frame already published on its EOI marker: ignore. */ + if (_uvc_mjpeg_payload_after_eoi(strmh, header_info, + payload + header_len, data_len)) + return; if ((strmh->fid != (header_info & UVC_STREAM_FID)) && strmh->got_bytes) { /* The frame ID bit was flipped, but we have image data sitting @@ -82,6 +80,26 @@ void _uvc_process_payload_bulk(uvc_stream_handle_t *strmh, const uint8_t *payloa strmh->fid = header_info & UVC_STREAM_FID; + if (UNLIKELY(header_info & UVC_STREAM_ERR)) { + /* A BFH ERR bit in a completed bulk payload is a device stream + * condition, not a USB endpoint halt. Do not issue synchronous + * control transfers from this hot path; true endpoint halts are + * reported separately as LIBUSB_TRANSFER_STALL. + * Per UVC 1.5 2.4.3.3 the frame is damaged: mark it so it is + * dropped at publish instead of decoded with a hole. */ + if (strmh->got_bytes || data_len) + strmh->bfh_err |= UVC_STREAM_ERR; + strmh->diag_bfh_err_packets++; + if (strmh->diag_bfh_err_packets == 1 + || !(strmh->diag_bfh_err_packets % 1000)) { + LOGI("libuvc bulk ERR bit set in payload header count=%u " + "(before_first_payload=%d got_bytes=%zu data_len=%zu)", + strmh->diag_bfh_err_packets, + !strmh->first_video_payload_received, + strmh->got_bytes, data_len); + } + } + if (header_info & UVC_STREAM_PTS) { // XXX saki some camera may send broken packet or failed to receive all data if (LIKELY(variable_offset + 4 <= header_len)) { @@ -113,7 +131,10 @@ void _uvc_process_payload_bulk(uvc_stream_handle_t *strmh, const uint8_t *payloa if (LIKELY(strmh->got_bytes + data_len <= strmh->size_buf)) { memcpy(strmh->outbuf + strmh->got_bytes, payload + header_len, data_len); strmh->got_bytes += data_len; - _uvc_mjpeg_note_payload_append(strmh); + if (_uvc_mjpeg_note_payload_append(strmh)) { + _uvc_mjpeg_publish_on_eoi(strmh, header_info, "bulk-eoi"); + return; + } } else { strmh->bfh_err |= UVC_STREAM_ERR; } @@ -146,7 +167,9 @@ uvc_error_t _uvc_stream_setup_bulk_transfers(uvc_stream_handle_t *strmh, strmh->transfer_bufs[transfer_id], strmh->cur_ctrl.dwMaxPayloadTransferSize, _uvc_stream_callback, (void *)strmh, LIBUVC_STREAM_XFER_TIMEOUT_MS); - + /* See stream_iso.c: build the URBs once, resubmit without allocating. */ + if (UNLIKELY(libusb_prealloc_bulk_urbs(transfer) != LIBUSB_SUCCESS)) + UVC_DEBUG("bulk transfer %d: URB prealloc failed, using slow path", transfer_id); } return UVC_SUCCESS; } diff --git a/Consolation/app/src/main/jni/libuvc/src/stream_iso.c b/Consolation/app/src/main/jni/libuvc/src/stream_iso.c index 74cf50d..cc759c2 100644 --- a/Consolation/app/src/main/jni/libuvc/src/stream_iso.c +++ b/Consolation/app/src/main/jni/libuvc/src/stream_iso.c @@ -4,18 +4,32 @@ #ifdef __ANDROID__ #include + #include #endif #include + #include + #include + #include #include + #include #include "libuvc/stream_log.h" #include "libuvc/libuvc.h" #include "libuvc/libuvc_internal.h" #include "libuvc/stream_internal.h" +/* + * Packets per ISO transfer sets the reap granularity: libusb only hands a + * transfer back once every packet's service interval has elapsed, so a frame + * whose EOF lands early in a transfer waits for the rest. At bInterval=1 on + * high speed one packet is 125 us: 32 packets held the EOF up to 4 ms (2 ms + * average); 8 packets caps that at 1 ms. The transfer count below is raised + * to keep the same 128 ms of bus time queued in the kernel. Cost is ~1000 + * reap ioctls/s instead of ~250, all on the pre-allocated URB path. + */ #ifndef LIBUVC_NUM_ISO_PACKETS_PER_XFER -#define LIBUVC_NUM_ISO_PACKETS_PER_XFER 32 +#define LIBUVC_NUM_ISO_PACKETS_PER_XFER 8 #endif /* * This is the number of libusb transfers submitted for ISO streaming, not the @@ -25,7 +39,7 @@ * time; otherwise ISO setup will write past the end of the stream handle. */ #ifndef LIBUVC_NUM_ISO_TRANSFER_BUFS -#define LIBUVC_NUM_ISO_TRANSFER_BUFS 32 +#define LIBUVC_NUM_ISO_TRANSFER_BUFS 128 #endif #if LIBUVC_NUM_ISO_TRANSFER_BUFS > LIBUVC_MAX_TRANSFER_BUFS #error "LIBUVC_NUM_ISO_TRANSFER_BUFS cannot exceed LIBUVC_MAX_TRANSFER_BUFS array capacity" @@ -122,6 +136,10 @@ void _uvc_diag_iso_frame_reset(uvc_stream_handle_t *strmh) { strmh->diag_iso_packet_len_hash = 2166136261u; } +#if UVC_RUNTIME_DIAG_ENABLED +/* Per-byte payload hash and packet-shape stats feed only _uvc_diag_mjpeg_publish, + * which is compiled out unless UVC_RUNTIME_DIAG_ENABLED. Keep them out of the + * USB reap path otherwise: the hash alone is a serial full pass over every frame. */ static void _uvc_diag_iso_payload_bytes(uvc_stream_handle_t *strmh, const uint8_t *data, size_t len) { size_t i; @@ -159,9 +177,97 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, strmh->diag_iso_packet_len_hash ^= (uint32_t)(len >> 16); strmh->diag_iso_packet_len_hash *= 16777619u; } +#endif /* UVC_RUNTIME_DIAG_ENABLED */ +/* + * High-bandwidth ISO IN (wMaxPacketSize mult > 1, i.e. 2 or 3 transactions per + * microframe) is corrupted by the Mentor MUSB host controller used on Unisoc + * (and some other low-cost) SoCs: a few bytes are inserted or dropped inside the + * packet at transaction boundaries, so an MJPEG frame still carries SOI/EOI but + * decodes shredded below the damage. Measured on a Unisoc ums9230 tablet: + * 10/300 frames damaged at 3072 B/uframe, 13/302 at 2048, 0/301 at 1024. + * Single-transaction endpoints (<= 1024 B/uframe) are reliable there, so on such + * hosts we restrict alt-setting selection to them. + * + * Detection: the sysfs bus symlink names the controller driver (readable by an + * app process), e.g. .../64900000.usb/musb-hdrc.1.auto/usb1. Fallback when the + * link cannot be read: Unisoc platform names (ro.board.platform ums.., sc98.., ud7..). + * Override for testing: adb shell setprop debug.consolation.iso_max_packet + * (N > 0 forces that cap; the property wins over auto-detection). + */ +#define LIBUVC_MUSB_ISO_PACKET_CAP 1024u + +/* This file's LOGI compiles out under LOG_NDEBUG; the cap decision must be + * visible in normal builds, so log it directly. */ +#ifdef __ANDROID__ +/* Tag must match the one apps allowlist via log.tag.* (libuvc/stream is + * filtered out on devices whose default log level is E). */ +#define UVC_HOSTCAP_LOGI(...) __android_log_print(ANDROID_LOG_INFO, "libUVCCamera", __VA_ARGS__) +#else +#define UVC_HOSTCAP_LOGI(...) LOGI(__VA_ARGS__) +#endif + +static unsigned int _uvc_iso_host_packet_cap(uvc_stream_handle_t *strmh) { +#ifdef __ANDROID__ + char value[PROP_VALUE_MAX] = {0}; + char path[64]; + char target[256]; + ssize_t n; + uint8_t bus; + const char *why = NULL; + + if (__system_property_get("debug.consolation.iso_max_packet", value) > 0 + && value[0]) { + const unsigned int forced = (unsigned int)atoi(value); + UVC_HOSTCAP_LOGI("libuvc iso host cap: forced by property %s -> %u", value, forced); + return forced; + } + + bus = libusb_get_bus_number(libusb_get_device(strmh->devh->usb_devh)); + if (bus) { + snprintf(path, sizeof(path), "/sys/bus/usb/devices/usb%u", (unsigned)bus); + n = readlink(path, target, sizeof(target) - 1); + if (n > 0) { + target[n] = 0; + if (strstr(target, "musb")) + why = "musb host controller"; + UVC_HOSTCAP_LOGI("libuvc iso host cap: bus %u -> %s", (unsigned)bus, target); + } else { + UVC_HOSTCAP_LOGI("libuvc iso host cap: readlink(%s) failed errno=%d", path, errno); + } + } + if (!why && bus == 0) { + /* Bus unknown (wrapped fd without a resolvable path): probe usb1..usb4. */ + unsigned int b; + for (b = 1; b <= 4 && !why; b++) { + snprintf(path, sizeof(path), "/sys/bus/usb/devices/usb%u", b); + n = readlink(path, target, sizeof(target) - 1); + if (n > 0) { + target[n] = 0; + if (strstr(target, "musb")) + why = "musb host controller (bus probe)"; + } + } + } + if (!why && __system_property_get("ro.board.platform", value) > 0) { + if (!strncmp(value, "ums", 3) || !strncmp(value, "sc98", 4) + || !strncmp(value, "sp98", 4) || !strncmp(value, "ud7", 3)) + why = "unisoc platform"; + } + if (why) { + UVC_HOSTCAP_LOGI("libuvc iso host cap: %s -> limiting ISO packets to %u B/uframe " + "(single transaction)", why, LIBUVC_MUSB_ISO_PACKET_CAP); + return LIBUVC_MUSB_ISO_PACKET_CAP; + } + return 0; +#else + (void)strmh; + return 0; +#endif +} + static void _uvc_process_payload_iso_packet(uvc_stream_handle_t *strmh, - const uint8_t *payload, size_t payload_len) { + const uint8_t *payload, size_t payload_len, int first_in_transfer) { size_t header_len; uint8_t header_info = 0; size_t data_len; @@ -182,9 +288,23 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, } data_len = payload_len - header_len; + if (header_len >= 2) + header_info = payload[1]; + + /* Diagnostic packet trace for the frame under assembly (see uvc_frame_t). */ + if (strmh->iso_trace_count < UVC_ISO_TRACE_MAX) { + const uint16_t i = strmh->iso_trace_count++; + strmh->iso_trace_len[i] = (uint16_t)(payload_len > 0xffff ? 0xffff : payload_len); + strmh->iso_trace_flags[i] = (uint8_t)((first_in_transfer ? 1u : 0u) + | ((header_info & UVC_STREAM_EOF) ? 2u : 0u) + | (header_len != 12 ? 4u : 0u) + | (data_len == 0 ? 8u : 0u)); + } if (UNLIKELY(header_len < 2)) { header_info = 0; + if (_uvc_mjpeg_eoi_pending(strmh)) + return; /* no status bits; never append past a held EOI */ } else { size_t variable_offset = 2; @@ -203,17 +323,37 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, data_len); } - if (UNLIKELY(header_info & UVC_STREAM_ERR)) { - LOGI("startup-diag:libuvc iso ERR bit set in header " - "(before_first_payload=%d)", - !strmh->first_video_payload_received); - } - + /* Tail of a frame already published on its EOI marker: ignore. */ + if (_uvc_mjpeg_payload_after_eoi(strmh, header_info, + payload + header_len, data_len)) + return; + if ((strmh->fid != (header_info & UVC_STREAM_FID)) && strmh->got_bytes) { _uvc_swap_buffers(strmh, "iso-fid"); } - + strmh->fid = header_info & UVC_STREAM_FID; + + if (UNLIKELY(header_info & UVC_STREAM_ERR)) { + /* UVC 1.5 2.4.3.3: the device hit an error transmitting this + * frame (typically its FIFO overran because the host link is too + * slow, e.g. a USB 2.0 port at 3072 B/uframe). The frame has a + * hole; an MJPEG with a hole still carries SOI/EOI and decodes + * as a shredded lower half. Mark it so it is dropped at publish. + * Only poison the frame in progress, not the next one, when the + * ERR arrives on an idle header between frames. */ + if (strmh->got_bytes || data_len) + strmh->bfh_err |= UVC_STREAM_ERR; + strmh->diag_bfh_err_packets++; + if (strmh->diag_bfh_err_packets == 1 + || !(strmh->diag_bfh_err_packets % 1000)) { + LOGI("libuvc iso ERR bit set in payload header count=%u " + "(before_first_payload=%d got_bytes=%zu data_len=%zu)", + strmh->diag_bfh_err_packets, + !strmh->first_video_payload_received, + strmh->got_bytes, data_len); + } + } if (header_info & UVC_STREAM_PTS) { if (LIKELY(variable_offset + 4 <= header_len)) { @@ -240,10 +380,15 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, strmh->first_video_payload_received = 1; _uvc_diag_first_payload(strmh, data_len, "iso"); if (LIKELY(strmh->got_bytes + data_len <= strmh->size_buf)) { +#if UVC_RUNTIME_DIAG_ENABLED _uvc_diag_iso_payload_bytes(strmh, payload + header_len, data_len); +#endif memcpy(strmh->outbuf + strmh->got_bytes, payload + header_len, data_len); strmh->got_bytes += data_len; - _uvc_mjpeg_note_payload_append(strmh); + if (_uvc_mjpeg_note_payload_append(strmh)) { + _uvc_mjpeg_publish_on_eoi(strmh, header_info, "iso-eoi"); + return; + } } else { strmh->diag_iso_overflow_count++; strmh->bfh_err |= UVC_STREAM_ERR; @@ -275,14 +420,19 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, } if (UNLIKELY(packet->actual_length <= 0)) { strmh->diag_iso_zero_packets++; - if (strmh->got_bytes) + /* A gap inside a frame is a hole; a gap after its EOI is just + * the device idling before the trailer or the next frame. */ + if (strmh->got_bytes && !_uvc_mjpeg_eoi_pending(strmh)) strmh->bfh_err |= UVC_STREAM_ERR; continue; } +#if UVC_RUNTIME_DIAG_ENABLED _uvc_diag_iso_packet_shape(strmh, packet->actual_length, packet->length); +#endif payload = libusb_get_iso_packet_buffer_simple(transfer, packet_id); - _uvc_process_payload_iso_packet(strmh, payload, packet->actual_length); + _uvc_process_payload_iso_packet(strmh, payload, packet->actual_length, + packet_id == 0); } } @@ -294,6 +444,7 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, const struct libusb_interface_descriptor *selected_altsetting = NULL; const struct libusb_endpoint_descriptor *selected_endpoint = NULL; unsigned int selected_packet_size = 0; + unsigned int packet_cap; uint32_t required_payload_size; int selected_satisfies_required = 0; int altsetting_id; @@ -306,6 +457,7 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, return UVC_ERROR_INVALID_PARAM; required_payload_size = _uvc_iso_required_payload_size(strmh, bandwidth_factor); + packet_cap = _uvc_iso_host_packet_cap(strmh); for (altsetting_id = 0; altsetting_id < interface->num_altsetting; ++altsetting_id) { const struct libusb_interface_descriptor *altsetting = @@ -321,14 +473,17 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, if (!packet_size) continue; - UVC_DIAG_LOGI("mjpeg-diag:iso-alt alt=%u ep=0x%02x packet_size=%u " - "wMaxPacketSize=0x%04x interval=%u required=%u", + UVC_HOSTCAP_LOGI("libuvc iso-alt alt=%u ep=0x%02x packet_size=%u " + "wMaxPacketSize=0x%04x interval=%u required=%u cap=%u", (unsigned)altsetting->bAlternateSetting, (unsigned)endpoint->bEndpointAddress, packet_size, (unsigned)endpoint->wMaxPacketSize, (unsigned)endpoint->bInterval, - (unsigned)required_payload_size); + (unsigned)required_payload_size, + packet_cap); + if (packet_cap && packet_size > packet_cap) + continue; #if LIBUVC_ISO_PREFER_MAX_PACKET_SIZE if (!selected_altsetting || packet_size > selected_packet_size) { @@ -371,7 +526,7 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, strmh->diag_selected_altsetting = selected_altsetting->bAlternateSetting; strmh->num_transfer_bufs = LIBUVC_NUM_ISO_TRANSFER_BUFS; - UVC_DIAG_LOGI("mjpeg-diag:iso-selected alt=%u ep=0x%02x packet_size=%u " + UVC_HOSTCAP_LOGI("libuvc iso-selected alt=%u ep=0x%02x packet_size=%u " "required=%u prefer_max=%u", (unsigned)selected_altsetting->bAlternateSetting, (unsigned)selected_endpoint->bEndpointAddress, @@ -401,7 +556,15 @@ static void _uvc_diag_iso_packet_shape(uvc_stream_handle_t *strmh, (void *)strmh, LIBUVC_STREAM_XFER_TIMEOUT_MS); libusb_set_iso_packet_lengths(transfer, selected_packet_size); - + /* Pre-build the kernel URBs once so every resubmit on the USB thread is + * a bookkeeping reset plus SUBMITURB ioctls, with no calloc/free. A + * failure just leaves the allocating slow path in place. */ + { + const int pre = libusb_prealloc_iso_urbs(transfer); + if (UNLIKELY(pre != LIBUSB_SUCCESS)) + UVC_HOSTCAP_LOGI("libuvc iso transfer %d: URB prealloc failed (%d), using slow path", + transfer_id, pre); + } } return UVC_SUCCESS; diff --git a/Consolation/app/src/main/jni/libuvc/src/stream_mjpeg.c b/Consolation/app/src/main/jni/libuvc/src/stream_mjpeg.c index a0a1917..691798a 100644 --- a/Consolation/app/src/main/jni/libuvc/src/stream_mjpeg.c +++ b/Consolation/app/src/main/jni/libuvc/src/stream_mjpeg.c @@ -2,6 +2,8 @@ * MJPEG-specific diagnostics and JPEG bitstream checks (libuvc stream path). *********************************************************************/ +#include + #include "libuvc/stream_internal.h" static uint32_t _uvc_diag_sample_hash(const uint8_t *data, size_t len) { @@ -97,11 +99,57 @@ static uint32_t _uvc_diag_mjpeg_header_hash(const uint8_t *data, size_t len, return hash; } -int _uvc_mjpeg_payload_has_markers(const uvc_stream_handle_t *strmh) { +void _uvc_mjpeg_scan_reset(uvc_stream_handle_t *strmh) { + if (!strmh) + return; + /* Note: mjpeg_eoi_skip_* deliberately survive this reset; they are + * cleared by the next FID flip (see _uvc_mjpeg_payload_after_eoi). */ + strmh->mjpeg_eoi_pending = 0; + strmh->mjpeg_scan_pos = 0; + strmh->mjpeg_scan_found_sos = 0; + strmh->mjpeg_scan_embedded_soi = 0; +} + +/* Advance the incremental marker scan over bytes appended to outbuf since the + * last call. Examines every (i, i+1) pair with i >= 2, carrying one byte across + * append boundaries, so the result at publish time equals a full scan of + * outbuf[2 .. got_bytes). Uses memchr to skip to 0xff candidates: entropy-coded + * JPEG data contains few 0xff bytes, so this runs near memcpy speed and the data + * is still cache-hot from the payload memcpy that preceded it. */ +static void _uvc_mjpeg_scan_advance(uvc_stream_handle_t *strmh) { const uint8_t *data = strmh->outbuf; const size_t len = strmh->got_bytes; size_t i; - int found_sos = 0; + + if (!data || len < 3) + return; + if (strmh->mjpeg_scan_embedded_soi) + return; /* already rejected; nothing further changes the verdict */ + + i = strmh->mjpeg_scan_pos < 2 ? 2 : strmh->mjpeg_scan_pos; + while (i + 1 < len) { + const uint8_t *ff = memchr(data + i, 0xff, len - 1 - i); + uint8_t next; + if (!ff) + break; + i = (size_t)(ff - data); + next = data[i + 1]; + if (next == 0xd8) { + strmh->mjpeg_scan_embedded_soi = 1; + break; + } + if (next == 0xda) + strmh->mjpeg_scan_found_sos = 1; + i++; + } + /* Next call resumes at the pair that starts on the current last byte, so a + * marker split across two appends is still seen. */ + strmh->mjpeg_scan_pos = len - 1; +} + +int _uvc_mjpeg_payload_has_markers(uvc_stream_handle_t *strmh) { + const uint8_t *data = strmh->outbuf; + const size_t len = strmh->got_bytes; if (strmh->frame_format != UVC_FRAME_FORMAT_MJPEG) return 1; @@ -111,26 +159,102 @@ int _uvc_mjpeg_payload_has_markers(const uvc_stream_handle_t *strmh) { && data[len - 2] == 0xff && data[len - 1] == 0xd9)) return 0; - for (i = 2; i + 1 < len - 2; i++) { - if (data[i] == 0xff && data[i + 1] == 0xd8) - return 0; - if (data[i] == 0xff && data[i + 1] == 0xda) - found_sos = 1; - } - - return found_sos; + /* Catch up in case bytes were appended without _uvc_mjpeg_note_payload_append. */ + _uvc_mjpeg_scan_advance(strmh); + return !strmh->mjpeg_scan_embedded_soi && strmh->mjpeg_scan_found_sos; } -void _uvc_mjpeg_note_payload_append(uvc_stream_handle_t *strmh) { +int _uvc_mjpeg_note_payload_append(uvc_stream_handle_t *strmh) { const uint8_t *data; const size_t len = strmh ? strmh->got_bytes : 0; if (!strmh || strmh->frame_format != UVC_FRAME_FORMAT_MJPEG || len < 2) - return; + return 0; + + _uvc_mjpeg_scan_advance(strmh); data = strmh->outbuf; - if (data && data[len - 2] == 0xff && data[len - 1] == 0xd9) + if (data && data[len - 2] == 0xff && data[len - 1] == 0xd9) { strmh->frame_complete_monotonic_ns = uvc_diag_now_ns(); + return 1; + } + return 0; +} + +/* + * Publish on EOI. A JPEG is complete at its EOI marker, so waiting for the + * UVC EOF bit (often carried by a later header-only packet, i.e. after the + * next transfer completes) or for the FID flip (a whole frame interval later + * on cameras that never set EOF) only adds latency. FFD9 cannot occur inside + * entropy-coded data (0xFF is always stuffed), so the tail check is exact. + * + * Complete is not the same as valid, though: a header-only trailer for this + * frame can still carry UVC_STREAM_ERR (the frame has a transmission hole but + * intact SOI/EOI). So the frame is only published straight away when the + * payload that carried the EOI also carried EOF. Otherwise it is held until + * a definitive boundary: a trailer with EOF, an FID flip, or the next frame's + * SOI. Idle ISO slots and past behaviour of the device prove nothing about + * whether a trailer is still coming, so neither releases a held frame. + */ +static void _uvc_mjpeg_eoi_publish_now(uvc_stream_handle_t *strmh, const char *reason) { + strmh->mjpeg_eoi_pending = 0; + strmh->mjpeg_eoi_skip_valid = 1; + strmh->mjpeg_eoi_skip_fid = strmh->fid; + _uvc_swap_buffers(strmh, reason); +} + +void _uvc_mjpeg_publish_on_eoi(uvc_stream_handle_t *strmh, uint8_t header_info, + const char *reason) { + if (!strmh || strmh->frame_format != UVC_FRAME_FORMAT_MJPEG) + return; + if (header_info & UVC_STREAM_EOF) { + _uvc_mjpeg_eoi_publish_now(strmh, reason); + return; + } + strmh->mjpeg_eoi_pending = 1; + strmh->mjpeg_eoi_pending_reason = reason; +} + +int _uvc_mjpeg_eoi_pending(const uvc_stream_handle_t *strmh) { + return strmh && strmh->mjpeg_eoi_pending; +} + +int _uvc_mjpeg_payload_after_eoi(uvc_stream_handle_t *strmh, uint8_t header_info, + const uint8_t *data, size_t data_len) { + const int new_frame = + ((header_info & UVC_STREAM_FID) != (strmh->mjpeg_eoi_pending + ? strmh->fid : strmh->mjpeg_eoi_skip_fid)) + /* New SOI under the same FID (camera does not toggle FID). */ + || (data_len >= 2 && data[0] == 0xff && data[1] == 0xd8); + + if (strmh->mjpeg_eoi_pending) { + if (new_frame) { + /* No trailer was sent: the held frame stands as assembled. */ + _uvc_mjpeg_eoi_publish_now(strmh, strmh->mjpeg_eoi_pending_reason); + strmh->mjpeg_eoi_skip_valid = 0; + return 0; /* this payload starts the next frame: process it */ + } + /* Trailer (or padding) of the held frame: its ERR bit still counts. */ + if (header_info & UVC_STREAM_ERR) { + strmh->bfh_err |= UVC_STREAM_ERR; /* dropped at publish */ + strmh->diag_bfh_err_packets++; + } + if (header_info & UVC_STREAM_EOF) { + _uvc_mjpeg_eoi_publish_now(strmh, strmh->mjpeg_eoi_pending_reason); + strmh->mjpeg_eoi_skip_valid = 0; /* explicit end of the frame */ + } + return 1; + } + + if (!strmh->mjpeg_eoi_skip_valid) + return 0; + if (new_frame) { + strmh->mjpeg_eoi_skip_valid = 0; /* next frame begins */ + return 0; + } + if (header_info & UVC_STREAM_EOF) + strmh->mjpeg_eoi_skip_valid = 0; /* explicit end of the published frame */ + return 1; } void _uvc_diag_mjpeg_drop(uvc_stream_handle_t *strmh, const char *reason) { diff --git a/Consolation/app/src/main/res/values/strings.xml b/Consolation/app/src/main/res/values/strings.xml index 628aad6..fdd58f2 100644 --- a/Consolation/app/src/main/res/values/strings.xml +++ b/Consolation/app/src/main/res/values/strings.xml @@ -19,6 +19,8 @@ Request USB permission Play Stop Video + Toggle 1:1 pixel scaling + 1:1 scaling: Each video pixel now matches one screen pixel. Dump USB diagnostics (Logcat) Settings Help diff --git a/Consolation/gradle/libs.versions.toml b/Consolation/gradle/libs.versions.toml index d6efd7a..39cf4e3 100644 --- a/Consolation/gradle/libs.versions.toml +++ b/Consolation/gradle/libs.versions.toml @@ -1,9 +1,9 @@ [versions] # Single minSdk for :app (vendored UVCCamera JNI + Java now live under app/src/main). -# Current product floor is API 34 (Android 14+); change here if you lower the app minimum. -minSdk = "34" -agp = "9.2.1" -kotlin = "2.2.0" +# Current product floor is API 28 (Android 9+); change here if you lower the app minimum. +minSdk = "28" +agp = "9.4.0" +kotlin = "2.2.10" playPublisher = "4.0.0" coreKtx = "1.10.1" junit = "4.13.2" diff --git a/Consolation/gradle/wrapper/gradle-wrapper.properties b/Consolation/gradle/wrapper/gradle-wrapper.properties index 8ba96de..654cbb8 100644 --- a/Consolation/gradle/wrapper/gradle-wrapper.properties +++ b/Consolation/gradle/wrapper/gradle-wrapper.properties @@ -1,8 +1,8 @@ #Sun May 03 09:22:19 MDT 2026 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=2ab2958f2a1e51120c326cad6f385153bb11ee93b3c216c5fccebfdfbb7ec6cb -distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip +distributionSha256Sum=bbaeb2fef8710818cf0e261201dab964c572f92b942812df0c3620d62a529a01 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/README.md b/README.md index ad0ddde..49ffbe9 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ Consolation has been tested by the developers on a Samsung Galaxy Tab S8 Ultra ( ### Running - Android device with a USB port -- Android OS 15 or higher +- Android OS 9 (API 28) or higher. Android 16 is the tested configuration; older + releases are supported on a best-effort basis. - A UVC-compliant video capture card ### Developer diff --git a/patches/libusb-1.0.30-prealloc/libusb_prealloc.h b/patches/libusb-1.0.30-prealloc/libusb_prealloc.h index 5e8bec0..1c01d42 100644 --- a/patches/libusb-1.0.30-prealloc/libusb_prealloc.h +++ b/patches/libusb-1.0.30-prealloc/libusb_prealloc.h @@ -11,7 +11,7 @@ #ifndef LIBUSB_PREALLOC_H #define LIBUSB_PREALLOC_H -#include +#include "libusb.h" /* relative to this header: works from any include path */ #ifdef __cplusplus extern "C" {