Expose hardware devices to filter graphs

av/codec/hwaccel.pxd

@@ -10,6 +10,12 @@ cdef class HWConfig:
 
 cdef HWConfig wrap_hwconfig(const lib.AVCodecHWConfig *ptr)
 
+cdef class HWDevice:
+    cdef int _device_type
+    cdef lib.AVBufferRef *ptr
+    cdef readonly dict options
+    cdef readonly int flags
+
 cdef class HWAccel:
     cdef str _device
     cdef readonly Codec codec

av/codec/hwaccel.py

@@ -105,6 +105,72 @@ def hwdevices_available():
     return result
 
 
+@cython.final
+@cython.cclass
+class HWDevice:
+    """HWDevice(device_type, device=None, options=None, flags=None)
+
+    A hardware device context which can be shared by components such as filter
+    graphs. The underlying FFmpeg buffer is reference counted; consumers take
+    their own references and this object never exposes the raw pointer.
+
+    :param device_type: The kind of device, e.g. ``"cuda"``, ``"vaapi"`` or
+        ``"vulkan"``. See :func:`hwdevices_available` for the types supported by
+        the loaded FFmpeg.
+    :type device_type: str or HWDeviceType
+    :param str device: An optional device identifier. Uses the default device if
+        ``None``.
+    :param dict options: Options passed to ``av_hwdevice_ctx_create``.
+    :param int flags: Flags passed to ``av_hwdevice_ctx_create``.
+    """
+
+    def __cinit__(self):
+        self.ptr = cython.NULL
+
+    def __init__(self, device_type, device=None, options=None, flags=None):
+        if isinstance(device_type, HWDeviceType):
+            self._device_type = int(device_type)
+        elif isinstance(device_type, str):
+            self._device_type = int(lib.av_hwdevice_find_type_by_name(device_type))
+            if self._device_type == lib.AV_HWDEVICE_TYPE_NONE:
+                raise ValueError(f"Unknown hardware device type: {device_type}")
+        elif isinstance(device_type, int):
+            self._device_type = device_type
+        else:
+            raise TypeError("device_type must be a string, integer, or HWDeviceType")
+
+        if self._device_type == lib.AV_HWDEVICE_TYPE_NONE:
+            raise ValueError("Hardware device type cannot be 'none'")
+
+        self.options = {} if not options else dict(options)
+        self.flags = 0 if flags is None else flags
+
+        c_device: cython.p_char = cython.NULL
+        device_name = None if device is None else f"{device}"
+        if device_name:
+            device_bytes = device_name.encode()
+            c_device = device_bytes
+        c_options: Dictionary = Dictionary(self.options)
+
+        err_check(
+            lib.av_hwdevice_ctx_create(
+                cython.address(self.ptr),
+                cython.cast(lib.AVHWDeviceType, self._device_type),
+                c_device,
+                c_options.ptr,
+                self.flags,
+            )
+        )
+
+    @property
+    def device_type(self):
+        return HWDeviceType(self._device_type)
+
+    def __dealloc__(self):
+        if self.ptr:
+            lib.av_buffer_unref(cython.address(self.ptr))
+
+
 @cython.final
 @cython.cclass
 class HWAccel:

av/codec/hwaccel.pyi

@@ -36,6 +36,20 @@ class HWConfig:
     @property
     def is_supported(self) -> bool: ...
 
+class HWDevice:
+    options: dict[str, object]
+    flags: int
+
+    def __init__(
+        self,
+        device_type: str | int | HWDeviceType,
+        device: str | int | None = None,
+        options: dict[str, object] | None = None,
+        flags: int | None = None,
+    ) -> None: ...
+    @property
+    def device_type(self) -> HWDeviceType: ...
+
 class HWAccel:
     options: dict[str, object]
 

av/filter/graph.pxd

@@ -1,5 +1,6 @@
 cimport libav as lib
 
+from av.codec.hwaccel cimport HWDevice
 from av.filter.context cimport FilterContext
 
 
@@ -8,6 +9,7 @@ cdef class Graph:
     # ints paired up, so there are no padding holes between them.
     cdef object __weakref__
     cdef lib.AVFilterGraph *ptr
+    cdef HWDevice _hw_device
     cdef dict _name_counts
     cdef dict[size_t, FilterContext] _context_by_ptr
     cdef dict[str, list[FilterContext]] _context_by_type

av/filter/graph.py

@@ -4,6 +4,7 @@
 from cython.cimports.av.audio.format import AudioFormat
 from cython.cimports.av.audio.frame import AudioFrame
 from cython.cimports.av.audio.layout import AudioLayout
+from cython.cimports.av.codec.hwaccel import HWDevice
 from cython.cimports.av.error import err_check
 from cython.cimports.av.filter.context import FilterContext, wrap_filter_context
 from cython.cimports.av.filter.filter import Filter, wrap_filter
@@ -15,8 +16,22 @@
 @cython.final
 @cython.cclass
 class Graph:
-    def __cinit__(self):
+    """Graph(hw_device=None)
+
+    A graph of audio and/or video filters.
+
+    :param HWDevice hw_device: An optional hardware device for filters such as
+        ``hwupload``. A reference is attached to every filter which declares
+        ``AVFILTER_FLAG_HWDEVICE`` before that filter is initialized.
+    """
+
+    def __cinit__(self, hw_device=None):
         self.ptr = lib.avfilter_graph_alloc()
+        if not self.ptr:
+            raise MemoryError("Could not allocate AVFilterGraph")
+        if hw_device is not None and not isinstance(hw_device, HWDevice):
+            raise TypeError("hw_device must be an HWDevice or None")
+        self._hw_device = hw_device
         self.configured = False
         self._name_counts = {}
         self._nb_filters_seen = 0
@@ -28,6 +43,11 @@ def __dealloc__(self):
             # This frees the graph, filter contexts, links, etc..
             lib.avfilter_graph_free(cython.address(self.ptr))
 
+    @property
+    def hw_device(self):
+        """The hardware device supplied when this graph was created."""
+        return self._hw_device
+
     @property
     def threads(self):
         """Maximum number of threads used by filters in this graph.
@@ -94,6 +114,14 @@ def add(self, filter, args=None, **kwargs):
         if not ptr:
             raise RuntimeError("Could not allocate AVFilterContext")
 
+        if (
+            self._hw_device is not None
+            and cy_filter.ptr.flags & lib.AVFILTER_FLAG_HWDEVICE
+        ):
+            ptr.hw_device_ctx = lib.av_buffer_ref(self._hw_device.ptr)
+            if not ptr.hw_device_ctx:
+                raise MemoryError("Could not reference graph hardware device")
+
         # Manually construct this context (so we can return it).
         ctx: FilterContext = wrap_filter_context(self, cy_filter, ptr)
         ctx.init(args, **kwargs)

av/filter/graph.pyi

@@ -5,6 +5,7 @@ from av.audio.format import AudioFormat
 from av.audio.frame import AudioFrame
 from av.audio.layout import AudioLayout
 from av.audio.stream import AudioStream
+from av.codec.hwaccel import HWDevice
 from av.rational import AVRational
 from av.video.format import VideoFormat
 from av.video.frame import VideoFrame
@@ -17,15 +18,17 @@ class Graph:
     configured: bool
     threads: int
 
-    def __init__(self) -> None: ...
+    def __init__(self, hw_device: HWDevice | None = None) -> None: ...
+    @property
+    def hw_device(self) -> HWDevice | None: ...
     def configure(self, auto_buffer: bool = True, force: bool = False) -> None: ...
     def link_nodes(self, *nodes: FilterContext) -> Graph: ...
     def add(
         self, filter: str | Filter, args: Any = None, **kwargs: str
     ) -> FilterContext: ...
     def add_buffer(
         self,
-        template: VideoStream | None = None,
+        template: VideoFrame | VideoStream | None = None,
         width: int | None = None,
         height: int | None = None,
         format: VideoFormat | str | None = None,

docs/api/codec.rst

@@ -141,6 +141,8 @@ Hardware Acceleration
 .. currentmodule:: av.codec.hwaccel
 .. automodule:: av.codec.hwaccel
 
+.. autoclass:: HWDevice
+
 .. autoclass:: HWAccel
 
 .. autofunction:: hwdevices_available
@@ -167,4 +169,3 @@ frames passed to ``encode`` are uploaded to the device automatically::
 
 See ``examples/basics/hw_decode.py`` for a complete example, including
 recommended device types per platform.
-

docs/api/filter.rst

@@ -12,6 +12,16 @@ Filters
     .. autoclass:: Graph
         :members:
 
+Hardware filters which create frames need an :class:`~av.codec.hwaccel.HWDevice`
+when the graph is constructed. The graph shares that device with every filter
+which requests one before the filter is initialized::
+
+    from av.codec.hwaccel import HWDevice
+    from av.filter import Graph
+
+    device = HWDevice("vaapi")
+    graph = Graph(hw_device=device)
+
 
 .. automodule:: av.filter.context
 
@@ -23,4 +33,3 @@ Filters
 
     .. autoclass:: FilterLink
         :members:
-

include/avfilter.pxd

@@ -36,6 +36,10 @@ cdef extern from "libavfilter/avfilter.h" nogil:
         AVFilterPad *output_pads
         AVFilterLink **outputs
 
+        AVBufferRef *hw_device_ctx
+
+    cdef int AVFILTER_FLAG_HWDEVICE
+
     cdef int avfilter_init_str(AVFilterContext *ctx, const char *args)
     cdef int avfilter_init_dict(AVFilterContext *ctx, AVDictionary **options)
 

tests/test_filters.py

@@ -2,6 +2,7 @@
 from fractions import Fraction
 
 import numpy as np
+import pytest
 
 import av
 from av import AudioFrame, AVRational, VideoFrame
@@ -52,6 +53,14 @@ def test_filter_descriptor(self) -> None:
         assert f.name == "testsrc"
         assert f.description == "Generate test pattern."
 
+    def test_graph_rejects_invalid_hw_device(self) -> None:
+        with pytest.raises(TypeError, match="hw_device must be an HWDevice"):
+            Graph(hw_device=object())  # type: ignore[arg-type]
+
+    def test_hw_device_rejects_unknown_type(self) -> None:
+        with pytest.raises(ValueError, match="Unknown hardware device type"):
+            av.codec.hwaccel.HWDevice("definitely-not-a-hardware-device")
+
     def test_generator_graph(self):
         graph = Graph()
         src = graph.add("testsrc")