Linux recordings pick a GPU encoder that actually works, then fall ba…

AutoCua/utils/video_compress.py

@@ -5,12 +5,17 @@
     python compress.py input.mp4 out.mp4
 
 Scales to 1080p and encodes with the GPU (VideoToolbox / NVENC / QuickSync / AMF / VAAPI); falls back to
-libx265 on the CPU. Uses the system ffmpeg, or the one bundled in the imageio-ffmpeg pip package.
+libx265 on the CPU. Uses the system ffmpeg, then the one bundled in the imageio-ffmpeg pip package, whose Linux
+build has no GPU encoders: on Linux the GPU needs the distro's ffmpeg (README, "Linux, from pip"). A GPU encoder
+is used only once a test clip it encodes decodes back right, so one the build lacks, one without its GPU and one
+that writes a broken picture are all skipped quietly.
 """
+import glob
 import os
 import shutil
 import subprocess
 import sys
+import tempfile
 import time
 
 
@@ -26,41 +31,101 @@ def ffmpeg():
     return imageio_ffmpeg.get_ffmpeg_exe()
 
 
+def bundled_ffmpeg():
+    """imageio-ffmpeg's ffmpeg, or None. It always has libx265, which a distro ffmpeg may lack (Fedora's
+    ffmpeg-free, openSUSE's own), so it backs the system one up."""
+    try:
+        import imageio_ffmpeg
+        return imageio_ffmpeg.get_ffmpeg_exe()
+    except Exception:
+        return None
+
+
+def vaapi_devices():
+    """The render nodes of the AMD and Intel GPUs, the ones VA-API encodes on, each tried in turn: the first may
+    have no HEVC encoder (a Radeon RX 6400/6500 has none) while the next does. Not a fixed renderD128: nodes are
+    numbered in driver load order, and on hybrid laptops renderD128 is often the NVIDIA card."""
+    nodes = []
+    for node in sorted(glob.glob("/sys/class/drm/renderD*")):
+        try:
+            with open(node + "/device/vendor") as f:
+                if f.read().strip() in ("0x1002", "0x8086"):  # AMD, Intel
+                    nodes.append("/dev/dri/" + os.path.basename(node))
+        except OSError:
+            pass
+    return nodes
+
+
 # encoder -> (input options, filter chain, encoder options); tried in order, first one that works wins
 Q = "26"  # CRF-style quality; VideoToolbox 55 is calibrated to match libx265 CRF 26 on screen content
 SCALE = "scale=w='if(gt(iw,ih),-2,min(1080,iw))':h='if(gt(iw,ih),min(1080,ih),-2)':flags=bicubic"
-VAAPI = "/dev/dri/renderD128"
 ENCODERS = {
     "darwin": [("hevc_videotoolbox", [], SCALE + ",format=yuv420p", ["-q:v", "55", "-profile:v", "main"])],
     "win32": [("hevc_nvenc", [], SCALE + ",format=yuv420p", ["-preset", "p4", "-rc", "vbr", "-cq", Q, "-b:v", "0", "-spatial-aq", "1"]),
               ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q]),
               ("hevc_amf", [], SCALE + ",format=nv12", ["-quality", "balanced", "-rc", "cqp", "-qp_i", Q, "-qp_p", Q])],
     "linux": [("hevc_nvenc", [], SCALE + ",format=yuv420p", ["-preset", "p4", "-rc", "vbr", "-cq", Q, "-b:v", "0", "-spatial-aq", "1"]),
-              ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q]),
-              ("hevc_vaapi", ["-init_hw_device", "vaapi=va:" + VAAPI, "-filter_hw_device", "va"],
-               SCALE + ",format=nv12,hwupload", ["-rc_mode", "CQP", "-qp", Q])],
+              ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q])] +
+             [("hevc_vaapi", ["-init_hw_device", "vaapi=va:" + dev, "-filter_hw_device", "va"],
+               SCALE + ",format=nv12,hwupload", ["-rc_mode", "CQP", "-qp", Q]) for dev in vaapi_devices()],
 }
 CPU = ("libx265", [], SCALE + ",format=yuv420p", ["-preset", "superfast", "-crf", Q, "-x265-params", "log-level=error"])
 NO_WINDOW = 0x08000000 if sys.platform == "win32" else 0  # no console window flashing up from the app on Windows
+TESTED = set()  # (ffmpeg, encoder, input options) whose test clip came back right: tested once a process
+
+
+def command(ff, inp, dst, enc, pre, vf, opts):
+    """The encode: scale, HEVC, AAC audio, hvc1 mp4. compress() runs it on the recording, usable() on a test clip."""
+    return ([ff, "-hide_banner", "-nostdin", "-y", "-loglevel", "error", "-stats"] + pre + inp +
+            ["-vf", vf, "-c:v", enc, "-g", "300"] + opts +  # keyframe every 10 s (30 fps)
+            ["-c:a", "aac", "-b:a", "160k", "-tag:v", "hvc1", "-movflags", "+faststart", "-f", "mp4", dst])
+
+
+def usable(ff, enc, pre, vf, opts):
+    """Whether ff encodes right with enc on this machine: a short flat gray clip goes through command() and is
+    decoded back, all output thrown away. That skips, without ffmpeg's errors after every recording, an encoder
+    the build lacks, one whose GPU is missing, and one that writes a broken picture: Mesa's VAAPI HEVC on AMD
+    writes its own SPS/PPS, which disagree with the hvc1 header ffmpeg stores, so every frame decodes as noise."""
+    key = (ff, enc) + tuple(pre)
+    if key not in TESTED:
+        quiet = dict(stderr=subprocess.DEVNULL, creationflags=NO_WINDOW, timeout=30)
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp:
+            out = os.path.join(tmp, "test.mp4")
+            try:
+                if subprocess.call(command(ff, ["-f", "lavfi", "-i", "color=c=gray:s=640x360:r=30:d=0.2"], out, enc, pre, vf, opts),
+                                   stdout=subprocess.DEVNULL, **quiet) == 0:
+                    y = subprocess.run([ff, "-v", "quiet", "-nostdin", "-i", out, "-f", "rawvideo", "-pix_fmt", "gray", "-"],
+                                       stdout=subprocess.PIPE, **quiet).stdout
+                    if y and 100 < min(y) and max(y) < 156:  # the flat gray came back
+                        TESTED.add(key)
+            except (OSError, subprocess.TimeoutExpired):
+                pass
+    return key in TESTED
 
 
 def compress(src, dst=None):
     """Compress src into dst (default <src>_1080p.mp4). Returns dst, or None when no encoder
     worked. The app calls this after each recording (ui/service.py); main() is the CLI."""
     dst = dst or os.path.splitext(src)[0] + "_1080p.mp4"
-    ff = ffmpeg()
-    for enc, pre, vf, opts in ENCODERS.get(sys.platform[:5] if sys.platform.startswith("linux") else sys.platform, []) + [CPU]:
+    exes = [ffmpeg()]
+    spare = bundled_ffmpeg()
+    if spare and os.path.realpath(spare) != os.path.realpath(exes[0]):
+        exes.append(spare)
+    gpus = ENCODERS.get(sys.platform[:5] if sys.platform.startswith("linux") else sys.platform, [])
+    plan = [(ff,) + e for e in gpus for ff in exes] + [(ff,) + CPU for ff in exes]
+    for i, (ff, enc, pre, vf, opts) in enumerate(plan):
+        if i < len(plan) - 1 and not usable(ff, enc, pre, vf, opts):
+            continue  # the last one runs untested, so when nothing works ffmpeg's errors still show
         print("encoding with %s ..." % enc, flush=True)
         t0 = time.time()
-        rc = subprocess.call([ff, "-hide_banner", "-nostdin", "-y", "-loglevel", "error", "-stats"] + pre + ["-i", src,
-                              "-vf", vf, "-c:v", enc, "-g", "300"] + opts +  # keyframe every 10 s (30 fps)
-                             ["-c:a", "aac", "-b:a", "160k", "-tag:v", "hvc1", "-movflags", "+faststart", "-f", "mp4", dst],
-                             creationflags=NO_WINDOW)
+        rc = subprocess.call(command(ff, ["-i", src], dst, enc, pre, vf, opts), creationflags=NO_WINDOW)
         if rc == 0 and os.path.getsize(dst) > 0:
             a, b = os.path.getsize(src), os.path.getsize(dst)
-            print("done: %s  %.1f MB -> %.1f MB (%.1fx smaller) in %.1f s" % (dst, a / 1e6, b / 1e6, a / b, time.time() - t0))
+            print("done: %s  %.1f MB -> %.1f MB (%.1fx smaller) in %.1f s" % (dst, a / 1e6, b / 1e6, a / b, time.time() - t0),
+                  flush=True)  # the app's stdout is a buffered wrapper, and it exits through os._exit
             return dst
-        print("  %s failed, trying the next encoder" % enc)
+        print("  %s failed, trying the next encoder" % enc, flush=True)
+        TESTED.discard((ff, enc) + tuple(pre))  # tested again next time
         if os.path.exists(dst):
             os.remove(dst)
     return None

README.md

@@ -212,6 +212,17 @@ sudo apt install python3-gi python3-cairo python3-gi-cairo python3-venv gir1.2-a
 On KDE, swap `xdg-desktop-portal-gnome` for `xdg-desktop-portal-kde`. Fedora, Arch and openSUSE
 ship the same packages under their own names; `setup_linux.sh` in the repository lists them.
 
+Optional, for faster recordings: also install your distro's ffmpeg. Each run's recording is then
+compressed on the GPU instead of the CPU, because the ffmpeg pip brings on Linux has no GPU encoders:
+
+```bash
+sudo apt install ffmpeg            # Debian, Ubuntu
+sudo pacman -S --needed ffmpeg     # Arch
+```
+
+Skip it on Fedora and openSUSE: their own ffmpeg builds leave out codecs the app needs (RPM Fusion
+and Packman have full ones). Without it, recordings are still compressed, on the CPU.
+
 Ubuntu and Debian refuse `pip install` into the system Python, so create a virtualenv first,
 inside the folder that holds your `main.py` and `.env` so the agent finds that `.env`. Build it on
 your **distro's own `python3`** (3.10 to 3.14): AutoCua loads the system PyGObject from inside

pyproject.toml

@@ -14,7 +14,7 @@ build-backend = "maturin"
 
 [project]
 name = "AutoCua"
-version = "0.1.2"
+version = "0.1.3"
 description = "AutoCua: an AI agent that operates your computer, shell, browser and iPhone"
 readme = "README.md"
 requires-python = ">=3.10"

setup_linux.sh

@@ -139,18 +139,34 @@ install_pkgs() {
     fi
 }
 
+# Optional, and apart from install_pkgs so a failure here costs nothing else:
+# a distro ffmpeg, which utils/video_compress.py prefers over imageio-ffmpeg's
+# Linux build. That build has no GPU encoders, so without this each recording
+# is compressed on the CPU, 2-3x slower. apt and pacman only: Fedora's
+# ffmpeg-free and openSUSE's own ffmpeg leave out codecs the app needs.
+optional_ffmpeg() {
+    command -v ffmpeg >/dev/null 2>&1 && return 0
+    if as_root "$@"; then
+        print_ok "ffmpeg installed: recordings can compress on the GPU"
+    else
+        print_info "ffmpeg not installed: recordings compress on the CPU instead"
+    fi
+}
+
 if command -v apt-get >/dev/null 2>&1; then
     # A fresh box may have stale lists; a failed refresh (an unreachable
     # third-party repo, say) must not block the install itself.
     as_root apt-get update -qq >/dev/null 2>&1 || true
     # shellcheck disable=SC2086
     install_pkgs "apt" apt-get install -y $APT_PKGS
+    optional_ffmpeg apt-get install -y ffmpeg
 elif command -v dnf >/dev/null 2>&1; then
     # shellcheck disable=SC2086
     install_pkgs "dnf" dnf install -y $DNF_PKGS
 elif command -v pacman >/dev/null 2>&1; then
     # shellcheck disable=SC2086
     install_pkgs "pacman" pacman -S --needed --noconfirm $PACMAN_PKGS
+    optional_ffmpeg pacman -S --needed --noconfirm ffmpeg
 elif command -v zypper >/dev/null 2>&1; then
     # shellcheck disable=SC2086
     install_pkgs "zypper" zypper install -y $ZYPPER_PKGS