From a20a81bd00a959b41aa97a68e5893566e8025736 Mon Sep 17 00:00:00 2001 From: auto-use Date: Sun, 18 Oct 2026 19:38:41 +0100 Subject: [PATCH] Linux recordings pick a GPU encoder that actually works, then fall back to CPU. Co-authored-by: Cursor --- AutoCua/utils/video_compress.py | 91 ++++++++++++++++++++++++++++----- README.md | 11 ++++ pyproject.toml | 2 +- setup_linux.sh | 16 ++++++ 4 files changed, 106 insertions(+), 14 deletions(-) diff --git a/AutoCua/utils/video_compress.py b/AutoCua/utils/video_compress.py index 61d6b59..df631ff 100644 --- a/AutoCua/utils/video_compress.py +++ b/AutoCua/utils/video_compress.py @@ -5,12 +5,17 @@ python compress.py input.mp4 out.mp4 Scales to 1080p and encodes with the GPU (VideoToolbox / NVENC / QuickSync / AMF / VAAPI); falls back to -libx265 on the CPU. Uses the system ffmpeg, or the one bundled in the imageio-ffmpeg pip package. +libx265 on the CPU. Uses the system ffmpeg, then the one bundled in the imageio-ffmpeg pip package, whose Linux +build has no GPU encoders: on Linux the GPU needs the distro's ffmpeg (README, "Linux, from pip"). A GPU encoder +is used only once a test clip it encodes decodes back right, so one the build lacks, one without its GPU and one +that writes a broken picture are all skipped quietly. """ +import glob import os import shutil import subprocess import sys +import tempfile import time @@ -26,41 +31,101 @@ def ffmpeg(): return imageio_ffmpeg.get_ffmpeg_exe() +def bundled_ffmpeg(): + """imageio-ffmpeg's ffmpeg, or None. It always has libx265, which a distro ffmpeg may lack (Fedora's + ffmpeg-free, openSUSE's own), so it backs the system one up.""" + try: + import imageio_ffmpeg + return imageio_ffmpeg.get_ffmpeg_exe() + except Exception: + return None + + +def vaapi_devices(): + """The render nodes of the AMD and Intel GPUs, the ones VA-API encodes on, each tried in turn: the first may + have no HEVC encoder (a Radeon RX 6400/6500 has none) while the next does. Not a fixed renderD128: nodes are + numbered in driver load order, and on hybrid laptops renderD128 is often the NVIDIA card.""" + nodes = [] + for node in sorted(glob.glob("/sys/class/drm/renderD*")): + try: + with open(node + "/device/vendor") as f: + if f.read().strip() in ("0x1002", "0x8086"): # AMD, Intel + nodes.append("/dev/dri/" + os.path.basename(node)) + except OSError: + pass + return nodes + + # encoder -> (input options, filter chain, encoder options); tried in order, first one that works wins Q = "26" # CRF-style quality; VideoToolbox 55 is calibrated to match libx265 CRF 26 on screen content SCALE = "scale=w='if(gt(iw,ih),-2,min(1080,iw))':h='if(gt(iw,ih),min(1080,ih),-2)':flags=bicubic" -VAAPI = "/dev/dri/renderD128" ENCODERS = { "darwin": [("hevc_videotoolbox", [], SCALE + ",format=yuv420p", ["-q:v", "55", "-profile:v", "main"])], "win32": [("hevc_nvenc", [], SCALE + ",format=yuv420p", ["-preset", "p4", "-rc", "vbr", "-cq", Q, "-b:v", "0", "-spatial-aq", "1"]), ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q]), ("hevc_amf", [], SCALE + ",format=nv12", ["-quality", "balanced", "-rc", "cqp", "-qp_i", Q, "-qp_p", Q])], "linux": [("hevc_nvenc", [], SCALE + ",format=yuv420p", ["-preset", "p4", "-rc", "vbr", "-cq", Q, "-b:v", "0", "-spatial-aq", "1"]), - ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q]), - ("hevc_vaapi", ["-init_hw_device", "vaapi=va:" + VAAPI, "-filter_hw_device", "va"], - SCALE + ",format=nv12,hwupload", ["-rc_mode", "CQP", "-qp", Q])], + ("hevc_qsv", [], SCALE + ",format=nv12", ["-preset", "faster", "-global_quality", Q])] + + [("hevc_vaapi", ["-init_hw_device", "vaapi=va:" + dev, "-filter_hw_device", "va"], + SCALE + ",format=nv12,hwupload", ["-rc_mode", "CQP", "-qp", Q]) for dev in vaapi_devices()], } CPU = ("libx265", [], SCALE + ",format=yuv420p", ["-preset", "superfast", "-crf", Q, "-x265-params", "log-level=error"]) NO_WINDOW = 0x08000000 if sys.platform == "win32" else 0 # no console window flashing up from the app on Windows +TESTED = set() # (ffmpeg, encoder, input options) whose test clip came back right: tested once a process + + +def command(ff, inp, dst, enc, pre, vf, opts): + """The encode: scale, HEVC, AAC audio, hvc1 mp4. compress() runs it on the recording, usable() on a test clip.""" + return ([ff, "-hide_banner", "-nostdin", "-y", "-loglevel", "error", "-stats"] + pre + inp + + ["-vf", vf, "-c:v", enc, "-g", "300"] + opts + # keyframe every 10 s (30 fps) + ["-c:a", "aac", "-b:a", "160k", "-tag:v", "hvc1", "-movflags", "+faststart", "-f", "mp4", dst]) + + +def usable(ff, enc, pre, vf, opts): + """Whether ff encodes right with enc on this machine: a short flat gray clip goes through command() and is + decoded back, all output thrown away. That skips, without ffmpeg's errors after every recording, an encoder + the build lacks, one whose GPU is missing, and one that writes a broken picture: Mesa's VAAPI HEVC on AMD + writes its own SPS/PPS, which disagree with the hvc1 header ffmpeg stores, so every frame decodes as noise.""" + key = (ff, enc) + tuple(pre) + if key not in TESTED: + quiet = dict(stderr=subprocess.DEVNULL, creationflags=NO_WINDOW, timeout=30) + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp: + out = os.path.join(tmp, "test.mp4") + try: + if subprocess.call(command(ff, ["-f", "lavfi", "-i", "color=c=gray:s=640x360:r=30:d=0.2"], out, enc, pre, vf, opts), + stdout=subprocess.DEVNULL, **quiet) == 0: + y = subprocess.run([ff, "-v", "quiet", "-nostdin", "-i", out, "-f", "rawvideo", "-pix_fmt", "gray", "-"], + stdout=subprocess.PIPE, **quiet).stdout + if y and 100 < min(y) and max(y) < 156: # the flat gray came back + TESTED.add(key) + except (OSError, subprocess.TimeoutExpired): + pass + return key in TESTED def compress(src, dst=None): """Compress src into dst (default _1080p.mp4). Returns dst, or None when no encoder worked. The app calls this after each recording (ui/service.py); main() is the CLI.""" dst = dst or os.path.splitext(src)[0] + "_1080p.mp4" - ff = ffmpeg() - for enc, pre, vf, opts in ENCODERS.get(sys.platform[:5] if sys.platform.startswith("linux") else sys.platform, []) + [CPU]: + exes = [ffmpeg()] + spare = bundled_ffmpeg() + if spare and os.path.realpath(spare) != os.path.realpath(exes[0]): + exes.append(spare) + gpus = ENCODERS.get(sys.platform[:5] if sys.platform.startswith("linux") else sys.platform, []) + plan = [(ff,) + e for e in gpus for ff in exes] + [(ff,) + CPU for ff in exes] + for i, (ff, enc, pre, vf, opts) in enumerate(plan): + if i < len(plan) - 1 and not usable(ff, enc, pre, vf, opts): + continue # the last one runs untested, so when nothing works ffmpeg's errors still show print("encoding with %s ..." % enc, flush=True) t0 = time.time() - rc = subprocess.call([ff, "-hide_banner", "-nostdin", "-y", "-loglevel", "error", "-stats"] + pre + ["-i", src, - "-vf", vf, "-c:v", enc, "-g", "300"] + opts + # keyframe every 10 s (30 fps) - ["-c:a", "aac", "-b:a", "160k", "-tag:v", "hvc1", "-movflags", "+faststart", "-f", "mp4", dst], - creationflags=NO_WINDOW) + rc = subprocess.call(command(ff, ["-i", src], dst, enc, pre, vf, opts), creationflags=NO_WINDOW) if rc == 0 and os.path.getsize(dst) > 0: a, b = os.path.getsize(src), os.path.getsize(dst) - print("done: %s %.1f MB -> %.1f MB (%.1fx smaller) in %.1f s" % (dst, a / 1e6, b / 1e6, a / b, time.time() - t0)) + print("done: %s %.1f MB -> %.1f MB (%.1fx smaller) in %.1f s" % (dst, a / 1e6, b / 1e6, a / b, time.time() - t0), + flush=True) # the app's stdout is a buffered wrapper, and it exits through os._exit return dst - print(" %s failed, trying the next encoder" % enc) + print(" %s failed, trying the next encoder" % enc, flush=True) + TESTED.discard((ff, enc) + tuple(pre)) # tested again next time if os.path.exists(dst): os.remove(dst) return None diff --git a/README.md b/README.md index 40a0451..98861c7 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,17 @@ sudo apt install python3-gi python3-cairo python3-gi-cairo python3-venv gir1.2-a On KDE, swap `xdg-desktop-portal-gnome` for `xdg-desktop-portal-kde`. Fedora, Arch and openSUSE ship the same packages under their own names; `setup_linux.sh` in the repository lists them. +Optional, for faster recordings: also install your distro's ffmpeg. Each run's recording is then +compressed on the GPU instead of the CPU, because the ffmpeg pip brings on Linux has no GPU encoders: + +```bash +sudo apt install ffmpeg # Debian, Ubuntu +sudo pacman -S --needed ffmpeg # Arch +``` + +Skip it on Fedora and openSUSE: their own ffmpeg builds leave out codecs the app needs (RPM Fusion +and Packman have full ones). Without it, recordings are still compressed, on the CPU. + Ubuntu and Debian refuse `pip install` into the system Python, so create a virtualenv first, inside the folder that holds your `main.py` and `.env` so the agent finds that `.env`. Build it on your **distro's own `python3`** (3.10 to 3.14): AutoCua loads the system PyGObject from inside diff --git a/pyproject.toml b/pyproject.toml index b0d4bd4..4a2e9b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ build-backend = "maturin" [project] name = "AutoCua" -version = "0.1.2" +version = "0.1.3" description = "AutoCua: an AI agent that operates your computer, shell, browser and iPhone" readme = "README.md" requires-python = ">=3.10" diff --git a/setup_linux.sh b/setup_linux.sh index 7808f7c..cdd9521 100755 --- a/setup_linux.sh +++ b/setup_linux.sh @@ -139,18 +139,34 @@ install_pkgs() { fi } +# Optional, and apart from install_pkgs so a failure here costs nothing else: +# a distro ffmpeg, which utils/video_compress.py prefers over imageio-ffmpeg's +# Linux build. That build has no GPU encoders, so without this each recording +# is compressed on the CPU, 2-3x slower. apt and pacman only: Fedora's +# ffmpeg-free and openSUSE's own ffmpeg leave out codecs the app needs. +optional_ffmpeg() { + command -v ffmpeg >/dev/null 2>&1 && return 0 + if as_root "$@"; then + print_ok "ffmpeg installed: recordings can compress on the GPU" + else + print_info "ffmpeg not installed: recordings compress on the CPU instead" + fi +} + if command -v apt-get >/dev/null 2>&1; then # A fresh box may have stale lists; a failed refresh (an unreachable # third-party repo, say) must not block the install itself. as_root apt-get update -qq >/dev/null 2>&1 || true # shellcheck disable=SC2086 install_pkgs "apt" apt-get install -y $APT_PKGS + optional_ffmpeg apt-get install -y ffmpeg elif command -v dnf >/dev/null 2>&1; then # shellcheck disable=SC2086 install_pkgs "dnf" dnf install -y $DNF_PKGS elif command -v pacman >/dev/null 2>&1; then # shellcheck disable=SC2086 install_pkgs "pacman" pacman -S --needed --noconfirm $PACMAN_PKGS + optional_ffmpeg pacman -S --needed --noconfirm ffmpeg elif command -v zypper >/dev/null 2>&1; then # shellcheck disable=SC2086 install_pkgs "zypper" zypper install -y $ZYPPER_PKGS